uhi-1.2.1/.git_archival.txt0000644000000000000000000000017515261523264012512 0ustar00node: $Format:%H$ node-date: $Format:%cI$ describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$ ref-names: $Format:%D$ uhi-1.2.1/.gitattributes0000644000000000000000000000004015261523264012121 0ustar00.git_archival.txt export-subst uhi-1.2.1/.pre-commit-config.yaml0000644000000000000000000000675515261523264013532 0ustar00ci: autoupdate_commit_msg: 'chore: update pre-commit hooks' autoupdate_schedule: 'monthly' repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 hooks: - id: check-added-large-files - id: check-case-conflict - id: check-merge-conflict - id: check-symlinks - id: check-yaml - id: debug-statements - id: end-of-file-fixer - id: mixed-line-ending - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit rev: "f12be1ebaa5351c1fc76472de98db2c3446c8253" # frozen: v0.16.10 hooks: - id: ruff-check args: ["--fix", "--show-fixes"] - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy rev: 2834ec6639549dd6796205c8f011dedcd587288b # frozen: v2.4.0 hooks: - id: mypy files: ^(src|tests) exclude: ^tests/utils args: [] additional_dependencies: - numpy<2.5 # 3.12+ types - pytest - importlib_resources - tomli - boost-histogram>=1.7.1 - hist>=2.10.1 - dependency-groups>=1.3 - repo: https://github.com/pre-commit/pygrep-hooks rev: 3a6eb0fadf60b3cccfd80bad9dbb6fae7e47b316 # frozen: v1.10.0 hooks: - id: rst-backticks - id: rst-directive-colons - id: rst-inline-touching-normal - repo: https://github.com/codespell-project/codespell rev: 57b21406f092110c18776e39b0bda50d37c945c8 # frozen: v2.4.3 hooks: - id: codespell args: ["-L", "hist,thist,ans,nd,gaus"] - repo: local hooks: - id: disallow-caps name: Disallow improper capitalization language: pygrep entry: PyBind|Numpy|Cmake|CCache|Github|PyTest|Weighed exclude: .pre-commit-config.yaml # The $id would make check-jsonschema fetch histogram.schema.json from # GitHub; a file:// base URI keeps the $ref local. - id: validate-histograms name: Validate Histogram examples language: python additional_dependencies: [check-jsonschema==0.38.0] entry: >- python -c "import pathlib, sys; from check_jsonschema import main; schema = pathlib.Path('src/uhi/resources/histograms.schema.json'); main(['--base-uri', schema.resolve().as_uri(), '--schemafile', str(schema), *sys.argv[1:]])" files: ^tests/resources/valid/.*\.json - repo: https://github.com/rbubley/mirrors-prettier rev: "ef4a397f916211b4a39ccf9d3d9cbb6562157251" # frozen: v3.9.9 hooks: - id: prettier types_or: [json] - repo: https://github.com/python-jsonschema/check-jsonschema rev: 4f85d46a92dc17713078e5de73f8a13190edf6fc # frozen: 0.38.2 hooks: - id: check-readthedocs - id: check-github-workflows - id: check-metaschema files: ^src/uhi/resources/histograms?\.schema\.json$ - id: check-jsonschema name: Validate single Histogram examples args: [--schemafile, src/uhi/resources/histogram.schema.json] files: ^tests/resources/valid_single/.*\.json - repo: https://github.com/henryiii/validate-pyproject-schema-store rev: ae9c8c7039f4ec4ae005660c80dbcb954f61da5a # frozen: 2026.10.02 hooks: - id: validate-pyproject - repo: https://github.com/zizmorcore/zizmor-pre-commit rev: fa412071e4f5d44d44f9e365f4676f9df92456a2 # frozen: v1.30.1 hooks: - id: zizmor files: "^\\.github" args: [--persona=pedantic] uhi-1.2.1/.readthedocs.yml0000644000000000000000000000110715261523264012321 0ustar00# Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details # Required version: 2 sphinx: configuration: docs/conf.py build: os: "ubuntu-24.04" tools: python: "3.13" jobs: post_checkout: - git fetch --unshallow || true pre_create_environment: - asdf plugin add uv - asdf install uv latest - asdf global uv latest create_environment: - uv venv "${READTHEDOCS_VIRTUALENV_PATH}" install: - UV_PROJECT_ENVIRONMENT="${READTHEDOCS_VIRTUALENV_PATH}" uv sync --group docs uhi-1.2.1/AGENTS.md0000644000000000000000000001441215261523264010541 0ustar00# UHI (Universal Histogram Interface) development ## Purpose UHI is a package for documenting histogram indexing and the PlottableHistogram Protocol, along with tools to support library authors working with histograms. It provides type-checking support and testing utilities for libraries that want to conform to the Universal Histogram Interface standard. It also documents cross-library histogram standards and best practices. ## Project layout Some key files: - `pyproject.toml`: Most configuration - `docs/`: Sphinx documentation - `docs/conf.py`: Sphinx configuration - `docs/index.rst`: Main documentation - `.pre-commit-config.yaml`: `prek` (pre-commit) configuration - `src/uhi/`: Core package code - `src/uhi/__init__.py`: Main package - `src/uhi/__main__.py`: The `uhi` CLI (`uhi add`, `uhi validate`) - `src/uhi/schema.py`: Histogram schema definitions - `src/uhi/tag.py`: Tag implementations - `src/uhi/numpy_plottable.py`: NumPy histogram compatibility - `src/uhi/typing/`: Protocol definitions for type checking - `src/uhi/io/`: Serialization and I/O utilities - `src/uhi/testing/`: Testing utilities and helpers - `tests/`: Test suite ## Architecture UHI is primarily a **standards + typing + testing** package, not a runtime dependency (the base package only needs numpy). The code splits into a few largely independent concerns under `src/uhi/`: 1. **Protocols (`typing/`)** — `typing/plottable.py` defines the `PlottableHistogram`/`PlottableAxis`/`PlottableTraits` Protocols (all `@runtime_checkable`, so `isinstance` works). `protocol_version` and the open-ended `Kind` string live here. `typing/serialization.py` defines the `TypedDict` "IR" (intermediate representation) types — `HistogramIR`, `AxisIR`, storage types — that the I/O layer reads and writes. These two files are the source of truth for the standard; changing them ripples into I/O, schema, and tests. 2. **Indexing tags (`tag.py`)** — the user-facing indexing vocabulary: `loc`, `at`, `rebin`, `Underflow`/`overflow`, all subclasses of `Locator` with offset arithmetic (`loc(1.0) + 2`). This is the spec implementation referenced by the docs (`docs/indexing.rst`). 3. **Serialization (`io/` + `schema.py` + `resources/`)** — the JSON Schemas in `resources/` are the contract: `histogram.schema.json` describes one histogram (the IR), and `histograms.schema.json` describes a JSON file (a single histogram or a dict of named histograms). `schema.py` compiles them with `fastjsonschema` (optional `[schema]` extra) and exposes `validate()` (a file), `validate_histogram()` (one histogram), and `load()` (read any supported file). `io/json.py`, `io/zip.py`, `io/hdf5.py` (optional `[hdf5]` extra), `io/root.py` (PyROOT, from conda-forge), and `io/uproot.py` (`[uproot]` extra) each serialize/deserialize the IR types. The two ROOT backends share the RNTuple layout in `io/_rntuple.py`. `io/_files.py` reads and writes whole files in any format, picked by suffix, with `file:path` specs. `io/ops.py` has `add` (bin-by-bin sum, like `hadd`). `io/__init__.py` holds format-agnostic helpers: `to_sparse`/`from_sparse` (dense⇄sparse storage conversion) and `remove_writer_info`. `io/_common.py` is shared internals. The serialization format is documented in `docs/serialization.md`. 4. **Adapters + conformance (`numpy_plottable.py`, `testing/`)** — `numpy_plottable.py` adapts NumPy-style histogram tuples into objects satisfying `PlottableHistogram` (`ensure_plottable_histogram`). `testing/indexing.py` provides `unittest`-based mixin classes (`Indexing1D`, `Indexing2D`, `Indexing3D`) that downstream libraries subclass to verify their histograms conform to UHI indexing semantics. 5. **CLI (`__main__.py`)** — the `uhi` command (also `python -m uhi`). `uhi add` sums histograms from several files into one; `uhi validate` checks files against the schema. Both use `io/_files.py`. The optional dependencies matter: `schema`/I/O code imports `fastjsonschema`, `h5py`, `ROOT`, and `uproot` lazily so the base package stays dependency-light. Downstream consumers: boost-histogram, hist, mplhep, uproot, histoprint. ## Dev environment The CLI tools `uv`, `prek`, and `nox` should be pre-installed as Python tools. - `uv sync` (along with all the run commands) will ensure a `.venv` folder with an environment is created and populated. - `uv run ` is a good way to run things like `pytest`. It will automatically make the `.venv` folder if it doesn't exist yet. - `prek -a` will check the formatting and style. - Docs are built with `nox -s docs --non-interactive`, which uses Sphinx to generate documentation from `.rst` files in the `docs/` directory. ## Testing instructions - `uv run pytest` is a good way to run tests. Args can be easily added, like for a specific test: `uv run pytest tests/test_serialization.py -k sparse`. - You can request a Python 3.15 alpha with `uv run --python 3.15 pytest`, remember it is "sticky", the `.venv` will be made with 3.15. - `tests/utils/` is on `pythonpath`/`mypy_path` (see `pyproject.toml`) and holds shared `helpers.py`. JSON fixtures live in `tests/resources/{valid,invalid}/`; invalid cases pair a `.json` with a `.error.txt` expected message. `tests/resources/valid_single/` holds files with one histogram at the top level (not a dict of named histograms). - `tests/test_root.py` needs ROOT 6.36+; run it and `tests/test_uproot.py` via `nox -s root_tests` (conda-based). - Validate a histogram file (JSON, zip, HDF5, or ROOT) with `uv run uhi validate `. - Always add/update tests. - Run `prek -a` to fixup style and look for linting issues. - When running `prek -a --quiet`, the linting rules are _very_ strict, so adding a local or global skip for a troublesome rule is fine if it makes the code better. ## Working on code - Update the `README.md` and documentation files in `docs/` if rules or options change. - Code is Python 3.10+, modern coding practices (like pattern matching) encouraged. - Readability is important. Feel free to comment blocks and docstrings to explain what is happening. - This is still a new project, feel free to refactor or change things to improve it. ## PR instructions - Titles follow Conventional Commits (like `feat: ...`) - Always run `prek -a` before committing. uhi-1.2.1/CITATION.cff0000644000000000000000000000177715261523264011142 0ustar00cff-version: 1.2.0 message: "Please cite the following works when using this software." type: software title: "uhi" abstract: "UHI is a library that helps connect other Histogramming libraries." authors: - family-names: "Schreiner" given-names: "Henry" orcid: "https://orcid.org/0000-0002-7833-783X" affiliation: "Princeton University" - given-names: Hans family-names: Dembinski email: hans.dembinski@gmail.com affiliation: TU Dortmund orcid: 'https://orcid.org/0000-0003-3337-3850' - family-names: "Pivarski" given-names: "Jim" affiliation: "Princeton University" orcid: "https://orcid.org/0000-0002-6649-343X" email: "jpivarski@gmail.com" - family-names: "Taider" given-names: "Silia" affiliation: "CERN" orcid: "https://orcid.org/0009-0003-5322-1499" email: "siliataider@gmail.com" doi: https://doi.org/10.5281/zenodo.13883674 repository-code: "https://github.com/scikit-hep/uhi" url: "https://uhi.readthedocs.io/" keywords: - python - histogram - scikit-hep license: "BSD-3-Clause" uhi-1.2.1/environment.yml0000644000000000000000000000021015261523264012314 0ustar00name: uhi channels: - conda-forge dependencies: - boost-histogram - dependency-groups - pip - pytest - root - uproot>=5.7 uhi-1.2.1/noxfile.py0000755000000000000000000000462215261523264011261 0ustar00#!/usr/bin/env -S uv run -q # /// script # dependencies = ["nox>=2025.2.9"] # /// from __future__ import annotations import argparse from pathlib import Path import nox nox.needs_version = ">=2025.2.9" nox.options.default_venv_backend = "uv|virtualenv" PYPROJECT = nox.project.load_toml() ALL_PYTHONS = nox.project.python_versions(PYPROJECT) DIR = Path(__file__).parent.resolve() @nox.session def lint(session): """ Run the linter. """ session.install("pre-commit") session.run("pre-commit", "run", "--all-files", *session.posargs) @nox.session(python=ALL_PYTHONS) def tests(session): """ Run the unit and regular tests. """ session.install("-e.[schema]", *nox.project.dependency_groups(PYPROJECT, "test")) session.run("pytest", *session.posargs) @nox.session(reuse_venv=True, default=False) def docs(session: nox.Session) -> None: """ Build the docs. Use "--non-interactive" to avoid serving. Pass "-b linkcheck" to check links. """ parser = argparse.ArgumentParser() parser.add_argument( "-b", dest="builder", default="html", help="Build target (default: html)" ) args, posargs = parser.parse_known_args(session.posargs) serve = args.builder == "html" and session.interactive extra_installs = ["sphinx-autobuild"] if serve else [] session.install( "-e.", *extra_installs, *nox.project.dependency_groups(PYPROJECT, "docs") ) session.chdir("docs") shared_args = ( "-n", # nitpicky mode "-T", # full tracebacks f"-b={args.builder}", ".", f"_build/{args.builder}", *posargs, ) if serve: session.run("sphinx-autobuild", "--open-browser", *shared_args) else: session.run("sphinx-build", "--keep-going", *shared_args) @nox.session(default=False) def build(session): """ Build an SDist and wheel. """ if session.venv_backend == "uv": session.run("uv", "build") else: session.install("build") session.run("python", "-m", "build") @nox.session(venv_backend="conda", default=False) def root_tests(session): """ Test against ROOT. """ session.conda_install( "--channel=conda-forge", "ROOT", "uproot>=5.7", "pytest", "boost-histogram" ) session.install("-e.") session.run("pytest", "tests/test_root.py", "tests/test_uproot.py") if __name__ == "__main__": nox.run() uhi-1.2.1/.github/CONTRIBUTING.md0000644000000000000000000000147715261523264013036 0ustar00See the [Scikit-HEP Developer introduction][skhep-dev-intro] for a detailed description of best practices for developing Scikit-HEP packages. [skhep-dev-intro]: https://scikit-hep.org/developer/intro # Quick run with Nox If you have nox, this project supports nox. These are the supplied sessions: ```console nox -s lint nox -s tests nox -s docs nox -s build ``` # Post setup You should prepare pre-commit, which will help you by checking that commits pass required checks: ```bash pip install pre-commit # or brew install pre-commit on macOS pre-commit install # Will install a pre-commit hook into the git repo ``` You can also/alternatively run `pre-commit run` (changes only) or `pre-commit run --all-files` to check even without installing the hook. # Testing Use pytest to run the unit checks: ```bash pytest ``` uhi-1.2.1/.github/dependabot.yml0000644000000000000000000000040515261523264013423 0ustar00version: 2 updates: # Maintain dependencies for GitHub Actions - package-ecosystem: "github-actions" directory: "/" schedule: interval: "monthly" groups: actions: patterns: - "*" cooldown: default-days: 7 uhi-1.2.1/.github/release.yml0000644000000000000000000000012615261523264012736 0ustar00changelog: exclude: authors: - dependabot[bot] - pre-commit-ci[bot] uhi-1.2.1/.github/zizmor.yaml0000644000000000000000000000024015261523264013006 0ustar00rules: concurrency-limits: ignore: # The Copilot setup workflow only runs on demand; concurrency is not applicable. - copilot-setup-steps.yml uhi-1.2.1/.github/workflows/cd.yml0000644000000000000000000000427015261523264013745 0ustar00name: CD on: workflow_dispatch: push: branches: - main release: types: - published schedule: - cron: "34 5 * * *" permissions: {} concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: dist: name: Build distributions runs-on: ubuntu-latest permissions: contents: read # to check out the repository steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - uses: hynek/build-and-inspect-python-package@2abe76da66d0a6a4a227101f9348ee855797cfa5 # v3.0.1 publish: name: Publish to PyPI needs: [dist] runs-on: ubuntu-latest if: github.event_name == 'release' && github.event.action == 'published' environment: name: pypi url: https://pypi.org/p/uhi permissions: id-token: write # to publish with trusted publishing attestations: write # to attach build provenance contents: read # to attach build provenance steps: - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: Packages path: dist - uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: "dist/*" - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 upload_nightly_wheels: name: Upload nightly wheels to Anaconda Cloud if: | (github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') && !github.event.repository.fork && github.ref == 'refs/heads/main' needs: [dist] runs-on: ubuntu-latest permissions: {} steps: - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: Packages path: dist - name: List all files run: ls -lh dist - name: Upload wheel to Anaconda Cloud as nightly uses: scientific-python/upload-nightly-action@16fa02eacee1655195143de09f03676e60ef2bf5 # 0.6.5 with: artifacts_path: dist anaconda_nightly_upload_token: ${{ secrets.ANACONDA_ORG_UPLOAD_TOKEN }} uhi-1.2.1/.github/workflows/ci.yml0000644000000000000000000000410215261523264013744 0ustar00name: CI on: workflow_dispatch: pull_request: push: branches: - main permissions: {} concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: checks: name: Check Python ${{ matrix.python-version }} on ${{ matrix.runs-on }} runs-on: ${{ matrix.runs-on }} permissions: contents: read # to check out the repository strategy: fail-fast: false matrix: python-version: ["3.10", "3.15"] runs-on: [ubuntu-latest, macos-latest, windows-latest] include: - python-version: pypy-3.11 runs-on: ubuntu-latest - python-version: "3.11" runs-on: ubuntu-latest - python-version: "3.12" runs-on: ubuntu-latest - python-version: "3.13" runs-on: ubuntu-latest - python-version: "3.14" runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} allow-prereleases: true - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 - name: Install package run: uv pip install --system -e.[schema] --group test - name: Test run: python -m pytest -ra root: name: ROOT test runs-on: ubuntu-latest permissions: contents: read # to check out the repository defaults: run: shell: bash -l {0} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 with: use-mamba: true environment-file: environment.yml - name: Install package run: pip install . - name: Test root run: pytest -ra tests/test_root.py tests/test_uproot.py uhi-1.2.1/.github/workflows/copilot-setup-steps.yml0000644000000000000000000000152515261523264017322 0ustar00name: "Copilot Setup Steps" permissions: {} on: workflow_dispatch: pull_request: paths: - .github/workflows/copilot-setup-steps.yml jobs: copilot-setup-steps: name: Copilot Setup Steps runs-on: ubuntu-latest permissions: contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" allow-prereleases: true - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 - name: Install tooling run: | uv tool install nox uv tool install prek - name: Pre-install checks run: | prek prepare-hooks uhi-1.2.1/docs/Makefile0000644000000000000000000000117215261523264011625 0ustar00# Minimal makefile for Sphinx documentation # # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = . BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) uhi-1.2.1/docs/changelog.md0000644000000000000000000000026315261523264012436 0ustar00# Changelog ```{changelog} :changelog-url: https://github.com/scikit-hep/uhi/releases :github: https://github.com/scikit-hep/uhi/releases :pypi: https://pypi.org/project/uhi ``` uhi-1.2.1/docs/conf.py0000644000000000000000000000572015261523264011467 0ustar00# Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html from __future__ import annotations import importlib.metadata import os from typing import Any import sphinx_github_changelog.changelog # -- Project information ----------------------------------------------------- project = "uhi" copyright = "2021, Henry Schreiner, Hans Dembinski, Jim Pivarski" author = "Henry Schreiner, Hans Dembinski, Jim Pivarski" # The full version, including alpha/beta/rc tags version = release = importlib.metadata.version("uhi") # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ "myst_parser", "sphinx-jsonschema", "sphinx.ext.napoleon", "sphinx_copybutton", "sphinx_github_changelog", "sphinx_llm.txt", ] source_suffix = [".rst", ".md"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] myst_enable_extensions = [ "colon_fence", ] # -- Options for LLM-friendly output ----------------------------------------- # The default is the full README, which is too long for the summary block llms_txt_description = ( "Documentation of histogram indexing, the PlottableHistogram Protocol, and" " the histogram serialization format, with tools for library authors." ) # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = "furo" html_theme_options = { "source_repository": "https://github.com/scikit-hep/uhi", "source_branch": "main", "source_directory": "docs/", } # -- Changelog builder ------------------------------------------------------- # GitHub release notes start at H2. The changelog extension parses them with # default docutils settings, so suppress_warnings in this file has no effect. _changelog_default_settings = sphinx_github_changelog.changelog.get_default_settings def _changelog_settings(*components: Any) -> Any: settings = _changelog_default_settings(*components) settings.myst_suppress_warnings = ["myst.header"] return settings sphinx_github_changelog.changelog.get_default_settings = _changelog_settings if "GITHUB_API_TOKEN" in os.environ: sphinx_github_changelog_token = os.environ["GITHUB_API_TOKEN"] commit = os.environ.get("READTHEDOCS_GIT_COMMIT_HASH", "main") code_url = "https://github.com/scikit-hep/uhi/blob" uhi-1.2.1/docs/index.rst0000644000000000000000000000311415261523264012024 0ustar00.. meta:: :description: UHI connects histogramming libraries with an indexing standard, the PlottableProtocol, and a serialization format. UHI: Unified Histogram Interface ================================= UHI is a library that helps connect other histogramming libraries. It is primarily intended to be a guide and static type check helper; you do not need a runtime dependency on UHI. It currently does so with the following components: UHI Indexing, which describes a powerful indexing system for histograms, designed to extend standard Array indexing for Histogram operations. UHI Indexing+ (referred to as UHI+ for short), which describes a set of extensions to the standard indexing that make it easier to use on the command line. The PlottableProtocol, which describes the minimal and complete set of requirements for a source library to produce and a plotting library to consume to plot a histogram, including error bars. The serialization format, which describes how to store histograms in JSON, ZIP, HDF5, and ROOT files, with a JSON schema and reference readers and writers in ``uhi.io``. The ``uhi`` command line tool, which can add histograms from several files (``uhi add``) and check files against the schema (``uhi validate``). The testing helpers in ``uhi.testing``, which a library can use to check that it follows UHI indexing. .. toctree:: :maxdepth: 2 :caption: Contents: indexing.rst indexing+.rst plotting.rst serialization.md testing.md changelog.md Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` uhi-1.2.1/docs/indexing+.rst0000644000000000000000000000356715261523264012611 0ustar00.. _usage-indexing+: Indexing+ ========= This is an extended version of UHI, called UHI+. This is not implemented in boost-histogram, but is implemented in Hist. Syntax extensions ----------------- UHI+ avoids using the standard tags found in UHI by using more advanced Python syntax. Location based slicing/access: numeric axes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can replace location based indexing ``loc(1.23) → 1.23j`` (a "j" suffix on a number literal). You can shift by an integer, just like with loc: ``2.3j + 1`` will be one bin past the one containing the location "2.3". .. code:: python3 v = h[2j] # Returns the bin containing "2.0" v = h[2j + 1] # Returns the bin above the one containing "2.0" h2 = h[2j:] # Slices starting with the bin containing "2.0" Location based slicing/access: string axis ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you have a string based axis, you can use a string directly ``loc("label") → "label"``. .. code:: python3 v = h["a"] # Returns the "a" bin (string category axis) Rebinning ^^^^^^^^^ You can replace ``rebin(2) → 2j`` in the third slot of a slice. .. code:: python3 h2 = h[::2j] # Modification operations (rebin) h2 = h[a:b:2j] # Modifications can combine with slices Named based indexing ^^^^^^^^^^^^^^^^^^^^ An optional extension to indexing is expected for histogram implementations that support names. If named axes are supported, any expression that refers to an axis by an integer can also refer to it by a name string. ``.project(*axis: int | str)`` is probably the most common place to see this, but you can also use strings in the UHI dict access, such as: .. code:: python3 h[{"a": np.s_[::2j]}] # rebin axis "a" by two h[{"x": np.s_[0:3.5j]}] # slice axis "x" from 0 to the data coordinate 3.5 h[{"other": np.s_[0:2:4j]}] # slice and rebin axis "other" uhi-1.2.1/docs/indexing.rst0000644000000000000000000002512415261523264012527 0ustar00.. _usage-indexing: Indexing ======== This is the design document for Unified Histogram Indexing (UHI). Much of the original plan is now implemented in boost-histogram. Other histogramming libraries can implement support for this as well, and the "tag" functors, like ``sum`` and ``loc`` can be used between libraries. Syntax ------ The following examples assume you have imported ``loc``, ``rebin``, ``underflow``, and ``overflow`` from boost-histogram or any other library that implements UHI. Access: ^^^^^^^ .. code:: python3 v = h[b] # Returns bin contents, indexed by bin number v = h[loc(b)] # Returns the bin containing the value v = h[loc(b) + 1] # Returns the bin above the one containing the value v = h[underflow] # Underflow and overflow can be accessed with special tags Indexing works like Python, with ``IndexError`` thrown if you are out of range. Slicing: ^^^^^^^^ .. code:: python3 h == h[:] # Slice over everything h2 = h[a:b] # Slice of histogram (includes flow bins) h2 = h[:b] # Leaving out endpoints is okay h2 = h[loc(v):] # Slices can be in data coordinates, too h2 = h[::rebin(2)] # Modification operations (rebin) h2 = h[a:b:rebin(2)] # Modifications can combine with slices v2 = h[::sum] # Projection operations # (name may change) v2 = h[a:b:sum] # Adding endpoints to projection operations v2 = h[0:len:sum] # removes under or overflow from the calculation h2 = h[v, a:b] # A single value v is like v:v+1:sum h2 = h[a:b, ...] # Ellipsis work just like normal numpy Setting ^^^^^^^ .. code:: python3 # Single values h[b] = v # Returns bin contents, indexed by bin number h[loc(b)] = v # Returns the bin containing the value h[underflow] = v # Underflow and overflow can be accessed with special tags h[...] = array(...) # Setting with an array or histogram sets the contents if the sizes match # Overflow can optionally be included if endpoints are left out # The number of dimensions for non-scalars should match (broadcasting works normally otherwise) All of this generalizes to multiple dimensions. ``loc(v)`` could return categorical bins, but slicing on categories would (currently) not be allowed. These all return histograms, so flow bins are always preserved - the one exception is projection; since this removes an axis, the only use for the slice edges is to be explicit on what part you are interested for the projection. So an explicit (non-empty) slice here will case the relevant flow bin to be excluded. ``loc`` and ``rebin`` live inside the histogramming package (like boost-histogram), but are completely general and can be created by a user using an explicit API (below). ``underflow`` and ``overflow`` also follow a general API. ``sum`` is just the Python built-in sum function. One drawback of the syntax listed above is that it is hard to select an action to run on an axis or a few axes out of many. For this use case, you can pass a dictionary to the index, and that has the syntax ``{axis:action}``. The actions are slices, and follow the rules listed above. All axes that are unmentioned are left alone, just like ``...`` or explicit ``:`` would do. This looks like: .. code:: python3 h[{0: slice(None, None, bh.rebin(2))}] # rebin axis 0 by two h[{1: slice(0, bh.loc(3.5))}] # slice axis 1 from 0 to the data coordinate 3.5 h[{7: slice(0, 2, bh.rebin(4))}] # slice and rebin axis 7 If you don't like manually building slices, you can use the ``np.s_`` utility to recover the original slicing syntax inside the dict: .. code:: python3 h[{0: np.s_[::rebin(2)]}] # rebin axis 0 by two h[{1: np.s_[0:loc(3.5)]}] # slice axis 1 from 0 to the data coordinate 3.5 h[{7: np.s_[0:2:rebin(4)]}] # slice and rebin axis 7 Invalid syntax: ^^^^^^^^^^^^^^^ .. code:: python3 h[1.0] # Floats are not allowed, just like numpy h[::2] # Skipping is not (currently) supported h[..., None] # None == np.newaxis is not supported Reordering axes ^^^^^^^^^^^^^^^ It is not possible to reorder axis with this syntax; libraries are expected to provide a ``.project(*axis: int)`` method which provides a way to reorder, as well as fast access to a small subset of a large histogram in a complementary way to the above indexing. Rejected proposals or proposals for future consideration, maybe ``hist``-only: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code:: python3 h2 = h[1.0j:2.5j + 1] # Adding a j suffix to a number could be used in place of ``loc(x)`` h2 = h[1.0] # Floats in place of ``loc(x)``: too easy to make a mistake Examples -------- For a histogram, the slice should be thought of like this: .. code:: python3 histogram[start:stop:action] The start and stop can be either a bin number (following Python rules), or a callable; the callable will get the axis being acted on and should return an extended bin number (``-1`` and ``len(ax)`` are flow bins). A provided callable is ``bh.loc``, which converts from axis data coordinates into bin number. The final argument, ``action``, is special. A general API is being worked on, but for now, ``bh.sum`` will “project out” or “integrate over” an axes, and ``bh.rebin(n)`` will rebin by a positive integral factor. Both work correctly with limits; ``bh.sum`` will remove flow bins if given a range. ``h[0:len:bh.sum]`` will sum without the flow bins. Here are a few examples that highlight the functionality of UHI: Example 1: ^^^^^^^^^^ You want to slice axis 0 from 0 to 20, axis 1 from .5 to 1.5 in data coordinates, axis 2 needs to have double size bins (rebin by 2), and axis 3 should be summed over. You have a 4D histogram. Solution: .. code:: python3 ans = h[:20, bh.loc(-.5):bh.loc(1.5), ::bh.rebin(2), ::bh.sum] Example 2: ^^^^^^^^^^ You want to set all bins above 4.0 in data coordinates to 0 on a 1D histogram. Solution: .. code:: python3 h[bh.loc(4.0):] = 0 You can set with an array, as well. The array can either be the same length as the range you give, or the same length as the range + under/overflows if the range is open ended (no limit given). For example: .. code:: python3 h = bh.Histogram(bh.axis.Regular(10, 0, 1)) h[:] = np.ones(10) # underflow/overflow still 0 h[:] = np.ones(12) # underflow/overflow now set too Note that for clarity, while basic NumPy broadcasting is supported, axis-adding broadcasting is not supported; you must set a 2D histogram with a 2D array or a scalar, not a 1D array. Example 3: ^^^^^^^^^^ You want to sum from -infinity to 2.4 in data coordinates in axis 1, leaving all other axes alone. You have an ND histogram, with N >= 2. Solution: .. code:: python3 ans = h[:, :bh.loc(2.4):bh.sum, ...] Notice that last example could be hard to write if the axis number, 1 in this case, was large or programmatically defined. In these cases, you can pass a dictionary of ``{axis:slice}`` into the indexing operation. A shortcut to quickly generate slices is provided, as well: .. code:: python3 ans = h[{1: slice(None,bh.loc(2.4),bh.sum)}] # Identical: ans = h[{1: np.s_[:bh.loc(2.4):bh.sum]}] Example 4: ^^^^^^^^^^ You want the underflow bin of a 1D histogram. Solution: .. code:: python3 val = h1[bh.underflow] -------------- Extensions ---------- Boost-histogram currently implements a few extra additions to this that are not yet required: * Passing ``sum`` directly acts as if it was given as the third argument (the action). * The ``rebin`` tag can be passed directly, as well. The inner workings of ``rebin`` are being worked on, and will be updated here when they are finalized. Details ------- Implementation notes ^^^^^^^^^^^^^^^^^^^^ loc, rebin, and sum are *not* unique tags, or special types, but rather APIs for classes. New versions of these could be added, and implementations could be shared among Histogram libraries. For clarity, the following code is written in Python 3.6+. `Prototype here `__. `Extra doc here `__. Note that the API comes in two forms; the ``__call__``/``__new__`` operator form is more powerful, slower, optional, and is currently not supported by boost-histogram. A fully conforming UHI implementation must allow the tag form without the operators. Basic implementation example (WIP): .. code:: python3 class loc: "When used in the start or stop of a Histogram's slice, x is taken to be the position in data coordinates." def __init__(self, value, offset): self.value = value self.offset = offset # supporting __add__ and __sub__ also recommended def __call__(self, axis): return axis.index(self.value) + self.offset # Other flags, such as callable functions, could be added and detected later. # UHI will perform a maximum performance sum when python's sum is encountered def underflow(axis): return -1 def overflow(axis): return len(axis) class rebin: """ When used in the step of a Histogram's slice, rebin(n) combines bins, scaling their widths by a factor of n. If the number of bins is not divisible by n, the remainder is added to the overflow bin. """ def __init__(self, factor): # Items with .factor are specially treated in boost-histogram, # performing a high performance rebinning self.factor = factor # Optional and not used by boost-histogram def __call__(self, binning, axis, counts): factor = self.factor if isinstance(binning, Regular): indexes = (numpy.arange(0, binning.num, factor),) num, remainder = divmod(binning.num, factor) high, hasover = binning.high, binning.hasover if binning.hasunder: indexes[0][:] += 1 indexes = ([0],) + indexes if remainder == 0: if binning.hasover: indexes = indexes + ([binning.num + int(binning.hasunder)],) else: high = binning.left(indexes[-1][-1]) hasover = True binning = Regular(num, binning.low, high, hasunder=binning.hasunder, hasover=hasover) counts = numpy.add.reduceat(counts, numpy.concatenate(indexes), axis=axis) return binning, counts else: raise NotImplementedError(type(binning)) uhi-1.2.1/docs/make.bat0000644000000000000000000000143315261523264011572 0ustar00@ECHO OFF pushd %~dp0 REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set SOURCEDIR=. set BUILDDIR=_build if "%1" == "" goto help %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.http://sphinx-doc.org/ exit /b 1 ) %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end popd uhi-1.2.1/docs/plotting.rst0000644000000000000000000000677315261523264012573 0ustar00.. _usage-plotting: Plotting ======== This is a description of the ``PlottableProtocol``. Any plotting library that accepts an object that follows the ``PlottableProtocol`` can plot object that follow this protocol, and libraries that follow this protocol are compatible with plotters. The Protocol is runtime checkable, though as usual, that will only check for the presence of the needed methods at runtime, not for the static types. Using the protocol: ^^^^^^^^^^^^^^^^^^^ Plotters should only depend on the methods and attributes listed below. In short, they are: * ``h.kind``: The ``bh.Kind`` of the histogram (COUNT or MEAN) * ``h.values()``: The value (as given by the kind) * ``h.variances()``: The variance in the value (None if an unweighed histogram was filled with weights) * ``h.counts()``: How many fills the bin received or the effective number of fills if the histogram is weighted * ``h.axes``: A Sequence of axes Axes have: * ``ax[i]``: A tuple of (lower, upper) bin, or the discrete bin value (integer or string) * ``len(ax)``: The number of bins * Iteration is supported * ``ax.traits.circular``: True if circular * ``ax.traits.discrete``: True if the bin represents a single value (e.g. Integer or Category axes) instead of an interval (e.g. Regular or Variable axes) A histogram can have no axes (a 0D histogram, like a counter or the sum of a 1D histogram). Then ``h.axes`` is empty, and ``.values()``, ``.variances()``, and ``.counts()`` return 0D arrays (shape ``()``). Plotters that support this case can show the single value without an x axis. Plotters should see if ``.counts()`` is None; no boost-histogram objects currently return None, but a future storage or different library could. Also check ``.variances``; if not None, this storage holds variance information and error bars should be included. Boost-histogram histograms will return something unless they know that this is an invalid assumption (a weighted fill was made on an unweighted histogram). To statically restrict yourself to valid API usage, use ``PlottableHistogram`` as the parameter type to your function (Not needed at runtime). Implementing the protocol: ^^^^^^^^^^^^^^^^^^^^^^^^^^ Add UHI to your MyPy environment; an example ``.pre-commit-config.yaml`` file: .. code:: yaml - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.812 hooks: - id: mypy files: src additional_dependencies: [uhi, numpy~=1.20.1] Then, check your library against the Protocol like this: .. code:: python3 from typing import TYPE_CHECKING, cast if TYPE_CHECKING: _: PlottableHistogram = cast(MyHistogram, None) Help for plotters ^^^^^^^^^^^^^^^^^ The module ``uhi.numpy_plottable`` has a utility to simplify the common use case of accepting a PlottableProtocol or other common formats, primarily a NumPy ``histogram``/``histogram2d``/``histogramdd`` tuple. The ``ensure_plottable_histogram`` function will take a histogram or NumPy tuple, or an object that implements ``.to_numpy()`` or ``.numpy()`` and convert it to a ``NumPyPlottableHistogram``, which is a minimal implementation of the Protocol. By calling this function on your input, you can then write your plotting function knowing that you always have a ``PlottableProtocol`` object, greatly simplifying your code. The full protocol version 1.2 follows: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (Also available as ``uhi.typing.plottable.PlottableHistogram``, for use in tests, etc.) .. literalinclude:: ../src/uhi/typing/plottable.py :language: python uhi-1.2.1/docs/serialization.md0000644000000000000000000004454415261523264013376 0ustar00# Serialization ## Introduction Histogram serialization has to cover a wide range of formats. As such, we describe a form for serialization that covers the metadata structure as JSON-like, with a provided JSON-schema. The data (bins and/or variable edges) is stored out-of-band in a binary format based on what type of data file you are in. For very small (primarily 1D) histograms, data is allowed inline as well. The following formats are being targeted: ``` ┌────────┐ ┌────────┐ ┌────────────┐ │ ROOT │ │ HDF5 │ │ ZIP/JSON │ └────────┘ └────────┘ └────────────┘ ``` Other formats can be used as well, assuming they support out-of-band data and text attributes or files for the metadata. We are working on a Zarr backend in the near future. ## Caveats This structure was based heavily on boost-histogram, but it is intended to be general, and can be expanded in the future as needed. As such, the following limitations are required: * Serialization followed by deserialization may cause axis changes. Axis types may change to an equivalent but less performant axis, growth status will be lost, etc. Libraries can record custom `"writer_info"` attributes to improve round trips, but histograms must always be openable without the extra info. * Metadata should also be reasonably sized; some formats like HDF5 may limit the size of attributes to 64K. * Floating point errors could be incurred on conversion, as the storage format uses a stable but different representation. * Axis `name` is only part of the metadata, and is not standardized. This is due to lack of support from boost-histogram. ## Design The following axes types are supported: * `"regular"`: A regularly spaced set of even bins. Boost-histogram's "integer" axes maps to this axis as well. Has `upper`, `lower`, `bins`, `underflow`, `overflow`, and `circular` properties. * `"variable"`: A continuous axis defined by bins+1 edges. Has `edges`, which is either an in-line list of numbers or a string pointing to an out-of-band data source. Also has `underflow`, `overflow`, and `circular` properties. * `"category_int"`: A list of integer bins, non-continuous. Has `categories`, which is an in-line list of integers. Also has `flow`. * `"category_str"`: A list of string bins. Has `categories`, which is an in-line list of strings. Also has `flow`. * `"boolean"`: A true/false axis. Axes with gaps are currently not supported. All axes support `metadata`, a dictionary of arbitrary data. The values can be strings, numbers, or booleans. Other value types are not supported. Libraries are encouraged to provide a way to indicate unserializable metadata; our recommendation is to avoid adding any metadata that starts with a `@` to the metadata dictionary. Libraries should not include keys with `None` values, as some formats might not support null values. The following storages are supported: * `"int"`: A collection of integers. Boost-histogram's `Int64` and `AtomicInt64` map to this, and sometimes `Unlimited`. * `"double"`: A collection of floating point values. Boost-histogram's `Double` storage maps to this, and sometimes `Unlimited`. * `"weighted"`: A collection of two arrays of floating point values, `"values"` and `"variances"`. Boost-histogram's `Weight` storage maps to this. * `"mean"`: A collection of three arrays of floating point values, `"counts"`, `"values"`, and `"variances"`. Boost-histogram's `Mean` storage maps to this. * `"weighted_mean"`: A collection of four arrays of floating point values, `"sum_of_weights"`, `"sum_of_weights_squared"`, `"values"`, and `"variances"`. Boost-histogram's `WeightedMean` storage maps to this. A histogram with no axes holds one bin. Its storage values can be a single number or an array of length 1. Readers must accept both forms. ```{versionchanged} 1.2 Storage values can be a single number for a histogram with no axes. ``` All storage types support **empty (metadata-only) histograms**, where the storage contains only the `"type"` field and no data arrays. This is useful for creating histograms that contain axes and metadata but no actual bin data. Empty storages can be filled later or used for schema/template purposes. For example: ```json { "storage": { "type": "int" } } ``` Empty storages are compatible with all serialization formats and can be converted to/from dense or sparse forms using the standard conversion functions. ```{versionadded} 1.1 Empty (metadata-only) histogram support. ``` Current backends (such as zip and hdf5) allow flexibility in the size of integer and floating point storage. It is allowed to use a smaller floating point or integer storage type for such storages; the reference is 64-bit floats and 64-bit integers. A library can fill the optional `"writer_info"` field with a key specific to the library containing library specific metadata anywhere a metadata field is allowed. There is one defined key at the Histogram level, `"version"`, which contains the version of the library that created the histogram. Libraries should include this key when creating a histogram. It is not required for reading histograms. Histogram libraries can put custom metadata here that they can use to record provenance information or help with same-library round trips. For example, a histogram created with boost-histogram might contain: ```json { "writer_info": { "boost-histogram": { "version": "1.0.0", } }, "...": "...", } ``` There is one more required key for each histogram: `"uhi_schema"`, which must be set to 1 currently. If there is a future revision with a backward incompatible change, this will be bumped to 2, and readers should always error on future schemas, and support all older schemas. This is hoped to be unlikely/rare, but this also serves as a check that this is in fact a uhi serialization object. Non-breaking changes like additions are allowed without bumping the schema. ### Named and single histograms A file can hold either a dictionary of named histograms, or a single histogram stored directly at the top level: ```json { "one": { "uhi_schema": 1, "axes": ["..."], "storage": { "...": "..." } }, "two": { "uhi_schema": 1, "axes": ["..."], "storage": { "...": "..." } } } ``` ```json { "uhi_schema": 1, "axes": ["..."], "storage": { "...": "..." } } ``` Readers can tell the two forms apart by the `"uhi_schema"` key: it is present (and required) in a single histogram, and a histogram name is never `"uhi_schema"` in a dictionary of histograms. The single form is the natural output of `json.dumps` on one histogram, and it matches the HDF5 layout, where each histogram is a group. Two schemas are provided. `histogram.schema.json` describes one histogram, the intermediate representation. `histograms.schema.json` describes a JSON file, in either form above, and refers to the first schema for each histogram. ```{versionadded} 1.2 The single histogram form is accepted, and the schema is split into `histogram.schema.json` (one histogram) and `histograms.schema.json` (a file). ``` ## Sparse storage For sparse histograms, storage contains an `index` key. This is a 2D array; the first dimension has the same number of entries as the number of axes, and the second dimension has the same number of entries as the number of filled bins. There should not be any duplicate entries. The values start at 0 for the underflow bin (if there is one), and are in the same order as the axes. The data in this case are 1D arrays, one for each bin. For example, take the following sparse histogram with three filled bins: ```json { "storage": { "index": [[0, 1, 2],[3, 3, 4]], "values": [5, 6, 7], } } ``` The `0, 3` bin is filled with 5, the `1, 3` bin is filled with 6, and the `2, 4` bin is filled with 7. If the first axes has `"underflow"` enabled, that first bin is an underflow bin. Empty (metadata-only) histograms are unaffected by sparse/dense conversions; they remain as-is since there is no data to convert. If a histogram library doesn't support sparse histograms, you can convert a sparse histogram to a dense one. UHI provides helpers `uhi.io.to_sparse` and `uhi.io.from_sparse` that can be used to support a library that doesn't support sparse histograms. Scalar histograms (with no axes) are always dense. ## Adding histograms `uhi.io.ops.add` sums histograms bin-by-bin, like ROOT's `hadd`. It works on the intermediate representation (or anything with `_to_uhi_`), so histograms from different libraries or files can be combined without a histogram library: ```python import uhi.io.ops total = uhi.io.ops.add(h1, h2, h3) ``` All inputs must have identical axes (metadata and `writer_info` are ignored when comparing) and the same storage type. Empty (metadata-only) storages count as zero. `"mean"` and `"weighted_mean"` storages are merged correctly, not just summed. The metadata and `writer_info` of the first histogram are kept. The result is sparse if every input is sparse, and dense otherwise. ```{versionadded} 1.2 ``` ## CLI/API The `uhi` command (also `python -m uhi`) provides `add`, which sums the histograms in several files into one file, like `hadd`: ```console $ uhi add total.zip run1.zip run2.zip run3.zip ``` Histograms are matched by name. A name that is only in some of the inputs is copied through. The format is chosen by the file extension: `.json` (a JSON object mapping names to histograms, or a single histogram), `.zip`, `.h5`/`.hdf5`/`.hdf` (every group with a `uhi_schema` attribute, at any depth), or `.root` (every RNTuple, at any depth). The input and output formats do not have to match, so this also converts between formats. Pass `-f`/`--force` to overwrite an existing output file. Add `:path` to an input to select one histogram by name, or a directory or group of histograms. If every input is a single histogram, the result is a single histogram; add `:name` to the output to name it (this is required for the zip and ROOT formats). For named histograms, `:dir` on the output prefixes every name: ```console $ uhi add total.json:h run1.json:h run2.zip:sub/h single.json $ uhi add out.root:merged run1.h5:results run2.h5:results ``` ```{versionadded} 1.2 ``` You can test a file against the schema with `uhi validate`. This needs the `schema` extra (`pip install uhi[schema]`). The format is selected by the file suffix: `.json`, `.zip`, `.h5`/`.hdf5`/`.hdf` (needs the `hdf5` extra), or `.root` (needs the `uproot` extra or PyROOT). Every histogram in a zip file (each `*.json` entry), HDF5 file (each group with a `uhi_schema` attribute), or ROOT file (each RNTuple, searched recursively) is checked: ```console $ uhi validate some/file.json some/other.zip some/data.h5 some/data.root ``` Add `:path` to restrict the check to one histogram by name, or to a group or directory of histograms (in JSON and zip files, all names that start with `path/`): ```console $ uhi validate some/file.json:h some/data.h5:run1/results some/data.root:analysis/main ``` ```{versionadded} 1.2 ``` Or with code: ```python import json import uhi.schema with filename.open(encoding="utf-8") as f: data = json.load(f) uhi.schema.validate(data) ``` `validate` checks a JSON file object, in either form. Use `uhi.schema.validate_histogram` to check a single histogram (the intermediate representation). `uhi.schema.load` reads any supported file into a JSON-compatible dict (or a single histogram), with an optional `path` keyword to select a histogram, group, or directory inside the file: ```python uhi.schema.validate(uhi.schema.load("data.root", path="analysis")) ``` ## Format specific details and helpers The `uhi` library contains reference implementations of target formats. You can implement these yourself, but if you are using or have access to Python, feel free to use `uhi` if that helps. If an object has a `_to_uhi_` method, that will be used to convert it to a dictionary following the schema. A `_from_uhi_` classmethod is also recommended; however, this is generally part of the histogram constructor; if a UHI object is passed, it should be converted automatically. ### JSON The simplest format, useful for writing tests. The JSON version is nearly identical to the intermediate representation; the only difference is that data is stored as nested lists. It is not intended for large histograms; the ZIP format (below) is nearly identical and builds on this with out-of-band data. Two utilities are provided; `uhi.io.json.default` and `uhi.io.json.object_hook`. These are used with Python's built-in json module to handle conversions. ```python import json import uhi.io.json ob = json.dumps(h, default=uhi.io.json.default) uhi_hist = json.loads(ob, object_hook=uhi.io.json.object_hook) ``` Above, `h` is a histogram that supports `_to_uhi_` or an intermediate representation,`ob` is a JSON string, and `uhi_hist` is an intermediate representation; you can pass it to `boost_histogram.Histogram` or `hist.Hist`. This stores a single histogram directly. To store several histograms in one file, pass a dictionary of histograms instead; both forms are valid. ### ZIP The zip format is very similar to JSON, but stores data in the numpy zip format. Arrays are replaced by strings, which represent the `.npy` file inside the zip file containing the array. The names are arbitrary; see the uhi code if you want to see how uhi creates names. The metadata is in a file with the name of the histogram and a `.json` extension. We provide `uhi.io.zip.write` and `uhi.io.zip.read`, which work with open `zipfile.ZipFile` objects from the standard library (or probably anything with a similar API). ```python import zipfile import uhi.io.zip with zipfile.ZipFile("myfile.zip", "w") as zip_file: uhi.io.zip.write(zip_file, "histogram", h) with zipfile.ZipFile("myfile.zip", "r") as zip_file: h2 = uhi.io.zip.read(zip_file, "histogram") ``` Above, `h` is a histogram that supports `_to_uhi_` or an intermediate representation, and `h2` is an intermediate representation; you can pass it to `boost_histogram.Histogram` or `hist.Hist`. The metadata name in the file is `"histogram.json"`. The contents of that file are identical to the JSON format, except arrays are replaced by string names to files inside the zipfile. ### HDF5 The HDF5 format is ideal for combining histograms with other data. You need the `h5py` library installed too to use this format. There is some extra structure here compared to the other formats. The groups are `"axes"`, `"ref_axes"`, `"metadata"`, and `"storage"`. Arrays for the axes are placed in `"ref_axes"`, since hdf5 doesn't have lists of arrays. Storage arrays are in-place. A Reference type is used to link the axes array with the data. `"edges"` and `"categories"` are datasets; the other axes values are attributes (or groups with attributes, like `"metadata"` and `"writer_info"`, which is a nested group). Likewise, the `"storage"` group sets `"type"` as an attribute, the others are datasets. We provide `uhi.io.hdf5.read` and `uhi.io.hdf5.write` to write to an open group. The structure is relative; you can place it anywhere inside a hdf5 file. ```python import h5py import uhi.io.hdf5 with h5py.File("myfile.hdf5", "w") as h5_file: uhi.io.hdf5.write(h5_file.create_group("histogram"), h) with h5py.File("myfile.hdf5", "r") as h5_file: h2 = uhi.io.hdf5.read(h5_file["histogram"]) ``` Above, `h` is a histogram that supports `_to_uhi_` or an intermediate representation, and `h2` is an intermediate representation; you can pass it to `boost_histogram.Histogram` or `hist.Hist`. You should create the group you want the histogram to be in. By default, we do not compress arrays smaller than 1,000 elements. You can control this by setting `min_compress_elements`; set it to 0 to compress all arrays. You can also pass through `compression` and `compression_opts`. ### ROOT The ROOT format stores each histogram as a single-entry [RNTuple](https://root.cern/doc/master/classROOT_1_1RNTuple.html). You can use [uproot](https://uproot.readthedocs.io) 5.7+ (`pip install uhi[uproot]`) or [ROOT](https://root.cern) 6.36+ (`conda install -c conda-forge root`) for this format; both write the same files. Install ROOT from conda-forge; the `ROOT` package on PyPI is not ready yet. The RNTuple has a `"uhi"` string field holding the IR as a string and arrays are replaced by the name of the field holding them. Storage arrays are stored flattened in `std::vector` fields named after their key (`"values"`, `"variances"`, ...). The shape of these arrays is recovered from the axes when reading. The `edges` of a variable axis is stored the same way in a field named `"axis_{i}_edges"`, where `i` is the axis index. We provide `uhi.io.root.read` and `uhi.io.root.write`, which work with an open `TFile` or other `TDirectory`. ```python import ROOT import uhi.io.root with ROOT.TFile.Open("myfile.root", "RECREATE") as root_file: uhi.io.root.write(root_file, "histogram", h) with ROOT.TFile.Open("myfile.root") as root_file: h2 = uhi.io.root.read(root_file, "histogram") ``` Above, `h` is a histogram that supports `_to_uhi_` or an intermediate representation, and `h2` is an intermediate representation; you can pass it to `ROOT.TH1*` or `boost_histogram.Histogram` or `hist.Hist`. ```{versionadded} 1.2 ``` With uproot, use `uhi.io.uproot.read` and `uhi.io.uproot.write` with an open uproot file or directory. Names can include a directory, like `"dir/histogram"`. ```python import uproot import uhi.io.uproot with uproot.recreate("myfile.root") as root_file: uhi.io.uproot.write(root_file, "histogram", h) with uproot.open("myfile.root") as root_file: h2 = uhi.io.uproot.read(root_file, "histogram") ``` The `uhi` command uses uproot if it is installed, and PyROOT otherwise. ## Schema A typing helper for the intermediate representation, `HistogramIR`, is provided in `uhi.typing.serialization` as a `TypedDict`. The schema, provided in `resources` as `histogram.schema.json`, also allows strings for data members, since some formats (like ZIP) put data into an optimized location and specify a reference to them. The JSON file schema, `histograms.schema.json`, wraps it. ### Rendered schema ```{jsonschema} ../src/uhi/resources/histograms.schema.json ``` ```{jsonschema} ../src/uhi/resources/histogram.schema.json ``` ### Full schema The full schemas are below: ```{literalinclude} ../src/uhi/resources/histograms.schema.json :language: json ``` ```{literalinclude} ../src/uhi/resources/histogram.schema.json :language: json ``` uhi-1.2.1/docs/testing.md0000644000000000000000000000242015261523264012161 0ustar00# Testing UHI has some testing helpers for use in test suites. This is primarily for library authors. ## Indexing You can see if your library passes the indexing test suite. We provide three sets of tests: 1D, 2D, and 3D tests. These can be implemented on your own library (`my.Histogram` in this example) like this: ```python import uhi.testing.indexing class TestAccess1D(uhi.testing.indexing.Indexing1D[my.Histogram]): @classmethod def make_histogram(cls) -> my.Histogram: return my.Histogram(cls.get_uhi()) class TestAccess2D(uhi.testing.indexing.Indexing2D[my.Histogram]): @classmethod def make_histogram(cls) -> my.Histogram: return my.Histogram(cls.get_uhi()) class TestAccess3D(uhi.testing.indexing.Indexing3D[my.Histogram]): @classmethod def make_histogram(cls) -> my.Histogram: return my.Histogram(cls.get_uhi()) ``` If you don't support serialization, then you can manually set the values with the UHI item, or check the docstrings to see what the correct parameters are. Import the module (`import uhi.testing.indexing`), as in the example above. Do not use `from uhi.testing.indexing import Indexing1D`: this puts the base classes in your test module, and some runners (such as unittest) then try to run them as tests too. uhi-1.2.1/docs/_templates/components/edit-this-page.html0000644000000000000000000000275615261523264020212 0ustar00{% extends "!components/edit-this-page.html" %} {#- Add a repo link next to the view/edit icons at the top of the page, using furo's `source_repository` option. Reuses furo's `.muted-link` styling so it matches and stays responsive. -#} {%- macro furo_github_button() -%} {%- if theme_source_repository -%} {%- endif -%} {%- endmacro -%} {% block link_available -%} {{ super() }} {{ furo_github_button() }} {%- endblock %} {% block link_not_available -%} {{ super() }} {{ furo_github_button() }} {%- endblock %} uhi-1.2.1/src/uhi/__init__.py0000644000000000000000000000015415261523264012721 0ustar00from __future__ import annotations from ._version import version as __version__ __all__ = ["__version__"] uhi-1.2.1/src/uhi/__main__.py0000644000000000000000000001070315261523264012703 0ustar00""" Command line interface for uhi. ``uhi add`` sums histograms from JSON, ZIP, HDF5, or ROOT files, like ROOT's ``hadd``. ``uhi validate`` checks histogram files (JSON, zip, HDF5, or ROOT) against the schema. """ from __future__ import annotations import argparse import sys from collections.abc import Mapping, Sequence from pathlib import Path from typing import Any import uhi.io.ops from uhi.io import _files __all__ = ["main"] def __dir__() -> list[str]: return __all__ _ERRORS = (OSError, ValueError, KeyError, TypeError, ImportError) def _read(spec: str, /) -> Any: """ Read a ``file[:path]`` spec: a dict of named histograms or a single one. """ try: file, path = _files.split_spec(spec) data = _files.load(file, path=path) except _ERRORS as e: msg = f"{spec}: {e}" raise SystemExit(msg) from None if not isinstance(data, Mapping): msg = f"{spec}: not a histogram or a dict of histograms" raise SystemExit(msg) if _files.is_single(data): return data if not data: msg = f"{spec}: no histograms found" raise SystemExit(msg) for name, hist in data.items(): if not _files.is_single(hist): msg = f"{spec}: {name!r} is not a histogram" raise SystemExit(msg) return data def _add(args: argparse.Namespace) -> None: try: target, target_path = _files.split_spec(args.target) _files.file_format(Path(target)) # Fail early on an unknown extension except ValueError as e: raise SystemExit(str(e)) from None if Path(target).exists() and not args.force: msg = f"{target} exists, use --force to overwrite" raise SystemExit(msg) inputs = [_read(spec) for spec in args.sources] singles = [_files.is_single(data) for data in inputs] if any(singles) and not all(singles): msg = "Cannot mix single histograms and files of named histograms" raise SystemExit(msg) try: if all(singles): result: Any = uhi.io.ops.add(*inputs) else: names = dict.fromkeys(name for hists in inputs for name in hists) result = { name: uhi.io.ops.add(*(h[name] for h in inputs if name in h)) for name in names } _files.write(target, result, path=target_path) except _ERRORS as e: raise SystemExit(str(e)) from None def _validate(args: argparse.Namespace) -> None: try: import fastjsonschema # noqa: F401, PLC0415 except ImportError: msg = "uhi validate needs fastjsonschema, install uhi[schema]" raise SystemExit(msg) from None from uhi.schema import main as validate_main # noqa: PLC0415 validate_main(*args.files) def main(argv: Sequence[str] | None = None) -> None: parser = argparse.ArgumentParser(prog="uhi", description=__doc__) subparsers = parser.add_subparsers(dest="command", required=True) add_parser = subparsers.add_parser( "add", help="add histograms from several files (like ROOT's hadd)", description="Sum the histograms in SOURCE files, bin-by-bin, into TARGET. " "Histograms are matched by name. The file extension sets the format " "(.json, .zip, .h5/.hdf5/.hdf, .root). Use file:name to select one " "histogram (or file:dir for a group or directory) in a SOURCE, and " "TARGET:name to name the output. TARGET must not exist unless -f is given.", ) add_parser.add_argument( "-f", "--force", action="store_true", help="overwrite TARGET if it exists" ) add_parser.add_argument("target", help="output file, optionally file:name") add_parser.add_argument("sources", nargs="+", help="input files (file[:path])") add_parser.set_defaults(func=_add) validate_parser = subparsers.add_parser( "validate", help="validate histogram files (JSON, zip, HDF5, or ROOT) against the schema", description="Validate histogram files against the uhi JSON schema. The " "file extension sets the format. Needs the uhi[schema] extra.", ) validate_parser.add_argument( "files", nargs="+", help="histogram files (.json, .zip, .h5/.hdf5/.hdf, .root); use file.h5:group or " "file.root:dir to select a group or directory inside the file", ) validate_parser.set_defaults(func=_validate) args = parser.parse_args(argv) args.func(args) if __name__ == "__main__": main(sys.argv[1:]) uhi-1.2.1/src/uhi/_version.py0000644000000000000000000000101015261523264012776 0ustar00# file generated by vcs-versioning # don't change, don't track in version control from __future__ import annotations __all__ = [ "__version__", "__version_tuple__", "version", "version_tuple", "__commit_id__", "commit_id", ] version: str __version__: str __version_tuple__: tuple[int | str, ...] version_tuple: tuple[int | str, ...] commit_id: str | None __commit_id__: str | None __version__ = version = '1.2.1' __version_tuple__ = version_tuple = (1, 2, 1) __commit_id__ = commit_id = None uhi-1.2.1/src/uhi/_version.pyi0000644000000000000000000000012215261523264013152 0ustar00version: str version_tuple: tuple[int, int, int] | tuple[int, int, int, str, str] uhi-1.2.1/src/uhi/numpy_plottable.py0000644000000000000000000003410015261523264014376 0ustar00""" This file holds the NumPyPlottableHistogram, meant to adapt any histogram that does not support the PlottableHistogram Protocol into a NumPy-backed standin for it, so plotting functions can remain simple and depend on having a PlottableHistogram regardless of the input. And this comes with an adaptor function, ensure_plottable_histogram, which will adapt common input types to a NumPyPlottibleHistogram, and pass a valid PlottibleHistogram through. Keep in mind, NumPyPlottableHistogram is a minimal PlottableHistogram instance, and does not provide any further functionality and is not intended to be used beyond plotting. Please see a full histogram library like boost-histogram or hist. """ from __future__ import annotations import abc import enum import typing from collections.abc import Iterator, Sequence from typing import TYPE_CHECKING, Any import numpy as np from uhi.typing.plottable import ( PlottableAxis, PlottableAxisGeneric, PlottableHistogram, PlottableTraits, ) if TYPE_CHECKING: from numpy.typing import ArrayLike else: ArrayLike = Any class Kind(str, enum.Enum): COUNT = "COUNT" MEAN = "MEAN" class Traits: __slots__ = ("circular", "discrete") def __init__(self, *, circular: bool = False, discrete: bool = False) -> None: self.circular = circular self.discrete = discrete if TYPE_CHECKING: _traits: PlottableTraits = typing.cast(Traits, None) class NumPyPlottableAxis: def __init__(self, edges: np.typing.NDArray[Any]) -> None: """ The vals should already be an Nx2 ndarray of edges. """ self.traits: PlottableTraits = Traits() assert edges.ndim == 2, "Must be 2D array of edges" assert edges.shape[1] == 2, "Second dimension must be 2 (lower, upper)" self.edges = edges def __repr__(self) -> str: """ Just to be nice for debugging. Not required for the Protocol. """ return f"{self.__class__.__name__}({self.edges!r})" def __getitem__(self, index: int) -> tuple[float, float]: """ Get the pair of edges (not discrete) or bin label (discrete). """ return tuple(self.edges[index]) def __len__(self) -> int: """ Return the number of bins (not counting flow bins, which are ignored for this Protocol currently). """ return self.edges.shape[0] # type: ignore[no-any-return] def __eq__(self, other: object) -> bool: """ Needed for the protocol (should be present to be stored in a Sequence). Returns NotImplemented when other has no .edges attribute (so Python can try the reflected operation), and False when shapes differ, to avoid broadcast errors inside np.allclose. """ other_edges = getattr(other, "edges", None) if other_edges is None: return NotImplemented other_edges = np.asarray(other_edges) if self.edges.shape != other_edges.shape: return False return bool(np.allclose(self.edges, other_edges)) def __iter__(self) -> Iterator[tuple[float, float]]: """ A useful part of the Protocol for easy access by plotters. """ return iter(self[t] for t in range(len(self))) __hash__ = None # type: ignore[assignment] if TYPE_CHECKING: _axis: PlottableAxisGeneric[tuple[float, float]] = typing.cast( NumPyPlottableAxis, None ) def _bin_helper(shape: int, bins: np.typing.NDArray[Any] | None) -> NumPyPlottableAxis: """ Returns a axis built from the input bins array, which can be None (0 to N), 2D lower, upper edges), or 1D (N+1 in length). """ if bins is None: return NumPyPlottableAxis( np.column_stack([np.arange(0, shape), np.arange(1, shape + 1)]) ) if bins.ndim == 2: if bins.shape != (shape, 2): msg = f"2D bins must have shape ({shape}, 2) to match the values, not {bins.shape}" raise ValueError(msg) return NumPyPlottableAxis(bins) if bins.ndim == 1: if len(bins) != shape + 1: msg = f"1D bins must have {shape + 1} edges to match {shape} values, not {len(bins)}" raise ValueError(msg) return NumPyPlottableAxis(np.column_stack([bins[:-1], bins[1:]])) msg = "Bins not understood, should be 2d array of min/max edges or 1D array of edges or None" raise ValueError(msg) class NumPyPlottableHistogram: def __init__( self, hist: np.typing.NDArray[Any], *bins: ( np.typing.NDArray[Any] | tuple[np.typing.NDArray[Any] | None, ...] | None ), variances: np.typing.NDArray[Any] | None = None, kind: Kind = Kind.COUNT, ) -> None: self._values = hist self._variances = variances if len(bins) == 1 and isinstance(bins[0], tuple): (bins,) = bins # type: ignore[assignment] if len(bins) == 0: bins = tuple([None] * len(hist.shape)) raw_bins: tuple[np.typing.NDArray[Any] | None, ...] = bins # type: ignore[assignment] if len(raw_bins) > len(hist.shape): msg = f"Too many bin arrays ({len(raw_bins)}) for histogram with {hist.ndim} dimension(s)" raise ValueError(msg) self.kind = kind # zip over only the last len(raw_bins) dimensions, so that a 2-D # "stacked" values array (e.g. multiple 1-D histograms sharing one # axis) is handled naturally: the leading dimension(s) are treated as # a stacking dimension and produce no axis entry. self.axes: Sequence[PlottableAxis] = [ _bin_helper(shape, b) for shape, b in zip(hist.shape[-len(raw_bins) :], raw_bins, strict=True) ] def __repr__(self) -> str: """ Just to be nice for debugging. Not required for the Protocol. """ axes = ", ".join(repr(s) for s in self.axes) return f"{self.__class__.__name__}({self._values!r}, <{axes}>)" def values(self) -> np.typing.NDArray[np.float64]: return self._values def counts(self) -> np.typing.NDArray[np.float64]: return self._values def variances(self) -> np.typing.NDArray[np.float64] | None: return self._variances if TYPE_CHECKING: # Verify that the above class is a valid PlottableHistogram _: PlottableHistogram = typing.cast(NumPyPlottableHistogram, None) def _roottarray_asnumpy( tarr: Any, shape: tuple[int, ...] | None = None ) -> np.typing.NDArray[Any]: llv = tarr.GetArray() arr: np.typing.NDArray[Any] = np.frombuffer( llv, dtype=llv.typecode, count=tarr.GetSize() ) if shape is not None: return np.reshape(arr, shape, order="F") return arr class ROOTAxis(abc.ABC): def __init__(self, tAxis: Any) -> None: self.tAx = tAxis def __len__(self) -> int: return self.tAx.GetNbins() # type: ignore[no-any-return] @abc.abstractmethod def __getitem__(self, index: int) -> Any: pass def __eq__(self, other: object) -> bool: if not isinstance(other, ROOTAxis): return NotImplemented return len(self) == len(other) and all( aEdges == bEdges for aEdges, bEdges in zip(self, other, strict=True) ) @abc.abstractmethod def __iter__(self) -> Iterator[tuple[float, float]] | Iterator[str]: pass @staticmethod def create(tAx: Any) -> DiscreteROOTAxis | ContinuousROOTAxis: nbins = tAx.GetNbins() if nbins and all(tAx.GetBinLabel(i + 1) for i in range(nbins)): return DiscreteROOTAxis(tAx) return ContinuousROOTAxis(tAx) __hash__ = None # type: ignore[assignment] class ContinuousROOTAxis(ROOTAxis): @property def traits(self) -> PlottableTraits: return Traits(circular=False, discrete=False) def __getitem__(self, index: int) -> tuple[float, float]: return (self.tAx.GetBinLowEdge(index + 1), self.tAx.GetBinUpEdge(index + 1)) def __iter__(self) -> Iterator[tuple[float, float]]: for i in range(len(self)): yield self[i] class DiscreteROOTAxis(ROOTAxis): @property def traits(self) -> PlottableTraits: return Traits(circular=False, discrete=True) def __getitem__(self, index: int) -> str: return self.tAx.GetBinLabel(index + 1) # type: ignore[no-any-return] def __iter__(self) -> Iterator[str]: for i in range(len(self)): yield self[i] class ROOTPlottableHistBase: """Common base for ROOT histograms and TProfile""" def __init__(self, thist: Any) -> None: self.thist: Any = thist nDim = thist.GetDimension() self._shape: tuple[int, ...] = tuple( getattr(thist, f"GetNbins{ax}")() + 2 for ax in "XYZ"[:nDim] ) self.axes: tuple[ContinuousROOTAxis | DiscreteROOTAxis, ...] = tuple( ROOTAxis.create(getattr(thist, f"Get{ax}axis")()) for ax in "XYZ"[:nDim] ) @property def name(self) -> str: return self.thist.GetName() # type: ignore[no-any-return] class ROOTPlottableHistogram(ROOTPlottableHistBase): def __init__(self, thist: Any) -> None: super().__init__(thist) @property def hasWeights(self) -> bool: return bool(self.thist.GetSumw2() and self.thist.GetSumw2N()) @property def kind(self) -> str: return Kind.COUNT def values(self) -> np.typing.NDArray[Any]: return _roottarray_asnumpy(self.thist, shape=self._shape)[ tuple([slice(1, -1)] * len(self._shape)) ] def variances(self) -> np.typing.NDArray[Any]: if self.hasWeights: return _roottarray_asnumpy(self.thist.GetSumw2(), shape=self._shape)[ tuple([slice(1, -1)] * len(self._shape)) ] return self.values() def counts(self) -> np.typing.NDArray[Any]: if not self.hasWeights: return self.values() sumw = self.values() result = np.zeros_like(sumw, dtype=np.float64) np.divide( sumw**2, self.variances(), out=result, where=sumw != 0, ) return result class ROOTPlottableProfile(ROOTPlottableHistBase): def __init__(self, thist: Any) -> None: super().__init__(thist) @property def kind(self) -> str: return Kind.MEAN def _per_cell(self, method: str) -> np.typing.NDArray[Any]: func = getattr(self.thist, method) return np.array([func(i) for i in range(self.thist.GetNcells())]).reshape( self._shape, order="F" )[tuple([slice(1, -1)] * len(self._shape))] def values(self) -> np.typing.NDArray[Any]: return self._per_cell("GetBinContent") def variances(self) -> np.typing.NDArray[Any]: # Computed from the sums, so the TProfile error option does not matter. # The sum of w*y**2 is stored in the Sumw2 array. sumw = self._per_cell("GetBinEntries") sumwy2 = _roottarray_asnumpy(self.thist.GetSumw2(), shape=self._shape)[ tuple([slice(1, -1)] * len(self._shape)) ] result = np.full_like(sumw, np.nan, dtype=np.float64) np.divide(sumwy2, sumw, out=result, where=sumw != 0) variances: np.typing.NDArray[Any] = result - self.values() ** 2 return variances def counts(self) -> np.typing.NDArray[Any]: return self._per_cell("GetBinEffectiveEntries") if TYPE_CHECKING: # Verify that the above class is a valid PlottableHistogram _axis = typing.cast(ContinuousROOTAxis, None) _axis2: PlottableAxisGeneric[str] = typing.cast(DiscreteROOTAxis, None) _ = typing.cast(ROOTPlottableHistogram, None) _ = typing.cast(ROOTPlottableProfile, None) def ensure_plottable_histogram(hist: Any) -> PlottableHistogram: """ Ensure a histogram follows the PlottableHistogram Protocol. Currently supports adapting the following inputs: * .to_numpy() objects * .numpy() objects (uproot3/ROOT) * A tuple of NumPy style input. If dd style tuple, must contain "np.typing.NDArray[Any]"s. It can contain None's instead of values, including just a single None for any number of axes. """ if isinstance(hist, PlottableHistogram): # Already satisfies the Protocol return hist if hasattr(hist, "to_numpy"): # Generic (possibly Uproot 4) _tup1: tuple[np.typing.NDArray[Any], ...] = hist.to_numpy(flow=False) return NumPyPlottableHistogram(*_tup1) if hasattr(hist, "numpy"): # uproot/TH1 - TODO: could support variances _tup2: tuple[np.typing.NDArray[Any], ...] = hist.numpy() return NumPyPlottableHistogram(*_tup2) if isinstance(hist, tuple): # NumPy histogram tuple if len(hist) == 1 and np.ndim(hist[0]) == 0: # 0D histogram, like boost-histogram's to_numpy() with no axes return NumPyPlottableHistogram(np.asarray(hist[0])) if len(hist) < 2: msg = "Can't be applied to less than 2D tuple" raise TypeError(msg) if ( len(hist) == 2 and isinstance(hist[1], list | tuple) and all(isinstance(h, np.ndarray) for h in hist[1]) ): # histogramdd tuple return NumPyPlottableHistogram( np.asarray(hist[0]), *(np.asarray(h) for h in hist[1]) ) if hist[1] is None: return NumPyPlottableHistogram( np.asarray(hist[0]), *(None for _ in np.asarray(hist[0]).shape) ) # Standard tuple — preserve None entries so _bin_helper can use the # default 0..N integer-edges axis for that dimension. return NumPyPlottableHistogram( np.asarray(hist[0]), *(None if b is None else np.asarray(b) for b in hist[1:]), ) if hasattr(hist, "InheritsFrom") and hist.InheritsFrom("TH1"): if any( hist.InheritsFrom(profCls) for profCls in ("TProfile", "TProfile2D", "TProfile3D") ): return ROOTPlottableProfile(hist) return ROOTPlottableHistogram(hist) msg = f"Can't be used on this type of object: {hist!r}" raise TypeError(msg) uhi-1.2.1/src/uhi/py.typed0000644000000000000000000000000015261523264012275 0ustar00uhi-1.2.1/src/uhi/schema.py0000644000000000000000000000662315261523264012431 0ustar00from __future__ import annotations import functools import json import sys from collections.abc import Callable from importlib import resources from pathlib import Path from typing import Any from .io import _files _resources = resources.files("uhi") / "resources" histogram_file = _resources / "histogram.schema.json" histograms_file = _resources / "histograms.schema.json" __all__ = [ "histogram_file", "histograms_file", "load", "validate", "validate_histogram", ] def __dir__() -> list[str]: return __all__ _BASE_URI = "https://raw.githubusercontent.com/scikit-hep/uhi/main/src/uhi/resources/" def _load_local(uri: str) -> dict[str, Any]: """ Resolve a ``$ref`` to a schema shipped in ``resources``. The ``$id`` makes refs absolute URLs; they are read from the package, never the network. """ name = uri.removeprefix(_BASE_URI) if "/" in name or ":" in name: msg = f"Cannot resolve {uri!r} without network access" raise ValueError(msg) with (_resources / name).open(encoding="utf-8") as f: return json.load(f) # type: ignore[no-any-return] @functools.cache def _compile(name: str) -> Callable[[dict[str, Any]], None]: import fastjsonschema # noqa: PLC0415 with (_resources / name).open(encoding="utf-8") as f: return fastjsonschema.compile( # type: ignore[no-any-return] json.load(f), handlers={"": _load_local, "https": _load_local} ) def validate(data: dict[str, Any]) -> None: """ Validate a JSON file object against the schema. This accepts a dictionary of named histograms or a single histogram. """ _compile("histograms.schema.json")(data) def validate_histogram(data: dict[str, Any]) -> None: """Validate a single histogram (the IR) against the schema.""" _compile("histogram.schema.json")(data) def _to_json_compatible(data: dict[str, Any]) -> dict[str, Any]: """Convert NumPy arrays and scalars so the schema validator can check them.""" from uhi.io.json import default # noqa: PLC0415 result: dict[str, Any] = json.loads(json.dumps(data, default=default)) return result def load(file: str | Path, /, *, path: str | None = None) -> dict[str, Any]: """ Load all histograms from a file as a ``{name: histogram}`` dict with JSON-compatible values. The format is selected by suffix: ``.json``, ``.zip``, ``.h5``/``.hdf5``/``.hdf``, or ``.root``. ``path`` restricts the search to a group or directory inside the file, or to a single histogram by name. If the file or ``path`` holds a single histogram, that histogram is returned instead of a dict. """ return _to_json_compatible(_files.load(file, path=path, raw=True)) def main(*files: str) -> None: """Validate histogram files.""" import fastjsonschema # noqa: PLC0415 retval = 0 for file in files: try: filename, path = _files.split_spec(file) validate(load(filename, path=path)) except fastjsonschema.JsonSchemaValueException as e: print(f"ERROR {file}: {e.message}") # noqa: T201 retval = 1 except (OSError, ValueError, TypeError, KeyError, ImportError) as e: print(f"ERROR {file}: {e}") # noqa: T201 retval = 1 else: print(f"OK {file}") # noqa: T201 if retval: raise SystemExit(retval) if __name__ == "__main__": main(*sys.argv[1:]) uhi-1.2.1/src/uhi/tag.py0000644000000000000000000000655415261523264011747 0ustar00from __future__ import annotations import copy import sys import typing from typing import Any if sys.version_info < (3, 11): from typing_extensions import Self else: from typing import Self from .typing.plottable import PlottableAxis __all__ = ["Locator", "Overflow", "Underflow", "at", "loc", "rebin"] def __dir__() -> list[str]: return __all__ class Locator: __slots__ = ("offset",) NAME = "" def __init__(self, offset: int = 0) -> None: if not isinstance(offset, int): msg = "The offset must be an integer" # type: ignore[unreachable] raise TypeError(msg) self.offset = offset def __add__(self, offset: int) -> Self: if not isinstance(offset, int): return NotImplemented other = copy.copy(self) other.offset += offset return other def __sub__(self, offset: int) -> Self: if not isinstance(offset, int): return NotImplemented other = copy.copy(self) other.offset -= offset return other def _print_self_(self) -> str: return "" def __repr__(self) -> str: s = self.NAME or self.__class__.__name__ s += self._print_self_() if self.offset != 0: s += " + " if self.offset > 0 else " - " s += str(abs(self.offset)) return s if typing.TYPE_CHECKING: # Type checkers think that this is required def __index__(self) -> int: return 42 class loc(Locator): __slots__ = ("value",) def __init__(self, value: str | float, offset: int = 0) -> None: super().__init__(offset) self.value = value def _print_self_(self) -> str: return f"({self.value})" # TODO: clarify that .index() is required def __call__(self, axis: Any) -> int: return axis.index(self.value) + self.offset # type: ignore[no-any-return] class Underflow(Locator): __slots__ = () NAME = "underflow" def __call__(self, axis: PlottableAxis) -> int: # noqa: ARG002 return -1 + self.offset underflow = Underflow() class Overflow(Locator): __slots__ = () NAME = "overflow" def __call__(self, axis: PlottableAxis) -> int: return len(axis) + self.offset overflow = Overflow() class at: __slots__ = ("value",) def __init__(self, value: int) -> None: self.value = value def __call__(self, axis: PlottableAxis) -> int: # noqa: ARG002 return self.value class rebin: """ When used in the step of a Histogram's slice, rebin(n) combines bins, scaling their widths by a factor of n. If the number of bins is not divisible by n, the remainder is added to the overflow bin. The factor must be a positive integer. """ __slots__ = ("factor",) def __init__(self, factor: int) -> None: if not isinstance(factor, int): msg = "The factor must be an integer" # type: ignore[unreachable] raise TypeError(msg) if factor < 1: msg = f"The factor must be positive, got {factor}" raise ValueError(msg) # Items with .factor are specially treated in boost-histogram, # performing a high performance rebinning self.factor = factor if typing.TYPE_CHECKING: # Type checkers think that this is required def __index__(self) -> int: return 42 uhi-1.2.1/src/uhi/io/__init__.py0000644000000000000000000001224515261523264013334 0ustar00from __future__ import annotations import copy from typing import Any, TypeVar import numpy as np from ..typing.serialization import AnyHistogramIR, AxisIR, HistogramIR __all__ = ["ARRAY_KEYS", "LIST_KEYS", "from_sparse", "remove_writer_info", "to_sparse"] ARRAY_KEYS = frozenset( [ "index", "values", "variances", "edges", "counts", "sum_of_weights", "sum_of_weights_squared", ] ) LIST_KEYS = frozenset( [ "categories", ] ) T = TypeVar("T", bound="dict[str, Any]") H = TypeVar("H", bound="dict[str, Any] | HistogramIR | AnyHistogramIR") def remove_writer_info(obj: T, /, *, library: str | None) -> T: """ Removes all ``writer_info`` for a library from a histogram dict, axes dict, or storage dict. Makes copies where required, and the outer dictionary is always copied. Specify a library name, or ``None`` to remove all. """ obj = copy.copy(obj) if library is None: obj.pop("writer_info", None) elif library in obj.get("writer_info", {}): obj["writer_info"] = copy.copy(obj["writer_info"]) del obj["writer_info"][library] if "axes" in obj: obj["axes"] = [remove_writer_info(ax, library=library) for ax in obj["axes"]] if "storage" in obj: obj["storage"] = remove_writer_info(obj["storage"], library=library) return obj def _compute_axis_length(axis: AxisIR) -> int: match axis["type"]: case "regular": return axis["bins"] + axis["underflow"] + axis["overflow"] case "variable": return len(axis["edges"]) - 1 + axis["underflow"] + axis["overflow"] case "category_str" | "category_int": return len(axis["categories"]) + axis["flow"] case "boolean": return 2 case unknown: # Reachable with input that does not match the schema msg = f"Unknown axis type {unknown!r}" # type: ignore[unreachable] raise ValueError(msg) def _empty_is_zero(storage_type: str, key: str) -> bool: """ Returns True if empty bins should be represented as zero (the common case). Returns False for weighted_mean variances, which use NaN to distinguish empty bins from bins with zero variance. """ return storage_type != "weighted_mean" or key != "variances" def to_sparse(hist: H, /) -> H: """ Convert a dense histogram to a sparse one. Leaves a sparse histogram alone. Leaves empty (metadata-only) histograms alone. """ storage = hist["storage"] storage_type = storage["type"] # Ignore histograms that have 0 dimensions, are already sparse, or are empty if "index" in storage or not hist["axes"] or len(storage) == 1: return hist # Get the arrays inside storage, ignoring "type" arrays = {k: np.asarray(v) for k, v in storage.items() if k != "type"} # Build mask of nonzero bins across *all* present keys mask = np.any( [ arr != 0 if _empty_is_zero(storage_type, k) else ~np.isnan(arr) for k, arr in arrays.items() ], axis=0, ) # Get the flat indices (or unravel them) nonzero_indices = np.nonzero(mask) # Pack indices into a single (ndim, n_nonzero) array index = np.vstack(nonzero_indices) # Build sparse storage dict sparse_storage = {"type": storage_type, "index": index} for k, arr in arrays.items(): sparse_storage[k] = arr[mask] # Return new histogram dict with modified storage sparse_hist = copy.copy(hist) sparse_hist["storage"] = sparse_storage # type: ignore[arg-type] return sparse_hist def from_sparse(sparse: H, /) -> H: """ Convert sparse histogram data back to dense format. If the histogram is already dense, just return it. """ storage = sparse["storage"] storage_type = storage["type"] index = storage.get("index") if index is None: return sparse # An empty index is read from JSON as float index = np.asarray(index, dtype=np.intp) ndim, _n_nonzero = index.shape shape = [_compute_axis_length(a) for a in sparse["axes"]] # type: ignore[arg-type] if len(shape) != ndim: msg = f"Shape {shape} does not match sparse index dimension {ndim}" raise ValueError(msg) dense_storage = {"type": storage["type"]} for k, arr1d in storage.items(): if k in {"index", "type"}: continue arr1dnp = np.asarray(arr1d) empty_is_zero = _empty_is_zero(storage_type, k) dtype = arr1dnp.dtype if arr1dnp.size == 0 and storage_type == "int" and dtype.kind == "f": # Empty JSON arrays read as float dtype = np.dtype(np.int64) elif not empty_is_zero and dtype.kind != "f": # NaN fill needs a float dtype dtype = np.dtype(np.float64) # Allocate a zeros (or nan) array of the original shape full = np.full(shape, 0 if empty_is_zero else np.nan, dtype=dtype) # Scatter sparse values back into dense array full[tuple(index)] = arr1dnp dense_storage[k] = full retval = copy.copy(sparse) retval["storage"] = dense_storage # type: ignore[arg-type] return retval uhi-1.2.1/src/uhi/io/_common.py0000644000000000000000000000256315261523264013226 0ustar00""" Common helpers for the different formats. """ from __future__ import annotations import typing from ..typing.serialization import AnyAxisIR, AnyHistogramIR, ToUHIHistogram __all__ = ["_check_uhi_schema_version", "_convert_input"] def _check_uhi_schema_version(uhi_schema: int, /) -> None: if uhi_schema != 1: msg = "Only uhi_schema=1 supported in this uhi version. Please update uhi." raise TypeError(msg) def _remove_empty_metadata(hist: AnyHistogramIR, /) -> AnyHistogramIR: if "metadata" in hist and not hist["metadata"]: hist = typing.cast( AnyHistogramIR, {k: v for k, v in hist.items() if k != "metadata"} ) # Copy the axes list before reassigning elements so the caller's list is # not mutated in place. axes = list(hist["axes"]) for i, axis in enumerate(axes): if "metadata" in axis and not axis["metadata"]: axes[i] = typing.cast( AnyAxisIR, {k: v for k, v in axis.items() if k != "metadata"} ) return typing.cast(AnyHistogramIR, {**hist, "axes": axes}) def _convert_input(hist: AnyHistogramIR | ToUHIHistogram, /) -> AnyHistogramIR: any_hist = typing.cast( AnyHistogramIR, hist._to_uhi_() if isinstance(hist, ToUHIHistogram) else hist ) _check_uhi_schema_version(any_hist["uhi_schema"]) return _remove_empty_metadata(any_hist) uhi-1.2.1/src/uhi/io/_files.py0000644000000000000000000002517315261523264013042 0ustar00""" Read and write whole histogram files in any supported format. A file holds either a dict of named histograms or a single histogram. A spec like ``file.h5:path`` selects a group, directory, or histogram inside the file. """ from __future__ import annotations import functools import importlib.metadata import json import re import zipfile from collections.abc import Mapping from pathlib import Path from typing import Any __all__ = ["file_format", "is_single", "load", "split_spec", "write"] def __dir__() -> list[str]: return __all__ _SPEC_RE = re.compile(r"^(.+?\.(?:json|zip|h5|hdf5|hdf|root)):(.*)$", re.IGNORECASE) _RNTUPLE_CLASSES = frozenset(["ROOT::RNTuple", "ROOT::Experimental::RNTuple"]) _DIRECTORY_CLASSES = frozenset(["TDirectory", "TDirectoryFile"]) _SUFFIXES = "expected .json, .zip, .h5/.hdf5/.hdf, or .root" def split_spec(spec: str, /) -> tuple[str, str | None]: """Split ``file.root:dir/name`` into ``("file.root", "dir/name")``.""" if match := _SPEC_RE.match(spec): if not match[2]: msg = f"Empty name after ':' in {spec!r}" raise ValueError(msg) return match[1], match[2] return spec, None def file_format(path: Path, /) -> str: match path.suffix.lower(): case ".json": return "json" case ".zip": return "zip" case ".h5" | ".hdf5" | ".hdf": return "hdf5" case ".root": return "root" case "": msg = f"No file extension in {str(path)!r}, {_SUFFIXES}" raise ValueError(msg) case suffix: msg = f"Unknown file format {suffix!r}, {_SUFFIXES}" raise ValueError(msg) def is_single(data: Any, /) -> bool: """True for a single histogram, False for a dict of named histograms.""" return isinstance(data, Mapping) and "uhi_schema" in data def _load_json(path: Path, subpath: str | None, *, raw: bool = False) -> Any: from .json import object_hook # noqa: PLC0415 with path.open(encoding="utf-8") as f: data = json.load(f, object_hook=None if raw else object_hook) if subpath is None: return data subpath = subpath.strip("/") if isinstance(data, Mapping) and not is_single(data): if subpath in data: return data[subpath] # Flat "dir/name" keys, as written by write(..., path="dir") prefix = f"{subpath}/" found = { k.removeprefix(prefix): v for k, v in data.items() if k.startswith(prefix) } if found: return found msg = f"{subpath!r} not found in {path}" raise KeyError(msg) def _load_zip(path: Path, subpath: str | None) -> Any: from . import zip as uhi_zip # noqa: PLC0415 try: zip_file = zipfile.ZipFile(path) except zipfile.BadZipFile as e: msg = f"{path}: {e}" raise ValueError(msg) from e with zip_file: names = [n[:-5] for n in zip_file.namelist() if n.endswith(".json")] if subpath is None: return {name: uhi_zip.read(zip_file, name) for name in names} subpath = subpath.strip("/") if subpath in names: return uhi_zip.read(zip_file, subpath) prefix = f"{subpath}/" found = { n.removeprefix(prefix): uhi_zip.read(zip_file, n) for n in names if n.startswith(prefix) } if not found: msg = f"{subpath!r} not found in {path}" raise KeyError(msg) return found def _load_hdf5(path: Path, subpath: str | None) -> Any: import h5py # noqa: PLC0415 from . import hdf5 # noqa: PLC0415 hists: dict[str, Any] = {} def visit(name: str, obj: Any) -> None: if isinstance(obj, h5py.Group) and "uhi_schema" in obj.attrs: hists[name] = hdf5.read(obj) with h5py.File(path, "r") as h5_file: try: start = h5_file[subpath] if subpath else h5_file except KeyError: msg = f"{subpath!r} not found in {path}" raise KeyError(msg) from None if not isinstance(start, h5py.Group): msg = f"{subpath!r} in {path} is not a histogram or group" raise ValueError(msg) # noqa: TRY004 if "uhi_schema" in start.attrs: return hdf5.read(start) start.visititems(visit) return hists def _import_root() -> Any: """Import PyROOT, the fallback when uproot 5.7+ is not installed.""" try: import ROOT # noqa: PLC0415 except ModuleNotFoundError as err: msg = ( "ROOT files need uproot 5.7+ (pip install 'uhi[uproot]') or " "PyROOT (conda install -c conda-forge root)" ) raise ModuleNotFoundError(msg, name=err.name) from err return ROOT @functools.cache def _uproot_available() -> bool: """Uproot 5.7+ is preferred for ROOT files; PyROOT is the fallback.""" try: version = importlib.metadata.version("uproot") except importlib.metadata.PackageNotFoundError: return False return tuple(int(v) for v in re.findall(r"\d+", version)[:2]) >= (5, 7) def _load_uproot(path: Path, subpath: str | None) -> Any: import uproot # noqa: PLC0415 from . import uproot as uhi_uproot # noqa: PLC0415 def visit(directory: Any) -> dict[str, Any]: return { name: uhi_uproot.read(directory, name) for name in directory.keys( recursive=True, cycle=False, filter_classname=_RNTUPLE_CLASSES ) } with uproot.open(path) as root_file: if not subpath: return visit(root_file) subpath = subpath.strip("/") if subpath not in root_file: msg = f"{subpath!r} not found in {path}" raise KeyError(msg) # classname_of fails for nested directories classname = root_file.key(subpath).fClassName if classname in _RNTUPLE_CLASSES: return uhi_uproot.read(root_file, subpath) if classname not in _DIRECTORY_CLASSES: msg = f"{subpath!r} in {path} is not a histogram or directory" raise ValueError(msg) return visit(root_file[subpath]) def _load_root(path: Path, subpath: str | None) -> Any: if _uproot_available(): return _load_uproot(path, subpath) ROOT = _import_root() from . import root # noqa: PLC0415 hists: dict[str, Any] = {} def visit(directory: Any, prefix: str) -> None: for key in directory.GetListOfKeys(): name = key.GetName() class_name = key.GetClassName() if class_name in _RNTUPLE_CLASSES: hists[f"{prefix}{name}"] = root.read(directory, name) elif class_name in _DIRECTORY_CLASSES: visit(directory.Get(name), f"{prefix}{name}/") root_file = ROOT.TFile.Open(str(path)) if not root_file: msg = f"Could not open {path} as a ROOT file" raise OSError(msg) with root_file: if not subpath: visit(root_file, "") return hists parent, _, name = subpath.rstrip("/").rpartition("/") directory = root_file.Get(parent) if parent else root_file key = directory.GetKey(name) if directory else None if not key: msg = f"{subpath!r} not found in {path}" raise KeyError(msg) class_name = key.GetClassName() if class_name in _DIRECTORY_CLASSES: visit(directory.Get(name), "") return hists if class_name not in _RNTUPLE_CLASSES: msg = f"{subpath!r} in {path} is not a histogram or directory" raise ValueError(msg) return root.read(directory, name) def load(file: str | Path, /, *, path: str | None = None, raw: bool = False) -> Any: """ Load a file as a ``{name: histogram}`` dict, or as a single histogram if the file (or ``path`` inside it) holds only one. Arrays are NumPy arrays, except that ``raw`` keeps JSON files exactly as written so that validation sees the original types. The format is selected by suffix. """ filepath = Path(file) match file_format(filepath): case "json": return _load_json(filepath, path, raw=raw) case "zip": return _load_zip(filepath, path) case "hdf5": return _load_hdf5(filepath, path) case _: return _load_root(filepath, path) def write(file: str | Path, data: Any, /, *, path: str | None = None) -> None: """ Write a ``{name: histogram}`` dict, or a single histogram, to a new file. ``path`` is a prefix for the names, or the name of a single histogram; it is required for a single histogram in the zip and ROOT formats. """ filepath = Path(file) fmt = file_format(filepath) prefix = f"{path.strip('/')}/" if path else "" if is_single(data): if path: data = {path.strip("/"): data} elif fmt in {"zip", "root"}: msg = f"A name is needed to write a single histogram, use {filepath}:name" raise ValueError(msg) else: data = {f"{prefix}{name}": hist for name, hist in data.items()} match fmt: case "json": from .json import default # noqa: PLC0415 with filepath.open("w", encoding="utf-8") as f: json.dump(data, f, default=default, indent=2) case "zip": from . import zip as uhi_zip # noqa: PLC0415 with zipfile.ZipFile(filepath, "w") as zip_file: for name, hist in data.items(): uhi_zip.write(zip_file, name, hist) case "hdf5": import h5py # noqa: PLC0415 from . import hdf5 # noqa: PLC0415 with h5py.File(filepath, "w") as h5_file: if is_single(data): hdf5.write(h5_file, data) else: for name, hist in data.items(): hdf5.write(h5_file.create_group(name), hist) case _ if _uproot_available(): import uproot # noqa: PLC0415 from . import uproot as uhi_uproot # noqa: PLC0415 with uproot.recreate(filepath) as root_file: for name, hist in data.items(): uhi_uproot.write(root_file, name, hist) case _: ROOT = _import_root() from . import root # noqa: PLC0415 with ROOT.TFile.Open(str(filepath), "RECREATE") as root_file: for name, hist in data.items(): directory, _, base = name.rpartition("/") target = ( root_file.mkdir(directory, "", True) if directory else root_file ) root.write(target, base, hist) uhi-1.2.1/src/uhi/io/_rntuple.py0000644000000000000000000000742415261523264013430 0ustar00""" Shared layout for the RNTuple format, used by both the ROOT and uproot backends. A histogram is one entry: a ``uhi`` string field holds the IR as JSON, with each array replaced by the name of the vector field holding it. """ from __future__ import annotations import json from collections.abc import Callable from typing import Any import numpy as np from ..typing.serialization import AnyHistogramIR, ToUHIHistogram from . import ARRAY_KEYS, _compute_axis_length from ._common import _check_uhi_schema_version, _convert_input __all__ = ["FIELD_TYPES", "from_fields", "to_fields"] def __dir__() -> list[str]: return __all__ # Supported array dtypes and their C++ RNTuple field types FIELD_TYPES = { "float64": "double", "float32": "float", "int64": "std::int64_t", "int32": "std::int32_t", "uint64": "std::uint64_t", "uint32": "std::uint32_t", } def to_fields( histogram: AnyHistogramIR | ToUHIHistogram, / ) -> tuple[str, dict[str, np.ndarray[Any, Any]]]: """ Split a histogram into the ``uhi`` JSON string and a dict of flat arrays. """ histogram = _convert_input(histogram) # Copy the histogram and the dicts/lists we mutate below so the caller's # arrays are not replaced with field names. histogram = histogram.copy() arrays: dict[str, np.ndarray[Any, Any]] = {} storage = histogram["storage"].copy() histogram["storage"] = storage for storage_key in ARRAY_KEYS & storage.keys(): arrays[storage_key] = np.asarray(storage[storage_key]) # type: ignore[literal-required] storage[storage_key] = storage_key # type: ignore[literal-required] axes = [axis.copy() for axis in histogram["axes"]] histogram["axes"] = axes for i, axis in enumerate(axes): for key in ARRAY_KEYS & axis.keys(): field = f"axis_{i}_{key}" arrays[field] = np.asarray(axis[key]) # type: ignore[literal-required] axis[key] = field # type: ignore[literal-required] for field, array in arrays.items(): # RNTuple has no small integer fields here; upcast like JSON would if array.dtype.kind in "bi" and array.dtype.itemsize < 4: array = array.astype(np.int64) # noqa: PLW2901 elif array.dtype.kind == "u" and array.dtype.itemsize < 4: array = array.astype(np.uint64) # noqa: PLW2901 elif array.dtype == np.float16: array = array.astype(np.float32) # noqa: PLW2901 if array.dtype.name not in FIELD_TYPES: msg = f"Unsupported array dtype {array.dtype} for {field}" raise TypeError(msg) # Awkward and RNTuple need native byte order arrays[field] = array.astype(array.dtype.newbyteorder("="), copy=False).ravel() return json.dumps(histogram), arrays def from_fields( uhi: str, get_array: Callable[[str], np.ndarray[Any, Any]], / ) -> dict[str, Any]: """ Rebuild a histogram from the ``uhi`` JSON string, reading each array field with ``get_array``. """ output: dict[str, Any] = json.loads(uhi) # Only storage and axes contain array references; metadata and writer # info may use the same keys for arbitrary strings. for obj in [output["storage"], *output["axes"]]: for key in ARRAY_KEYS & obj.keys(): if isinstance(obj[key], str): obj[key] = get_array(obj[key]) _check_uhi_schema_version(output["uhi_schema"]) # Arrays are stored flattened; the shape is recovered from the axes storage = output["storage"] if "index" in storage: storage["index"] = storage["index"].reshape(len(output["axes"]), -1) else: shape = [_compute_axis_length(axis) for axis in output["axes"]] for key in ARRAY_KEYS & storage.keys(): storage[key] = storage[key].reshape(shape) return output uhi-1.2.1/src/uhi/io/hdf5.py0000644000000000000000000001627615261523264012433 0ustar00from __future__ import annotations import typing from typing import Any import h5py import numpy as np from ..typing.serialization import ( AnyAxisIR, AnyHistogramIR, AnyStorageIR, HistogramIR, SupportedMetadata, ToUHIHistogram, ) from . import ARRAY_KEYS from ._common import _check_uhi_schema_version, _convert_input __all__ = ["read", "write"] def __dir__() -> list[str]: return __all__ def _handle_metadata_writer_info( grp: h5py.Group, metadata: dict[str, SupportedMetadata] | None, writer_info: dict[str, dict[str, SupportedMetadata]] | None, ) -> None: # Metadata if metadata: metadata_grp = grp.create_group("metadata") for key, val1 in metadata.items(): metadata_grp.attrs[key] = val1 # Writer info if writer_info: writer_info_grp = grp.create_group("writer_info") for key, value in writer_info.items(): inner_wi_grp = writer_info_grp.create_group(key) for k, v in value.items(): inner_wi_grp.attrs[k] = v def _create_dataset( group: h5py.Group, name: str, data: Any, *, compression: str, compression_opts: int, min_compress_elements: int, ) -> None: """ Create an HDF5 dataset, applying compression only when the element count meets the minimum threshold. ``data`` may be a NumPy array, a scalar, or a (possibly nested) Python list (e.g. string categories). The element count comes from ``np.asarray(data).size``, but the original ``data`` goes to h5py unchanged. """ arr = np.asarray(data) # h5py cannot compress scalar (0-d) datasets if arr.ndim == 0 or arr.size < min_compress_elements: group.create_dataset(name, data=data) else: group.create_dataset( name, data=data, compression=compression, compression_opts=compression_opts, ) def write( grp: h5py.Group, /, histogram: AnyHistogramIR | ToUHIHistogram, *, compression: str = "gzip", compression_opts: int = 4, min_compress_elements: int = 1_000, ) -> None: """ Write a histogram to an HDF5 group. Arrays larger than `min_compress_elements` will be compressed; set to 0 to compress all arrays. The `compression` and `compression_opts` arguments are passed through. """ histogram = _convert_input(histogram) # All referenced objects will be stored inside of /{name}/ref_axes hist_folder_storage = grp.create_group("ref_axes") # UHI version number grp.attrs["uhi_schema"] = histogram["uhi_schema"] _handle_metadata_writer_info( grp, histogram.get("metadata"), histogram.get("writer_info") ) # Axes axes_dataset = grp.create_dataset( "axes", len(histogram["axes"]), dtype=h5py.special_dtype(ref=h5py.Reference) ) for i, axis in enumerate(histogram["axes"]): # Iterating through the axes, calling `create_axes_object` for each of them, # creating references to new groups and appending it to the `items` dataset defined above ax_group = hist_folder_storage.create_group(f"axis_{i}") ax_info = axis.copy() ax_edges_raw = ax_info.pop("edges", None) ax_edges = np.asarray(ax_edges_raw) if ax_edges_raw is not None else None ax_cats: list[int] | list[str] | None = ax_info.pop("categories", None) _handle_metadata_writer_info( ax_group, ax_info.pop("metadata", None), ax_info.pop("writer_info", None) ) for key, val2 in ax_info.items(): ax_group.attrs[key] = val2 if ax_edges is not None: _create_dataset( ax_group, "edges", ax_edges, compression=compression, compression_opts=compression_opts, min_compress_elements=min_compress_elements, ) if ax_cats is not None: _create_dataset( ax_group, "categories", ax_cats, compression=compression, compression_opts=compression_opts, min_compress_elements=min_compress_elements, ) axes_dataset[i] = ax_group.ref # Storage storage_grp = grp.create_group("storage") storage_type = histogram["storage"]["type"] storage_grp.attrs["type"] = storage_type for key, val3 in histogram["storage"].items(): if key == "type": continue _create_dataset( storage_grp, key, val3, compression=compression, compression_opts=compression_opts, min_compress_elements=min_compress_elements, ) def _convert_item(name: str, item: Any, /) -> Any: """ Convert an HDF5 item to a native Python type. """ if isinstance(item, bytes): return item.decode("utf-8") if name == "metadata": return {k: _convert_item("", v) for k, v in item.items()} if name in ARRAY_KEYS: return item if isinstance(item, np.generic): return item.item() return item def _read_metadata_writer_info( output: AnyHistogramIR | AnyAxisIR, group: h5py.Group | h5py.Dataset | h5py.Datatype ) -> None: if "metadata" in group: output["metadata"] = _convert_item("metadata", group["metadata"].attrs) if "writer_info" in group: output["writer_info"] = { k: _convert_item("metadata", v.attrs) for k, v in group["writer_info"].items() } def _convert_axes(group: h5py.Group | h5py.Dataset | h5py.Datatype) -> AnyAxisIR: """ Convert an HDF5 axis reference to a dictionary. """ assert isinstance(group, h5py.Group) axis = typing.cast( AnyAxisIR, {k: _convert_item(k, v) for k, v in group.attrs.items()} ) if "edges" in group: edges = group["edges"] assert isinstance(edges, h5py.Dataset) axis["edges"] = np.asarray(edges) if "categories" in group: categories = group["categories"] assert isinstance(categories, h5py.Dataset) axis["categories"] = [_convert_item("", c) for c in categories] _read_metadata_writer_info(axis, group) return axis def read(grp: h5py.Group, /) -> HistogramIR: """ Read a histogram from an HDF5 group. """ uhi_schema = _convert_item("", grp.attrs["uhi_schema"]) _check_uhi_schema_version(uhi_schema) axes_grp = grp["axes"] axes_ref = grp["ref_axes"] assert isinstance(axes_ref, h5py.Group) assert isinstance(axes_grp, h5py.Dataset) # Dereference the ordered ``axes`` dataset rather than iterating the # ``ref_axes`` group, which h5py yields in alphabetical (not numeric) order. axes = [_convert_axes(axes_ref[ref]) for ref in axes_grp] storage_grp = grp["storage"] assert isinstance(storage_grp, h5py.Group) storage = AnyStorageIR(type=_convert_item("", storage_grp.attrs["type"])) for key in storage_grp: storage[key] = np.asarray(storage_grp[key]) # type: ignore[literal-required] histogram_dict = AnyHistogramIR(uhi_schema=uhi_schema, axes=axes, storage=storage) _read_metadata_writer_info(histogram_dict, grp) return histogram_dict # type: ignore[return-value] uhi-1.2.1/src/uhi/io/json.py0000644000000000000000000000150415261523264012542 0ustar00from __future__ import annotations from typing import Any import numpy as np from . import ARRAY_KEYS from ._common import _convert_input __all__ = ["default", "object_hook"] def __dir__() -> list[str]: return __all__ def default(obj: Any, /) -> Any: if hasattr(obj, "_to_uhi_"): return _convert_input(obj) if isinstance(obj, np.ndarray): return obj.tolist() # Convert ndarray to list if isinstance(obj, np.generic): return obj.item() msg = f"Object of type {type(obj)} is not JSON serializable" raise TypeError(msg) def object_hook(dct: dict[str, Any], /) -> dict[str, Any]: """ Decode a histogram from a dictionary. """ for item in ARRAY_KEYS & dct.keys(): if isinstance(dct[item], list): dct[item] = np.asarray(dct[item]) return dct uhi-1.2.1/src/uhi/io/ops.py0000644000000000000000000001503215261523264012373 0ustar00""" Operations on histograms in the intermediate representation. """ from __future__ import annotations import copy import functools from typing import Any import numpy as np from ..typing.serialization import ( AnyAxisIR, AnyHistogramIR, AnyStorageIR, ToUHIHistogram, ) from . import from_sparse, to_sparse from ._common import _convert_input __all__ = ["add"] def __dir__() -> list[str]: return __all__ def _strip_axis(axis: AnyAxisIR, /) -> dict[str, Any]: return {k: v for k, v in axis.items() if k not in {"metadata", "writer_info"}} def _axes_equal(a: AnyAxisIR, b: AnyAxisIR, /) -> bool: """ Compare two axes, ignoring metadata and writer_info. """ a_dict = _strip_axis(a) b_dict = _strip_axis(b) if a_dict.keys() != b_dict.keys(): return False return all( np.array_equal(a_dict[k], b_dict[k]) if k == "edges" else a_dict[k] == b_dict[k] for k in a_dict ) def _add_mean(a: AnyStorageIR, b: AnyStorageIR, /) -> AnyStorageIR: """ Merge two mean storages. ``variances`` holds the sample variance, ``M2 / (counts - 1)``, so it is converted back to ``M2`` (the sum of squared deviations) before merging with the parallel-variance formula. """ n_a, n_b = np.asarray(a["counts"]), np.asarray(b["counts"]) m_a, m_b = np.asarray(a["values"]), np.asarray(b["values"]) v_a, v_b = np.asarray(a["variances"]), np.asarray(b["variances"]) n = n_a + n_b with np.errstate(divide="ignore", invalid="ignore"): m = np.divide( n_a * m_a + n_b * m_b, n, out=np.zeros_like(n, dtype=float), where=n != 0 ) # The variance is undefined (0/0) for fewer than two entries m2_a = np.where(n_a > 1, v_a * (n_a - 1), 0.0) m2_b = np.where(n_b > 1, v_b * (n_b - 1), 0.0) m2 = ( m2_a + m2_b + np.divide( (m_a - m_b) ** 2 * n_a * n_b, n, out=np.zeros_like(n, dtype=float), where=n != 0, ) ) v = m2 / (n - 1) return { "type": "mean", "counts": np.asarray(n), "values": np.asarray(m), "variances": np.asarray(v), } def _add_weighted_mean(a: AnyStorageIR, b: AnyStorageIR, /) -> AnyStorageIR: """ Merge two weighted_mean storages. ``variances`` holds ``M2 / (sum_of_weights - sum_of_weights_squared / sum_of_weights)``, so it is converted back to ``M2`` before merging. """ w_a, w_b = np.asarray(a["sum_of_weights"]), np.asarray(b["sum_of_weights"]) w2_a, w2_b = ( np.asarray(a["sum_of_weights_squared"]), np.asarray(b["sum_of_weights_squared"]), ) m_a, m_b = np.asarray(a["values"]), np.asarray(b["values"]) v_a, v_b = np.asarray(a["variances"]), np.asarray(b["variances"]) w = w_a + w_b w2 = w2_a + w2_b with np.errstate(divide="ignore", invalid="ignore"): m = np.divide( w_a * m_a + w_b * m_b, w, out=np.zeros_like(w, dtype=float), where=w != 0 ) d_a = w_a - np.divide( w2_a, w_a, out=np.zeros_like(w, dtype=float), where=w_a != 0 ) d_b = w_b - np.divide( w2_b, w_b, out=np.zeros_like(w, dtype=float), where=w_b != 0 ) # The variance is undefined (0/0) when the denominator is zero m2_a = np.where(d_a != 0, v_a * d_a, 0.0) m2_b = np.where(d_b != 0, v_b * d_b, 0.0) m2 = ( m2_a + m2_b + np.divide( (m_a - m_b) ** 2 * w_a * w_b, w, out=np.zeros_like(w, dtype=float), where=w != 0, ) ) v = m2 / (w - w2 / w) return { "type": "weighted_mean", "sum_of_weights": np.asarray(w), "sum_of_weights_squared": np.asarray(w2), "values": np.asarray(m), "variances": np.asarray(v), } def _add_storage(a: AnyStorageIR, b: AnyStorageIR, /) -> AnyStorageIR: """ Add two dense, non-empty storages of the same type. Results are always arrays; NumPy scalars (from zero-dimensional histograms) are not JSON serializable. """ match a["type"]: case "int" | "double": return { "type": a["type"], "values": np.asarray(np.add(a["values"], b["values"])), } case "weighted": return { "type": "weighted", "values": np.asarray(np.add(a["values"], b["values"])), "variances": np.asarray(np.add(a["variances"], b["variances"])), } case "mean": return _add_mean(a, b) case "weighted_mean": return _add_weighted_mean(a, b) case unknown: # Reachable with input that does not match the schema msg = f"Unknown storage type {unknown!r}" # type: ignore[unreachable] raise ValueError(msg) def add( first: AnyHistogramIR | ToUHIHistogram, /, *rest: AnyHistogramIR | ToUHIHistogram ) -> AnyHistogramIR: """ Add histograms bin-by-bin, like ROOT's ``hadd``. All histograms must have identical axes (metadata and ``writer_info`` are ignored in the comparison) and the same storage type. Empty (metadata-only) storages count as zero. Metadata and ``writer_info`` are taken from the first histogram. The result is sparse if every input is sparse, and dense otherwise. .. versionadded:: 1.2 """ hists = [_convert_input(h) for h in (first, *rest)] base = hists[0] for i, hist in enumerate(hists[1:], start=1): if hist["storage"]["type"] != base["storage"]["type"]: msg = f"Histogram {i} has storage type {hist['storage']['type']!r}, expected {base['storage']['type']!r}" raise ValueError(msg) # zip() is shadowed by the uhi.io.zip submodule, so index instead if len(hist["axes"]) != len(base["axes"]) or not all( _axes_equal(hist["axes"][j], base["axes"][j]) for j in range(len(base["axes"])) ): msg = f"Histogram {i} has axes that do not match histogram 0" raise ValueError(msg) all_sparse = all("index" in h["storage"] for h in hists) storages = [from_sparse(h)["storage"] for h in hists] non_empty = [s for s in storages if len(s) > 1] if non_empty: storage = functools.reduce(_add_storage, non_empty) else: storage = AnyStorageIR(type=base["storage"]["type"]) result = copy.copy(base) result["storage"] = storage return to_sparse(result) if all_sparse else result uhi-1.2.1/src/uhi/io/root.py0000644000000000000000000000305015261523264012552 0ustar00from __future__ import annotations from typing import Any import numpy as np import ROOT from ..typing.serialization import AnyHistogramIR, ToUHIHistogram from ._rntuple import FIELD_TYPES, from_fields, to_fields __all__ = ["read", "write"] def __dir__() -> list[str]: return __all__ def write( directory: ROOT.TDirectory, /, name: str, histogram: AnyHistogramIR | ToUHIHistogram, ) -> None: """ Write a histogram to a ROOT directory as a single-entry RNTuple. The histogram JSON is stored in the ``uhi`` field; each array is stored flattened in a vector field of the same name. """ uhi, arrays = to_fields(histogram) model = ROOT.RNTupleModel.Create() model.MakeField["std::string"]("uhi") vectors = {} for field, array in arrays.items(): ctype = FIELD_TYPES[array.dtype.name] model.MakeField[f"std::vector<{ctype}>"](field) vectors[field] = ROOT.std.vector[ctype](array) with ROOT.RNTupleWriter.Append(model, name, directory) as writer: entry = writer.CreateEntry() entry["uhi"] = uhi for field, vector in vectors.items(): entry[field] = ROOT.std.move(vector) writer.Fill(entry) def read(directory: ROOT.TDirectory, /, name: str) -> dict[str, Any]: """ Read a histogram from a ROOT directory. """ with ROOT.RNTupleReader.Open(directory.Get(name)) as reader: entry = reader.CreateEntry() reader.LoadEntry(0, entry) return from_fields(str(entry["uhi"]), lambda field: np.array(entry[field])) uhi-1.2.1/src/uhi/io/uproot.py0000644000000000000000000000204415261523264013121 0ustar00from __future__ import annotations from typing import Any import awkward as ak import numpy as np import uproot from ..typing.serialization import AnyHistogramIR, ToUHIHistogram from ._rntuple import from_fields, to_fields __all__ = ["read", "write"] def __dir__() -> list[str]: return __all__ def write( directory: uproot.WritableDirectory, /, name: str, histogram: AnyHistogramIR | ToUHIHistogram, ) -> None: """ Write a histogram to an uproot directory as a single-entry RNTuple, in the same layout as :func:`uhi.io.root.write`. """ uhi, arrays = to_fields(histogram) data = {"uhi": ak.Array([uhi])} for field, array in arrays.items(): data[field] = ak.unflatten(array, [len(array)]) directory.mkrntuple(name, data) def read(directory: uproot.ReadOnlyDirectory, /, name: str) -> dict[str, Any]: """ Read a histogram from an uproot directory. """ entry = directory[name].arrays(entry_stop=1)[0] return from_fields(entry["uhi"], lambda field: np.asarray(entry[field])) uhi-1.2.1/src/uhi/io/zip.py0000644000000000000000000000467515261523264012407 0ustar00from __future__ import annotations import json import zipfile from typing import Any import numpy as np from ..typing.serialization import AnyHistogramIR, ToUHIHistogram from . import ARRAY_KEYS from ._common import _check_uhi_schema_version, _convert_input __all__ = ["read", "write"] def __dir__() -> list[str]: return __all__ def write( zip_file: zipfile.ZipFile, /, name: str, histogram: AnyHistogramIR | ToUHIHistogram, ) -> None: """ Write a histogram to a zip file. """ histogram = _convert_input(histogram) # Copy the histogram and the dicts/lists we mutate below so the caller's # arrays are not replaced with path strings. histogram = histogram.copy() # Write out numpy arrays to files in the zipfile storage = histogram["storage"].copy() histogram["storage"] = storage for storage_key in ARRAY_KEYS & storage.keys(): path = f"{name}_storage_{storage_key}.npy" with zip_file.open(path, "w") as f: np.save(f, storage[storage_key]) # type: ignore[literal-required] storage[storage_key] = path # type: ignore[literal-required] axes = [axis.copy() for axis in histogram["axes"]] histogram["axes"] = axes for i, axis in enumerate(axes): for key in ARRAY_KEYS & axis.keys(): path = f"{name}_axis_{i}_{key}.npy" with zip_file.open(path, "w") as f: np.save(f, axis[key]) # type: ignore[literal-required] axis[key] = path # type: ignore[literal-required] hist_json = json.dumps(histogram) zip_file.writestr(f"{name}.json", hist_json) def _load_arrays(dct: dict[str, Any], /, *, zip_file: zipfile.ZipFile) -> None: """ Replace the stored paths with the arrays they point at, in-place. """ for item in ARRAY_KEYS & dct.keys(): if isinstance(dct[item], str): dct[item] = np.load(zip_file.open(dct[item])) def read(zip_file: zipfile.ZipFile, /, name: str) -> dict[str, Any]: """ Read histograms from a zip file. """ with zip_file.open(f"{name}.json") as f: output: dict[str, Any] = json.load(f) _check_uhi_schema_version(output["uhi_schema"]) # Only storage and axes hold arrays; metadata and writer_info can contain # any keys, including ones that look like array keys. _load_arrays(output["storage"], zip_file=zip_file) for axis in output["axes"]: _load_arrays(axis, zip_file=zip_file) return output uhi-1.2.1/src/uhi/resources/histogram.schema.json0000644000000000000000000002727215261523264016763 0ustar00{ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://raw.githubusercontent.com/scikit-hep/uhi/main/src/uhi/resources/histogram.schema.json", "title": "Histogram", "description": "A single histogram, the intermediate representation (IR). See histograms.schema.json for the JSON file format.", "type": "object", "required": ["uhi_schema", "axes", "storage"], "additionalProperties": false, "properties": { "uhi_schema": { "const": 1, "description": "The schema version number" }, "writer_info": { "$ref": "#/$defs/writer_info" }, "metadata": { "$ref": "#/$defs/metadata" }, "axes": { "type": "array", "description": "A list of the axes of the histogram.", "items": { "oneOf": [ { "$ref": "#/$defs/regular_axis" }, { "$ref": "#/$defs/variable_axis" }, { "$ref": "#/$defs/category_str_axis" }, { "$ref": "#/$defs/category_int_axis" }, { "$ref": "#/$defs/boolean_axis" } ] } }, "storage": { "description": "The storage of the bins of the histogram.", "oneOf": [ { "$ref": "#/$defs/int_storage" }, { "$ref": "#/$defs/double_storage" }, { "$ref": "#/$defs/weighted_storage" }, { "$ref": "#/$defs/mean_storage" }, { "$ref": "#/$defs/weighted_mean_storage" } ] } }, "$defs": { "supported_metadata": { "oneOf": [ { "type": "string" }, { "type": "number" }, { "type": "boolean" } ] }, "metadata": { "type": "object", "description": "Arbitrary metadata dictionary.", "additionalProperties": false, "patternProperties": { ".+": { "$ref": "#/$defs/supported_metadata" } } }, "writer_info": { "type": "object", "description": "Information from the library that created the histogram.", "additionalProperties": false, "patternProperties": { ".+": { "type": "object", "description": "Library specific information.", "additionalProperties": false, "patternProperties": { ".+": { "$ref": "#/$defs/supported_metadata" } } } } }, "ndarray": { "type": "array", "items": { "oneOf": [{ "type": "number" }, { "$ref": "#/$defs/ndarray" }] }, "description": "A ND (nested) array of numbers." }, "int_ndarray": { "type": "array", "items": { "oneOf": [{ "type": "integer" }, { "$ref": "#/$defs/int_ndarray" }] }, "description": "A ND (nested) array of integers." }, "int_data_array": { "oneOf": [ { "type": "string", "description": "A path (similar to URI) to the integer bin data" }, { "$ref": "#/$defs/int_ndarray" }, { "type": "integer", "description": "A single integer, for a histogram with no axes" } ] }, "data_array": { "oneOf": [ { "type": "string", "description": "A path (similar to URI) to the floating point bin data" }, { "$ref": "#/$defs/ndarray" }, { "type": "number", "description": "A single number, for a histogram with no axes" } ] }, "sparse_array": { "oneOf": [ { "type": "string", "description": "A path (similar to URI) to the index data" }, { "type": "array", "items": { "type": "array", "items": { "type": "integer", "description": "Bin indexes, 0 is the underflow if present" }, "description": "A 2D array of integers." }, "description": "The 2D index array, outer dimension is axis number, inner matches the 1D data." } ] }, "regular_axis": { "type": "object", "description": "An evenly spaced set of continuous bins.", "required": [ "type", "lower", "upper", "bins", "underflow", "overflow", "circular" ], "additionalProperties": false, "properties": { "type": { "type": "string", "const": "regular" }, "lower": { "type": "number", "description": "Lower edge of the axis." }, "upper": { "type": "number", "description": "Upper edge of the axis." }, "bins": { "type": "integer", "minimum": 0, "description": "Number of bins in the axis." }, "underflow": { "type": "boolean", "description": "True if there is a bin for underflow." }, "overflow": { "type": "boolean", "description": "True if there is a bin for overflow." }, "circular": { "type": "boolean", "description": "True if the axis wraps around." }, "metadata": { "$ref": "#/$defs/metadata" }, "writer_info": { "$ref": "#/$defs/writer_info" } } }, "variable_axis": { "type": "object", "description": "A variably spaced set of continuous bins.", "required": ["type", "edges", "underflow", "overflow", "circular"], "additionalProperties": false, "properties": { "type": { "type": "string", "const": "variable" }, "edges": { "oneOf": [ { "type": "array", "items": { "type": "number" }, "minItems": 2, "uniqueItems": true }, { "type": "string", "description": "A path (URI?) to the edges data." } ] }, "underflow": { "type": "boolean" }, "overflow": { "type": "boolean" }, "circular": { "type": "boolean" }, "metadata": { "$ref": "#/$defs/metadata" }, "writer_info": { "$ref": "#/$defs/writer_info" } } }, "category_str_axis": { "type": "object", "description": "A set of string categorical bins.", "required": ["type", "categories", "flow"], "additionalProperties": false, "properties": { "type": { "type": "string", "const": "category_str" }, "categories": { "type": "array", "items": { "type": "string" }, "uniqueItems": true }, "flow": { "type": "boolean", "description": "True if flow bin (at the overflow position) present." }, "metadata": { "$ref": "#/$defs/metadata" }, "writer_info": { "$ref": "#/$defs/writer_info" } } }, "category_int_axis": { "type": "object", "description": "A set of integer categorical bins in any order.", "required": ["type", "categories", "flow"], "additionalProperties": false, "properties": { "type": { "type": "string", "const": "category_int" }, "categories": { "type": "array", "items": { "type": "integer" }, "uniqueItems": true }, "flow": { "type": "boolean", "description": "True if flow bin (at the overflow position) present." }, "metadata": { "$ref": "#/$defs/metadata" }, "writer_info": { "$ref": "#/$defs/writer_info" } } }, "boolean_axis": { "type": "object", "description": "A simple true/false axis with no flow.", "required": ["type"], "additionalProperties": false, "properties": { "type": { "type": "string", "const": "boolean" }, "metadata": { "$ref": "#/$defs/metadata" }, "writer_info": { "$ref": "#/$defs/writer_info" } } }, "int_storage": { "type": "object", "description": "A storage holding integer counts. Must be empty (only type), dense (type + values), or sparse (type + index + values).", "additionalProperties": false, "properties": { "type": { "type": "string", "const": "int" }, "index": { "$ref": "#/$defs/sparse_array" }, "values": { "$ref": "#/$defs/int_data_array" } }, "oneOf": [ { "required": ["type"], "maxProperties": 1 }, { "required": ["type", "values"], "not": { "required": ["index"] } }, { "required": ["type", "index", "values"] } ] }, "double_storage": { "type": "object", "description": "A storage holding floating point counts. Must be empty (only type), dense (type + values), or sparse (type + index + values).", "additionalProperties": false, "properties": { "type": { "type": "string", "const": "double" }, "index": { "$ref": "#/$defs/sparse_array" }, "values": { "$ref": "#/$defs/data_array" } }, "oneOf": [ { "required": ["type"], "maxProperties": 1 }, { "required": ["type", "values"], "not": { "required": ["index"] } }, { "required": ["type", "index", "values"] } ] }, "weighted_storage": { "type": "object", "description": "A storage holding floating point counts and variances. Must be empty, dense (type + values + variances), or sparse (type + index + values + variances).", "additionalProperties": false, "properties": { "type": { "type": "string", "const": "weighted" }, "index": { "$ref": "#/$defs/sparse_array" }, "values": { "$ref": "#/$defs/data_array" }, "variances": { "$ref": "#/$defs/data_array" } }, "oneOf": [ { "required": ["type"], "maxProperties": 1 }, { "required": ["type", "values", "variances"], "not": { "required": ["index"] } }, { "required": ["type", "index", "values", "variances"] } ] }, "mean_storage": { "type": "object", "description": "A storage holding 'profile'-style floating point counts, values, and variances. Must be empty, dense (type + counts + values + variances), or sparse (type + index + counts + values + variances).", "additionalProperties": false, "properties": { "type": { "type": "string", "const": "mean" }, "index": { "$ref": "#/$defs/sparse_array" }, "counts": { "$ref": "#/$defs/data_array" }, "values": { "$ref": "#/$defs/data_array" }, "variances": { "$ref": "#/$defs/data_array" } }, "oneOf": [ { "required": ["type"], "maxProperties": 1 }, { "required": ["type", "counts", "values", "variances"], "not": { "required": ["index"] } }, { "required": ["type", "index", "counts", "values", "variances"] } ] }, "weighted_mean_storage": { "type": "object", "description": "A storage holding 'profile'-style floating point ∑weights, ∑weights², values, and variances. Must be empty, dense (all data fields), or sparse (index + all data fields).", "additionalProperties": false, "properties": { "type": { "type": "string", "const": "weighted_mean" }, "index": { "$ref": "#/$defs/sparse_array" }, "sum_of_weights": { "$ref": "#/$defs/data_array" }, "sum_of_weights_squared": { "$ref": "#/$defs/data_array" }, "values": { "$ref": "#/$defs/data_array" }, "variances": { "$ref": "#/$defs/data_array" } }, "oneOf": [ { "required": ["type"], "maxProperties": 1 }, { "required": [ "type", "sum_of_weights", "sum_of_weights_squared", "values", "variances" ], "not": { "required": ["index"] } }, { "required": [ "type", "index", "sum_of_weights", "sum_of_weights_squared", "values", "variances" ] } ] } } } uhi-1.2.1/src/uhi/resources/histograms.schema.json0000644000000000000000000000120315261523264017130 0ustar00{ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://raw.githubusercontent.com/scikit-hep/uhi/main/src/uhi/resources/histograms.schema.json", "title": "Histograms", "description": "A JSON file holding either a dictionary of named histograms, or a single histogram stored directly. A single histogram is detected by its `uhi_schema` key. Each histogram follows histogram.schema.json.", "type": "object", "if": { "required": ["uhi_schema"] }, "then": { "$ref": "histogram.schema.json" }, "else": { "additionalProperties": false, "patternProperties": { ".+": { "$ref": "histogram.schema.json" } } } } uhi-1.2.1/src/uhi/testing/__init__.py0000644000000000000000000000000015261523264014364 0ustar00uhi-1.2.1/src/uhi/testing/indexing.py0000644000000000000000000006716215261523264014460 0ustar00from __future__ import annotations import abc import typing import unittest from typing import Any import numpy as np import uhi.tag from uhi.typing.serialization import HistogramIR T = typing.TypeVar("T", bound=Any) __all__ = ["Indexing1D", "Indexing2D", "Indexing3D"] if typing.TYPE_CHECKING: # This is a workaround for the issue with type checkers # and generics with custom __getitem__ methods. sum = 42 def __dir__() -> list[str]: return __all__ class Indexing(abc.ABC, unittest.TestCase): """ This super class provides the basic structure for indexing tests. """ def bin_to_value(self, bin: Any) -> Any: """ Allow downstream classes to handle more complex bin objects. """ return bin def value_to_bin(self, value: Any) -> Any: """ Inverse of bin_to_value, for downstream classes with complex bins. """ return value def values_to_bins(self, values: Any) -> Any: """ Loops over values and converts them to bins using value_to_bin. """ return [self.value_to_bin(v) for v in values] def sum_to_value(self, bin: Any) -> Any: """ Allow downstream classes to handle more summed bin objects. E.g. it can be a projection from a Histo2D to Histo1D. """ return bin def assertEqualBinValue(self, bin: Any, value: Any) -> None: self.assertEqual(self.bin_to_value(bin), value) def assertEqualSum(self, bin: Any, value: Any) -> None: self.assertEqual(self.sum_to_value(bin), value) class Indexing1D(Indexing, typing.Generic[T]): """ This test requires a histogram to be created first. h is a 1D histogram with 10 bins from 0 to 1. Each bin has 2 more than the one before, starting with 0. The overflow bin has 1. The underflow bin has 3. You can access the UHI serialized version with `.uhi`. """ h: T tag = uhi.tag @staticmethod def get_uhi() -> HistogramIR: return { "uhi_schema": 1, "axes": [ { "type": "regular", "lower": 0.0, "upper": 1.0, "bins": 10, "underflow": True, "overflow": True, "circular": False, } ], "storage": { "type": "double", "values": np.array( [3.0, 0.0, 2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 14.0, 16.0, 18.0, 1.0] ), }, } @classmethod @abc.abstractmethod def make_histogram(cls) -> T: pass @classmethod def setUpClass(cls) -> None: cls.h = cls.make_histogram() def test_access_integer(self) -> None: for i in range(10): with self.subTest(i=i): self.assertEqualBinValue(self.h[i], 2 * i) with self.assertRaises(IndexError): self.h[10] self.assertEqualBinValue(self.h[-1], 18) def test_access_integer_flow(self) -> None: self.assertEqualBinValue(self.h[self.tag.underflow], 3) self.assertEqualBinValue(self.h[self.tag.overflow], 1) def test_access_loc(self) -> None: self.assertEqualBinValue(self.h[self.tag.loc(0.05)], 0) self.assertEqualBinValue(self.h[self.tag.loc(0.15)], 2) self.assertEqualBinValue(self.h[self.tag.loc(0.95)], 18) self.assertEqualBinValue(self.h[self.tag.loc(-1)], 3) self.assertEqualBinValue(self.h[self.tag.loc(2)], 1) def test_access_loc_addition(self) -> None: self.assertEqualBinValue(self.h[self.tag.loc(0.05) + 1], 2) self.assertEqualBinValue(self.h[self.tag.loc(0.55) + 2], 14) self.assertEqualBinValue(self.h[self.tag.loc(0.55) - 2], 6) def test_slicing_all(self) -> None: self.assertEqual(self.h[:], self.h) self.assertEqual(self.h[...], self.h) def test_slicing_closed(self) -> None: h = self.h[2:4] self.assertEqualBinValue(h[self.tag.underflow], 5) self.assertEqualBinValue(h[0], 4) self.assertEqualBinValue(h[1], 6) self.assertEqualBinValue(h[self.tag.overflow], 79) with self.assertRaises(IndexError): h[2] def test_slicing_open_upper(self) -> None: h = self.h[5:] self.assertEqualBinValue(h[self.tag.underflow], 23) self.assertEqualBinValue(h[0], 10) self.assertEqualBinValue(h[4], 18) self.assertEqualBinValue(h[self.tag.overflow], 1) with self.assertRaises(IndexError): h[5] def test_slicing_open_lower(self) -> None: h = self.h[:5] self.assertEqualBinValue(h[self.tag.underflow], 3) self.assertEqualBinValue(h[0], 0) self.assertEqualBinValue(h[4], 8) self.assertEqualBinValue(h[self.tag.overflow], 71) with self.assertRaises(IndexError): h[5] def test_slicing_loc_closed(self) -> None: h = self.h[self.tag.loc(0.2) : self.tag.loc(0.4)] self.assertEqualBinValue(h[self.tag.underflow], 5) self.assertEqualBinValue(h[0], 4) self.assertEqualBinValue(h[1], 6) self.assertEqualBinValue(h[self.tag.overflow], 79) with self.assertRaises(IndexError): h[2] def test_slicing_loc_open_upper(self) -> None: h = self.h[self.tag.loc(0.5) :] self.assertEqualBinValue(h[self.tag.underflow], 23) self.assertEqualBinValue(h[0], 10) self.assertEqualBinValue(h[4], 18) self.assertEqualBinValue(h[self.tag.overflow], 1) with self.assertRaises(IndexError): h[5] def test_slicing_loc_open_lower(self) -> None: h = self.h[: self.tag.loc(0.5)] self.assertEqualBinValue(h[self.tag.underflow], 3) self.assertEqualBinValue(h[0], 0) self.assertEqualBinValue(h[4], 8) self.assertEqualBinValue(h[self.tag.overflow], 71) with self.assertRaises(IndexError): h[5] def test_slicing_loc_mixed(self) -> None: h = self.h[2 : self.tag.loc(0.4) + 1] self.assertEqualBinValue(h[self.tag.underflow], 5) self.assertEqualBinValue(h[0], 4) self.assertEqualBinValue(h[1], 6) self.assertEqualBinValue(h[2], 8) self.assertEqualBinValue(h[self.tag.overflow], 71) with self.assertRaises(IndexError): h[3] def test_rebinning(self) -> None: # Boost-histogram allows the `::` to be skipped. h = self.h[:: self.tag.rebin(2)] self.assertEqualBinValue(h[0], 2) self.assertEqualBinValue(h[1], 10) self.assertEqualBinValue(h[2], 18) self.assertEqualBinValue(h[4], 34) with self.assertRaises(IndexError): h[5] def test_rebinning_with_endpoints(self) -> None: h = self.h[1 : 5 : self.tag.rebin(2)] self.assertEqualBinValue(h[0], 6) self.assertEqualBinValue(h[1], 14) with self.assertRaises(IndexError): h[2] def test_rebinning_with_endpoints_mixed(self) -> None: h = self.h[: self.tag.loc(0.55) : self.tag.rebin(2)] self.assertEqualBinValue(h[0], 2) self.assertEqualBinValue(h[1], 10) with self.assertRaises(IndexError): h[2] def test_full_integration(self) -> None: # boost-histogram allows the `::` to be skipped. v = self.h[::sum] self.assertEqualSum(v, 94) def test_non_flow_integration(self) -> None: v = self.h[0:len:sum] self.assertEqualSum(v, 90) def test_ranged_integration(self) -> None: v = self.h[2:5:sum] self.assertEqualSum(v, 18) def test_open_lower_integration(self) -> None: v = self.h[:4:sum] self.assertEqualSum(v, 15) def test_open_upper_integration(self) -> None: v = self.h[4::sum] self.assertEqualSum(v, 79) def test_setting_single_value(self) -> None: h = self.make_histogram() h[0] = self.value_to_bin(42) self.assertEqualBinValue(h[0], 42) self.assertEqualBinValue(h[1], 2) self.assertEqualBinValue(h[-1], 18) h[-1] = self.value_to_bin(99) self.assertEqualBinValue(h[-1], 99) self.assertEqualBinValue(h[0], 42) self.assertEqualBinValue(h[1], 2) def test_setting_single_value_loc(self) -> None: h = self.make_histogram() h[self.tag.loc(0.05)] = self.value_to_bin(42) self.assertEqualBinValue(h[0], 42) self.assertEqualBinValue(h[1], 2) def test_setting_underflow(self) -> None: h = self.make_histogram() h[self.tag.underflow] = self.value_to_bin(42) self.assertEqualBinValue(h[self.tag.underflow], 42) self.assertEqualBinValue(h[0], 0) def test_setting_overflow(self) -> None: h = self.make_histogram() h[self.tag.overflow] = self.value_to_bin(42) self.assertEqualBinValue(h[self.tag.overflow], 42) self.assertEqualBinValue(h[-1], 18) def test_setting_array(self) -> None: h = self.make_histogram() h[1:3] = self.value_to_bin(42) # TODO: this is broken, fix! # self.assertEqual(h[0], 0) self.assertEqualBinValue(h[1], 42) self.assertEqualBinValue(h[2], 42) # self.assertEqual(h[3], 6) def test_setting_array_slice(self) -> None: h = self.make_histogram() h[1:3] = self.values_to_bins([42, 42]) self.assertEqualBinValue(h[0], 0) self.assertEqualBinValue(h[1], 42) self.assertEqualBinValue(h[2], 42) self.assertEqualBinValue(h[3], 6) def test_setting_array_without_underflow(self) -> None: h = self.make_histogram() h[:3] = self.values_to_bins([42, 43, 44]) self.assertEqualBinValue(h[self.tag.underflow], 3) self.assertEqualBinValue(h[0], 42) self.assertEqualBinValue(h[1], 43) self.assertEqualBinValue(h[2], 44) self.assertEqualBinValue(h[3], 6) def test_setting_array_with_underflow(self) -> None: h = self.make_histogram() h[:3] = self.values_to_bins([41, 42, 43, 44]) self.assertEqualBinValue(h[self.tag.underflow], 41) self.assertEqualBinValue(h[0], 42) self.assertEqualBinValue(h[1], 43) self.assertEqualBinValue(h[2], 44) self.assertEqualBinValue(h[3], 6) def test_setting_array_without_overflow(self) -> None: h = self.make_histogram() h[7:] = self.values_to_bins([42, 43, 44]) self.assertEqualBinValue(h[6], 12) self.assertEqualBinValue(h[7], 42) self.assertEqualBinValue(h[8], 43) self.assertEqualBinValue(h[9], 44) self.assertEqualBinValue(h[self.tag.overflow], 1) def test_setting_array_with_overflow(self) -> None: h = self.make_histogram() h[7:] = self.values_to_bins([42, 43, 44, 45]) self.assertEqualBinValue(h[6], 12) self.assertEqualBinValue(h[7], 42) self.assertEqualBinValue(h[8], 43) self.assertEqualBinValue(h[9], 44) self.assertEqualBinValue(h[self.tag.overflow], 45) def test_setting_whole_array(self) -> None: h = self.make_histogram() h[:] = self.values_to_bins(range(10)) self.assertEqualBinValue(h[self.tag.underflow], 3) self.assertEqualBinValue(h[0], 0) self.assertEqualBinValue(h[1], 1) self.assertEqualBinValue(h[2], 2) self.assertEqualBinValue(h[3], 3) self.assertEqualBinValue(h[4], 4) self.assertEqualBinValue(h[5], 5) self.assertEqualBinValue(h[6], 6) self.assertEqualBinValue(h[7], 7) self.assertEqualBinValue(h[8], 8) self.assertEqualBinValue(h[9], 9) self.assertEqualBinValue(h[self.tag.overflow], 1) def test_setting_whole_array_with_flow(self) -> None: h = self.make_histogram() h[:] = self.values_to_bins(range(12)) self.assertEqualBinValue(h[self.tag.underflow], 0) self.assertEqualBinValue(h[0], 1) self.assertEqualBinValue(h[1], 2) self.assertEqualBinValue(h[2], 3) self.assertEqualBinValue(h[3], 4) self.assertEqualBinValue(h[4], 5) self.assertEqualBinValue(h[5], 6) self.assertEqualBinValue(h[6], 7) self.assertEqualBinValue(h[7], 8) self.assertEqualBinValue(h[8], 9) self.assertEqualBinValue(h[9], 10) self.assertEqualBinValue(h[self.tag.overflow], 11) def test_setting_len_mismatch(self) -> None: h = self.make_histogram() with self.assertRaises(ValueError): h[:] = self.values_to_bins(range(9)) with self.assertRaises(ValueError): h[:] = self.values_to_bins(range(11)) with self.assertRaises(ValueError): h[:] = self.values_to_bins(range(13)) with self.assertRaises(ValueError): h[1:4] = self.values_to_bins(range(2)) with self.assertRaises(ValueError): h[1:4] = self.values_to_bins(range(4)) with self.assertRaises(ValueError): h[1:4] = self.values_to_bins(range(5)) class Indexing2D(Indexing, typing.Generic[T]): """ This test requires histograms to be created first. h is a 2D histogram with [2,5] bins. The contents are x+2y, where x and y are the bin indices. """ h: T tag = uhi.tag @staticmethod def get_uhi() -> HistogramIR: x, y = np.mgrid[0:2, 0:5] data = np.pad(x + 2 * y, 1, mode="constant") return { "uhi_schema": 1, "axes": [ { "type": "regular", "lower": 0.0, "upper": 2.0, "bins": 2, "underflow": True, "overflow": True, "circular": False, }, { "type": "regular", "lower": 0.0, "upper": 5.0, "bins": 5, "underflow": True, "overflow": True, "circular": False, }, ], "storage": { "type": "double", "values": data, }, } @classmethod @abc.abstractmethod def make_histogram(cls) -> T: pass @classmethod def setUpClass(cls) -> None: cls.h = cls.make_histogram() def test_access_integer(self) -> None: for i in range(2): for j in range(5): with self.subTest(i=i, j=j): self.assertEqualBinValue(self.h[i, j], i + 2 * j) with self.assertRaises(IndexError): self.h[2, 0] with self.assertRaises(IndexError): self.h[0, 5] self.assertEqualBinValue(self.h[-1, -1], 9) def test_access_loc(self) -> None: self.assertEqualBinValue(self.h[self.tag.loc(0.5), self.tag.loc(0.5)], 0) self.assertEqualBinValue(self.h[self.tag.loc(0.5), self.tag.loc(1.5)], 2) self.assertEqualBinValue(self.h[self.tag.loc(0.5), self.tag.loc(4.5)], 8) def test_access_loc_underflow(self) -> None: self.assertEqualBinValue(self.h[self.tag.loc(-1), self.tag.loc(0.5)], 0) def test_access_loc_mixed(self) -> None: self.assertEqualBinValue(self.h[self.tag.loc(0.5), 0], 0) self.assertEqualBinValue(self.h[0, self.tag.loc(1.5)], 2) self.assertEqualBinValue(self.h[1, self.tag.loc(4.5)], 9) def test_slicing_all(self) -> None: self.assertEqual(self.h[:, :], self.h) self.assertEqual(self.h[...], self.h) self.assertEqual(self.h[:, ...], self.h) self.assertEqual(self.h[..., :], self.h) def test_slicing_loc(self) -> None: h = self.h[ 1, self.tag.loc(0.5) : self.tag.loc(2.5), ] self.assertEqualBinValue(h[0], 1) self.assertEqualBinValue(h[1], 3) with self.assertRaises(IndexError): h[2] def test_rebinning(self) -> None: h = self.h[:: self.tag.rebin(2), :: self.tag.rebin(2)] self.assertEqualBinValue(h[0, 0], 6) self.assertEqualBinValue(h[0, 1], 22) with self.assertRaises(IndexError): h[0, 2] with self.assertRaises(IndexError): h[1, 0] def test_full_integration(self) -> None: v = self.h[::sum, ::sum] self.assertEqualSum(v, 45) def test_mixed_integration(self) -> None: h = self.h[:2:sum, 1:3] self.assertEqualSum(h[0], 5) self.assertEqualSum(h[1], 9) with self.assertRaises(IndexError): h[2] def test_mixed_single_integration(self) -> None: v = self.h[1, ::sum] self.assertEqualSum(v, 25) def test_mixed_single_integration_dict(self) -> None: v = self.h[{0: 1, 1: np.s_[::sum]}] self.assertEqualSum(v, 25) def test_ellipsis_integration(self) -> None: h = self.h[..., ::sum] self.assertEqualSum(h[0], 20) self.assertEqualSum(h[1], 25) with self.assertRaises(IndexError): h[2] def test_ellipsis_integration_dict(self) -> None: h = self.h[{0: np.s_[::sum]}] self.assertEqualSum(h[0], 1) self.assertEqualSum(h[1], 5) self.assertEqualSum(h[4], 17) with self.assertRaises(IndexError): h[5] def test_setting_single_value(self) -> None: h = self.make_histogram() h[0, 0] = self.value_to_bin(42) self.assertEqualBinValue(h[0, 0], 42) self.assertEqualBinValue(h[1, 1], 3) def test_setting_underflow(self) -> None: h = self.make_histogram() h[self.tag.underflow, ...] = self.value_to_bin(42) self.assertEqualBinValue(h[self.tag.underflow, 0], 42) def test_setting_array(self) -> None: h = self.make_histogram() h[0:2, 0:2] = np.array( [self.values_to_bins([42, 43]), self.values_to_bins([44, 45])] ) self.assertEqualBinValue(h[0, 0], 42) self.assertEqualBinValue(h[0, 1], 43) self.assertEqualBinValue(h[1, 0], 44) self.assertEqualBinValue(h[1, 1], 45) self.assertEqualBinValue(h[1, 2], 5) def test_setting_array_broadcast(self) -> None: h = self.make_histogram() h[0:2, 0:2] = np.array([self.values_to_bins([42]), self.values_to_bins([3])]) self.assertEqualBinValue(h[0, 0], 42) self.assertEqualBinValue(h[0, 1], 42) self.assertEqualBinValue(h[1, 0], 3) self.assertEqualBinValue(h[1, 1], 3) self.assertEqualBinValue(h[1, 2], 5) def test_setting_dict(self) -> None: h = self.make_histogram() h[{0: 1, 1: 0}] = self.value_to_bin(42) self.assertEqualBinValue(h[1, 0], 42) self.assertEqualBinValue(h[0, 1], 2) def test_setting_dict_slice(self) -> None: h = self.make_histogram() h[{0: 1, 1: slice(2, 4)}] = self.values_to_bins(range(42, 44)) self.assertEqualBinValue(h[1, 1], 3) self.assertEqualBinValue(h[1, 2], 42) self.assertEqualBinValue(h[1, 3], 43) self.assertEqualBinValue(h[1, 4], 9) def test_setting_dict_slicer(self) -> None: h = self.make_histogram() h[{0: 1, 1: np.s_[3:5]}] = self.values_to_bins(range(42, 44)) self.assertEqualBinValue(h[1, 2], 5) self.assertEqualBinValue(h[1, 3], 42) self.assertEqualBinValue(h[1, 4], 43) class Indexing3D(Indexing, typing.Generic[T]): """ This test requires histograms to be created first. h is a 3D histogram with [2,5,10] bins. The contents are x+2y+3z, where x, y, and z are the bin indices. """ h: T tag = uhi.tag @staticmethod def get_uhi() -> HistogramIR: x, y, z = np.mgrid[0:2, 0:5, 0:10] data = np.pad(x + 2 * y + 3 * z, 1, mode="constant") return { "uhi_schema": 1, "writer_info": {"boost-histogram": {"version": "1.6.1"}}, "axes": [ { "type": "regular", "lower": 0.0, "upper": 2.0, "bins": 2, "underflow": True, "overflow": True, "circular": False, "metadata": {}, }, { "type": "regular", "lower": 0.0, "upper": 5.0, "bins": 5, "underflow": True, "overflow": True, "circular": False, "metadata": {}, }, { "type": "regular", "lower": 0.0, "upper": 10.0, "bins": 10, "underflow": True, "overflow": True, "circular": False, "metadata": {}, }, ], "storage": { "type": "double", "values": data, }, "metadata": {"_variance_known": True}, } @classmethod @abc.abstractmethod def make_histogram(cls) -> T: pass @classmethod def setUpClass(cls) -> None: cls.h = cls.make_histogram() def test_access_integer(self) -> None: for i in range(2): for j in range(5): for k in range(10): with self.subTest(i=i, j=j, k=k): self.assertEqualBinValue(self.h[i, j, k], i + 2 * j + 3 * k) with self.assertRaises(IndexError): self.h[2, 0, 0] with self.assertRaises(IndexError): self.h[0, 5, 0] with self.assertRaises(IndexError): self.h[0, 0, 10] self.assertEqualBinValue(self.h[-1, -1, -1], 36) def test_access_loc(self) -> None: self.assertEqualBinValue( self.h[self.tag.loc(0.5), self.tag.loc(0.5), self.tag.loc(0.5)], 0 ) self.assertEqualBinValue( self.h[self.tag.loc(0.5), self.tag.loc(1.5), self.tag.loc(0.5)], 2 ) self.assertEqualBinValue( self.h[self.tag.loc(0.5), self.tag.loc(4.5), self.tag.loc(9.5)], 35 ) self.assertEqualBinValue( self.h[self.tag.loc(-1), self.tag.loc(0.5), self.tag.loc(0.5)], 0 ) def test_access_loc_mixed(self) -> None: self.assertEqualBinValue(self.h[self.tag.loc(0.5), 0, self.tag.loc(0.5)], 0) self.assertEqualBinValue(self.h[0, self.tag.loc(1.5), self.tag.loc(0.5)], 2) self.assertEqualBinValue(self.h[self.tag.loc(0.5), self.tag.loc(4.5), 9], 35) self.assertEqualBinValue(self.h[1, self.tag.loc(4.5), 9], 36) def test_slicing_all(self) -> None: self.assertEqual(self.h[:, :, :], self.h) self.assertEqual(self.h[...], self.h) self.assertEqual(self.h[:, ...], self.h) self.assertEqual(self.h[..., :], self.h) def test_slicing_loc(self) -> None: h = self.h[ 1, 3, self.tag.loc(0.5) : self.tag.loc(2.5), ] self.assertEqualBinValue(h[0], 7) self.assertEqualBinValue(h[1], 10) with self.assertRaises(IndexError): h[2] def test_rebinning(self) -> None: h = self.h[:: self.tag.rebin(2), :: self.tag.rebin(2), :: self.tag.rebin(2)] self.assertEqualBinValue(h[0, 0, 0], 24) self.assertEqualBinValue(h[0, 0, 1], 72) self.assertEqualBinValue(h[0, 1, 0], 56) self.assertEqualBinValue(h[0, 1, 4], 248) with self.assertRaises(IndexError): h[0, 1, 5] with self.assertRaises(IndexError): h[0, 2, 4] with self.assertRaises(IndexError): h[1, 1, 4] def test_full_integration(self) -> None: v = self.h[::sum, ::sum, ::sum] self.assertEqualSum(v, 1800) def test_mixed_integration(self) -> None: h = self.h[::sum, :2:sum, 1:3] self.assertEqualSum(h[0], 18) self.assertEqualSum(h[1], 30) with self.assertRaises(IndexError): h[2] def test_mixed_single_integration(self) -> None: h = self.h[1, ::sum, 1:3] self.assertEqualSum(h[0], 40) self.assertEqualSum(h[1], 55) with self.assertRaises(IndexError): h[2] def test_mixed_single_integration_dict(self) -> None: h = self.h[{0: 1, 1: np.s_[::sum], 2: np.s_[1:3]}] self.assertEqualSum(h[0], 40) self.assertEqualSum(h[1], 55) with self.assertRaises(IndexError): h[2] def test_ellipsis_integration(self) -> None: h = self.h[::sum, ..., ::sum] self.assertEqualSum(h[0], 280) self.assertEqualSum(h[1], 320) self.assertEqualSum(h[4], 440) with self.assertRaises(IndexError): h[5] def test_ellipsis_integration_dict(self) -> None: h = self.h[{0: np.s_[::sum], 2: np.s_[::sum]}] self.assertEqualSum(h[0], 280) self.assertEqualSum(h[1], 320) self.assertEqualSum(h[4], 440) with self.assertRaises(IndexError): h[5] def test_setting_single_value(self) -> None: h = self.make_histogram() h[0, 0, 0] = self.value_to_bin(42) self.assertEqualBinValue(h[0, 0, 0], 42) self.assertEqualBinValue(h[1, 1, 1], 6) def test_setting_underflow(self) -> None: h = self.make_histogram() h[self.tag.underflow, ...] = self.value_to_bin(42) self.assertEqualBinValue(h[self.tag.underflow, 0, 0], 42) def test_setting_array(self) -> None: h = self.make_histogram() h[0:2, 0:2, 0:2] = np.array( [ [self.values_to_bins([42, 43]), self.values_to_bins([44, 45])], [self.values_to_bins([46, 47]), self.values_to_bins([48, 49])], ] ) self.assertEqualBinValue(h[0, 0, 0], 42) self.assertEqualBinValue(h[0, 0, 1], 43) self.assertEqualBinValue(h[0, 1, 0], 44) self.assertEqualBinValue(h[0, 1, 1], 45) self.assertEqualBinValue(h[1, 0, 0], 46) self.assertEqualBinValue(h[1, 0, 1], 47) self.assertEqualBinValue(h[1, 1, 0], 48) self.assertEqualBinValue(h[1, 1, 1], 49) self.assertEqualBinValue(h[1, 1, 2], 9) def test_setting_array_broadcast(self) -> None: h = self.make_histogram() h[0:2, 0:2, 0:2] = np.array( [ [self.values_to_bins([42]), self.values_to_bins([3])], [self.values_to_bins([46]), self.values_to_bins([4])], ] ) self.assertEqualBinValue(h[0, 0, 0], 42) self.assertEqualBinValue(h[0, 0, 1], 42) self.assertEqualBinValue(h[0, 1, 0], 3) self.assertEqualBinValue(h[0, 1, 1], 3) self.assertEqualBinValue(h[1, 0, 0], 46) self.assertEqualBinValue(h[1, 0, 1], 46) self.assertEqualBinValue(h[1, 1, 0], 4) self.assertEqualBinValue(h[1, 1, 1], 4) self.assertEqualBinValue(h[1, 1, 2], 9) def test_setting_dict(self) -> None: h = self.make_histogram() h[{0: 1, 1: 0, 2: 3}] = self.value_to_bin(3) self.assertEqualBinValue(h[1, 0, 3], 3) self.assertEqualBinValue(h[1, 0, 4], 13) def test_setting_dict_slice(self) -> None: h = self.make_histogram() h[{0: 1, 1: 0, 2: slice(3, 5)}] = self.values_to_bins(range(42, 44)) self.assertEqualBinValue(h[1, 0, 2], 7) self.assertEqualBinValue(h[1, 0, 3], 42) self.assertEqualBinValue(h[1, 0, 4], 43) self.assertEqualBinValue(h[1, 0, 5], 16) def test_setting_dict_slicer(self) -> None: h = self.make_histogram() h[{0: 1, 1: 0, 2: np.s_[3:5]}] = self.values_to_bins(range(42, 44)) self.assertEqualBinValue(h[1, 0, 2], 7) self.assertEqualBinValue(h[1, 0, 3], 42) self.assertEqualBinValue(h[1, 0, 4], 43) self.assertEqualBinValue(h[1, 0, 5], 16) uhi-1.2.1/src/uhi/typing/__init__.py0000644000000000000000000000000015261523264014221 0ustar00uhi-1.2.1/src/uhi/typing/plottable.py0000644000000000000000000001301615261523264014463 0ustar00""" Using the protocol: Producers: use isinstance(myhist, PlottableHistogram) in your tests; part of the protocol is checkable at runtime, though ideally you should use MyPy; if your histogram class supports PlottableHistogram, this will pass. Consumers: Make your functions accept the PlottableHistogram static type, and MyPy will force you to only use items in the Protocol. """ from __future__ import annotations from collections.abc import Iterator, Sequence from typing import Protocol, TypeVar, runtime_checkable # NumPy 1.20+ will work much, much better than previous versions when type checking import numpy as np protocol_version = (1, 2) # Known kinds of histograms. A Producer can add Kinds not defined here; a # Consumer should check for known types if it matters. A simple plotter could # just use .value and .variance if non-None and ignore .kind. # # Could have been Kind = Literal["COUNT", "MEAN"] - left as a generic string so # it can be extendable. Kind = str # Implementations are highly encouraged to use the following construct: # class Kind(str, enum.Enum): # COUNT = "COUNT" # MEAN = "MEAN" # Then return and use Kind.COUNT or Kind.MEAN. @runtime_checkable class PlottableTraits(Protocol): @property def circular(self) -> bool: """ True if the axis "wraps around" """ @property def discrete(self) -> bool: """ True if each bin is discrete - Integer, Boolean, or Category, for example """ T_co = TypeVar("T_co", covariant=True) @runtime_checkable class PlottableAxisGeneric(Protocol[T_co]): # noqa: PLW1641 # name: str - Optional, not part of Protocol # label: str - Optional, not part of Protocol # # Plotters are encouraged to plot label if it exists and is not None, and # name otherwise if it exists and is not None, but these properties are not # available on all histograms and not part of the Protocol. @property def traits(self) -> PlottableTraits: ... def __getitem__(self, index: int) -> T_co: """ Get the pair of edges (not discrete) or bin label (discrete). """ def __len__(self) -> int: """ Return the number of bins (not counting flow bins, which are ignored for this Protocol currently). """ def __eq__(self, other: object) -> bool: """ Required to be sequence-like. """ def __iter__(self) -> Iterator[T_co]: """ Useful element of a Sequence to include. """ PlottableAxisContinuous = PlottableAxisGeneric[tuple[float, float]] PlottableAxisInt = PlottableAxisGeneric[int] PlottableAxisStr = PlottableAxisGeneric[str] PlottableAxis = PlottableAxisContinuous | PlottableAxisInt | PlottableAxisStr @runtime_checkable class PlottableHistogram(Protocol): @property def axes(self) -> Sequence[PlottableAxis]: ... @property def kind(self) -> Kind: ... # All methods can have a flow=False argument - not part of this Protocol. # If this is included, it should return an array with flow bins added, # normal ordering. def values(self) -> np.typing.NDArray[np.float64]: """ Returns the accumulated values. The counts for simple histograms, the sum of weights for weighted histograms, the mean for profiles, etc. If counts is equal to 0, the value in that cell is undefined if kind == "MEAN". """ def variances(self) -> np.typing.NDArray[np.float64] | None: """ Returns the estimated variance of the accumulated values. The sum of squared weights for weighted histograms, the variance of samples for profiles, etc. For an unweighed histogram where kind == "COUNT", this should return the same as values if the histogram was not filled with weights, and None otherwise. If counts is equal to 1 or less, the variance in that cell is undefined if kind == "MEAN". If kind == "MEAN", the counts can be used to compute the error on the mean as sqrt(variances / counts), this works whether or not the entries are weighted if the weight variance was tracked by the implementation. """ def counts(self) -> np.typing.NDArray[np.float64] | None: """ Returns the number of entries in each bin for an unweighted histogram or profile and an effective number of entries (defined below) for a weighted histogram or profile. An exotic generalized histogram could have no sensible .counts, so this is Optional and should be checked by Consumers. If kind == "MEAN", counts (effective or not) can and should be used to determine whether the mean value and its variance should be displayed (see documentation of values and variances, respectively). The counts should also be used to compute the error on the mean (see documentation of variances). For a weighted histogram, counts is defined as sum_of_weights ** 2 / sum_of_weights_squared. It is equal or less than the number of times the bin was filled, the equality holds when all filled weights are equal. The larger the spread in weights, the smaller it is, but it is always 0 if filled 0 times, and 1 if filled once, and more than 1 otherwise. A suggested implementation is: return np.divide( sum_of_weights**2, sum_of_weights_squared, out=np.zeros_like(sum_of_weights, dtype=np.float64), where=sum_of_weights_squared != 0) """ uhi-1.2.1/src/uhi/typing/serialization.py0000644000000000000000000001216015261523264015351 0ustar00"""Serialization types for UHI. Two types of dictionaries are defined here: 1. ``AnyAxis``, ``AnyStorage``, and ``AnyHistogram`` are used for inputs. They represent the merger of all possible types. 2. ``Axis``, ``Storage``, and ``histogram`` are used for outputs. These have precise entries defined for each Literal type. There's also a Protocol, `ToUHIHistogram`, for anything that supports conversion. """ from __future__ import annotations import sys import typing from collections.abc import Sequence from typing import Any, Literal, Protocol, TypedDict import numpy as np from numpy.typing import NDArray if sys.version_info < (3, 11): from typing_extensions import NotRequired, Required else: from typing import NotRequired, Required __all__ = [ "AnyAxisIR", "AnyHistogramIR", "AnyStorageIR", "AxisIR", "BooleanAxisIR", "CategoryIntAxisIR", "CategoryStrAxisIR", "DoubleStorageIR", "HistogramIR", "IntStorageIR", "MeanStorageIR", "RegularAxisIR", "StorageIR", "ToUHIHistogram", "VariableAxisIR", "WeightedMeanStorageIR", "WeightedStorageIR", ] SupportedMetadata = float | str | bool ArrayLikeFloat = Sequence[float] | NDArray[np.float64] def __dir__() -> list[str]: return __all__ class RegularAxisIR(TypedDict): type: Literal["regular"] lower: float upper: float bins: int underflow: bool overflow: bool circular: bool metadata: NotRequired[dict[str, SupportedMetadata]] writer_info: NotRequired[dict[str, dict[str, SupportedMetadata]]] class VariableAxisIR(TypedDict): type: Literal["variable"] edges: ArrayLikeFloat underflow: bool overflow: bool circular: bool metadata: NotRequired[dict[str, SupportedMetadata]] writer_info: NotRequired[dict[str, dict[str, SupportedMetadata]]] class CategoryStrAxisIR(TypedDict): type: Literal["category_str"] categories: list[str] flow: bool metadata: NotRequired[dict[str, SupportedMetadata]] writer_info: NotRequired[dict[str, dict[str, SupportedMetadata]]] class CategoryIntAxisIR(TypedDict): type: Literal["category_int"] categories: list[int] flow: bool metadata: NotRequired[dict[str, SupportedMetadata]] writer_info: NotRequired[dict[str, dict[str, SupportedMetadata]]] class BooleanAxisIR(TypedDict): type: Literal["boolean"] metadata: NotRequired[dict[str, SupportedMetadata]] writer_info: NotRequired[dict[str, dict[str, SupportedMetadata]]] class IntStorageIR(TypedDict): type: Literal["int"] values: NotRequired[NDArray[np.integer[Any]]] index: NotRequired[NDArray[np.integer[Any]]] class DoubleStorageIR(TypedDict): type: Literal["double"] values: NotRequired[NDArray[np.float64]] index: NotRequired[NDArray[np.integer[Any]]] class WeightedStorageIR(TypedDict): type: Literal["weighted"] values: NotRequired[NDArray[np.float64]] variances: NotRequired[NDArray[np.float64]] index: NotRequired[NDArray[np.integer[Any]]] class MeanStorageIR(TypedDict): type: Literal["mean"] counts: NotRequired[NDArray[np.float64]] values: NotRequired[NDArray[np.float64]] variances: NotRequired[NDArray[np.float64]] index: NotRequired[NDArray[np.integer[Any]]] class WeightedMeanStorageIR(TypedDict): type: Literal["weighted_mean"] sum_of_weights: NotRequired[NDArray[np.float64]] sum_of_weights_squared: NotRequired[NDArray[np.float64]] values: NotRequired[NDArray[np.float64]] variances: NotRequired[NDArray[np.float64]] index: NotRequired[NDArray[np.integer[Any]]] StorageIR = ( IntStorageIR | DoubleStorageIR | WeightedStorageIR | MeanStorageIR | WeightedMeanStorageIR ) AxisIR = ( RegularAxisIR | VariableAxisIR | CategoryStrAxisIR | CategoryIntAxisIR | BooleanAxisIR ) class AnyStorageIR(TypedDict, total=False): type: Required[Literal["int", "double", "weighted", "mean", "weighted_mean"]] index: NDArray[np.integer[Any]] values: NDArray[np.number[Any]] variances: NDArray[np.float64] sum_of_weights: NDArray[np.float64] sum_of_weights_squared: NDArray[np.float64] counts: NDArray[np.float64] class AnyAxisIR(TypedDict, total=False): type: Required[ Literal["regular", "variable", "category_str", "category_int", "boolean"] ] metadata: dict[str, SupportedMetadata] writer_info: dict[str, dict[str, SupportedMetadata]] lower: float upper: float bins: int edges: ArrayLikeFloat categories: list[str] | list[int] underflow: bool overflow: bool flow: bool circular: bool class HistogramIR(TypedDict): uhi_schema: int axes: list[AxisIR] storage: StorageIR metadata: NotRequired[dict[str, SupportedMetadata]] writer_info: NotRequired[dict[str, dict[str, SupportedMetadata]]] class AnyHistogramIR(TypedDict): uhi_schema: int axes: list[AnyAxisIR] storage: AnyStorageIR metadata: NotRequired[dict[str, SupportedMetadata]] writer_info: NotRequired[dict[str, dict[str, SupportedMetadata]]] @typing.runtime_checkable class ToUHIHistogram(Protocol): def _to_uhi_(self) -> dict[str, Any]: ... uhi-1.2.1/tests/conftest.py0000644000000000000000000000472315261523264012603 0ustar00from __future__ import annotations import contextlib import importlib.metadata import sys import sysconfig from pathlib import Path import pytest from dependency_groups import resolve from packaging.requirements import Requirement if sys.version_info < (3, 11): import tomli as tomllib else: import tomllib DIR = Path(__file__).parent.resolve() VALID_FILES = DIR.glob("resources/valid/*.json") INVALID_FILES = DIR.glob("resources/invalid/*.json") VALID_SINGLE_FILES = DIR.glob("resources/valid_single/*.json") @pytest.fixture(scope="session") def resources() -> Path: return DIR / "resources" @pytest.fixture(params=VALID_FILES, ids=lambda p: p.name) def valid(request: pytest.FixtureRequest) -> Path: return request.param # type: ignore[no-any-return] @pytest.fixture(params=INVALID_FILES, ids=lambda p: p.name) def invalid(request: pytest.FixtureRequest) -> Path: return request.param # type: ignore[no-any-return] @pytest.fixture(params=VALID_SINGLE_FILES, ids=lambda p: p.name) def valid_single(request: pytest.FixtureRequest) -> Path: """A single histogram stored directly, without a name.""" return request.param # type: ignore[no-any-return] @pytest.fixture(params=[False, True], ids=["dense", "sparse"]) def sparse(request: pytest.FixtureRequest) -> bool: return request.param # type: ignore[no-any-return] def pytest_report_header() -> str: with DIR.parent.joinpath("pyproject.toml").open("rb") as f: pyproject = tomllib.load(f) project = pyproject.get("project", {}) pkgs = project.get("dependencies", []) pkgs += [p for ps in project.get("optional-dependencies", {}).values() for p in ps] # Process dependency groups, resolving {include-group = "..."} references dep_groups = pyproject.get("dependency-groups", {}) if dep_groups: pkgs += resolve(dep_groups, *dep_groups.keys()) if "name" in project: pkgs.append(project["name"]) interesting_packages = {Requirement(p).name for p in pkgs} interesting_packages.add("pip") interesting_packages.add("numpy") valid = [] for package in sorted(interesting_packages): with contextlib.suppress(ModuleNotFoundError): valid.append(f"{package}=={importlib.metadata.version(package)}") reqs = " ".join(valid) lines = [ f"installed packages of interest: {reqs}", ] if sysconfig.get_config_var("Py_GIL_DISABLED"): lines.append("free-threaded Python build") return "\n".join(lines) uhi-1.2.1/tests/test_add.py0000644000000000000000000004720615261523264012550 0ustar00from __future__ import annotations import json import re import sys import typing import zipfile from collections.abc import Mapping from pathlib import Path from typing import Any import boost_histogram as bh import numpy as np import pytest import uhi.io import uhi.io._files import uhi.io.json import uhi.io.zip from uhi.__main__ import main from uhi.io import ARRAY_KEYS, to_sparse from uhi.io.ops import add from uhi.typing.serialization import AnyHistogramIR STORAGES = [ bh.storage.Int64(), bh.storage.Double(), bh.storage.Weight(), bh.storage.Mean(), bh.storage.WeightedMean(), ] RNG = np.random.default_rng(42) def _ir(h: bh.Histogram[Any]) -> AnyHistogramIR: return typing.cast(AnyHistogramIR, h._to_uhi_()) def _fill(h: bh.Histogram[Any], n: int) -> bh.Histogram[Any]: x = RNG.normal(size=n) y = RNG.normal(size=n) kwargs: dict[str, Any] = {} if h.kind == bh.Kind.MEAN: kwargs["sample"] = RNG.normal(size=n) if h.storage_type in {bh.storage.Weight, bh.storage.WeightedMean}: kwargs["weight"] = RNG.uniform(0.5, 2, size=n) h.fill(x, y, **kwargs) return h def _defined_variance(storage: Mapping[str, Any]) -> Any: """ Mask of bins where the variance of a mean storage is well defined. With a single entry the variance is 0/0, and rounding in boost-histogram's incremental update turns that into arbitrary values (inf, nan, or finite garbage), so those bins are not compared. """ match storage["type"]: case "mean": return storage["counts"] > 1 case "weighted_mean": w = storage["sum_of_weights"] with np.errstate(divide="ignore", invalid="ignore"): return w - storage["sum_of_weights_squared"] / w > 1e-9 * w case _: return np.ones_like(storage["values"], dtype=bool) def _assert_storage_equal(a: Mapping[str, Any], b: Mapping[str, Any]) -> None: assert a["type"] == b["type"] assert a.keys() == b.keys() mask = _defined_variance(b) for key in ARRAY_KEYS & a.keys(): if key == "variances": np.testing.assert_allclose(a[key][mask], b[key][mask], equal_nan=True) else: np.testing.assert_allclose(a[key], b[key], equal_nan=True) @pytest.mark.parametrize("storage", STORAGES, ids=lambda s: type(s).__name__) @pytest.mark.parametrize("n", [0, 1, 2, 50]) def test_add_matches_boost_histogram(storage: bh.storage.Storage, n: int) -> None: axes = (bh.axis.Regular(3, -1, 1), bh.axis.Regular(4, -2, 2)) h1 = _fill(bh.Histogram(*axes, storage=storage), 100) h2 = _fill(bh.Histogram(*axes, storage=storage), n) h3 = _fill(bh.Histogram(*axes, storage=storage), 30) expected = _ir(h1 + h2 + h3) result = add(_ir(h1), h2, _ir(h3)) assert result["axes"] == expected["axes"] _assert_storage_equal(result["storage"], expected["storage"]) # Result round-trips into boost-histogram (exact == fails on float rounding) _assert_storage_equal( bh.Histogram(dict(result))._to_uhi_()["storage"], expected["storage"] ) def test_add_valid_fixture(valid: Path, sparse: bool) -> None: hists = json.loads( valid.read_text(encoding="utf-8"), object_hook=uhi.io.json.object_hook ) for dense in hists.values(): hist = to_sparse(dense) if sparse else dense result = add(hist, hist) assert ("index" in result["storage"]) == ("index" in hist["storage"]) assert result["axes"] == hist["axes"] assert result.get("metadata") == hist.get("metadata") assert result.get("writer_info") == hist.get("writer_info") if hist["storage"]["type"] in {"int", "double", "weighted"} and hist["axes"]: expected = uhi.io.from_sparse(hist)["storage"] got: Mapping[str, Any] = uhi.io.from_sparse(result)["storage"] for key in ARRAY_KEYS & expected.keys(): np.testing.assert_array_equal(got[key], 2 * np.asarray(expected[key])) def test_add_keeps_int_dtype() -> None: h = bh.Histogram(bh.axis.Integer(0, 3), storage=bh.storage.Int64()) h.fill([0, 1, 1]) result = add(h, h) assert np.issubdtype(result["storage"]["values"].dtype, np.integer) np.testing.assert_array_equal(result["storage"]["values"], [0, 2, 4, 0, 0]) def test_add_empty_sparse_keeps_uint64() -> None: h = bh.Histogram(bh.axis.Integer(0, 3), storage=bh.storage.Int64()) empty = to_sparse(_ir(h)) empty["storage"]["values"] = np.zeros(0, dtype=np.uint64) full = _ir(h) full["storage"]["values"] = np.asarray([0, 2**53 + 1, 0, 0, 0], dtype=np.uint64) result = add(empty, full)["storage"]["values"] assert result.dtype == np.uint64 assert result[1] == 2**53 + 1 def test_add_scalar_int_is_json_serializable() -> None: h = bh.Histogram(storage=bh.storage.Int64()) h.fill() result = add(h, h) assert isinstance(result["storage"]["values"], np.ndarray) assert json.loads(json.dumps(result, default=uhi.io.json.default))["storage"] == { "type": "int", "values": 2, } @pytest.mark.parametrize("bad", [np.inf, np.nan]) def test_add_mean_keeps_nonfinite_variance(bad: float) -> None: """A populated bin with a nonfinite variance must not become zero.""" h = bh.Histogram(bh.axis.Integer(0, 2), storage=bh.storage.Mean()) h.fill([0, 0], sample=[1.0, 3.0]) ir = _ir(h) ir["storage"]["variances"] = np.asarray([0.0, bad, 0.0, 0.0]) result = add(ir, ir) assert np.isnan(result["storage"]["variances"][1]) == np.isnan(bad) assert np.isinf(result["storage"]["variances"][1]) == np.isinf(bad) w = bh.Histogram(bh.axis.Integer(0, 2), storage=bh.storage.WeightedMean()) w.fill([0, 0], sample=[1.0, 3.0], weight=[0.5, 2.0]) wir = _ir(w) wir["storage"]["variances"] = np.asarray([0.0, bad, 0.0, 0.0]) wresult = add(wir, wir) assert np.isnan(wresult["storage"]["variances"][1]) == np.isnan(bad) assert np.isinf(wresult["storage"]["variances"][1]) == np.isinf(bad) def test_add_mixed_sparse_gives_dense() -> None: h = bh.Histogram(bh.axis.Integer(0, 3)) h.fill([0, 1, 1]) dense = _ir(h) sparse = to_sparse(dense) assert "index" in sparse["storage"] assert "index" not in add(dense, sparse)["storage"] assert "index" not in add(sparse, dense)["storage"] assert "index" in add(sparse, sparse)["storage"] np.testing.assert_array_equal( uhi.io.from_sparse(add(sparse, sparse))["storage"]["values"], [0, 2, 4, 0, 0] ) def test_add_empty_storage() -> None: h = bh.Histogram(bh.axis.Integer(0, 3)) h.fill([0, 1, 1]) full = _ir(h) empty: Any = {**full, "storage": {"type": "double"}} assert add(empty, empty)["storage"] == {"type": "double"} np.testing.assert_array_equal( add(empty, full)["storage"]["values"], full["storage"]["values"] ) np.testing.assert_array_equal( add(full, empty)["storage"]["values"], full["storage"]["values"] ) def test_add_ignores_axis_metadata_and_keeps_first() -> None: h1 = bh.Histogram( bh.axis.Regular(3, 0, 1, metadata={"name": "a"}), metadata={"hist": 1} ) h2 = bh.Histogram( bh.axis.Regular(3, 0, 1, metadata={"name": "b"}), metadata={"hist": 2} ) result = add(h1, h2) assert result["metadata"] == h1._to_uhi_()["metadata"] assert result["axes"][0]["metadata"] == h1._to_uhi_()["axes"][0]["metadata"] def test_add_mismatched_axes() -> None: h1 = bh.Histogram(bh.axis.Regular(3, 0, 1)) h2 = bh.Histogram(bh.axis.Regular(4, 0, 1)) h3 = bh.Histogram(bh.axis.Regular(3, 0, 1), bh.axis.Regular(3, 0, 1)) h4 = bh.Histogram(bh.axis.Variable([0, 0.5, 1])) h5 = bh.Histogram(bh.axis.Variable([0, 0.25, 1])) with pytest.raises(ValueError, match="Histogram 1 has axes"): add(h1, h2) with pytest.raises(ValueError, match="Histogram 1 has axes"): add(h1, h3) with pytest.raises(ValueError, match="Histogram 1 has axes"): add(h1, h4) with pytest.raises(ValueError, match="Histogram 1 has axes"): add(h4, h5) def test_add_mismatched_storage() -> None: h1 = bh.Histogram(bh.axis.Regular(3, 0, 1), storage=bh.storage.Int64()) h2 = bh.Histogram(bh.axis.Regular(3, 0, 1), storage=bh.storage.Double()) with pytest.raises(ValueError, match="storage type 'double', expected 'int'"): add(h1, h2) # CLI def _make_files( tmp_path: Path, suffix: str ) -> tuple[list[Path], dict[str, bh.Histogram[Any]]]: """Write two files; ``only_in_first`` is present in the first file only.""" a = _fill(bh.Histogram(bh.axis.Regular(3, -1, 1), bh.axis.Regular(4, -2, 2)), 100) b = _fill(bh.Histogram(bh.axis.Regular(3, -1, 1), bh.axis.Regular(4, -2, 2)), 100) m1 = _fill( bh.Histogram( bh.axis.Regular(3, -1, 1), bh.axis.Regular(4, -2, 2), storage=bh.storage.Mean(), ), 100, ) m2 = _fill( bh.Histogram( bh.axis.Regular(3, -1, 1), bh.axis.Regular(4, -2, 2), storage=bh.storage.Mean(), ), 100, ) only = _fill(bh.Histogram(bh.axis.Regular(3, -1, 1), bh.axis.Regular(4, -2, 2)), 10) files = [tmp_path / f"in1{suffix}", tmp_path / f"in2{suffix}"] _write_file(files[0], {"h": a, "sub/mean": m1, "only_in_first": only}) _write_file(files[1], {"h": b, "sub/mean": m2}) return files, {"h": a + b, "sub/mean": m1 + m2, "only_in_first": only} def _write_file(path: Path, hists: dict[str, bh.Histogram[Any]]) -> None: match path.suffix: case ".json": path.write_text( json.dumps(hists, default=uhi.io.json.default), encoding="utf-8" ) case ".zip": with zipfile.ZipFile(path, "w") as zf: for name, hist in hists.items(): uhi.io.zip.write(zf, name, hist) case _: h5py = pytest.importorskip("h5py") from uhi.io import hdf5 with h5py.File(path, "w") as f: for name, hist in hists.items(): hdf5.write(f.create_group(name), hist) def _read_file(path: Path, name: str | None = None) -> Any: return uhi.io._files.load(path, path=name) @pytest.mark.parametrize("in_suffix", [".json", ".zip", ".h5"]) @pytest.mark.parametrize("out_suffix", [".json", ".zip", ".h5"]) def test_cli_add(tmp_path: Path, in_suffix: str, out_suffix: str) -> None: if ".h5" in {in_suffix, out_suffix}: pytest.importorskip("h5py") files, expected = _make_files(tmp_path, in_suffix) target = tmp_path / f"out{out_suffix}" main(["add", str(target), *map(str, files)]) result = _read_file(target) assert result.keys() == expected.keys() for name, hist in expected.items(): _assert_storage_equal(result[name]["storage"], hist._to_uhi_()["storage"]) def test_cli_add_scalar(tmp_path: Path) -> None: h = bh.Histogram(storage=bh.storage.Int64()) h.fill() src = tmp_path / "in.json" uhi.io._files.write(src, {"one": h}) target = tmp_path / "out.json" main(["add", str(target), str(src), str(src)]) assert _read_file(target)["one"]["storage"]["values"] == 2 def test_cli_add_sparse_all_zero(tmp_path: Path) -> None: """An empty sparse index reads back from JSON as float.""" ir = to_sparse(_ir(bh.Histogram(bh.axis.Regular(3, -1, 1)))) src = tmp_path / "in.json" src.write_text(json.dumps(ir, default=uhi.io.json.default), encoding="utf-8") target = tmp_path / "out.json" main(["add", str(target), str(src), str(src)]) result = uhi.io.from_sparse(_read_file(target)) np.testing.assert_array_equal(result["storage"]["values"], np.zeros(5)) def test_cli_add_force(tmp_path: Path) -> None: files, _ = _make_files(tmp_path, ".json") target = tmp_path / "out.json" target.write_text("{}", encoding="utf-8") with pytest.raises(SystemExit, match="use --force"): main(["add", str(target), *map(str, files)]) assert target.read_text(encoding="utf-8") == "{}" main(["add", "--force", str(target), *map(str, files)]) assert "only_in_first" in _read_file(target) def test_cli_add_unknown_extension(tmp_path: Path) -> None: files, _ = _make_files(tmp_path, ".json") with pytest.raises(SystemExit, match=re.escape("Unknown file format '.txt'")): main(["add", str(tmp_path / "out.txt"), *map(str, files)]) def _write_single(path: Path, hist: bh.Histogram[Any]) -> None: path.write_text(json.dumps(hist, default=uhi.io.json.default), encoding="utf-8") @pytest.mark.parametrize("out_suffix", [".json", ".h5"]) def test_cli_add_single(tmp_path: Path, out_suffix: str) -> None: """Unnamed (top-level) JSON histograms sum to an unnamed histogram.""" if out_suffix == ".h5": pytest.importorskip("h5py") a = _fill(bh.Histogram(bh.axis.Regular(3, -1, 1), bh.axis.Regular(4, -2, 2)), 100) b = _fill(bh.Histogram(bh.axis.Regular(3, -1, 1), bh.axis.Regular(4, -2, 2)), 100) _write_single(tmp_path / "a.json", a) _write_single(tmp_path / "b.json", b) target = tmp_path / f"out{out_suffix}" main(["add", str(target), str(tmp_path / "a.json"), str(tmp_path / "b.json")]) result = _read_file(target) assert uhi.io._files.is_single(result) _assert_storage_equal(result["storage"], (a + b)._to_uhi_()["storage"]) @pytest.mark.parametrize("out_suffix", [".json", ".zip", ".h5"]) def test_cli_add_named_output(tmp_path: Path, out_suffix: str) -> None: """``target:name`` names a single result; ``file:name`` selects inputs.""" if out_suffix == ".h5": pytest.importorskip("h5py") files, expected = _make_files(tmp_path, ".json") _write_single(tmp_path / "c.json", expected["only_in_first"]) target = tmp_path / f"out{out_suffix}" main( [ "add", f"{target}:sub/total", f"{files[0]}:h", f"{files[1]}:h", str(tmp_path / "c.json"), ] ) result = _read_file(target) assert result.keys() == {"sub/total"} total = expected["h"] + expected["only_in_first"] _assert_storage_equal(result["sub/total"]["storage"], total._to_uhi_()["storage"]) _assert_storage_equal( _read_file(target, "sub/total")["storage"], total._to_uhi_()["storage"] ) def test_cli_add_named_output_prefix(tmp_path: Path) -> None: """``target:dir`` prefixes every name when the result is a dict.""" files, expected = _make_files(tmp_path, ".json") target = tmp_path / "out.zip" main(["add", f"{target}:run", *map(str, files)]) result = _read_file(target) assert result.keys() == {f"run/{name}" for name in expected} assert _read_file(target, "run").keys() == expected.keys() def test_cli_add_single_needs_name(tmp_path: Path) -> None: a = _fill(bh.Histogram(bh.axis.Regular(3, -1, 1), bh.axis.Regular(4, -2, 2)), 10) _write_single(tmp_path / "a.json", a) with pytest.raises(SystemExit, match="A name is needed"): main(["add", str(tmp_path / "out.zip"), str(tmp_path / "a.json")]) assert not (tmp_path / "out.zip").exists() def test_cli_add_mixed_single_and_named(tmp_path: Path) -> None: files, expected = _make_files(tmp_path, ".json") _write_single(tmp_path / "c.json", expected["h"]) with pytest.raises(SystemExit, match="Cannot mix"): main( ["add", str(tmp_path / "out.json"), str(files[0]), str(tmp_path / "c.json")] ) def test_cli_add_missing_input_name(tmp_path: Path) -> None: files, _ = _make_files(tmp_path, ".json") with pytest.raises(SystemExit, match="'nope' not found"): main(["add", str(tmp_path / "out.json"), f"{files[0]}:nope"]) def test_cli_add_mismatched_axes(tmp_path: Path) -> None: a = _fill(bh.Histogram(bh.axis.Regular(3, -1, 1), bh.axis.Regular(4, -2, 2)), 10) b = _fill(bh.Histogram(bh.axis.Regular(3, -1, 1), bh.axis.Regular(5, -2, 2)), 10) _write_single(tmp_path / "a.json", a) _write_single(tmp_path / "b.json", b) with pytest.raises(SystemExit, match="axes that do not match"): main( [ "add", str(tmp_path / "out.json"), str(tmp_path / "a.json"), str(tmp_path / "b.json"), ] ) def test_cli_add_prefix_json_roundtrip(tmp_path: Path) -> None: """``out.json:dir`` writes ``dir/name`` keys that ``load(path="dir")`` finds.""" files, expected = _make_files(tmp_path, ".json") target = tmp_path / "out.json" main(["add", f"{target}:run", *map(str, files)]) assert _read_file(target, "run").keys() == expected.keys() assert _read_file(target, "run/").keys() == expected.keys() assert _read_file(target, "run/sub").keys() == {"mean"} assert _read_file(target, "run/h")["uhi_schema"] == 1 with pytest.raises(KeyError, match="'nope' not found"): _read_file(target, "nope") @pytest.mark.parametrize( ("content", "match"), [ ("[1, 2]", "not a histogram or a dict of histograms"), ('{"x": 1}', "'x' is not a histogram"), ("{}", "no histograms found"), ], ) def test_cli_add_bad_source(tmp_path: Path, content: str, match: str) -> None: src = tmp_path / "bad.json" src.write_text(content, encoding="utf-8") target = tmp_path / "out.json" with pytest.raises(SystemExit, match=match): main(["add", str(target), str(src)]) assert not target.exists() def test_cli_add_bad_uhi_schema(tmp_path: Path) -> None: h = bh.Histogram(bh.axis.Regular(3, -1, 1)) data = json.loads(json.dumps({"h": h}, default=uhi.io.json.default)) data["h"]["uhi_schema"] = 2 src = tmp_path / "in.json" src.write_text(json.dumps(data), encoding="utf-8") with pytest.raises(SystemExit, match="Only uhi_schema=1"): main(["add", str(tmp_path / "out.json"), str(src)]) def test_cli_add_missing_output_dir(tmp_path: Path) -> None: files, _ = _make_files(tmp_path, ".json") with pytest.raises(SystemExit, match="No such file or directory"): main(["add", str(tmp_path / "nope" / "out.json"), *map(str, files)]) def test_cli_add_missing_writer( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: files, _ = _make_files(tmp_path, ".json") monkeypatch.setitem(sys.modules, "h5py", None) with pytest.raises(SystemExit, match="h5py"): main(["add", str(tmp_path / "out.h5"), *map(str, files)]) def test_cli_add_empty_target_name(tmp_path: Path) -> None: files, _ = _make_files(tmp_path, ".json") with pytest.raises(SystemExit, match="Empty name"): main(["add", f"{tmp_path / 'out.zip'}:", *map(str, files)]) def test_cli_add_no_extension(tmp_path: Path) -> None: files, _ = _make_files(tmp_path, ".json") with pytest.raises(SystemExit, match="No file extension"): main(["add", str(tmp_path / "out"), *map(str, files)]) def test_cli_add_hdf5_errors(tmp_path: Path) -> None: pytest.importorskip("h5py") files, _ = _make_files(tmp_path, ".h5") target = tmp_path / "out.json" with pytest.raises(SystemExit, match="'nope' not found"): main(["add", str(target), f"{files[0]}:nope"]) with pytest.raises(SystemExit, match="is not a histogram or group"): main(["add", str(target), f"{files[0]}:h/storage/values"]) def test_cli_add_bad_zip(tmp_path: Path) -> None: bad = tmp_path / "bad.zip" bad.write_text("x", encoding="utf-8") with pytest.raises(SystemExit, match="not a zip file"): main(["add", str(tmp_path / "out.json"), str(bad)]) def test_cli_add_unknown_storage(tmp_path: Path) -> None: src = tmp_path / "in.json" src.write_text( json.dumps( {"uhi_schema": 1, "axes": [], "storage": {"type": "unlimited", "values": 3}} ), encoding="utf-8", ) with pytest.raises(SystemExit, match="Unknown storage type 'unlimited'"): main(["add", str(tmp_path / "out.json"), str(src), str(src)]) uhi-1.2.1/tests/test_cli.py0000644000000000000000000002530515261523264012563 0ustar00from __future__ import annotations import json import re import sys import zipfile from pathlib import Path from typing import Any import fastjsonschema import pytest import uhi.io._files import uhi.io.json import uhi.io.zip import uhi.schema from uhi.__main__ import main def test_cli_validate(resources: Path, capsys: pytest.CaptureFixture[str]) -> None: main(["validate", str(resources / "valid" / "reg.json")]) assert capsys.readouterr().out.startswith("OK") main(["validate", str(resources / "valid_single" / "reg.json")]) assert capsys.readouterr().out.startswith("OK") with pytest.raises(SystemExit): main(["validate", str(resources / "invalid" / "missing_axis.json")]) assert capsys.readouterr().out.startswith("ERROR") def test_cli_validate_unknown_format( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: bad = tmp_path / "hist.txt" bad.write_text("{}", encoding="utf-8") with pytest.raises(SystemExit): main(["validate", str(bad)]) assert "Unknown file format '.txt'" in capsys.readouterr().out def test_cli_validate_zip( valid: Path, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: hists = json.loads( valid.read_text(encoding="utf-8"), object_hook=uhi.io.json.object_hook ) tmp_file = tmp_path / "test.zip" with zipfile.ZipFile(tmp_file, "w") as zip_file: for name, hist in hists.items(): uhi.io.zip.write(zip_file, name, hist) main(["validate", str(tmp_file)]) assert capsys.readouterr().out.startswith("OK") def test_cli_validate_zip_invalid( resources: Path, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: hists = json.loads( (resources / "valid" / "reg.json").read_text(encoding="utf-8"), object_hook=uhi.io.json.object_hook, ) tmp_file = tmp_path / "test.zip" with zipfile.ZipFile(tmp_file, "w") as zip_file: for name, hist in hists.items(): uhi.io.zip.write(zip_file, name, hist) # A bare JSON entry without the required keys zip_file.writestr("broken.json", json.dumps({"uhi_schema": 1, "axes": []})) with pytest.raises(SystemExit): main(["validate", str(tmp_file)]) out = capsys.readouterr().out assert out.startswith("ERROR") assert "storage" in out def test_cli_validate_hdf5( valid: Path, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: h5py = pytest.importorskip("h5py") uhi_io_hdf5 = pytest.importorskip("uhi.io.hdf5") hists = json.loads( valid.read_text(encoding="utf-8"), object_hook=uhi.io.json.object_hook ) tmp_file = tmp_path / "test.h5" with h5py.File(tmp_file, "w") as h5_file: # Nest one level to check that groups are discovered recursively grp = h5_file.create_group("nested") for name, hist in hists.items(): uhi_io_hdf5.write(grp.create_group(name), hist) main(["validate", str(tmp_file)]) assert capsys.readouterr().out.startswith("OK") def test_cli_validate_hdf5_invalid( resources: Path, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: h5py = pytest.importorskip("h5py") uhi_io_hdf5 = pytest.importorskip("uhi.io.hdf5") hists = json.loads( (resources / "valid" / "reg.json").read_text(encoding="utf-8"), object_hook=uhi.io.json.object_hook, ) tmp_file = tmp_path / "test.h5" with h5py.File(tmp_file, "w") as h5_file: for name, hist in hists.items(): grp = h5_file.create_group(name) uhi_io_hdf5.write(grp, hist) grp["storage"].attrs["type"] = "not_a_storage" with pytest.raises(SystemExit): main(["validate", str(tmp_file)]) assert capsys.readouterr().out.startswith("ERROR") @pytest.mark.parametrize( ("spec", "expected"), [ ("data.json", ("data.json", None)), ("data.root:sub/dir", ("data.root", "sub/dir")), ("some/dir/data.h5:grp", ("some/dir/data.h5", "grp")), ("C:\\dir\\data.ROOT:grp", ("C:\\dir\\data.ROOT", "grp")), ("C:\\dir\\data.root", ("C:\\dir\\data.root", None)), ], ) def test_split_spec(spec: str, expected: tuple[str, str | None]) -> None: assert uhi.io._files.split_spec(spec) == expected def test_cli_validate_json_path( resources: Path, capsys: pytest.CaptureFixture[str] ) -> None: reg = resources / "valid" / "reg.json" single = uhi.schema.load(reg, path="one") assert single["uhi_schema"] == 1 main(["validate", f"{reg}:one"]) assert capsys.readouterr().out.startswith("OK") with pytest.raises(SystemExit): main(["validate", f"{reg}:missing"]) assert "'missing' not found" in capsys.readouterr().out with pytest.raises(SystemExit): main(["validate", f"{resources / 'valid_single' / 'reg.json'}:one"]) assert "'one' not found" in capsys.readouterr().out def test_cli_validate_zip_path( resources: Path, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: hists = json.loads( (resources / "valid" / "reg.json").read_text(encoding="utf-8"), object_hook=uhi.io.json.object_hook, ) tmp_file = tmp_path / "test.zip" with zipfile.ZipFile(tmp_file, "w") as zip_file: for name, hist in hists.items(): uhi.io.zip.write(zip_file, f"good/{name}", hist) broken: Any = { **hist, "storage": {**hist["storage"], "type": "not_a_storage"}, } uhi.io.zip.write(zip_file, f"bad/{name}", broken) assert set(uhi.schema.load(tmp_file, path="good")) == set(hists) assert set(uhi.schema.load(tmp_file, path="good/")) == set(hists) single = uhi.schema.load(tmp_file, path="good/one") assert single["uhi_schema"] == 1 main(["validate", f"{tmp_file}:good", f"{tmp_file}:good/one"]) assert capsys.readouterr().out.count("OK") == 2 with pytest.raises(SystemExit): main(["validate", f"{tmp_file}:bad"]) assert capsys.readouterr().out.startswith("ERROR") with pytest.raises(SystemExit): main(["validate", f"{tmp_file}:missing"]) assert "'missing' not found" in capsys.readouterr().out def test_cli_validate_hdf5_path( resources: Path, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: h5py = pytest.importorskip("h5py") uhi_io_hdf5 = pytest.importorskip("uhi.io.hdf5") hists = json.loads( (resources / "valid" / "reg.json").read_text(encoding="utf-8"), object_hook=uhi.io.json.object_hook, ) tmp_file = tmp_path / "test.h5" with h5py.File(tmp_file, "w") as h5_file: good = h5_file.create_group("good") for name, hist in hists.items(): uhi_io_hdf5.write(good.create_group(name), hist) bad = h5_file.create_group("bad") for name, hist in hists.items(): grp = bad.create_group(name) uhi_io_hdf5.write(grp, hist) grp["storage"].attrs["type"] = "not_a_storage" assert set(uhi.schema.load(tmp_file, path="good")) == set(hists) assert set(uhi.schema.load(tmp_file, path="/good/")) == set(hists) single = uhi.schema.load(tmp_file, path="good/one") assert single["uhi_schema"] == 1 main(["validate", f"{tmp_file}:good", f"{tmp_file}:good/one"]) assert capsys.readouterr().out.count("OK") == 2 with pytest.raises(SystemExit): main(["validate", f"{tmp_file}:bad"]) assert capsys.readouterr().out.startswith("ERROR") with pytest.raises(SystemExit): main(["validate", f"{tmp_file}:missing"]) assert capsys.readouterr().out.startswith("ERROR") def test_cli_validate_json_keeps_raw_types( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: """Loading for validation must not coerce JSON values into arrays.""" bad = { "one": { "uhi_schema": 1, "axes": [{"type": "boolean"}], "storage": {"type": "double", "values": [True, 1]}, } } tmp_file = tmp_path / "bad.json" tmp_file.write_text(json.dumps(bad), encoding="utf-8") with pytest.raises(fastjsonschema.exceptions.JsonSchemaException): uhi.schema.validate(uhi.schema.load(tmp_file)) with pytest.raises(SystemExit): main(["validate", str(tmp_file)]) assert capsys.readouterr().out.startswith("ERROR") def test_cli_validate_no_fastjsonschema( resources: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setitem(sys.modules, "fastjsonschema", None) with pytest.raises(SystemExit, match=re.escape("uhi[schema]")): main(["validate", str(resources / "valid" / "reg.json")]) def test_cli_validate_empty_name( resources: Path, capsys: pytest.CaptureFixture[str] ) -> None: with pytest.raises(SystemExit): main(["validate", f"{resources / 'valid' / 'reg.json'}:"]) assert "Empty name" in capsys.readouterr().out def test_split_spec_empty_name() -> None: with pytest.raises(ValueError, match="Empty name"): uhi.io._files.split_spec("out.zip:") def test_file_format_no_extension() -> None: with pytest.raises(ValueError, match="No file extension"): uhi.io._files.file_format(Path("out")) def test_cli_validate_hdf5_dataset( resources: Path, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: h5py = pytest.importorskip("h5py") uhi_io_hdf5 = pytest.importorskip("uhi.io.hdf5") hists = uhi.io._files.load(resources / "valid" / "reg.json") tmp_file = tmp_path / "test.h5" with h5py.File(tmp_file, "w") as h5_file: uhi_io_hdf5.write(h5_file.create_group("one"), hists["one"]) with pytest.raises(ValueError, match="not a histogram or group"): uhi.io._files.load(tmp_file, path="one/storage/values") with pytest.raises(KeyError, match="'missing' not found in"): uhi.io._files.load(tmp_file, path="missing") with pytest.raises(SystemExit): main(["validate", f"{tmp_file}:one/storage/values"]) assert "not a histogram or group" in capsys.readouterr().out def test_root_without_backend( resources: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(uhi.io._files, "_uproot_available", lambda: False) monkeypatch.setitem(sys.modules, "ROOT", None) hists = uhi.io._files.load(resources / "valid" / "reg.json") with pytest.raises(ModuleNotFoundError, match=r"uhi\[uproot\]"): uhi.io._files.write(tmp_path / "test.root", hists) with pytest.raises(ModuleNotFoundError, match=r"uhi\[uproot\]"): uhi.io._files.load(tmp_path / "test.root") def test_cli_validate_bad_zip( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: bad = tmp_path / "bad.zip" bad.write_text("x", encoding="utf-8") with pytest.raises(SystemExit): main(["validate", str(bad)]) assert "not a zip file" in capsys.readouterr().out uhi-1.2.1/tests/test_ensure.py0000644000000000000000000002061115261523264013310 0ustar00from __future__ import annotations import numpy as np import pytest from pytest import approx from uhi.numpy_plottable import NumPyPlottableHistogram, ensure_plottable_histogram from uhi.typing.plottable import PlottableHistogram def test_from_numpy() -> None: hist1 = ((1, 2, 3), (0, 1, 2, 3)) h = ensure_plottable_histogram(hist1) assert h.values() == approx(np.array(hist1[0])) assert len(h.axes) == 1 assert len(h.axes[0]) == 3 assert h.axes[0][0] == (0.0, 1.0) assert h.axes[0][1] == (1.0, 2.0) assert h.axes[0][2] == (2.0, 3.0) def test_from_numpy_2d() -> None: np.random.seed(42) x = np.random.normal(1, 2, 1000) y = np.random.normal(-1, 1, 1000) result = np.histogram2d(x, y) h = ensure_plottable_histogram(result) assert h.values() == approx(result[0]) assert len(h.axes) == 2 assert len(h.axes[0]) == 10 assert h.axes[0][0] == approx(result[1][0:2]) assert h.axes[0][1] == approx(result[1][1:3]) assert h.axes[1][0] == approx(result[2][0:2]) assert h.axes[1][1] == approx(result[2][1:3]) def test_from_numpy_dd() -> None: np.random.seed(42) x = np.random.normal(1, 2, 1000) y = np.random.normal(-1, 1, 1000) z = np.random.normal(3, 3, 1000) result = np.histogramdd((x, y, z)) h = ensure_plottable_histogram(result) assert h.values() == approx(result[0]) assert len(h.axes) == 3 assert len(h.axes[0]) == 10 assert h.axes[0][0] == approx(result[1][0][0:2]) assert h.axes[0][1] == approx(result[1][0][1:3]) assert h.axes[1][0] == approx(result[1][1][0:2]) assert h.axes[1][1] == approx(result[1][1][1:3]) assert h.axes[2][0] == approx(result[1][2][0:2]) assert h.axes[2][1] == approx(result[1][2][1:3]) def test_from_bh_regular() -> None: bh = pytest.importorskip("boost_histogram") h1 = bh.Histogram(bh.axis.Regular(5, 0, 5)) h1[...] = (3, 2, 1, 2, 3) h = ensure_plottable_histogram(h1) assert h is h1 assert h.values() == approx(np.array((3, 2, 1, 2, 3))) assert len(h.axes) == 1 assert len(h.axes[0]) == 5 assert h.axes[0][0] == approx(np.array((0, 1))) assert h.axes[0][1] == approx(np.array((1, 2))) assert h.axes[0][2] == approx(np.array((2, 3))) def test_from_bh_integer() -> None: bh = pytest.importorskip("boost_histogram") h1 = bh.Histogram(bh.axis.Integer(1, 6)) h1[...] = (3, 2, 1, 2, 3) h = ensure_plottable_histogram(h1) assert h is h1 assert h.values() == approx(np.array((3, 2, 1, 2, 3))) assert len(h.axes) == 1 assert len(h.axes[0]) == 5 assert h.axes[0][0] == 1 assert h.axes[0][1] == 2 assert h.axes[0][2] == 3 def test_from_numpy_stacked() -> None: """Regression test for issue #233. A tuple of (list_of_value_arrays, bin_edges) — a stacked histogram format used by histoprint — was mishandled: the list was converted to a 2D array but only one bin dimension was supplied, causing zip() to raise ValueError. """ rng = np.random.default_rng(42) bins = np.linspace(-5, 5, 16) # 15 bins → 16 edges values = [rng.integers(0, 300, size=15) for _ in range(3)] hist = (values, bins) h = ensure_plottable_histogram(hist) assert h.values() == approx(np.asarray(values)) assert len(h.axes) == 1 assert len(h.axes[0]) == 15 assert h.axes[0][0] == approx(bins[0:2]) assert h.axes[0][-1] == approx(bins[-2:]) def test_from_numpy_excess_bins_raises() -> None: """If more bin arrays are provided than histogram dimensions, raise. This exercises the new guard that prevents passing e.g. two bin arrays for a 1-D values array. """ values = np.arange(5) bins1 = np.linspace(0, 5, 6) bins2 = np.linspace(0, 5, 6) # Provide two bin arrays for a 1-D histogram hist = (values, (bins1, bins2)) with pytest.raises(ValueError, match="Too many bin arrays"): ensure_plottable_histogram(hist) def test_from_numpy_mixed_none_bins() -> None: """Mixed None entries in a standard tuple should use the default integer axis. A tuple like (vals, edges0, None) was broken: np.asarray(None) produced a 0-d object array, which bypassed _bin_helper's ``bins is None`` check and raised ValueError. The fix preserves None values so _bin_helper produces a 0..N integer-edges axis for those dimensions. """ values = np.zeros((3, 4)) edges0 = np.array([0.0, 1.0, 2.0, 3.0]) hist = (values, edges0, None) h = ensure_plottable_histogram(hist) assert h.values() == approx(values) assert len(h.axes) == 2 # Axis 0 should use the provided edges assert len(h.axes[0]) == 3 assert h.axes[0][0] == approx((0.0, 1.0)) assert h.axes[0][2] == approx((2.0, 3.0)) # Axis 1 should be the default integer axis (0..N) assert len(h.axes[1]) == 4 assert h.axes[1][0] == approx((0.0, 1.0)) assert h.axes[1][3] == approx((3.0, 4.0)) def test_axis_eq_non_axis_object() -> None: """Comparing a NumPyPlottableAxis to a non-axis object must not raise. __eq__ should return NotImplemented (which Python reflects to False) when the other side has no .edges attribute. """ from uhi.numpy_plottable import NumPyPlottableAxis edges = np.array([[0.0, 1.0], [1.0, 2.0], [2.0, 3.0]]) axis = NumPyPlottableAxis(edges) # Must not raise — Python reflects NotImplemented to False assert (axis == "foo") is False assert (axis == 42) is False assert (axis == None) is False def test_axis_eq_different_shape() -> None: """Comparing axes with different numbers of bins should return False, not raise.""" from uhi.numpy_plottable import NumPyPlottableAxis edges_3 = np.array([[0.0, 1.0], [1.0, 2.0], [2.0, 3.0]]) edges_4 = np.array([[0.0, 1.0], [1.0, 2.0], [2.0, 3.0], [3.0, 4.0]]) axis_3 = NumPyPlottableAxis(edges_3) axis_4 = NumPyPlottableAxis(edges_4) assert axis_3 != axis_4 def test_axis_eq_list_edges() -> None: """Comparing to an object whose .edges is a list should not raise.""" from types import SimpleNamespace from uhi.numpy_plottable import NumPyPlottableAxis axis = NumPyPlottableAxis(np.array([[0.0, 1.0], [1.0, 2.0]])) assert axis == SimpleNamespace(edges=[[0.0, 1.0], [1.0, 2.0]]) assert axis != SimpleNamespace(edges=[[0.0, 1.0]]) def test_from_bh_str_cat() -> None: bh = pytest.importorskip("boost_histogram") h1 = bh.Histogram(bh.axis.StrCategory(["hi", "ho"])) h1.fill(["hi", "hi", "hi", "ho"]) h = ensure_plottable_histogram(h1) assert h is h1 assert h.values() == approx(np.array((3, 1))) assert len(h.axes) == 1 assert len(h.axes[0]) == 2 assert h.axes[0][0] == "hi" assert h.axes[0][1] == "ho" def test_0d_plottable() -> None: h = NumPyPlottableHistogram(np.array(3.0), variances=np.array(2.0)) assert isinstance(h, PlottableHistogram) assert ensure_plottable_histogram(h) is h assert len(h.axes) == 0 assert h.values().shape == () assert h.values() == approx(3.0) assert h.variances() == approx(2.0) def test_0d_from_bh() -> None: bh = pytest.importorskip("boost_histogram") h = bh.Histogram() h.fill() h.fill() ph = ensure_plottable_histogram(h) assert isinstance(ph, PlottableHistogram) assert len(ph.axes) == 0 assert np.asarray(ph.values()).shape == () assert ph.values() == approx(2.0) def test_0d_from_tuple() -> None: # boost-histogram's to_numpy() returns a 1-tuple for a 0D histogram ph = ensure_plottable_histogram((np.array(5.0),)) assert isinstance(ph, PlottableHistogram) assert len(ph.axes) == 0 assert ph.values() == approx(5.0) def test_1_tuple_with_axes_raises() -> None: with pytest.raises(TypeError, match="less than 2D tuple"): ensure_plottable_histogram((np.zeros(3),)) @pytest.mark.parametrize( "hist", [ (np.zeros(3), np.arange(10.0)), (np.zeros(3), np.arange(3.0)), (np.zeros((3, 4)), np.arange(4.0), np.arange(4.0)), (np.zeros(3), (np.arange(5.0),)), (np.zeros(3), np.zeros((4, 2))), (np.zeros(3), np.zeros((3, 3))), ], ) def test_edges_mismatch_raises(hist: tuple[object, ...]) -> None: with pytest.raises(ValueError, match="bins"): ensure_plottable_histogram(hist) def test_2d_edges() -> None: edges = np.array([[0.0, 1.0], [1.0, 3.0], [5.0, 6.0]]) h = NumPyPlottableHistogram(np.zeros(3), edges) assert len(h.axes[0]) == 3 assert h.axes[0][1] == approx((1.0, 3.0)) uhi-1.2.1/tests/test_hdf5.py0000644000000000000000000002501015261523264012633 0ustar00from __future__ import annotations import importlib.metadata import json from pathlib import Path from typing import Any import numpy as np import packaging.version import pytest from helpers import convert_histogram_to_32bit import uhi.io.json from uhi.io import to_sparse h5py = pytest.importorskip("h5py", reason="h5py is not installed") uhi_io_hdf5 = pytest.importorskip("uhi.io.hdf5") BHVERSION = packaging.version.Version(importlib.metadata.version("boost_histogram")) HISTVERSION = packaging.version.Version(importlib.metadata.version("hist")) def test_valid_json(valid: Path, tmp_path: Path, sparse: bool) -> None: data = valid.read_text(encoding="utf-8") hists = json.loads(data, object_hook=uhi.io.json.object_hook) if sparse: hists = {name: to_sparse(hist) for name, hist in hists.items()} tmp_file = tmp_path / "test.h5" with h5py.File(tmp_file, "w") as h5_file: for name, hist in hists.items(): uhi_io_hdf5.write(h5_file.create_group(name), hist) with h5py.File(tmp_file, "r") as h5_file: rehists = {name: uhi_io_hdf5.read(h5_file[name]) for name in hists} assert hists.keys() == rehists.keys() for name in hists: hist = hists[name] rehist = rehists[name] # Check that the JSON representation is the same data = json.dumps(hist, default=uhi.io.json.default, sort_keys=True) redata = json.dumps(rehist, default=uhi.io.json.default, sort_keys=True) redata = redata.replace(" ", "").replace("\n", "") data = data.replace(" ", "").replace("\n", "") assert redata == data def test_reg_load(tmp_path: Path, resources: Path) -> None: data = resources / "valid/reg.json" hists = json.loads( data.read_text(encoding="utf-8"), object_hook=uhi.io.json.object_hook ) tmp_file = tmp_path / "test.h5" with h5py.File(tmp_file, "w") as h5_file: uhi_io_hdf5.write( h5_file.create_group("one"), hists["one"], min_compress_elements=0 ) with h5py.File(tmp_file, "r") as h5_file: one = uhi_io_hdf5.read(h5_file["one"]) assert one["metadata"] == {"one": True, "two": 2, "three": "three"} assert len(one["axes"]) == 1 assert one["axes"][0]["type"] == "regular" assert one["axes"][0]["lower"] == pytest.approx(0) assert one["axes"][0]["upper"] == pytest.approx(5) assert one["axes"][0]["bins"] == 3 assert one["axes"][0]["underflow"] assert one["axes"][0]["overflow"] assert not one["axes"][0]["circular"] assert one["storage"]["type"] == "int" assert one["storage"]["values"] == pytest.approx([1, 2, 3, 4, 5]) def test_scalar_storage(tmp_path: Path) -> None: hist: dict[str, Any] = { "uhi_schema": 1, "axes": [], "storage": {"type": "double", "values": 3.0}, } tmp_file = tmp_path / "test.h5" with h5py.File(tmp_file, "w") as h5_file: uhi_io_hdf5.write(h5_file.create_group("scalar"), hist) with h5py.File(tmp_file, "r") as h5_file: rehist = uhi_io_hdf5.read(h5_file["scalar"]) assert rehist["axes"] == [] assert rehist["storage"]["values"] == pytest.approx(3.0) def test_scalar_storage_compress_all(tmp_path: Path) -> None: hist: dict[str, Any] = { "uhi_schema": 1, "axes": [], "storage": {"type": "int", "values": 5}, } tmp_file = tmp_path / "test.h5" with h5py.File(tmp_file, "w") as h5_file: uhi_io_hdf5.write(h5_file.create_group("h"), hist, min_compress_elements=0) with h5py.File(tmp_file, "r") as h5_file: rehist = uhi_io_hdf5.read(h5_file["h"]) assert rehist["storage"]["values"] == 5 def test_fixed_length_storage_type(tmp_path: Path) -> None: hist: dict[str, Any] = { "uhi_schema": 1, "axes": [], "storage": {"type": "int", "values": 5}, } tmp_file = tmp_path / "test.h5" with h5py.File(tmp_file, "w") as h5_file: uhi_io_hdf5.write(h5_file.create_group("h"), hist) # C and Fortran writers often use fixed-length strings h5_file["h/storage"].attrs["type"] = np.bytes_("int") with h5py.File(tmp_file, "r") as h5_file: rehist = uhi_io_hdf5.read(h5_file["h"]) assert rehist["storage"]["type"] == "int" assert isinstance(rehist["storage"]["type"], str) def test_create_dataset_nested_list_size(tmp_path: Path) -> None: with h5py.File(tmp_path / "test.h5", "w") as h5_file: uhi_io_hdf5._create_dataset( h5_file, "nested", [[1, 2], [3, 4]], compression="gzip", compression_opts=4, min_compress_elements=3, ) assert h5_file["nested"].compression == "gzip" def test_axis_order_many_axes(tmp_path: Path) -> None: """Axis order must be preserved past 10 axes (issue #241). h5py yields group members alphabetically (axis_0, axis_1, axis_10, axis_11, axis_2, ...), so reading must dereference the ordered ``axes`` dataset. """ n_axes = 12 uppers = [float(i + 1) for i in range(n_axes)] hist = { "uhi_schema": 1, "axes": [ { "type": "regular", "lower": 0.0, "upper": upper, "bins": 1, "underflow": False, "overflow": False, "circular": False, } for upper in uppers ], "storage": {"type": "double", "values": np.ones((1,) * n_axes)}, } tmp_file = tmp_path / "test.h5" with h5py.File(tmp_file, "w") as h5_file: uhi_io_hdf5.write(h5_file.create_group("h"), hist) with h5py.File(tmp_file, "r") as h5_file: rehist = uhi_io_hdf5.read(h5_file["h"]) assert [ax["upper"] for ax in rehist["axes"]] == pytest.approx(uppers) @pytest.mark.skipif( packaging.version.Version("1.6.1") > BHVERSION, reason="Requires boost-histogram 1.6+", ) def test_convert_bh(tmp_path: Path) -> None: import boost_histogram as bh h = bh.Histogram( bh.axis.Regular(3, 13, 10, __dict__={"name": "x"}), storage=bh.storage.Weight() ) tmp_file = tmp_path / "test.h5" with h5py.File(tmp_file, "w") as h5_file: uhi_io_hdf5.write(h5_file.create_group("histogram"), h) with h5py.File(tmp_file, "r") as h5_file: rehist = uhi_io_hdf5.read(h5_file["histogram"]) h2 = bh.Histogram(rehist) assert h == h2 @pytest.mark.skipif( packaging.version.Version("1.7.2") > BHVERSION, reason="Requires boost-histogram 1.7.2+ for keep_storage=False support", ) def test_convert_bh_no_storage(tmp_path: Path) -> None: """Test HDF5 serialization with keep_storage=False (structure-only histograms).""" import boost_histogram as bh import boost_histogram.serialization h = bh.Histogram( bh.axis.Regular(3, 0, 10, __dict__={"name": "x"}), storage=bh.storage.Weight() ) h.fill([0.1, 0.3, 0.5], weight=[1.5, 2.5, 3.5]) # Convert to UHI format without storage values uhi_dict = boost_histogram.serialization.to_uhi(h, keep_storage=False) # Verify storage has type but no values assert "storage" in uhi_dict assert uhi_dict["storage"]["type"] == "weighted" assert "values" not in uhi_dict["storage"] # Write to HDF5 tmp_file = tmp_path / "test.h5" with h5py.File(tmp_file, "w") as h5_file: uhi_io_hdf5.write(h5_file.create_group("histogram"), uhi_dict) # Read back from HDF5 with h5py.File(tmp_file, "r") as h5_file: rehist = uhi_io_hdf5.read(h5_file["histogram"]) # Should be able to reconstruct a histogram with the same structure h2 = bh.Histogram(rehist) assert h.axes == h2.axes assert isinstance(h2.storage_type(), bh.storage.Weight) @pytest.mark.skipif( packaging.version.Version("1.6.1") > BHVERSION or packaging.version.Version("2.9.0") > HISTVERSION, reason="Requires boost-histogram 1.6+ / Hist 2.9+", ) def test_convert_hist(tmp_path: Path) -> None: import hist h = hist.Hist( hist.axis.Regular(10, 0, 1, name="a", label="A"), hist.axis.Integer(7, 13, overflow=False, name="b", label="B"), storage=hist.storage.Weight(), name="h", label="H", ) tmp_file = tmp_path / "test.h5" with h5py.File(tmp_file, "w") as h5_file: uhi_io_hdf5.write(h5_file.create_group("histogram"), h) with h5py.File(tmp_file, "r") as h5_file: rehist = uhi_io_hdf5.read(h5_file["histogram"]) h2 = hist.Hist(rehist) assert h == h2 @pytest.mark.skipif( packaging.version.Version("1.6.1") > BHVERSION, reason="Requires boost-histogram 1.6+", ) @pytest.mark.parametrize( "storage_type", [ pytest.param("int", id="int_storage"), pytest.param("double", id="double_storage"), pytest.param("weighted", id="weighted_storage"), pytest.param("mean", id="mean_storage"), ], ) def test_convert_bh_32bit_hdf5(tmp_path: Path, storage_type: str) -> None: """Test serialization of 32-bit histograms via HDF5.""" import boost_histogram as bh # Create histogram with appropriate storage type axis = bh.axis.Regular(5, 0, 1, __dict__={"name": "x"}) h: bh.Histogram[Any] if storage_type == "int": h = bh.Histogram(axis, storage=bh.storage.Int64()) # Fill with some data for i in range(5): h.fill([0.1 + i * 0.15] * 10) elif storage_type == "double": h = bh.Histogram(axis, storage=bh.storage.Double()) h.fill([0.1, 0.3, 0.5, 0.7, 0.9]) elif storage_type == "weighted": h = bh.Histogram(axis, storage=bh.storage.Weight()) h.fill([0.1, 0.3, 0.5], weight=[1.5, 2.5, 3.5]) elif storage_type == "mean": h = bh.Histogram(axis, storage=bh.storage.Mean()) h.fill([0.1, 0.3, 0.5], sample=[10.0, 20.0, 30.0]) else: msg = f"Unknown storage type: {storage_type}" raise ValueError(msg) # Convert to UHI format uhi_dict = h._to_uhi_() # Convert to 32-bit uhi_32bit = convert_histogram_to_32bit(uhi_dict) # Write to HDF5 tmp_file = tmp_path / "test_32bit.h5" with h5py.File(tmp_file, "w") as h5_file: uhi_io_hdf5.write(h5_file.create_group("histogram"), uhi_32bit) # Read back from HDF5 with h5py.File(tmp_file, "r") as h5_file: rehist_32bit = uhi_io_hdf5.read(h5_file["histogram"]) # Verify storage type and data integrity assert rehist_32bit["storage"]["type"] == storage_type assert "values" in rehist_32bit["storage"] # Verify JSON representation is consistent redata = json.dumps(rehist_32bit, default=uhi.io.json.default, sort_keys=True) assert len(redata) > 0 uhi-1.2.1/tests/test_histogram_schema.py0000644000000000000000000000617215261523264015332 0ustar00from __future__ import annotations import json import re from pathlib import Path import fastjsonschema import pytest import uhi.schema def test_valid_schemas(valid: Path) -> None: with valid.open(encoding="utf-8") as f: data = json.load(f) uhi.schema.validate(data) def test_valid_single_schemas(valid_single: Path) -> None: with valid_single.open(encoding="utf-8") as f: data = json.load(f) uhi.schema.validate(data) uhi.schema.validate_histogram(data) def test_named_not_a_histogram(valid: Path) -> None: with valid.open(encoding="utf-8") as f: data = json.load(f) with pytest.raises(fastjsonschema.exceptions.JsonSchemaException): uhi.schema.validate_histogram(data) def test_invalid_schemas(invalid: Path) -> None: with invalid.open(encoding="utf-8") as f: data = json.load(f) try: errmsg = invalid.with_suffix(".error.txt").read_text(encoding="utf-8").strip() except FileNotFoundError: errmsg = "NO ERROR MESSAGE FILE FOUND" with pytest.raises( fastjsonschema.exceptions.JsonSchemaException, match=re.escape(errmsg) ): uhi.schema.validate(data) def test_invalid_single_schemas(invalid: Path) -> None: """Each named invalid fixture is also invalid as a single histogram.""" with invalid.open(encoding="utf-8") as f: data = json.load(f) if "axes" in data: pytest.skip("already a single histogram") (name, hist), *_ = data.items() errmsg = invalid.with_suffix(".error.txt").read_text(encoding="utf-8").strip() errmsg = errmsg.replace(f"data.{name}", "data", 1) with pytest.raises( fastjsonschema.exceptions.JsonSchemaException, match=re.escape(errmsg) ): uhi.schema.validate(hist) def test_invalid_named_entry(resources: Path) -> None: with resources.joinpath("valid/reg.json").open(encoding="utf-8") as f: data = json.load(f) del data["two"]["storage"] with pytest.raises( fastjsonschema.exceptions.JsonSchemaException, match=re.escape("data.two must contain ['storage'] properties"), ): uhi.schema.validate(data) def test_missing_uhi_schema() -> None: with pytest.raises( fastjsonschema.exceptions.JsonSchemaException, match=re.escape("data.bad must contain ['uhi_schema'] properties"), ): uhi.schema.validate( {"bad": {"axes": [], "storage": {"type": "int", "values": [1]}}} ) def test_named_entry_not_object() -> None: with pytest.raises( fastjsonschema.exceptions.JsonSchemaException, match=re.escape("data.bad must be object"), ): uhi.schema.validate({"bad": 1}) def test_refs_resolve_locally(valid: Path, monkeypatch: pytest.MonkeyPatch) -> None: def no_network(*args: object, **kwargs: object) -> None: msg = "network access attempted" raise AssertionError(msg) monkeypatch.setattr("urllib.request.urlopen", no_network) uhi.schema._compile.cache_clear() try: with valid.open(encoding="utf-8") as f: uhi.schema.validate(json.load(f)) finally: uhi.schema._compile.cache_clear() uhi-1.2.1/tests/test_json.py0000644000000000000000000002133415261523264012763 0ustar00from __future__ import annotations import importlib.metadata import json from pathlib import Path from typing import Any import numpy as np import packaging.version import pytest from helpers import convert_histogram_to_32bit import uhi.io.json import uhi.schema BHVERSION = packaging.version.Version(importlib.metadata.version("boost_histogram")) HISTVERSION = packaging.version.Version(importlib.metadata.version("hist")) def test_valid_json(valid: Path) -> None: data = valid.read_text(encoding="utf-8") hist = json.loads(data, object_hook=uhi.io.json.object_hook) redata = json.dumps(hist, default=uhi.io.json.default) rehist = json.loads(redata, object_hook=uhi.io.json.object_hook) assert redata.replace(" ", "").replace("\n", "") == data.replace(" ", "").replace( "\n", "" ) assert hist.keys() == rehist.keys() def test_valid_single_json(valid_single: Path) -> None: data = valid_single.read_text(encoding="utf-8") hist = json.loads(data, object_hook=uhi.io.json.object_hook) redata = json.dumps(hist, default=uhi.io.json.default) assert redata.replace(" ", "").replace("\n", "") == data.replace(" ", "").replace( "\n", "" ) assert hist["storage"]["values"] == pytest.approx([1, 2, 3, 4, 5]) def test_reg_load(resources: Path) -> None: data = resources / "valid/reg.json" hists = json.loads( data.read_text(encoding="utf-8"), object_hook=uhi.io.json.object_hook ) one = hists["one"] two = hists["two"] assert one["metadata"] == {"one": True, "two": 2, "three": "three"} assert len(one["axes"]) == 1 assert one["axes"][0]["type"] == "regular" assert one["axes"][0]["lower"] == pytest.approx(0) assert one["axes"][0]["upper"] == pytest.approx(5) assert one["axes"][0]["bins"] == 3 assert one["axes"][0]["underflow"] assert one["axes"][0]["overflow"] assert not one["axes"][0]["circular"] assert one["storage"]["type"] == "int" assert one["storage"]["values"] == pytest.approx([1, 2, 3, 4, 5]) assert len(two["axes"]) == 1 assert two["axes"][0]["type"] == "regular" assert two["axes"][0]["lower"] == pytest.approx(0) assert two["axes"][0]["upper"] == pytest.approx(5) assert two["axes"][0]["bins"] == 5 assert two["axes"][0]["underflow"] assert two["axes"][0]["overflow"] assert not two["axes"][0]["circular"] assert two["storage"]["type"] == "double" assert two["storage"]["values"] == pytest.approx([1, 2, 3, 4, 5, 6, 7]) @pytest.mark.skipif( packaging.version.Version("1.6.1") > BHVERSION, reason="Requires boost-histogram 1.6+", ) def test_convert_bh() -> None: import boost_histogram as bh h = bh.Histogram( bh.axis.Regular(3, 13, 10, __dict__={"name": "x"}), storage=bh.storage.Weight() ) redata = json.dumps(h, default=uhi.io.json.default) rehist = json.loads(redata, object_hook=uhi.io.json.object_hook) h2 = bh.Histogram(rehist) assert h == h2 @pytest.mark.skipif( packaging.version.Version("1.7.2") > BHVERSION, reason="Requires boost-histogram 1.7.2+ for keep_storage=False support", ) def test_convert_bh_no_storage() -> None: """Test serialization with keep_storage=False (structure-only histograms).""" import boost_histogram as bh import boost_histogram.serialization h = bh.Histogram( bh.axis.Regular(3, 0, 10, __dict__={"name": "x"}), storage=bh.storage.Weight() ) h.fill([0.1, 0.3, 0.5], weight=[1.5, 2.5, 3.5]) # Convert to UHI format without storage values uhi_dict = boost_histogram.serialization.to_uhi(h, keep_storage=False) # Verify storage has type but no values assert "storage" in uhi_dict assert uhi_dict["storage"]["type"] == "weighted" assert "values" not in uhi_dict["storage"] assert "variances" not in uhi_dict["storage"] # Should be able to serialize to JSON redata = json.dumps(uhi_dict, default=uhi.io.json.default) rehist = json.loads(redata, object_hook=uhi.io.json.object_hook) # Should be able to reconstruct a histogram with the same structure h2 = bh.Histogram(rehist) assert h.axes == h2.axes assert isinstance(h2.storage_type(), bh.storage.Weight) @pytest.mark.skipif( packaging.version.Version("1.6.1") > BHVERSION or packaging.version.Version("2.9.0") > HISTVERSION, reason="Requires boost-histogram 1.6+ / Hist 2.9+", ) def test_convert_hist() -> None: import hist h = hist.Hist( hist.axis.Regular(10, 0, 1, name="a", label="A"), hist.axis.Integer(7, 13, overflow=False, name="b", label="B"), storage=hist.storage.Weight(), name="h", label="H", ) redata = json.dumps(h, default=uhi.io.json.default) rehist = json.loads(redata, object_hook=uhi.io.json.object_hook) h2 = hist.Hist(rehist) assert h == h2 def test_empty_storage_roundtrip() -> None: """Test that empty storages can be read and written with JSON.""" data = { "empty_int": { "uhi_schema": 1, "axes": [ { "type": "regular", "lower": 0, "upper": 10, "bins": 5, "underflow": False, "overflow": False, "circular": False, } ], "storage": {"type": "int"}, } } # Dump to JSON json_str = json.dumps(data, default=uhi.io.json.default) # Load back from JSON loaded = json.loads(json_str, object_hook=uhi.io.json.object_hook) # Verify structure is preserved assert loaded == data assert loaded["empty_int"]["storage"]["type"] == "int" assert "values" not in loaded["empty_int"]["storage"] @pytest.mark.skipif( packaging.version.Version("1.6.1") > BHVERSION, reason="Requires boost-histogram 1.6+", ) @pytest.mark.parametrize( "storage_type", [ pytest.param("int", id="int_storage"), pytest.param("double", id="double_storage"), pytest.param("weighted", id="weighted_storage"), pytest.param("mean", id="mean_storage"), ], ) def test_convert_bh_32bit(storage_type: str) -> None: """Test serialization of 32-bit histograms via JSON.""" import boost_histogram as bh # Create histogram with appropriate storage type axis = bh.axis.Regular(5, 0, 1, __dict__={"name": "x"}) h: bh.Histogram[Any] if storage_type == "int": h = bh.Histogram(axis, storage=bh.storage.Int64()) # Fill with some data for i in range(5): h.fill([0.1 + i * 0.15] * 10) elif storage_type == "double": h = bh.Histogram(axis, storage=bh.storage.Double()) h.fill([0.1, 0.3, 0.5, 0.7, 0.9]) elif storage_type == "weighted": h = bh.Histogram(axis, storage=bh.storage.Weight()) h.fill([0.1, 0.3, 0.5], weight=[1.5, 2.5, 3.5]) elif storage_type == "mean": h = bh.Histogram(axis, storage=bh.storage.Mean()) h.fill([0.1, 0.3, 0.5], sample=[10.0, 20.0, 30.0]) else: msg = f"Unknown storage type: {storage_type}" raise ValueError(msg) # Convert to UHI format uhi_dict = h._to_uhi_() # Convert to 32-bit uhi_32bit = convert_histogram_to_32bit(uhi_dict) # Serialize via JSON redata = json.dumps(uhi_32bit, default=uhi.io.json.default) # Deserialize rehist_32bit = json.loads(redata, object_hook=uhi.io.json.object_hook) # Verify storage type and data integrity assert rehist_32bit["storage"]["type"] == storage_type assert "values" in rehist_32bit["storage"] # Verify values can be serialized again without error redata2 = json.dumps(rehist_32bit, default=uhi.io.json.default) assert len(redata2) > 0 @pytest.mark.skipif( packaging.version.Version("1.6.1") > BHVERSION, reason="Requires boost-histogram 1.6+", ) def test_named_round_trip() -> None: import boost_histogram as bh h1 = bh.Histogram(bh.axis.Regular(3, 0, 1), storage=bh.storage.Weight()) h1.fill([0.1, 0.5], weight=[2, 3]) h2 = bh.Histogram(bh.axis.Integer(0, 4), bh.axis.Boolean()) h2.fill([1, 2], [True, False]) data = json.dumps({"a": h1, "b": h2}, default=uhi.io.json.default) uhi.schema.validate(json.loads(data)) rehists = json.loads(data, object_hook=uhi.io.json.object_hook) assert rehists.keys() == {"a", "b"} assert bh.Histogram(rehists["a"]) == h1 assert bh.Histogram(rehists["b"]) == h2 @pytest.mark.parametrize("value", [np.int64(5), np.float32(5.0)]) def test_numpy_scalar_storage(value: Any) -> None: hist = {"uhi_schema": 1, "axes": [], "storage": {"type": "double", "values": value}} text = json.dumps(hist, default=uhi.io.json.default) rehist = json.loads(text, object_hook=uhi.io.json.object_hook) assert rehist["storage"]["values"] == 5 uhi-1.2.1/tests/test_root.py0000644000000000000000000004773715261523264013014 0ustar00from __future__ import annotations import importlib.metadata import json from pathlib import Path from typing import Any import numpy as np import packaging.version import pytest from helpers import convert_histogram_to_32bit, scalar_no_axis_storage from pytest import approx import uhi.io._files import uhi.io.json import uhi.io.ops import uhi.schema from uhi.io import ARRAY_KEYS, to_sparse from uhi.numpy_plottable import ( ContinuousROOTAxis, ROOTAxis, ROOTPlottableProfile, ensure_plottable_histogram, ) ROOT = pytest.importorskip("ROOT") uhi_io_root = pytest.importorskip("uhi.io.root") BHVERSION = packaging.version.Version(importlib.metadata.version("boost_histogram")) def test_root_imported() -> None: assert ROOT.TString("hi") == "hi" def test_root_th1f_convert() -> None: th = ROOT.TH1F("h1", "h1", 50, -2.5, 2.5) th.FillRandom("gaus", 10000) h = ensure_plottable_histogram(th) assert all(th.GetBinContent(i + 1) == approx(iv) for i, iv in enumerate(h.values())) var = h.variances() assert var is not None assert all(th.GetBinError(i + 1) == approx(ie) for i, ie in enumerate(np.sqrt(var))) def test_root_th2f_convert() -> None: th = ROOT.TH2F("h2", "h2", 50, -2.5, 2.5, 50, -2.5, 2.5) _ = ROOT.TF2("xyg", "xygaus", -2.5, 2.5, -2.5, 2.5) th.FillRandom("xyg", 10000) h = ensure_plottable_histogram(th) assert all( th.GetBinContent(i + 1, j + 1) == approx(iv) for i, row in enumerate(h.values()) for j, iv in enumerate(row) ) var = h.variances() assert var is not None assert all( th.GetBinError(i + 1, j + 1) == approx(ie) for i, row in enumerate(np.sqrt(var)) for j, ie in enumerate(row) ) # Newer ROOT supports the Protocol natively, so use the adapter directly def test_root_profile_adapter_unweighted() -> None: tp = ROOT.TProfile("p1", "p1", 1, 0, 1) for y in (1, 2, 3, 6): tp.Fill(0.5, y) h = ROOTPlottableProfile(tp) assert h.values() == approx([3]) assert h.counts() == approx([4]) assert h.variances() == approx([np.var([1, 2, 3, 6])]) def test_root_profile_adapter_weighted() -> None: tp = ROOT.TProfile("p2", "p2", 2, 0, 2) tp.Fill(0.5, 1, 1) tp.Fill(0.5, 3, 1) tp.Fill(1.5, 2, 2) tp.Fill(1.5, 4, 1) h = ROOTPlottableProfile(tp) assert h.values() == approx([2, 8 / 3]) # Effective entries: sum(w)**2 / sum(w**2) assert h.counts() == approx([2, 9 / 5]) assert h.variances() == approx([1, 8 / 9]) # Error on the mean assert np.sqrt(h.variances() / h.counts()) == approx( [tp.GetBinError(1), tp.GetBinError(2)] ) tp.SetErrorOption("s") assert h.variances() == approx([1, 8 / 9]) def test_root_axis_zero_bins() -> None: assert isinstance(ROOTAxis.create(ROOT.TAxis(0, 0, 1)), ContinuousROOTAxis) # Serialization def test_valid_json(valid: Path, tmp_path: Path, sparse: bool) -> None: data = valid.read_text(encoding="utf-8") hists = json.loads(data, object_hook=uhi.io.json.object_hook) if sparse: hists = {name: to_sparse(hist) for name, hist in hists.items()} tmp_file = tmp_path / "test.root" with ROOT.TFile.Open(str(tmp_file), "RECREATE") as root_file: for name, hist in hists.items(): uhi_io_root.write(root_file, name, hist) with ROOT.TFile.Open(str(tmp_file)) as root_file: rehists = {name: uhi_io_root.read(root_file, name) for name in hists} assert hists.keys() == rehists.keys() for name in hists: hist = hists[name] rehist = rehists[name] scalar_no_axis_storage(hist) # Check that the JSON representation is the same data = json.dumps(hist, default=uhi.io.json.default, sort_keys=True) redata = json.dumps(rehist, default=uhi.io.json.default, sort_keys=True) redata = redata.replace(" ", "").replace("\n", "") data = data.replace(" ", "").replace("\n", "") assert redata == data def test_reg_load(tmp_path: Path, resources: Path) -> None: data = resources / "valid/reg.json" hists = uhi.io._files.load(data) tmp_file = tmp_path / "test.root" with ROOT.TFile.Open(str(tmp_file), "RECREATE") as root_file: for name, hist in hists.items(): uhi_io_root.write(root_file, name, hist) with ROOT.TFile.Open(str(tmp_file)) as root_file: rehists = {name: uhi_io_root.read(root_file, name) for name in hists} # One single-entry RNTuple per histogram with ROOT.RNTupleReader.Open(root_file.Get("one")) as reader: fields = { field.GetFieldName() for field in reader.GetDescriptor().GetTopLevelFields() } assert fields == {"uhi", "values"} assert reader.GetNEntries() == 1 entry = reader.CreateEntry() reader.LoadEntry(0, entry) native_one = json.loads(str(entry["uhi"])) assert native_one["storage"]["values"] == "values" one = rehists["one"] two = rehists["two"] assert one["metadata"] == {"one": True, "two": 2, "three": "three"} assert len(one["axes"]) == 1 assert one["axes"][0]["type"] == "regular" assert one["axes"][0]["lower"] == pytest.approx(0) assert one["axes"][0]["upper"] == pytest.approx(5) assert one["axes"][0]["bins"] == 3 assert one["axes"][0]["underflow"] assert one["axes"][0]["overflow"] assert not one["axes"][0]["circular"] assert one["storage"]["type"] == "int" assert one["storage"]["values"] == pytest.approx([1, 2, 3, 4, 5]) assert two["storage"]["type"] == "double" assert two["storage"]["values"] == pytest.approx([1, 2, 3, 4, 5, 6, 7]) @pytest.mark.parametrize("value", ["description", "values", "axis_0_edges"]) def test_metadata_array_keys(tmp_path: Path, value: str) -> None: metadata = dict.fromkeys(ARRAY_KEYS, value) writer_info = {"test": metadata} hist: dict[str, Any] = { "uhi_schema": 1, "metadata": metadata, "writer_info": writer_info, "axes": [ { "type": "variable", "edges": np.array([0.0, 1.0, 2.0]), "underflow": False, "overflow": False, "circular": False, "metadata": metadata, "writer_info": writer_info, } ], "storage": {"type": "double", "values": np.array([1.0, 2.0])}, } tmp_file = tmp_path / "metadata.root" with ROOT.TFile.Open(str(tmp_file), "RECREATE") as root_file: uhi_io_root.write(root_file, "histogram", hist) with ROOT.TFile.Open(str(tmp_file)) as root_file: rehist = uhi_io_root.read(root_file, "histogram") assert rehist["metadata"] == metadata assert rehist["writer_info"] == writer_info assert rehist["axes"][0]["metadata"] == metadata assert rehist["axes"][0]["writer_info"] == writer_info assert rehist["axes"][0]["edges"] == pytest.approx(hist["axes"][0]["edges"]) assert rehist["storage"]["values"] == pytest.approx(hist["storage"]["values"]) def test_two_variable_axes(tmp_path: Path) -> None: """Two variable axes must not collide on their field names.""" edges_a = np.array([0.0, 1.0, 2.0, 3.0]) edges_b = np.array([0.0, 10.0, 20.0]) def axis(edges: np.ndarray) -> dict[str, Any]: return { "type": "variable", "edges": edges, "underflow": False, "overflow": False, "circular": False, } hist: dict[str, Any] = { "uhi_schema": 1, "axes": [axis(edges_a), axis(edges_b)], "storage": { "type": "double", "values": np.arange(6, dtype=np.float64).reshape(3, 2), }, } tmp_file = tmp_path / "test.root" with ROOT.TFile.Open(str(tmp_file), "RECREATE") as root_file: uhi_io_root.write(root_file, "h", hist) # write() must not replace the caller's arrays with field names. assert isinstance(hist["axes"][0]["edges"], np.ndarray) assert isinstance(hist["storage"]["values"], np.ndarray) with ROOT.TFile.Open(str(tmp_file)) as root_file: rehist = uhi_io_root.read(root_file, "h") assert rehist["axes"][0]["edges"] == pytest.approx(edges_a) assert rehist["axes"][1]["edges"] == pytest.approx(edges_b) assert rehist["storage"]["values"].shape == (3, 2) assert rehist["storage"]["values"] == pytest.approx(hist["storage"]["values"]) def test_subdirectory(tmp_path: Path, resources: Path) -> None: data = resources / "valid/2d.json" hists = uhi.io._files.load(data) tmp_file = tmp_path / "test.root" with ROOT.TFile.Open(str(tmp_file), "RECREATE") as root_file: uhi_io_root.write(root_file.mkdir("sub"), "main", hists["main"]) with ROOT.TFile.Open(str(tmp_file)) as root_file: rehist = uhi_io_root.read(root_file.Get("sub"), "main") assert rehist["storage"]["values"] == pytest.approx( hists["main"]["storage"]["values"] ) @pytest.mark.skipif( packaging.version.Version("1.6.1") > BHVERSION, reason="Requires boost-histogram 1.6+", ) def test_convert_bh(tmp_path: Path) -> None: import boost_histogram as bh h = bh.Histogram( bh.axis.Regular(3, 13, 10, __dict__={"name": "x"}), storage=bh.storage.Weight() ) tmp_file = tmp_path / "test.root" with ROOT.TFile.Open(str(tmp_file), "RECREATE") as root_file: uhi_io_root.write(root_file, "histogram", h) with ROOT.TFile.Open(str(tmp_file)) as root_file: rehist = uhi_io_root.read(root_file, "histogram") h2 = bh.Histogram(rehist) assert h == h2 @pytest.mark.skipif( packaging.version.Version("1.7.2") > BHVERSION, reason="Requires boost-histogram 1.7.2+ for keep_storage=False support", ) def test_convert_bh_no_storage(tmp_path: Path) -> None: """Test ROOT serialization with keep_storage=False (structure-only histograms).""" import boost_histogram as bh import boost_histogram.serialization h = bh.Histogram( bh.axis.Regular(3, 0, 10, __dict__={"name": "x"}), storage=bh.storage.Weight() ) h.fill([0.1, 0.3, 0.5], weight=[1.5, 2.5, 3.5]) uhi_dict = boost_histogram.serialization.to_uhi(h, keep_storage=False) assert uhi_dict["storage"]["type"] == "weighted" assert "values" not in uhi_dict["storage"] tmp_file = tmp_path / "test.root" with ROOT.TFile.Open(str(tmp_file), "RECREATE") as root_file: uhi_io_root.write(root_file, "histogram", uhi_dict) with ROOT.TFile.Open(str(tmp_file)) as root_file: rehist = uhi_io_root.read(root_file, "histogram") h2 = bh.Histogram(rehist) assert h.axes == h2.axes assert isinstance(h2.storage_type(), bh.storage.Weight) @pytest.mark.skipif( packaging.version.Version("1.6.1") > BHVERSION, reason="Requires boost-histogram 1.6+", ) def test_convert_hist(tmp_path: Path) -> None: h: Any try: import hist h = hist.Hist( hist.axis.Regular(10, 0, 1, name="a", label="A"), hist.axis.Integer(7, 13, overflow=False, name="b", label="B"), storage=hist.storage.Weight(), name="h", label="H", ) except ImportError: # Fall back to boost-histogram (as in the ROOT CI environment) import boost_histogram as bh h = bh.Histogram( bh.axis.Regular(10, 0, 1, __dict__={"name": "a", "label": "A"}), bh.axis.Integer( 7, 13, overflow=False, __dict__={"name": "b", "label": "B"} ), storage=bh.storage.Weight(), ) tmp_file = tmp_path / "test.root" with ROOT.TFile.Open(str(tmp_file), "RECREATE") as root_file: uhi_io_root.write(root_file, "histogram", h) with ROOT.TFile.Open(str(tmp_file)) as root_file: rehist = uhi_io_root.read(root_file, "histogram") h2 = type(h)(rehist) assert h == h2 @pytest.mark.skipif( packaging.version.Version("1.6.1") > BHVERSION, reason="Requires boost-histogram 1.6+", ) @pytest.mark.parametrize( "storage_type", [ pytest.param("int", id="int_storage"), pytest.param("double", id="double_storage"), pytest.param("weighted", id="weighted_storage"), pytest.param("mean", id="mean_storage"), ], ) def test_convert_bh_32bit_root(tmp_path: Path, storage_type: str) -> None: """Test serialization of 32-bit histograms via ROOT.""" import boost_histogram as bh axis = bh.axis.Regular(5, 0, 1, __dict__={"name": "x"}) h: Any if storage_type == "int": h = bh.Histogram(axis, storage=bh.storage.Int64()) for i in range(5): h.fill([0.1 + i * 0.15] * 10) elif storage_type == "double": h = bh.Histogram(axis, storage=bh.storage.Double()) h.fill([0.1, 0.3, 0.5, 0.7, 0.9]) elif storage_type == "weighted": h = bh.Histogram(axis, storage=bh.storage.Weight()) h.fill([0.1, 0.3, 0.5], weight=[1.5, 2.5, 3.5]) elif storage_type == "mean": h = bh.Histogram(axis, storage=bh.storage.Mean()) h.fill([0.1, 0.3, 0.5], sample=[10.0, 20.0, 30.0]) else: msg = f"Unknown storage type: {storage_type}" raise ValueError(msg) uhi_32bit = convert_histogram_to_32bit(h._to_uhi_()) tmp_file = tmp_path / "test_32bit.root" with ROOT.TFile.Open(str(tmp_file), "RECREATE") as root_file: uhi_io_root.write(root_file, "histogram", uhi_32bit) with ROOT.TFile.Open(str(tmp_file)) as root_file: rehist_32bit = uhi_io_root.read(root_file, "histogram") # The 32-bit dtypes are preserved by the field types assert rehist_32bit["storage"]["type"] == storage_type assert ( rehist_32bit["storage"]["values"].dtype == uhi_32bit["storage"]["values"].dtype ) assert rehist_32bit["storage"]["values"] == pytest.approx( uhi_32bit["storage"]["values"] ) @pytest.mark.parametrize("writer", ["root", "uproot"]) def test_uproot_compat(valid: Path, tmp_path: Path, sparse: bool, writer: str) -> None: """Files written by ROOT read with uproot, and the reverse.""" uproot = pytest.importorskip("uproot") import uhi.io.uproot hists = uhi.io._files.load(valid) if sparse: hists = {name: to_sparse(hist) for name, hist in hists.items()} tmp_file = tmp_path / "test.root" if writer == "root": with ROOT.TFile.Open(str(tmp_file), "RECREATE") as root_file: for name, hist in hists.items(): uhi_io_root.write(root_file, name, hist) with uproot.open(tmp_file) as root_file: rehists = {name: uhi.io.uproot.read(root_file, name) for name in hists} else: with uproot.recreate(tmp_file) as root_file: for name, hist in hists.items(): uhi.io.uproot.write(root_file, name, hist) with ROOT.TFile.Open(str(tmp_file)) as root_file: rehists = {name: uhi_io_root.read(root_file, name) for name in hists} for name, hist in hists.items(): scalar_no_axis_storage(hist) data = json.dumps(hist, default=uhi.io.json.default, sort_keys=True) redata = json.dumps(rehists[name], default=uhi.io.json.default, sort_keys=True) assert redata == data # CLI @pytest.fixture(params=["pyroot", "uproot"]) def files_backend( request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch ) -> None: """Run the file-level helpers with each backend.""" if request.param == "uproot": pytest.importorskip("uproot") else: monkeypatch.setattr(uhi.io._files, "_uproot_available", lambda: False) @pytest.mark.usefixtures("files_backend") def test_cli_validate_root( valid: Path, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: from uhi.__main__ import main hists = uhi.io._files.load(valid) tmp_file = tmp_path / "test.root" with ROOT.TFile.Open(str(tmp_file), "RECREATE") as root_file: # Nest one level to check that directories are searched recursively directory = root_file.mkdir("nested") for name, hist in hists.items(): uhi_io_root.write(directory, name, hist) main(["validate", str(tmp_file)]) assert capsys.readouterr().out.startswith("OK") @pytest.mark.usefixtures("files_backend") def test_cli_validate_root_invalid( resources: Path, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: from uhi.__main__ import main hists = uhi.io._files.load(resources / "valid" / "reg.json") tmp_file = tmp_path / "test.root" with ROOT.TFile.Open(str(tmp_file), "RECREATE") as root_file: for name, hist in hists.items(): hist = dict(hist) # noqa: PLW2901 hist["storage"] = {**hist["storage"], "type": "not_a_storage"} uhi_io_root.write(root_file, name, hist) with pytest.raises(SystemExit): main(["validate", str(tmp_file)]) assert capsys.readouterr().out.startswith("ERROR") @pytest.mark.usefixtures("files_backend") def test_cli_validate_root_path( resources: Path, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: from uhi.__main__ import main hists = uhi.io._files.load(resources / "valid" / "reg.json") tmp_file = tmp_path / "test.root" with ROOT.TFile.Open(str(tmp_file), "RECREATE") as root_file: good = root_file.mkdir("good") bad = root_file.mkdir("bad") for name, hist in hists.items(): uhi_io_root.write(good, name, hist) broken = {**hist, "storage": {**hist["storage"], "type": "not_a_storage"}} uhi_io_root.write(bad, name, broken) assert set(uhi.schema.load(tmp_file, path="good")) == set(hists) assert set(uhi.schema.load(tmp_file, path="good/")) == set(hists) single = uhi.schema.load(tmp_file, path="good/one") assert single["uhi_schema"] == 1 main(["validate", f"{tmp_file}:good", f"{tmp_file}:good/one"]) assert capsys.readouterr().out.count("OK") == 2 with pytest.raises(SystemExit): main(["validate", f"{tmp_file}:bad"]) assert capsys.readouterr().out.startswith("ERROR") with pytest.raises(SystemExit): main(["validate", f"{tmp_file}:missing"]) assert capsys.readouterr().out.startswith("ERROR") def _root_add_inputs(resources: Path, tmp_path: Path) -> tuple[list[Path], list[Any]]: """Write the same histograms to two ROOT files, nested one level deep.""" hists = uhi.io._files.load(resources / "valid/reg.json") files = [tmp_path / "in1.root", tmp_path / "in2.root"] for file in files: with ROOT.TFile.Open(str(file), "RECREATE") as root_file: directory = root_file.mkdir("sub") for name, hist in hists.items(): uhi_io_root.write(directory, name, hist) return files, [f"sub/{name}" for name in hists] @pytest.mark.usefixtures("files_backend") @pytest.mark.parametrize("out_suffix", [".root", ".json"]) def test_cli_add_root(resources: Path, tmp_path: Path, out_suffix: str) -> None: from uhi.__main__ import main files, names = _root_add_inputs(resources, tmp_path) target = tmp_path / f"out{out_suffix}" main(["add", str(target), *map(str, files)]) inputs = [uhi.io._files.load(file) for file in files] result = uhi.io._files.load(target) assert result.keys() == {*names} for name in names: expected = uhi.io.ops.add(*(hists[name] for hists in inputs)) assert result[name]["storage"]["values"] == approx( expected["storage"]["values"] ) @pytest.mark.usefixtures("files_backend") def test_cli_validate_root_non_rntuple( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: from uhi.__main__ import main tmp_file = tmp_path / "test.root" with ROOT.TFile.Open(str(tmp_file), "RECREATE") as root_file: th = ROOT.TH1D("th1", "th1", 3, 0, 3) root_file.WriteObject(th, "th1") root_file.mkdir("sub").WriteObject(th, "th1") for path in ("th1", "sub/th1"): with pytest.raises(ValueError, match="not a histogram or directory"): uhi.io._files.load(tmp_file, path=path) with pytest.raises(SystemExit): main(["validate", f"{tmp_file}:th1"]) assert "not a histogram or directory" in capsys.readouterr().out uhi-1.2.1/tests/test_serialization.py0000644000000000000000000001217215261523264014667 0ustar00from __future__ import annotations import dataclasses from typing import Any import numpy as np from uhi.io import from_sparse, remove_writer_info, to_sparse from uhi.io._common import _convert_input from uhi.io._rntuple import from_fields, to_fields from uhi.typing.serialization import AnyHistogramIR, AnyStorageIR def test_remove_writer_info() -> None: d = {"uhi_schema": 1, "writer_info": {"a": {"foo": "bar"}, "b": {"FOO": "BAR"}}} assert remove_writer_info(d, library=None) == {"uhi_schema": 1} assert remove_writer_info(d, library="a") == { "uhi_schema": 1, "writer_info": {"b": {"FOO": "BAR"}}, } assert remove_writer_info(d, library="b") == { "uhi_schema": 1, "writer_info": {"a": {"foo": "bar"}}, } assert remove_writer_info(d, library="c") == d # A histogram without writer_info must not raise (issue #241). no_wi = {"uhi_schema": 1} assert remove_writer_info(no_wi, library=None) == {"uhi_schema": 1} assert remove_writer_info(no_wi, library="a") == {"uhi_schema": 1} @dataclasses.dataclass class _Simple: value: dict[str, Any] def _to_uhi_(self) -> dict[str, Any]: return self.value def test_remove_empty_metadata() -> None: d = { "uhi_schema": 1, "writer_info": {"boost-histogram": {"version": "1.6.1"}}, "axes": [ { "type": "regular", "lower": 0.0, "upper": 1.0, "bins": 4, "underflow": True, "overflow": True, "circular": False, "metadata": {}, } ], "storage": { "type": "weighted_mean", "sum_of_weights": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "sum_of_weights_squared": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "variances": [float("nan")] * 6, }, "metadata": {}, } h = _Simple(d) ir = _convert_input(h) assert "metadata" not in ir assert "metadata" not in ir["axes"][0] def test_empty_storage_conversion() -> None: """Test that empty storages are handled correctly during conversion.""" # Test each storage type with empty arrays storage_types: list[tuple[str, dict[str, Any]]] = [ ("int", {}), ("double", {}), ("weighted", {}), ("mean", {}), ("weighted_mean", {}), ] for storage_type, arrays in storage_types: d = { "uhi_schema": 1, "axes": [ { "type": "regular", "lower": 0.0, "upper": 1.0, "bins": 4, "underflow": False, "overflow": False, "circular": False, } ], "storage": {"type": storage_type, **arrays}, } h = _Simple(d) ir = _convert_input(h) storage: AnyStorageIR = ir["storage"] assert storage["type"] == storage_type # Verify no array fields are present array_keys = { "values", "variances", "counts", "sum_of_weights", "sum_of_weights_squared", "index", } assert not any(k in storage for k in array_keys) def test_to_sparse_empty_storage() -> None: """Test that to_sparse leaves empty storages alone.""" d = { "uhi_schema": 1, "axes": [ { "type": "regular", "lower": 0.0, "upper": 1.0, "bins": 4, "underflow": False, "overflow": False, "circular": False, } ], "storage": {"type": "int"}, } result = to_sparse(d) # Should return the same histogram unchanged assert result == d storage_result: AnyStorageIR = result["storage"] # type: ignore[assignment] assert storage_result["type"] == "int" assert "values" not in storage_result def test_from_sparse_empty_storage() -> None: """Test that from_sparse leaves empty storages alone.""" d = { "uhi_schema": 1, "axes": [ { "type": "regular", "lower": 0.0, "upper": 1.0, "bins": 4, "underflow": False, "overflow": False, "circular": False, } ], "storage": {"type": "double"}, } result = from_sparse(d) # Should return the same histogram unchanged assert result == d storage_result: AnyStorageIR = result["storage"] # type: ignore[assignment] assert storage_result["type"] == "double" assert "values" not in storage_result def test_rntuple_fields_0d() -> None: hist: AnyHistogramIR = { "uhi_schema": 1, "axes": [], "storage": {"type": "double", "values": np.array(12.0)}, } uhi, arrays = to_fields(hist) rehist = from_fields(uhi, arrays.__getitem__) assert rehist["storage"]["values"].shape == () assert rehist["storage"]["values"] == 12.0 uhi-1.2.1/tests/test_sparse.py0000644000000000000000000001640115261523264013306 0ustar00from __future__ import annotations import json from pathlib import Path from typing import Any import numpy as np import pytest import uhi.io.json from uhi.io import from_sparse, to_sparse from uhi.typing.serialization import HistogramIR, WeightedStorageIR def test_to_from_sparse_roundtrip() -> None: # Original dense data hist: HistogramIR = { "uhi_schema": 1, "storage": { "type": "weighted", "values": np.array([[0, 1, 0], [0, 2, 0]], dtype=float), "variances": np.array([[0, 0.1, 0], [0, 0, 0]], dtype=float), }, "axes": [ {"type": "boolean"}, { "type": "regular", "bins": 3, "overflow": False, "underflow": False, "lower": 0, "upper": 1, "circular": False, }, ], } # Convert to sparse shist = to_sparse(hist) sparse: WeightedStorageIR = shist["storage"] # type: ignore[assignment] # Basic checks on sparse structure assert "index" in sparse index = sparse["index"] assert index.shape[0] == 2 assert index.shape[1] == 2 # Verify sparse arrays align with mask assert np.all(sparse["values"] == np.array([1.0, 2.0])) assert np.all(sparse["variances"] == np.array([0.1, 0.0])) # Convert back to dense dense = from_sparse(shist) # Check round-trip reconstruction for key, value in hist["storage"].items(): if key == "type": continue assert np.allclose(dense["storage"][key], value) # type: ignore[literal-required,arg-type] def test_all_valid(valid: Path) -> None: data = valid.read_text(encoding="utf-8") hists = json.loads(data, object_hook=uhi.io.json.object_hook) hists = { k: from_sparse(v) if "index" in v.get("storage", {}) else v for k, v in hists.items() } shists = {k: to_sparse(v) for k, v in hists.items()} for h in shists.values(): # Empty storages (metadata-only) won't get an index since there's no data has_data = len(h["storage"]) > 1 # More than just "type" if h["axes"] and has_data: assert "index" in h["storage"] dhists = {k: from_sparse(v) for k, v in shists.items()} for h in dhists.values(): assert "index" not in h["storage"] assert hists.keys() == dhists.keys() for v, dv in zip(hists.values(), dhists.values(), strict=True): assert v.keys() == dv.keys() assert v["axes"] == dv["axes"] assert v["storage"].keys() == dv["storage"].keys() # Empty storages won't have values to compare if "values" in v["storage"] and "values" in dv["storage"]: assert v["storage"]["values"] == pytest.approx(dv["storage"]["values"]) if v["storage"]["type"] == "weighted_mean": v_var = v["storage"].get("variances") dv_var = dv["storage"].get("variances") if v_var is not None and dv_var is not None: assert np.all(np.isnan(v_var) == np.isnan(dv_var)) assert v_var[~np.isnan(v_var)] == pytest.approx( dv_var[~np.isnan(dv_var)] ) assert v["storage"].get("sum_of_weights") == pytest.approx( dv["storage"].get("sum_of_weights") ) assert v["storage"].get("sum_of_weights_squared") == pytest.approx( dv["storage"].get("sum_of_weights_squared") ) else: assert v["storage"].get("variances") == pytest.approx( dv["storage"].get("variances") ) assert v["storage"].get("counts") == pytest.approx( dv["storage"].get("counts") ) @pytest.mark.parametrize("name", ["mean", "weighted_mean"]) def test_mean(resources: Path, name: str) -> None: data = resources.joinpath("valid/mean.json").read_text() hist = json.loads(data, object_hook=uhi.io.json.object_hook)[name] sparse_hist = to_sparse(hist) assert len(sparse_hist["storage"]["values"]) == 2 assert sparse_hist["storage"]["index"].shape == (1, 2) def test_from_sparse_empty_index_json_roundtrip() -> None: """An all-zero histogram has an empty index; JSON reads it back as float.""" hist: HistogramIR = { "uhi_schema": 1, "storage": {"type": "double", "values": np.zeros(3)}, "axes": [ { "type": "regular", "bins": 3, "overflow": False, "underflow": False, "lower": 0, "upper": 1, "circular": False, }, ], } text = json.dumps(to_sparse(hist), default=uhi.io.json.default) shist = json.loads(text, object_hook=uhi.io.json.object_hook) assert shist["storage"]["index"].shape == (1, 0) dense = from_sparse(shist) np.testing.assert_array_equal(dense["storage"]["values"], np.zeros(3)) def test_from_sparse_weighted_mean_int_variances() -> None: hist = json.loads( """{ "uhi_schema": 1, "axes": [{"type": "regular", "lower": 0, "upper": 1, "bins": 2, "underflow": false, "overflow": false, "circular": false}], "storage": {"type": "weighted_mean", "index": [[1]], "sum_of_weights": [1], "sum_of_weights_squared": [1], "values": [1], "variances": [1]} }""", object_hook=uhi.io.json.object_hook, ) dense = from_sparse(hist) np.testing.assert_array_equal(dense["storage"]["variances"], [np.nan, 1.0]) np.testing.assert_array_equal(dense["storage"]["values"], [0, 1]) @pytest.mark.parametrize( ("storage_type", "dtype"), [("int", np.int64), ("double", np.float64)] ) def test_from_sparse_empty_json_dtype(storage_type: str, dtype: type) -> None: hist = json.loads( f"""{{ "uhi_schema": 1, "axes": [{{"type": "regular", "lower": 0, "upper": 1, "bins": 2, "underflow": false, "overflow": false, "circular": false}}], "storage": {{"type": "{storage_type}", "index": [[]], "values": []}} }}""", object_hook=uhi.io.json.object_hook, ) values = from_sparse(hist)["storage"]["values"] assert values.dtype == dtype np.testing.assert_array_equal(values, [0, 0]) @pytest.mark.parametrize( ("storage_type", "dtype"), [("int", np.uint64), ("int", np.int32), ("double", np.float32)], ) def test_from_sparse_empty_keeps_dtype(storage_type: str, dtype: type) -> None: hist: Any = { "uhi_schema": 1, "axes": [ { "type": "regular", "lower": 0, "upper": 1, "bins": 2, "underflow": False, "overflow": False, "circular": False, } ], "storage": {"type": storage_type, "values": np.zeros(2, dtype=dtype)}, } values = from_sparse(to_sparse(hist))["storage"]["values"] assert values.dtype == dtype def test_from_sparse_unknown_axis() -> None: hist: Any = { "uhi_schema": 1, "axes": [{"type": "integer"}], "storage": {"type": "int", "index": [[0]], "values": [3]}, } with pytest.raises(ValueError, match="Unknown axis type 'integer'"): from_sparse(hist) uhi-1.2.1/tests/test_tag.py0000644000000000000000000000222615261523264012564 0ustar00from __future__ import annotations import pytest from uhi.tag import loc, overflow, rebin, underflow def test_locator_offset_arithmetic() -> None: assert (loc(1.0) + 2).offset == 2 assert (loc(1.0) - 2).offset == -2 assert (overflow + 1).offset == 1 assert (underflow - 1).offset == -1 @pytest.mark.parametrize("offset", [0.5, 1.0, "1"]) def test_locator_offset_must_be_int(offset: object) -> None: with pytest.raises(TypeError): loc(0.5, offset) # type: ignore[arg-type] with pytest.raises(TypeError): loc(0.5) + offset # type: ignore[operator] with pytest.raises(TypeError): loc(0.5) - offset # type: ignore[operator] with pytest.raises(TypeError): overflow + offset # type: ignore[operator] def test_rebin() -> None: assert rebin(1).factor == 1 assert rebin(3).factor == 3 def test_rebin_factor_must_be_int() -> None: with pytest.raises(TypeError): rebin(2.0) # type: ignore[arg-type] @pytest.mark.parametrize("factor", [0, -1, -2]) def test_rebin_factor_must_be_positive(factor: int) -> None: with pytest.raises(ValueError, match="positive"): rebin(factor) uhi-1.2.1/tests/test_testing.py0000644000000000000000000000510215261523264013462 0ustar00from __future__ import annotations from typing import Any import boost_histogram as bh import numpy as np import uhi.testing.indexing from uhi.typing.serialization import HistogramIR class TestAccess1D(uhi.testing.indexing.Indexing1D[bh.Histogram[bh.storage.Double]]): @classmethod def make_histogram(cls) -> bh.Histogram[bh.storage.Double]: return bh.Histogram(dict(cls.get_uhi())) class TestAccess2D(uhi.testing.indexing.Indexing2D[bh.Histogram[bh.storage.Double]]): @classmethod def make_histogram(cls) -> bh.Histogram[bh.storage.Double]: return bh.Histogram(dict(cls.get_uhi())) class TestAccess3D(uhi.testing.indexing.Indexing3D[bh.Histogram[bh.storage.Double]]): @classmethod def make_histogram(cls) -> bh.Histogram[bh.storage.Double]: return bh.Histogram(dict(cls.get_uhi())) class TestAccessBHTag1D(TestAccess1D): tag = bh.tag class TestAccessBHTag2D(TestAccess2D): tag = bh.tag class TestAccessBHTag3D(TestAccess3D): tag = bh.tag WEIGHT_DTYPE = np.dtype([("value", "f8"), ("variance", "f8")]) def make_weighted(ir: HistogramIR) -> bh.Histogram[bh.storage.Weight]: h: bh.Histogram[bh.storage.Double] = bh.Histogram(dict(ir)) weighted = bh.Histogram(*h.axes, storage=bh.storage.Weight()) weighted.view(flow=True).value = h.values(flow=True) weighted.view(flow=True).variance = h.values(flow=True) return weighted class WeightMixin(uhi.testing.indexing.Indexing): """Run the suites with a storage whose bins are not plain numbers.""" def bin_to_value(self, bin: Any) -> Any: return bin.value def value_to_bin(self, value: Any) -> Any: return np.array((value, value), dtype=WEIGHT_DTYPE) def sum_to_value(self, bin: Any) -> Any: if isinstance(bin, bh.Histogram): total: Any = bin.sum(flow=True) return total.value return bin.value class TestAccessWeight1D( WeightMixin, uhi.testing.indexing.Indexing1D[bh.Histogram[bh.storage.Weight]] ): @classmethod def make_histogram(cls) -> bh.Histogram[bh.storage.Weight]: return make_weighted(cls.get_uhi()) class TestAccessWeight2D( WeightMixin, uhi.testing.indexing.Indexing2D[bh.Histogram[bh.storage.Weight]] ): @classmethod def make_histogram(cls) -> bh.Histogram[bh.storage.Weight]: return make_weighted(cls.get_uhi()) class TestAccessWeight3D( WeightMixin, uhi.testing.indexing.Indexing3D[bh.Histogram[bh.storage.Weight]] ): @classmethod def make_histogram(cls) -> bh.Histogram[bh.storage.Weight]: return make_weighted(cls.get_uhi()) uhi-1.2.1/tests/test_uhi.py0000644000000000000000000000026515261523264012577 0ustar00from __future__ import annotations import importlib.metadata from uhi import __version__ def test_version() -> None: assert __version__ == importlib.metadata.version("uhi") uhi-1.2.1/tests/test_uproot.py0000644000000000000000000002707715261523264013354 0ustar00from __future__ import annotations import json from pathlib import Path from typing import Any import numpy as np import pytest from helpers import convert_histogram_to_32bit, scalar_no_axis_storage from pytest import approx import uhi.io._files import uhi.io.json import uhi.io.ops import uhi.schema from uhi.io import ARRAY_KEYS, to_sparse uproot = pytest.importorskip("uproot") uhi_io_uproot = pytest.importorskip("uhi.io.uproot") def _roundtrip(tmp_path: Path, hists: dict[str, Any]) -> dict[str, Any]: tmp_file = tmp_path / "test.root" with uproot.recreate(tmp_file) as root_file: for name, hist in hists.items(): uhi_io_uproot.write(root_file, name, hist) with uproot.open(tmp_file) as root_file: return {name: uhi_io_uproot.read(root_file, name) for name in hists} def test_valid_json(valid: Path, tmp_path: Path, sparse: bool) -> None: hists = uhi.io._files.load(valid) if sparse: hists = {name: to_sparse(hist) for name, hist in hists.items()} rehists = _roundtrip(tmp_path, hists) assert hists.keys() == rehists.keys() for name, hist in hists.items(): scalar_no_axis_storage(hist) data = json.dumps(hist, default=uhi.io.json.default, sort_keys=True) redata = json.dumps(rehists[name], default=uhi.io.json.default, sort_keys=True) assert redata == data def test_reg_load(tmp_path: Path, resources: Path) -> None: hists = uhi.io._files.load(resources / "valid/reg.json") rehists = _roundtrip(tmp_path, hists) # One single-entry RNTuple per histogram with uproot.open(tmp_path / "test.root") as root_file: ntuple = root_file["one"] assert set(ntuple.keys()) == {"uhi", "values"} assert ntuple.num_entries == 1 native_one = json.loads(ntuple.arrays()["uhi"][0]) assert native_one["storage"]["values"] == "values" one = rehists["one"] assert one["metadata"] == {"one": True, "two": 2, "three": "three"} assert one["axes"][0]["type"] == "regular" assert one["axes"][0]["bins"] == 3 assert one["storage"]["type"] == "int" assert one["storage"]["values"] == approx([1, 2, 3, 4, 5]) assert rehists["two"]["storage"]["values"] == approx([1, 2, 3, 4, 5, 6, 7]) @pytest.mark.parametrize("value", ["description", "values", "axis_0_edges"]) def test_metadata_array_keys(tmp_path: Path, value: str) -> None: metadata = dict.fromkeys(ARRAY_KEYS, value) writer_info = {"test": metadata} hist: dict[str, Any] = { "uhi_schema": 1, "metadata": metadata, "writer_info": writer_info, "axes": [ { "type": "variable", "edges": np.array([0.0, 1.0, 2.0]), "underflow": False, "overflow": False, "circular": False, "metadata": metadata, "writer_info": writer_info, } ], "storage": {"type": "double", "values": np.array([1.0, 2.0])}, } rehist = _roundtrip(tmp_path, {"histogram": hist})["histogram"] assert rehist["metadata"] == metadata assert rehist["writer_info"] == writer_info assert rehist["axes"][0]["metadata"] == metadata assert rehist["axes"][0]["edges"] == approx(hist["axes"][0]["edges"]) assert rehist["storage"]["values"] == approx(hist["storage"]["values"]) def test_subdirectory(tmp_path: Path, resources: Path) -> None: hists = uhi.io._files.load(resources / "valid/2d.json") rehist = _roundtrip(tmp_path, {"sub/main": hists["main"]})["sub/main"] with uproot.open(tmp_path / "test.root") as root_file: assert root_file.classname_of("sub") == "TDirectory" rehist_sub = uhi_io_uproot.read(root_file["sub"], "main") for h in (rehist, rehist_sub): assert h["storage"]["values"] == approx(hists["main"]["storage"]["values"]) def test_0d_scalar_storage(tmp_path: Path) -> None: hist: dict[str, Any] = { "uhi_schema": 1, "axes": [], "storage": { "type": "weighted", "values": np.array(6.0), "variances": np.array(8.0), }, } rehist = _roundtrip(tmp_path, {"h": hist})["h"] for key in ("values", "variances"): assert rehist["storage"][key].shape == () assert rehist["storage"][key] == hist["storage"][key] def test_unsupported_dtype(tmp_path: Path) -> None: hist: dict[str, Any] = { "uhi_schema": 1, "axes": [], "storage": {"type": "double", "values": np.array(1.0, dtype=np.complex64)}, } with ( uproot.recreate(tmp_path / "test.root") as root_file, pytest.raises(TypeError, match="complex64"), ): uhi_io_uproot.write(root_file, "h", hist) @pytest.mark.parametrize( ("dtype", "expected"), [ (np.int8, np.int64), (np.int16, np.int64), (np.uint8, np.uint64), (np.uint16, np.uint64), (np.bool_, np.int64), ], ) def test_small_int_dtype(tmp_path: Path, dtype: type, expected: type) -> None: hist: dict[str, Any] = { "uhi_schema": 1, "axes": [ { "type": "regular", "lower": 0.0, "upper": 1.0, "bins": 2, "underflow": False, "overflow": False, "circular": False, } ], "storage": {"type": "int", "values": np.array([1, 0], dtype=dtype)}, } values = _roundtrip(tmp_path, {"h": hist})["h"]["storage"]["values"] assert values.dtype == expected assert values.tolist() == [1, 0] def test_float16_dtype(tmp_path: Path) -> None: hist: dict[str, Any] = { "uhi_schema": 1, "axes": [], "storage": {"type": "double", "values": np.array(1.5, dtype=np.float16)}, } values = _roundtrip(tmp_path, {"h": hist})["h"]["storage"]["values"] assert values.dtype == np.float32 assert values == 1.5 @pytest.mark.parametrize("storage_type", ["int", "double", "weighted", "mean"]) def test_convert_bh_32bit(tmp_path: Path, storage_type: str) -> None: bh = pytest.importorskip("boost_histogram") axis = bh.axis.Regular(5, 0, 1, __dict__={"name": "x"}) h: Any match storage_type: case "int": h = bh.Histogram(axis, storage=bh.storage.Int64()) h.fill([0.1, 0.3, 0.3]) case "double": h = bh.Histogram(axis, storage=bh.storage.Double()) h.fill([0.1, 0.3, 0.5, 0.7, 0.9]) case "weighted": h = bh.Histogram(axis, storage=bh.storage.Weight()) h.fill([0.1, 0.3, 0.5], weight=[1.5, 2.5, 3.5]) case _: h = bh.Histogram(axis, storage=bh.storage.Mean()) h.fill([0.1, 0.3, 0.5], sample=[10.0, 20.0, 30.0]) uhi_32bit = convert_histogram_to_32bit(h._to_uhi_()) rehist = _roundtrip(tmp_path, {"h": uhi_32bit})["h"] storage: dict[str, Any] = dict(uhi_32bit["storage"]) for key in ARRAY_KEYS & storage.keys(): assert rehist["storage"][key].dtype == storage[key].dtype np.testing.assert_array_equal(rehist["storage"][key], storage[key]) def test_convert_hist(tmp_path: Path) -> None: hist = pytest.importorskip("hist") h = hist.Hist( hist.axis.Regular(10, 0, 1, name="a", label="A"), hist.axis.Integer(7, 13, overflow=False, name="b", label="B"), storage=hist.storage.Weight(), name="h", label="H", ) h.fill(a=[0.1, 0.5], b=[8, 12]) rehist = _roundtrip(tmp_path, {"histogram": h})["histogram"] assert hist.Hist(rehist) == h # CLI def test_cli_validate_path( resources: Path, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: from uhi.__main__ import main hists = uhi.io._files.load(resources / "valid" / "reg.json") tmp_file = tmp_path / "test.root" with uproot.recreate(tmp_file) as root_file: for name, hist in hists.items(): uhi_io_uproot.write(root_file, f"good/{name}", hist) broken = {**hist, "storage": {**hist["storage"], "type": "not_a_storage"}} uhi_io_uproot.write(root_file, f"bad/{name}", broken) assert set(uhi.schema.load(tmp_file, path="good")) == set(hists) assert set(uhi.schema.load(tmp_file, path="good/")) == set(hists) assert uhi.schema.load(tmp_file, path="good/one")["uhi_schema"] == 1 main(["validate", f"{tmp_file}:good", f"{tmp_file}:good/one"]) assert capsys.readouterr().out.count("OK") == 2 with pytest.raises(SystemExit): main(["validate", str(tmp_file)]) assert "ERROR" in capsys.readouterr().out with pytest.raises(SystemExit): main(["validate", f"{tmp_file}:missing"]) assert capsys.readouterr().out.startswith("ERROR") def test_load_non_rntuple(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: from uhi.__main__ import main tmp_file = tmp_path / "test.root" with uproot.recreate(tmp_file) as root_file: root_file["th1"] = np.histogram([1, 2, 3]) with pytest.raises(ValueError, match="not a histogram or directory"): uhi.io._files.load(tmp_file, path="th1") with pytest.raises(SystemExit): main(["validate", f"{tmp_file}:th1"]) assert "not a histogram or directory" in capsys.readouterr().out @pytest.mark.parametrize("out_suffix", [".root", ".json"]) def test_cli_add(resources: Path, tmp_path: Path, out_suffix: str) -> None: from uhi.__main__ import main hists = uhi.io._files.load(resources / "valid/reg.json") files = [tmp_path / "in1.root", tmp_path / "in2.root"] for file in files: with uproot.recreate(file) as root_file: for name, hist in hists.items(): uhi_io_uproot.write(root_file, f"sub/{name}", hist) target = tmp_path / f"out{out_suffix}" main(["add", str(target), *map(str, files)]) names = {f"sub/{name}" for name in hists} inputs = [uhi.io._files.load(file) for file in files] result = uhi.io._files.load(target) assert result.keys() == names for name in names: expected = uhi.io.ops.add(*(h[name] for h in inputs)) assert result[name]["storage"]["values"] == approx( expected["storage"]["values"] ) def test_write_non_native_byte_order(tmp_path: Path) -> None: swapped = np.dtype("f8").newbyteorder() hist = { "uhi_schema": 1, "axes": [ { "type": "regular", "lower": 0.0, "upper": 1.0, "bins": 2, "underflow": False, "overflow": False, "circular": False, } ], "storage": {"type": "double", "values": np.array([1.0, 2.0], dtype=swapped)}, } rehist = _roundtrip(tmp_path, {"h": hist})["h"] assert rehist["storage"]["values"] == approx([1.0, 2.0]) @pytest.mark.parametrize( ("version", "expected"), [ ("5.7.0", True), ("5.7.0rc1", True), ("6.0.0", True), ("5.6.3", False), ("4.3.7", False), ], ) def test_uproot_version_check( monkeypatch: pytest.MonkeyPatch, version: str, expected: bool ) -> None: monkeypatch.setattr("importlib.metadata.version", lambda _: version) uhi.io._files._uproot_available.cache_clear() assert uhi.io._files._uproot_available() is expected uhi.io._files._uproot_available.cache_clear() def test_load_nested_directory(tmp_path: Path, resources: Path) -> None: hist = uhi.io._files.load(resources / "valid" / "reg.json")["one"] tmp_file = tmp_path / "test.root" uhi.io._files.write(tmp_file, {"a/b/h": hist}) assert set(uhi.io._files.load(tmp_file, path="a/b")) == {"h"} assert uhi.io._files.load(tmp_file, path="a/b/h")["uhi_schema"] == 1 uhi-1.2.1/tests/test_zip.py0000644000000000000000000002662315261523264012622 0ustar00from __future__ import annotations import importlib.metadata import json import typing import zipfile from pathlib import Path from typing import Any import numpy as np import packaging.version import pytest from helpers import convert_histogram_to_32bit import uhi.io.json import uhi.io.zip from uhi.io import ARRAY_KEYS, to_sparse from uhi.typing.serialization import AnyHistogramIR BHVERSION = packaging.version.Version(importlib.metadata.version("boost_histogram")) HISTVERSION = packaging.version.Version(importlib.metadata.version("hist")) def test_valid_json(valid: Path, tmp_path: Path, sparse: bool) -> None: data = valid.read_text(encoding="utf-8") hists = json.loads(data, object_hook=uhi.io.json.object_hook) if sparse: hists = {name: to_sparse(hist) for name, hist in hists.items()} tmp_file = tmp_path / "test.zip" with zipfile.ZipFile(tmp_file, "w") as zip_file: for name, hist in hists.items(): # No deepcopy: write() must not mutate the caller's histogram. uhi.io.zip.write(zip_file, name, hist) with zipfile.ZipFile(tmp_file, "r") as zip_file: rehists = {name: uhi.io.zip.read(zip_file, name) for name in hists} assert hists.keys() == rehists.keys() # write() must not replace the caller's arrays with path strings. for hist in hists.values(): for key in ARRAY_KEYS & hist["storage"].keys(): assert not isinstance(hist["storage"][key], str) for axis in hist["axes"]: for key in ARRAY_KEYS & axis.keys(): assert not isinstance(axis[key], str) for name in hists: hist = hists[name] rehist = rehists[name] # Check that the JSON representation is the same redata = json.dumps(hist, default=uhi.io.json.default) data = json.dumps(rehist, default=uhi.io.json.default) assert redata.replace(" ", "").replace("\n", "") == data.replace( " ", "" ).replace("\n", "") def test_reg_load(tmp_path: Path, resources: Path) -> None: data = resources / "valid/reg.json" hists = json.loads( data.read_text(encoding="utf-8"), object_hook=uhi.io.json.object_hook ) tmp_file = tmp_path / "test.zip" with zipfile.ZipFile(tmp_file, "w") as zip_file: for name, hist in hists.items(): uhi.io.zip.write(zip_file, name, hist) with zipfile.ZipFile(tmp_file, "r") as zip_file: names = zip_file.namelist() rehists = { name[:-5]: uhi.io.zip.read(zip_file, name[:-5]) for name in names if name.endswith(".json") } with zip_file.open("one.json") as f: native_one = json.load(f) assert set(names) == { "one_storage_values.npy", "one.json", "two_storage_values.npy", "two.json", } assert native_one["storage"]["values"] == "one_storage_values.npy" one = rehists["one"] two = rehists["two"] assert one.get("metadata", {}) == {"one": True, "two": 2, "three": "three"} assert len(one["axes"]) == 1 assert one["axes"][0]["type"] == "regular" assert one["axes"][0]["lower"] == pytest.approx(0) assert one["axes"][0]["upper"] == pytest.approx(5) assert one["axes"][0]["bins"] == 3 assert one["axes"][0]["underflow"] assert one["axes"][0]["overflow"] assert not one["axes"][0]["circular"] assert one["storage"]["type"] == "int" assert one["storage"]["values"] == pytest.approx([1, 2, 3, 4, 5]) assert len(two["axes"]) == 1 assert two["axes"][0]["type"] == "regular" assert two["axes"][0]["lower"] == pytest.approx(0) assert two["axes"][0]["upper"] == pytest.approx(5) assert two["axes"][0]["bins"] == 5 assert two["axes"][0]["underflow"] assert two["axes"][0]["overflow"] assert not two["axes"][0]["circular"] assert two["storage"]["type"] == "double" assert two["storage"]["values"] == pytest.approx([1, 2, 3, 4, 5, 6, 7]) def test_two_variable_axes(tmp_path: Path) -> None: """Two variable axes must not collide on their zip filenames (issue #241).""" edges_a = np.array([0.0, 1.0, 2.0, 3.0]) edges_b = np.array([0.0, 10.0, 20.0]) def axis(edges: np.ndarray) -> dict[str, Any]: return { "type": "variable", "edges": edges, "underflow": False, "overflow": False, "circular": False, } hist: dict[str, Any] = { "uhi_schema": 1, "axes": [axis(edges_a), axis(edges_b)], "storage": { "type": "double", "values": np.zeros((len(edges_a) - 1, len(edges_b) - 1)), }, } tmp_file = tmp_path / "test.zip" with zipfile.ZipFile(tmp_file, "w") as zip_file: uhi.io.zip.write(zip_file, "h", typing.cast(AnyHistogramIR, hist)) names = zip_file.namelist() # Distinct filenames per axis (no duplicate-name UserWarning, no overwrite). edge_names = [n for n in names if "_axis_" in n] assert len(edge_names) == 2 assert len(set(edge_names)) == 2 with zipfile.ZipFile(tmp_file, "r") as zip_file: rehist = uhi.io.zip.read(zip_file, "h") # Each axis must round-trip with its own edges, not the other axis's. assert rehist["axes"][0]["edges"] == pytest.approx(edges_a) assert rehist["axes"][1]["edges"] == pytest.approx(edges_b) @pytest.mark.skipif( packaging.version.Version("1.6.1") > BHVERSION, reason="Requires boost-histogram 1.6+", ) def test_convert_bh(tmp_path: Path) -> None: import boost_histogram as bh h = bh.Histogram( bh.axis.Regular(3, 13, 10, __dict__={"name": "x"}), storage=bh.storage.Weight() ) tmp_file = tmp_path / "test.zip" with zipfile.ZipFile(tmp_file, "w") as zip_file: uhi.io.zip.write(zip_file, "histogram", h) with zipfile.ZipFile(tmp_file, "r") as zip_file: rehist = uhi.io.zip.read(zip_file, "histogram") h2 = bh.Histogram[bh.storage.Weight](rehist) assert h == h2 @pytest.mark.skipif( packaging.version.Version("1.7.2") > BHVERSION, reason="Requires boost-histogram 1.7.2+ for keep_storage=False support", ) def test_convert_bh_no_storage(tmp_path: Path) -> None: """Test ZIP serialization with keep_storage=False (structure-only histograms).""" import typing import boost_histogram as bh import boost_histogram.serialization from uhi.typing.serialization import AnyHistogramIR h = bh.Histogram( bh.axis.Regular(3, 0, 10, __dict__={"name": "x"}), storage=bh.storage.Weight() ) h.fill([0.1, 0.3, 0.5], weight=[1.5, 2.5, 3.5]) # Convert to UHI format without storage values uhi_untyped = boost_histogram.serialization.to_uhi(h, keep_storage=False) uhi_dict = typing.cast(AnyHistogramIR, uhi_untyped) # Verify storage has type but no values assert "storage" in uhi_dict assert uhi_dict["storage"]["type"] == "weighted" assert "values" not in uhi_dict["storage"] # Write to ZIP tmp_file = tmp_path / "test.zip" with zipfile.ZipFile(tmp_file, "w") as zip_file: uhi.io.zip.write(zip_file, "histogram", uhi_dict) # Read back from ZIP with zipfile.ZipFile(tmp_file, "r") as zip_file: rehist = uhi.io.zip.read(zip_file, "histogram") # Should be able to reconstruct a histogram with the same structure h2: bh.Histogram[bh.storage.Weight] = bh.Histogram(rehist) assert h.axes == h2.axes assert isinstance(h2.storage_type(), bh.storage.Weight) @pytest.mark.skipif( packaging.version.Version("1.6.1") > BHVERSION or packaging.version.Version("2.9.0") > HISTVERSION, reason="Requires boost-histogram 1.6+ / Hist 2.9+", ) def test_convert_hist(tmp_path: Path) -> None: import hist h = hist.Hist( hist.axis.Regular(10, 0, 1, name="a", label="A"), hist.axis.Integer(7, 13, overflow=False, name="b", label="B"), storage=hist.storage.Weight(), name="h", label="H", ) tmp_file = tmp_path / "test.zip" with zipfile.ZipFile(tmp_file, "w") as zip_file: uhi.io.zip.write(zip_file, "histogram", h) with zipfile.ZipFile(tmp_file, "r") as zip_file: rehist = uhi.io.zip.read(zip_file, "histogram") h2 = hist.Hist[hist.storage.Double](rehist) assert h == h2 @pytest.mark.skipif( packaging.version.Version("1.6.1") > BHVERSION, reason="Requires boost-histogram 1.6+", ) @pytest.mark.parametrize( "storage_type", [ pytest.param("int", id="int_storage"), pytest.param("double", id="double_storage"), pytest.param("weighted", id="weighted_storage"), pytest.param("mean", id="mean_storage"), ], ) def test_convert_bh_32bit_zip(tmp_path: Path, storage_type: str) -> None: """Test serialization of 32-bit histograms via ZIP.""" import boost_histogram as bh # Create histogram with appropriate storage type axis = bh.axis.Regular(5, 0, 1, __dict__={"name": "x"}) h: bh.Histogram[Any] if storage_type == "int": h = bh.Histogram(axis, storage=bh.storage.Int64()) # Fill with some data for i in range(5): h.fill([0.1 + i * 0.15] * 10) elif storage_type == "double": h = bh.Histogram(axis, storage=bh.storage.Double()) h.fill([0.1, 0.3, 0.5, 0.7, 0.9]) elif storage_type == "weighted": h = bh.Histogram(axis, storage=bh.storage.Weight()) h.fill([0.1, 0.3, 0.5], weight=[1.5, 2.5, 3.5]) elif storage_type == "mean": h = bh.Histogram(axis, storage=bh.storage.Mean()) h.fill([0.1, 0.3, 0.5], sample=[10.0, 20.0, 30.0]) else: msg = f"Unknown storage type: {storage_type}" raise ValueError(msg) # Convert to UHI format uhi_dict = h._to_uhi_() # Convert to 32-bit uhi_32bit = convert_histogram_to_32bit(uhi_dict) # Write to ZIP tmp_file = tmp_path / "test_32bit.zip" with zipfile.ZipFile(tmp_file, "w") as zip_file: uhi.io.zip.write(zip_file, "histogram", uhi_32bit) # Read back from ZIP with zipfile.ZipFile(tmp_file, "r") as zip_file: rehist_32bit = uhi.io.zip.read(zip_file, "histogram") # Verify storage type and data integrity assert rehist_32bit["storage"]["type"] == storage_type assert "values" in rehist_32bit["storage"] # Verify JSON representation is consistent redata = json.dumps(rehist_32bit, default=uhi.io.json.default) assert len(redata) > 0 def test_metadata_with_array_key_names(tmp_path: Path) -> None: """ Metadata keys can collide with array key names; they must not be treated as references to stored arrays. """ hist: Any = { "uhi_schema": 1, "metadata": {"values": "description", "edges": [1, 2, 3]}, "axes": [ { "type": "regular", "lower": 0.0, "upper": 1.0, "bins": 1, "underflow": False, "overflow": False, "circular": False, "metadata": {"counts": "label"}, } ], "storage": {"type": "int", "values": np.array([1])}, } tmp_file = tmp_path / "meta.zip" with zipfile.ZipFile(tmp_file, "w") as zip_file: uhi.io.zip.write(zip_file, "h", hist) with zipfile.ZipFile(tmp_file, "r") as zip_file: rehist = uhi.io.zip.read(zip_file, "h") assert rehist["metadata"] == {"values": "description", "edges": [1, 2, 3]} assert rehist["axes"][0]["metadata"] == {"counts": "label"} np.testing.assert_array_equal(rehist["storage"]["values"], [1]) uhi-1.2.1/tests/resources/invalid/int_storage_float_values.error.txt0000644000000000000000000000006515261523264023012 0ustar00data.storage must be valid exactly by one definition uhi-1.2.1/tests/resources/invalid/int_storage_float_values.json0000644000000000000000000000040015261523264022005 0ustar00{ "uhi_schema": 1, "axes": [ { "type": "regular", "lower": 0, "upper": 1, "bins": 2, "underflow": false, "overflow": false, "circular": false } ], "storage": { "type": "int", "values": [1.5, 2] } } uhi-1.2.1/tests/resources/invalid/metadata_nested.error.txt0000644000000000000000000000007615261523264021054 0ustar00data.one.metadata.one must be valid exactly by one definition uhi-1.2.1/tests/resources/invalid/metadata_nested.json0000644000000000000000000000022115261523264020046 0ustar00{ "one": { "uhi_schema": 1, "metadata": { "one": { "two": 2 } }, "axes": [], "storage": { "type": "int", "values": [1] } } } uhi-1.2.1/tests/resources/invalid/missing_axis.error.txt0000644000000000000000000000005215261523264020421 0ustar00data.one must contain ['axes'] properties uhi-1.2.1/tests/resources/invalid/missing_axis.json0000644000000000000000000000013015261523264017420 0ustar00{ "one": { "uhi_schema": 1, "storage": { "type": "int", "values": [1] } } } uhi-1.2.1/tests/resources/invalid/missing_storage.error.txt0000644000000000000000000000005515261523264021124 0ustar00data.one must contain ['storage'] properties uhi-1.2.1/tests/resources/invalid/missing_storage.json0000644000000000000000000000006715261523264020131 0ustar00{ "one": { "uhi_schema": 1, "axes": [] } } uhi-1.2.1/tests/resources/invalid/single_bad_axis.error.txt0000644000000000000000000000006515261523264021043 0ustar00data.axes[0] must be valid exactly by one definition uhi-1.2.1/tests/resources/invalid/single_bad_axis.json0000644000000000000000000000021415261523264020041 0ustar00{ "uhi_schema": 1, "axes": [{ "type": "regular", "lower": 0, "upper": 5 }], "storage": { "type": "int", "values": [1, 2, 3, 4, 5] } } uhi-1.2.1/tests/resources/invalid/single_missing_storage.error.txt0000644000000000000000000000005115261523264022461 0ustar00data must contain ['storage'] properties uhi-1.2.1/tests/resources/invalid/single_missing_storage.json0000644000000000000000000000031215261523264021463 0ustar00{ "uhi_schema": 1, "axes": [ { "type": "regular", "lower": 0, "upper": 5, "bins": 3, "underflow": true, "overflow": true, "circular": false } ] } uhi-1.2.1/tests/resources/invalid/variable_one_edge.error.txt0000644000000000000000000000006515261523264021342 0ustar00data.axes[0] must be valid exactly by one definition uhi-1.2.1/tests/resources/invalid/variable_one_edge.json0000644000000000000000000000031615261523264020343 0ustar00{ "uhi_schema": 1, "axes": [ { "type": "variable", "edges": [1.0], "underflow": false, "overflow": false, "circular": false } ], "storage": { "type": "int" } } uhi-1.2.1/tests/resources/invalid/variable_repeated_edges.error.txt0000644000000000000000000000006515261523264022535 0ustar00data.axes[0] must be valid exactly by one definition uhi-1.2.1/tests/resources/invalid/variable_repeated_edges.json0000644000000000000000000000032315261523264021534 0ustar00{ "uhi_schema": 1, "axes": [ { "type": "variable", "edges": [1.0, 1.0], "underflow": false, "overflow": false, "circular": false } ], "storage": { "type": "int" } } uhi-1.2.1/tests/resources/valid/2d.json0000644000000000000000000000076615261523264014720 0ustar00{ "main": { "uhi_schema": 1, "axes": [ { "type": "variable", "edges": [1, 2, 3, 4], "underflow": true, "overflow": false, "circular": false }, { "type": "category_str", "categories": ["a", "b", "c"], "flow": false } ], "storage": { "type": "double", "values": [ [1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0], [10.0, 11.0, 12.0] ] } } } uhi-1.2.1/tests/resources/valid/empty_storage.json0000644000000000000000000000317115261523264017266 0ustar00{ "empty_int": { "uhi_schema": 1, "metadata": { "description": "metadata-only histogram with int storage" }, "axes": [ { "type": "regular", "lower": 0, "upper": 10, "bins": 5, "underflow": false, "overflow": false, "circular": false } ], "storage": { "type": "int" } }, "empty_double": { "uhi_schema": 1, "metadata": { "description": "metadata-only histogram with double storage" }, "axes": [ { "type": "regular", "lower": 0, "upper": 10, "bins": 5, "underflow": true, "overflow": true, "circular": false } ], "storage": { "type": "double" } }, "empty_weighted": { "uhi_schema": 1, "metadata": { "description": "metadata-only histogram with weighted storage" }, "axes": [ { "type": "variable", "edges": [0.0, 2.5, 5.0, 7.5, 10.0], "underflow": false, "overflow": false, "circular": false } ], "storage": { "type": "weighted" } }, "empty_mean": { "uhi_schema": 1, "metadata": { "description": "metadata-only histogram with mean storage" }, "axes": [ { "type": "category_str", "categories": ["a", "b", "c"], "flow": false } ], "storage": { "type": "mean" } }, "empty_weighted_mean": { "uhi_schema": 1, "metadata": { "description": "metadata-only histogram with weighted_mean storage" }, "axes": [ { "type": "boolean" } ], "storage": { "type": "weighted_mean" } } } uhi-1.2.1/tests/resources/valid/mean.json0000644000000000000000000000263715261523264015332 0ustar00{ "weighted_mean": { "uhi_schema": 1, "writer_info": { "boost-histogram": { "version": "1.6.1" } }, "axes": [ { "type": "regular", "lower": 0.0, "upper": 1.0, "bins": 4, "underflow": true, "overflow": true, "circular": false } ], "storage": { "type": "weighted_mean", "sum_of_weights": [0.0, 3.9000000000000004, 0.6, 0.0, 0.0, 0.0], "sum_of_weights_squared": [0.0, 8.73, 0.36, 0.0, 0.0, 0.0], "values": [0.0, 5.6923076923076925, 7.000000000000001, 0.0, 0.0, 0.0], "variances": [NaN, 0.4999999999999996, -Infinity, NaN, NaN, NaN] }, "metadata": { "_variance_known": true, "fill": "h.fill([.1, .1, .4], weight=[1.2, 2.7, .6], sample=[5, 6, 7])" } }, "mean": { "uhi_schema": 1, "writer_info": { "boost-histogram": { "version": "1.6.1" } }, "axes": [ { "type": "regular", "lower": 0.0, "upper": 1.0, "bins": 4, "underflow": true, "overflow": true, "circular": false } ], "storage": { "type": "mean", "counts": [0.0, 3.9000000000000004, 0.6, 0.0, 0.0, 0.0], "values": [0.0, 5.6923076923076925, 7.000000000000001, 0.0, 0.0, 0.0], "variances": [ -0.0, 0.28647214854111386, 9.325873406851315e-15, -0.0, -0.0, -0.0 ] }, "metadata": { "_variance_known": true } } } uhi-1.2.1/tests/resources/valid/noaxes.json0000644000000000000000000000015015261523264015673 0ustar00{ "one": { "uhi_schema": 1, "axes": [], "storage": { "type": "int", "values": [1] } } } uhi-1.2.1/tests/resources/valid/noaxes_scalar.json0000644000000000000000000000017715261523264017231 0ustar00{ "one": { "uhi_schema": 1, "axes": [], "storage": { "type": "weighted", "values": 2.5, "variances": 3.0 } } } uhi-1.2.1/tests/resources/valid/reg.json0000644000000000000000000000155115261523264015161 0ustar00{ "one": { "uhi_schema": 1, "metadata": { "one": true, "two": 2, "three": "three" }, "axes": [ { "type": "regular", "lower": 0, "upper": 5, "bins": 3, "underflow": true, "overflow": true, "circular": false, "metadata": { "simple": 1 }, "writer_info": { "uhi": { "bool": true } } } ], "storage": { "type": "int", "values": [1, 2, 3, 4, 5] } }, "two": { "uhi_schema": 1, "writer_info": { "uhi": { "version": "1.0.0", "other_info": "Something" } }, "axes": [ { "type": "regular", "lower": 0, "upper": 5, "bins": 5, "underflow": true, "overflow": true, "circular": false } ], "storage": { "type": "double", "values": [1, 2, 3, 4, 5, 6, 7] } } } uhi-1.2.1/tests/resources/valid/sparse.json0000644000000000000000000000072015261523264015676 0ustar00{ "first": { "uhi_schema": 1, "metadata": { "one": true, "two": 2, "three": "three" }, "axes": [ { "type": "regular", "lower": 0, "upper": 5, "bins": 3, "underflow": true, "overflow": true, "circular": false, "metadata": { "simple": 1 }, "writer_info": { "uhi": { "bool": true } } } ], "storage": { "type": "int", "index": [[1, 2]], "values": [3, 5] } } } uhi-1.2.1/tests/resources/valid_single/reg.json0000644000000000000000000000050015261523264016513 0ustar00{ "uhi_schema": 1, "metadata": { "one": true, "two": 2, "three": "three" }, "axes": [ { "type": "regular", "lower": 0, "upper": 5, "bins": 3, "underflow": true, "overflow": true, "circular": false } ], "storage": { "type": "int", "values": [1, 2, 3, 4, 5] } } uhi-1.2.1/tests/utils/__init__.py0000644000000000000000000000013715261523264013650 0ustar00"""Test utilities and helpers.""" from __future__ import annotations __all__: list[str] = [] uhi-1.2.1/tests/utils/helpers.py0000644000000000000000000001430315261523264013553 0ustar00"""Helper functions for histogram testing.""" from __future__ import annotations import sys import typing from collections.abc import Mapping from typing import Any import numpy as np if sys.version_info < (3, 11): from typing import TypeAlias else: from typing import TypeAlias from uhi.typing.serialization import AnyHistogramIR, AnyStorageIR __all__ = ["convert_histogram_to_32bit", "scalar_no_axis_storage"] # Type alias for arrays that can be converted ArrayLike: TypeAlias = ( np.ndarray | list[typing.Any] | int | np.integer | float | np.floating | None ) def _convert_array_to_32bit(arr: ArrayLike) -> ArrayLike: """Recursively convert numeric arrays to 32-bit precision. Uses pattern matching to handle different array types. """ match arr: case np.ndarray(): if np.issubdtype(arr.dtype, np.floating): return arr.astype(np.float32) if np.issubdtype(arr.dtype, np.signedinteger): return arr.astype(np.int32) if np.issubdtype(arr.dtype, np.unsignedinteger): return arr.astype(np.uint32) return arr case list(): return [_convert_array_to_32bit(item) for item in arr] case int(n) if n > 2147483647 or n < -2147483648: # numpy's stubs type the scalar constructors as returning Any return typing.cast("np.int32", np.int32(n)) case _ as other: return other def convert_histogram_to_32bit(hist: Mapping[str, Any]) -> AnyHistogramIR: """Convert a histogram dictionary to use 32-bit values where applicable. This processes the storage arrays and converts float64 to float32 and int64 to int32 (where appropriate). Args: hist: A histogram in UHI format Returns: A histogram with 32-bit precision storage arrays """ hist_copy: AnyHistogramIR = typing.cast(AnyHistogramIR, {**hist}) if "storage" in hist_copy: storage: AnyStorageIR = typing.cast(AnyStorageIR, {**hist_copy["storage"]}) storage_type = storage.get("type") match storage_type: case "int": # Convert integer values to int32 if "values" in storage: storage["values"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["values"]) ) if "index" in storage: storage["index"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["index"]) ) case "double": # Convert float values to float32 if "values" in storage: storage["values"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["values"]) ) if "index" in storage: storage["index"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["index"]) ) case "weighted": # Convert float values and variances to float32 if "values" in storage: storage["values"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["values"]) ) if "variances" in storage: storage["variances"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["variances"]) ) if "index" in storage: storage["index"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["index"]) ) case "mean": # Convert counts, values, and variances to float32 if "counts" in storage: storage["counts"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["counts"]) ) if "values" in storage: storage["values"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["values"]) ) if "variances" in storage: storage["variances"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["variances"]) ) if "index" in storage: storage["index"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["index"]) ) case "weighted_mean": # Convert all float arrays to float32 if "sum_of_weights" in storage: storage["sum_of_weights"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["sum_of_weights"]), ) if "sum_of_weights_squared" in storage: storage["sum_of_weights_squared"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["sum_of_weights_squared"]), ) if "values" in storage: storage["values"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["values"]) ) if "variances" in storage: storage["variances"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["variances"]) ) if "index" in storage: storage["index"] = typing.cast( np.ndarray, _convert_array_to_32bit(storage["index"]) ) hist_copy["storage"] = storage return hist_copy def scalar_no_axis_storage(hist: dict[str, Any]) -> None: """Make storage arrays scalars in place if the histogram has no axes. RNTuple fields are flat and the shape comes from the axes, so a no-axis storage of length 1 reads back from ROOT files as a scalar. """ from uhi.io import ARRAY_KEYS if not hist["axes"]: storage = hist["storage"] for key in ARRAY_KEYS & storage.keys(): storage[key] = np.reshape(storage[key], ()) uhi-1.2.1/tests/utils/py.typed0000644000000000000000000000000015261523264013223 0ustar00uhi-1.2.1/.gitignore0000644000000000000000000000417715261523264011235 0ustar00# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ cover/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder .pybuilder/ target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: # .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # Cython debug symbols cython_debug/ .pdm.toml # Version file _version.py # Editor output .vscode # Lockfiles *.lock pylock*.toml # Symlink to AGENTS.md CLAUDE.md .claude/ uhi-1.2.1/LICENSE0000644000000000000000000000276515261523264010253 0ustar00BSD 3-Clause License Copyright (c) 2021, Henry Schreiner. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the vector package developers nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. uhi-1.2.1/README.md0000644000000000000000000001036315261523264010516 0ustar00# UHI: Universal Histogram Interface [![Actions Status][actions-badge]][actions-link] [![Documentation Status][rtd-badge]][rtd-link] [![pre-commit.ci Status][pre-commit-badge]][pre-commit-link] [![Code style: black][black-badge]][black-link] [![PyPI version][pypi-version]][pypi-link] [![Conda-forge version][conda-badge]][conda-link] [![PyPI platforms][pypi-platforms]][pypi-link] [![GitHub Discussion][github-discussions-badge]][github-discussions-link] [![Gitter][gitter-badge]][gitter-link] [![Zenodo][zenodo-badge]][zenodo-link] [![Scikit-HEP][sk-badge]](https://scikit-hep.org/) [![SPEC 4 — Using and Creating Nightly Wheels][spec4-badge]][spec4-link] This is a package meant primarily for [documenting][rtd-link] histogram indexing and the PlottableHistogram Protocol and any future cross-library standards. It also contains the code for the PlottableHistogram Protocol, to be used in type checking libraries wanting to conform to the protocol, and `uhi.testing`, a set of tools for testing conformance to UHI indexing. It is not usually a runtime dependency, but only a type checking, testing, and/or docs dependency in support of other libraries (such as [boost-histogram][] 0.13+, [hist][] 2.1+, [mplhep][] 0.2.15+, [uproot][] 4+, and [histoprint][] 2+). There are a few useful runtime usable components (listed below). Older versions are available for Python 3.6+. [See what's new](https://github.com/scikit-hep/uhi/releases). To assist plotting libraries in accepting Histograms from classic sources, see `uhi.numpy_plottable.ensure_plottable_histogram`, which will adapt NumPy style tuples into a simple PlottableHistogram. The `uhi.io` module can read and write histograms in JSON, ZIP, HDF5, and ROOT (with uproot or PyROOT), and `uhi.io.ops.add` sums histograms bin-by-bin. The `uhi add` command does the same for files, like ROOT's `hadd`. The `uhi validate` command checks files against the schema (needs the `schema` extra). The Protocols provided do support runtime checking, so `isinstance(h, uhi.typing.plottable.PlottableHistogram)` is valid at runtime and might be simpler than manually checking for the expected methods. [actions-badge]: https://github.com/Scikit-HEP/uhi/workflows/CI/badge.svg [actions-link]: https://github.com/Scikit-HEP/uhi/actions [black-badge]: https://img.shields.io/badge/code%20style-black-000000.svg [black-link]: https://github.com/psf/black [conda-badge]: https://img.shields.io/conda/vn/conda-forge/uhi [conda-link]: https://github.com/conda-forge/uhi-feedstock [github-discussions-badge]: https://img.shields.io/static/v1?label=Discussions&message=Ask&color=blue&logo=github [github-discussions-link]: https://github.com/Scikit-HEP/uhi/discussions [gitter-badge]: https://badges.gitter.im/https://github.com/Scikit-HEP/uhi/community.svg [gitter-link]: https://gitter.im/https://github.com/Scikit-HEP/uhi/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge [zenodo-badge]: https://zenodo.org/badge/DOI/10.5281/zenodo.13883674.svg [zenodo-link]: https://doi.org/10.5281/zenodo.13883674 [pre-commit-badge]: https://results.pre-commit.ci/badge/github/scikit-hep/uhi/main.svg [pre-commit-link]: https://results.pre-commit.ci/repo/github/309772485 [pypi-link]: https://pypi.org/project/uhi/ [pypi-platforms]: https://img.shields.io/pypi/pyversions/uhi [pypi-version]: https://badge.fury.io/py/uhi.svg [rtd-badge]: https://readthedocs.org/projects/uhi/badge/?version=latest [rtd-link]: https://uhi.readthedocs.io/en/latest/?badge=latest [sk-badge]: https://scikit-hep.org/assets/images/Scikit--HEP-Project-blue.svg [spec4-badge]: https://img.shields.io/badge/SPEC-4-green?labelColor=%23004811&color=%235CA038 [spec4-link]: https://scientific-python.org/specs/spec-0004/ [boost-histogram]: https://github.com/scikit-hep/boost-histogram [hist]: https://github.com/scikit-hep/hist [mplhep]: https://github.com/scikit-hep/mplhep [uproot]: https://github.com/scikit-hep/uproot5 [histoprint]: https://github.com/scikit-hep/histoprint uhi-1.2.1/pyproject.toml0000644000000000000000000001042115261523264012146 0ustar00[build-system] requires = ["hatchling", "hatch-vcs"] build-backend = "hatchling.build" [project] name = "uhi" description = "Unified Histogram Interface: tools to help library authors work with histograms" authors = [ {name="Henry Schreiner", email="henryschreineriii@gmail.com"}, {name="Hans Dembinski", email="hans.dembinski@gmail.com"}, {name="Jim Pivarski", email="jpivarski@gmail.com"}, {name="Silia Taider", email="siliataider@gmail.com"}, ] maintainers = [ {name="The Scikit-HEP admins", email="scikit-hep-admins@googlegroups.com"}, ] requires-python = ">=3.10" readme = "README.md" license = "bsd-3-clause" license-files =["LICENSE"] dependencies = [ "numpy>=1.19.3", "typing_extensions>=4; python_version<'3.11'", ] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Operating System :: OS Independent", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Programming Language :: Python :: 3.15", "Topic :: Scientific/Engineering", "Typing :: Typed", ] dynamic = ["version"] [project.urls] Homepage = "https://github.com/Scikit-HEP/uhi" Repository = "https://github.com/Scikit-HEP/uhi" Documentation = "https://uhi.readthedocs.io/en/latest/" Changelog = "https://github.com/scikit-hep/uhi/releases" [project.scripts] uhi = "uhi.__main__:main" [project.optional-dependencies] schema = [ "fastjsonschema", ] hdf5 = [ "h5py", ] uproot = [ "awkward>=2", "uproot>=5.7", ] [dependency-groups] docs = [ "sphinx>=4.0", "furo", "sphinx-copybutton>=0.3.1", "sphinx-jsonschema", "myst-parser", "sphinx_github_changelog", "sphinx-llm", ] test-core = [ "pytest>=7", "boost-histogram>=1.6.1", "hist>=2.6", "fastjsonschema", "packaging", "tomli; python_version<'3.11'", "dependency-groups>=1.3", ] test = [ { include-group = "test-core" }, "h5py; platform_python_implementation == 'CPython' and python_version < '3.15'", # No PyPy or 3.15 wheels yet "uproot>=5.7; platform_python_implementation == 'CPython'", # No PyPy wheels for awkward-cpp ] dev = [{ include-group = "test" }] [tool.uv] exclude-newer = "7 days" [tool.hatch] version.source = "vcs" build.hooks.vcs.version-file = "src/uhi/_version.py" [tool.pytest.ini_options] minversion = "7.0" pythonpath = ["tests/utils"] addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"] xfail_strict = true filterwarnings = [ "error", # NumPy's PyPy wheels warn on import of numpy.random "ignore:cython.collection_type only works on PyPy:RuntimeWarning", ] testpaths = ["tests"] log_level = "INFO" [tool.mypy] files = ["src", "tests"] mypy_path = "tests/utils" python_version = "3.10" warn_unused_configs = true strict = true warn_unreachable = true enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] [[tool.mypy.overrides]] module = ["awkward", "fastjsonschema", "h5py", "ROOT", "uproot"] ignore_missing_imports = true [tool.ruff.lint] extend-select = [ "B", # flake8-bugbear "I", # isort "ARG", # flake8-unused-arguments "C4", # flake8-comprehensions "EM", # flake8-errmsg "ICN", # flake8-import-conventions "ISC", # flake8-implicit-str-concat "G", # flake8-logging-format "PGH", # pygrep-hooks "PIE", # flake8-pie "PL", # pylint "PT", # flake8-pytest-style "PTH", # flake8-use-pathlib "RET", # flake8-return "RUF", # Ruff-specific "SIM", # flake8-simplify "T20", # flake8-print "UP", # pyupgrade "YTT", # flake8-2020 "EXE", # flake8-executable "NPY", # NumPy specific rules "PD", # pandas-vet ] ignore = [ "PLR09", # Design related pylint codes "PLR2004", # Magic value in comparison "PT013", # Incorrect import of pytest ] flake8-unused-arguments.ignore-variadic-names = true isort.required-imports = ["from __future__ import annotations"] [tool.ruff.lint.per-file-ignores] "tests/**" = ["T20", "PLC0415"] "noxfile.py" = ["T20"] "tests/test_ensure.py" = ["NPY002"] "src/**" = ["PT"] uhi-1.2.1/PKG-INFO0000644000000000000000000001356715261523264010345 0ustar00Metadata-Version: 2.5 Name: uhi Version: 1.2.1 Summary: Unified Histogram Interface: tools to help library authors work with histograms Project-URL: Homepage, https://github.com/Scikit-HEP/uhi Project-URL: Repository, https://github.com/Scikit-HEP/uhi Project-URL: Documentation, https://uhi.readthedocs.io/en/latest/ Project-URL: Changelog, https://github.com/scikit-hep/uhi/releases Author-email: Henry Schreiner , Hans Dembinski , Jim Pivarski , Silia Taider Maintainer-email: The Scikit-HEP admins License-Expression: BSD-3-Clause License-File: LICENSE Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: Intended Audience :: Science/Research Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: 3.14 Classifier: Programming Language :: Python :: 3.15 Classifier: Topic :: Scientific/Engineering Classifier: Typing :: Typed Requires-Python: >=3.10 Requires-Dist: numpy>=1.19.3 Requires-Dist: typing-extensions>=4; python_version < '3.11' Provides-Extra: hdf5 Requires-Dist: h5py; extra == 'hdf5' Provides-Extra: schema Requires-Dist: fastjsonschema; extra == 'schema' Provides-Extra: uproot Requires-Dist: awkward>=2; extra == 'uproot' Requires-Dist: uproot>=5.7; extra == 'uproot' Description-Content-Type: text/markdown # UHI: Universal Histogram Interface [![Actions Status][actions-badge]][actions-link] [![Documentation Status][rtd-badge]][rtd-link] [![pre-commit.ci Status][pre-commit-badge]][pre-commit-link] [![Code style: black][black-badge]][black-link] [![PyPI version][pypi-version]][pypi-link] [![Conda-forge version][conda-badge]][conda-link] [![PyPI platforms][pypi-platforms]][pypi-link] [![GitHub Discussion][github-discussions-badge]][github-discussions-link] [![Gitter][gitter-badge]][gitter-link] [![Zenodo][zenodo-badge]][zenodo-link] [![Scikit-HEP][sk-badge]](https://scikit-hep.org/) [![SPEC 4 — Using and Creating Nightly Wheels][spec4-badge]][spec4-link] This is a package meant primarily for [documenting][rtd-link] histogram indexing and the PlottableHistogram Protocol and any future cross-library standards. It also contains the code for the PlottableHistogram Protocol, to be used in type checking libraries wanting to conform to the protocol, and `uhi.testing`, a set of tools for testing conformance to UHI indexing. It is not usually a runtime dependency, but only a type checking, testing, and/or docs dependency in support of other libraries (such as [boost-histogram][] 0.13+, [hist][] 2.1+, [mplhep][] 0.2.15+, [uproot][] 4+, and [histoprint][] 2+). There are a few useful runtime usable components (listed below). Older versions are available for Python 3.6+. [See what's new](https://github.com/scikit-hep/uhi/releases). To assist plotting libraries in accepting Histograms from classic sources, see `uhi.numpy_plottable.ensure_plottable_histogram`, which will adapt NumPy style tuples into a simple PlottableHistogram. The `uhi.io` module can read and write histograms in JSON, ZIP, HDF5, and ROOT (with uproot or PyROOT), and `uhi.io.ops.add` sums histograms bin-by-bin. The `uhi add` command does the same for files, like ROOT's `hadd`. The `uhi validate` command checks files against the schema (needs the `schema` extra). The Protocols provided do support runtime checking, so `isinstance(h, uhi.typing.plottable.PlottableHistogram)` is valid at runtime and might be simpler than manually checking for the expected methods. [actions-badge]: https://github.com/Scikit-HEP/uhi/workflows/CI/badge.svg [actions-link]: https://github.com/Scikit-HEP/uhi/actions [black-badge]: https://img.shields.io/badge/code%20style-black-000000.svg [black-link]: https://github.com/psf/black [conda-badge]: https://img.shields.io/conda/vn/conda-forge/uhi [conda-link]: https://github.com/conda-forge/uhi-feedstock [github-discussions-badge]: https://img.shields.io/static/v1?label=Discussions&message=Ask&color=blue&logo=github [github-discussions-link]: https://github.com/Scikit-HEP/uhi/discussions [gitter-badge]: https://badges.gitter.im/https://github.com/Scikit-HEP/uhi/community.svg [gitter-link]: https://gitter.im/https://github.com/Scikit-HEP/uhi/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge [zenodo-badge]: https://zenodo.org/badge/DOI/10.5281/zenodo.13883674.svg [zenodo-link]: https://doi.org/10.5281/zenodo.13883674 [pre-commit-badge]: https://results.pre-commit.ci/badge/github/scikit-hep/uhi/main.svg [pre-commit-link]: https://results.pre-commit.ci/repo/github/309772485 [pypi-link]: https://pypi.org/project/uhi/ [pypi-platforms]: https://img.shields.io/pypi/pyversions/uhi [pypi-version]: https://badge.fury.io/py/uhi.svg [rtd-badge]: https://readthedocs.org/projects/uhi/badge/?version=latest [rtd-link]: https://uhi.readthedocs.io/en/latest/?badge=latest [sk-badge]: https://scikit-hep.org/assets/images/Scikit--HEP-Project-blue.svg [spec4-badge]: https://img.shields.io/badge/SPEC-4-green?labelColor=%23004811&color=%235CA038 [spec4-link]: https://scientific-python.org/specs/spec-0004/ [boost-histogram]: https://github.com/scikit-hep/boost-histogram [hist]: https://github.com/scikit-hep/hist [mplhep]: https://github.com/scikit-hep/mplhep [uproot]: https://github.com/scikit-hep/uproot5 [histoprint]: https://github.com/scikit-hep/histoprint