pax_global_header00006660000000000000000000000064152175464400014522gustar00rootroot0000000000000052 comment=2b313fd3fa047bc5a1767cdb78628645088e56ab bumps-bumps-2b313fd/000077500000000000000000000000001521754644000144025ustar00rootroot00000000000000bumps-bumps-2b313fd/.git-blame-ignore-revs000066400000000000000000000001521521754644000205000ustar00rootroot00000000000000# .git-blame-ignore-revs # Added formatting to backend (PR #193) 155b502bfdd4863919322774023bfdbca731731a bumps-bumps-2b313fd/.gitattributes000066400000000000000000000002211521754644000172700ustar00rootroot00000000000000*.py text eol=lf *.c text eol=lf *.h text eol=lf *.cpp text eol=lf *.rst text eol=lf *.bat text eol=crlf *.BAT text eol=crlf *.ps1 text eol=crlf bumps-bumps-2b313fd/.github/000077500000000000000000000000001521754644000157425ustar00rootroot00000000000000bumps-bumps-2b313fd/.github/dependabot.yaml000066400000000000000000000003301521754644000207270ustar00rootroot00000000000000# Set update schedule for GitHub Actions version: 2 updates: - package-ecosystem: "github-actions" directory: "/" schedule: # Check for updates to GitHub Actions every week interval: "monthly" bumps-bumps-2b313fd/.github/release.yml000066400000000000000000000002141521754644000201020ustar00rootroot00000000000000# .github/release.yml changelog: exclude: labels: - ignore-for-release authors: - dependabot - dependabot[bot] bumps-bumps-2b313fd/.github/workflows/000077500000000000000000000000001521754644000177775ustar00rootroot00000000000000bumps-bumps-2b313fd/.github/workflows/build-distributables.yml000066400000000000000000000165161521754644000246500ustar00rootroot00000000000000name: Build conda-packed distribution on: release: types: [published] workflow_dispatch: env: branch_name: master PACKAGE_NAME: bumps PYTHON_VERSION: 3.12 jobs: build_and_publish: runs-on: ${{ matrix.config.os }} defaults: run: shell: bash -el {0} strategy: fail-fast: false matrix: config: - { os: ubuntu-latest } - { os: windows-latest } - { os: macos-latest } - { os: macos-15-intel } steps: - uses: actions/checkout@v6 with: ref: ${{ env.branch_name }} fetch-depth: 0 - uses: conda-incubator/setup-miniconda@v4 with: auto-update-conda: true python-version: ${{ env.PYTHON_VERSION }} miniforge-version: latest activate-environment: builder - name: Build conda-packed (all platforms) run: | conda install -y versioningit nodejs conda activate base ./extra/build_conda_packed.sh - name: Create MacOS App if: startsWith(matrix.config.os, 'macos') env: MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE_ISA }} MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_ISA_PWD }} KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }} NOTARIZATION_USERNAME: ${{ secrets.MACOS_NOTARIZATION_USERNAME }} NOTARIZATION_PASSWORD: ${{ secrets.MACOS_NOTARIZATION_PASSWORD }} DEVELOPER_IDENTITY: "Developer ID Application: The International Scattering Alliance (8CX8K63BQM)" run: | echo "$MACOS_CERTIFICATE" | base64 --decode > certificate.p12 security create-keychain -p "$KEYCHAIN_PASSWORD" build.keychain security default-keychain -s build.keychain security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain security import certificate.p12 -k build.keychain -P "$MACOS_CERTIFICATE_PWD" -T /usr/bin/codesign security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$KEYCHAIN_PASSWORD" build.keychain security find-identity -p codesigning echo "Creating MacOS App" # This assumes that conda-packed enviroment exists at "conda_packed" VERSION=$(versioningit) MAJOR_MINOR_PATCH=$(sed -E 's/(^[0-9]+\.[0-9]+\.[0-9]+).*$/\1/' <<< "$VERSION") PRODUCT_NAME="$PACKAGE_NAME-$VERSION" FULL_PACKAGE_NAME="$PRODUCT_NAME-$(uname -s)-$(uname -m)" BUILD_DIR="${PACKAGE_NAME}_installer" mkdir -p $BUILD_DIR APP_FOLDER="$BUILD_DIR/$PRODUCT_NAME" APP="$APP_FOLDER/${PACKAGE_NAME}.app" PYTHON_FRAMEWORK="$APP/Contents/Frameworks/python.framework" PYTHON_ENV="$PYTHON_FRAMEWORK/Resources/env" ln -s /Applications "$BUILD_DIR/Applications" cp -R -P "extra/platform_resources/macos/app_folder_template" "$APP_FOLDER" cp -R -P "conda_packed" "$PYTHON_ENV" plutil -replace CFBundleVersion -string "$VERSION" "$APP/Contents/Info.plist" plutil -replace CFBundleShortVersionString -string "$MAJOR_MINOR_PATCH" "$APP/Contents/Info.plist" plutil -replace NSHumanReadableCopyright -string "Copyright $(date +'%Y') the $PACKAGE_NAME developers" "$APP/Contents/Info.plist" sign_file() { codesign --force --timestamp --options=runtime --verify --verbose=4 --sign "${DEVELOPER_IDENTITY}" "$1" } SRC_ENTITLEMENTS="extra/platform_resources/macos/app_folder_template/${PACKAGE_NAME}.app/Contents/Frameworks/python.framework/Resources/Entitlements.plist" sign_with_entitlements() { codesign --verify --options=runtime --entitlements "${SRC_ENTITLEMENTS}" --timestamp --verbose=4 --force --sign "${DEVELOPER_IDENTITY}" "$1" } # Sign shared libraries find "$PYTHON_ENV" -name "*\.so" -print0 | while IFS= read -r -d '' file; do sign_file "$file"; done; find "$PYTHON_ENV" -name "*\.dylib" -print0 | while IFS= read -r -d '' file; do sign_file "$file"; done; find "$PYTHON_ENV/bin" -type f -perm +111 -print0 | while IFS= read -r -d '' file; do sign_file "$file"; done; # Sign python framework with entitlements sign_with_entitlements "$PYTHON_FRAMEWORK/Versions/A/Resources/Entitlements.plist" PYTHON_EXECUTABLE="$(realpath $PYTHON_ENV/bin/python)" sign_with_entitlements "$PYTHON_EXECUTABLE" sign_with_entitlements "$PYTHON_FRAMEWORK" # Sign apps codesign --verify --options=runtime --timestamp --verbose=4 --force --sign "${DEVELOPER_IDENTITY}" "$APP/Contents/MacOS/${PACKAGE_NAME}_webview" codesign --verify --options=runtime --timestamp --verbose=4 --force --sign "${DEVELOPER_IDENTITY}" "$APP" codesign --verify --options=runtime --timestamp --verbose=4 --force --sign "${DEVELOPER_IDENTITY}" "$APP_FOLDER/${PACKAGE_NAME}_shell.app" # After signing, ensure all writes are flushed sync sleep 2 DMG="artifacts/$FULL_PACKAGE_NAME.dmg" mkdir -p artifacts hdiutil create $DMG -srcfolder "$BUILD_DIR" -ov -format UDZO codesign -s "${DEVELOPER_IDENTITY}" $DMG # Clean up signing keys security delete-keychain build.keychain rm certificate.p12 # Notarize xcrun notarytool submit --wait --apple-id "$NOTARIZATION_USERNAME" --password "$NOTARIZATION_PASSWORD" --team-id 8CX8K63BQM $DMG # Staple xcrun stapler staple $DMG - name: Create Windows Installer if: startsWith(matrix.config.os, 'windows') run: | echo "Creating Windows Installer" # This assumes that conda-packed enviroment exists at "conda_packed" VERSION=$(versioningit) MAJOR_MINOR_PATCH=$(sed -E 's/(^[0-9]+\.[0-9]+\.[0-9]+).*$/\1/' <<< "$VERSION") FULL_PACKAGE_NAME="$PACKAGE_NAME-$VERSION-Windows-$(uname -m)" # clean up junk from conda-pack operation: find conda_packed -name "*.conda_trash" -delete conda install -y nsis # create the installer: PRODUCT_NAME="$PACKAGE_NAME-$VERSION" $CONDA_PREFIX/NSIS/makensis.exe -DPRODUCT_NAME="$PRODUCT_NAME" -DPRODUCT_VERSION="$MAJOR_MINOR_PATCH.0" extra/installer.nsi mkdir -p artifacts mv extra/BumpsWebviewSetup.exe "artifacts/$FULL_PACKAGE_NAME-installer.exe" - name: Create Linux Installer if: startsWith(matrix.config.os, 'ubuntu') run: | echo "Creating Linux Installer" conda info VERSION=$(versioningit) FULL_PACKAGE_NAME="$PACKAGE_NAME-$VERSION-$(uname -s)-$(uname -m)" PRODUCT_NAME="$PACKAGE_NAME-$VERSION" mkdir -p "$PRODUCT_NAME" cp extra/platform_resources/linux/* "$PRODUCT_NAME" mv conda_packed/ "$PRODUCT_NAME/env" mkdir -p artifacts tar -czf "artifacts/$FULL_PACKAGE_NAME.tar.gz" "$PRODUCT_NAME" - name: Upload artifacts uses: actions/upload-artifact@v7 with: name: artifacts-${{ matrix.config.os }}-${{ matrix.config.py }} path: artifacts/* - name: Enumerate artifacts run: | # Collect the distributables { echo 'DISTRIBUTABLES<> "$GITHUB_ENV" - name: Update current release if: startsWith(github.ref, 'refs/tags') uses: johnwbyrd/update-release@v1.0.0 with: token: ${{ secrets.GITHUB_TOKEN }} files: | ${{ env.DISTRIBUTABLES }} bumps-bumps-2b313fd/.github/workflows/build-webview.yml000066400000000000000000000066201521754644000232730ustar00rootroot00000000000000name: Build webview on: workflow_dispatch: env: branch_name: master jobs: test_and_build: runs-on: ${{ matrix.config.os }} strategy: matrix: config: #- { os: ubuntu-latest, py: 3.8, doc: 1 } - { os: windows-latest, py: "3.11", exe: 1, whl: 1 } #- { os: macos-latest, py: 3.8, whl: 1 } # all using to stable abi steps: - uses: actions/checkout@v6 with: ref: ${{ env.branch_name }} - name: Set up Python ${{ matrix.config.py }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.config.py }} - uses: conda-incubator/setup-miniconda@v4 with: auto-update-conda: true python-version: ${{ matrix.config.py }} - name: Build installer run: | pwsh -command ".\$GITHUB_WORKSPACE\extra\build_conda_packed.ps1" mkdir unstable pwd dir dist dir extra mv dist\bumps*.tar.gz "unstable\Bumps-windows-exe-$($env:branch_name).tar.gz" # See the following for how to upload to a release # https://eugene-babichenko.github.io/blog/2020/05/09/github-actions-cross-platform-auto-releases/ - name: Archive artifacts uses: actions/upload-artifact@v7 with: name: artifacts path: | unstable/* updateUnstable: needs: test_and_build runs-on: ubuntu-latest steps: - name: Retrieve all artifacts uses: actions/download-artifact@v8 with: name: artifacts - name: show files run: | ls * -l - name: repack self-extracting run: | sudo apt-get install -y p7zip-full mkdir self_extracting curl -L https://www.7-zip.org/a/7z2106-x64.exe --output 7z.exe 7z e 7z.exe -aoa -oself_extracting 7z.sfx tar -xzf "Bumps-windows-exe-$branch_name.tar.gz" -C self_extracting cd self_extracting && 7z a -mhe=on -mx=1 -sfx7z.sfx "../Bumps-$branch_name-self-extracting.exe" bumps*/ - name: Update release assets and text uses: actions/github-script@v9 with: github-token: ${{secrets.GITHUB_TOKEN}} script: | const fs = require('fs'); const { owner, repo } = context.repo; let sid_release = await github.rest.repos.getReleaseByTag({ owner, repo, tag: "sid" }); await github.rest.repos.updateRelease({ owner, repo, release_id: sid_release.data.id, body: "A persistent prerelease where build artifacts for the current tip will be deposited\n\n## Last updated: " + (new Date()).toDateString() }); // delete existing release assets (if needed) and upload new ones: const to_update = ["Bumps-windows-exe-${{ env.branch_name }}.tar.gz", "Bumps-${{ env.branch_name }}-self-extracting.exe"]; for (let fn of to_update) { let asset_id = (sid_release.data.assets.find((a) => (a.name == fn)) ?? {}).id; if (asset_id) { await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id }); } await github.rest.repos.uploadReleaseAsset({ owner, repo, release_id: sid_release.data.id, name: fn, data: await fs.readFileSync(fn) }); } bumps-bumps-2b313fd/.github/workflows/test-publish.yml000066400000000000000000000120511521754644000231440ustar00rootroot00000000000000name: Test and Publish to PyPI on: push: branches: [master] pull_request: release: types: [published] workflow_dispatch: defaults: run: shell: bash jobs: # Build a pure Python wheel and upload as an artifact build-wheel: runs-on: ubuntu-latest outputs: versions: ${{ steps.set-matrix.outputs.versions }} default_version: ${{ steps.set-matrix.outputs.default_version }} steps: - name: Checkout code uses: actions/checkout@v6 - name: Checkout Random123 uses: actions/checkout@v6 with: repository: "DEShawResearch/random123" ref: v1.14.0 path: bumps/dream/random123 - name: Install uv uses: astral-sh/setup-uv@v7 - name: Get Filtered Minor Versions id: set-matrix run: | MIN_VERSION=$(grep "requires-python" pyproject.toml | grep -oE '[0-9]+\.[0-9]+' | head -1) # Get all versions as numeric arrays, sorted RAW_JSON=$(uv python list --all-versions --output-format json | jq -c --arg min "$MIN_VERSION" ' ($min | split(".") | map(tonumber)) as $min_parts | [ .[] | # Only select versions that do not have a dash or letter suffixes for pre-releases select(.version | test("^\\d+\\.\\d+\\.\\d+$")) | .version | split(".") | map(tonumber) | .[0:2] ] | unique | map(select(. >= $min_parts)) | sort ') # 1. Full Matrix (everything) VERSIONS=$(echo "$RAW_JSON" | jq -c 'map(join("."))') # 2. Extract (n-1) version for "stable" default # If only one version exists, it defaults to that one DEFAULT_VERSION=$(echo "$RAW_JSON" | jq -r 'if length > 1 then .[-2] else .[-1] end | join(".")') echo "versions=$VERSIONS" >> $GITHUB_OUTPUT echo "default_version=$DEFAULT_VERSION" >> $GITHUB_OUTPUT - name: setup bun uses: oven-sh/setup-bun@v2 - name: build webview run: | cd bumps/webview/client bun install bun run build - name: Create the wheel run: uv build --python ${{ steps.set-matrix.outputs.default_version }} - name: Upload the wheel and source distribution artifacts uses: actions/upload-artifact@v7 with: name: artifacts path: | dist/bumps-*-py3-none-any.whl dist/bumps-*.tar.gz # Test the wheel on different platforms, test webview, and check docs build test: runs-on: ${{ matrix.os }} needs: build-wheel env: # Use the n-1 version as the "stable" baseline STABLE_PYTHON: ${{ needs.build-wheel.outputs.default_version }} strategy: matrix: os: ["ubuntu-latest"] py: ${{ fromJson(needs.build-wheel.outputs.versions) }} include: - os: windows-latest py: ${{ needs.build-wheel.outputs.default_version }} - os: macos-latest py: ${{ needs.build-wheel.outputs.default_version }} - os: ubuntu-latest py: docs fail-fast: false steps: - uses: actions/checkout@v6 - name: Install uv uses: astral-sh/setup-uv@v7 - name: Download the wheel uses: actions/download-artifact@v8 with: name: artifacts path: dist - name: Install the wheel id: install-step # This allows the job to proceed even if wheels are missing for dependencies continue-on-error: true run: | PY_VER="${{ (matrix.py == 'docs' && env.STABLE_PYTHON) || matrix.py }}" uv venv --python=$PY_VER for whl in dist/bumps-*-py3-none-any.whl; do uv pip install --no-build "$whl[dev]" done # Emit a warning annotation if the install failed, but don't fail the job - name: Notify Missing Wheels if: steps.install-step.outcome == 'failure' run: | echo "::warning title=Incomplete Test Run::Python ${{ matrix.py }} could not be fully tested because some dependencies lack wheels." - name: Run tests # Only run tests if the install actually succeeded if: steps.install-step.outcome == 'success' && matrix.extra != 'docs' run: uv run python -m pytest -v - name: Check that the docs build (linux only) # Use the stable n-1 version for doc builds if: matrix.extra == 'docs' run: | source .venv/bin/activate make -j 4 -C doc SPHINXOPTS="-W --keep-going -n" html # Upload wheel to PyPI only when a tag is pushed, and its name begins with 'v' upload-to-pypi: runs-on: ubuntu-latest needs: test if: startsWith(github.ref, 'refs/tags/v') permissions: id-token: write steps: - name: Retrieve all artifacts uses: actions/download-artifact@v8 with: name: artifacts path: dist - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: dist/ bumps-bumps-2b313fd/.github/workflows/test-webview-client.yaml000066400000000000000000000026241521754644000245700ustar00rootroot00000000000000name: Test Webview Client on: pull_request defaults: run: working-directory: ./bumps/webview/client jobs: # Test that app can build without issues test-build: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v6 - name: Set up Bun uses: oven-sh/setup-bun@v2 - name: Install packages run: bun install - name: Run test run: bun run build # Test that app is properly linted test-lint: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v6 - name: Set up Bun uses: oven-sh/setup-bun@v2 - name: Install packages run: bun install - name: Run test run: bun run test:lint # Test that app is properly formatted test-format: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v6 - name: Set up Bun uses: oven-sh/setup-bun@v2 - name: Install packages run: bun install - name: Run test run: bun run test:format # Test that app has no typescript errors test-types: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v6 - name: Set up Bun uses: oven-sh/setup-bun@v2 - name: Install packages run: bun install - name: Run test run: bun run test:types bumps-bumps-2b313fd/.github/workflows/webview-client-publish.yml000066400000000000000000000013651521754644000251170ustar00rootroot00000000000000name: Publish Package to npmjs on: push: # Pattern matched against refs/tags tags: - 'client-v*' # Push events to every tag like client-v0.6.15 workflow_dispatch: jobs: build: runs-on: ubuntu-latest permissions: contents: read id-token: write steps: - uses: actions/checkout@v6 # Setup .npmrc file to publish to npm - uses: actions/setup-node@v6 with: node-version: '20.x' registry-url: 'https://registry.npmjs.org' - run: | cd bumps/webview/client npm i npm run build_prod - run: cd bumps/webview/client && npm publish --provenance --access public env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} bumps-bumps-2b313fd/.gitignore000066400000000000000000000011761521754644000163770ustar00rootroot00000000000000# bumps specific ignores bumps/webview/client/node_modules/ bumps/webview/client/dist # virtual environment venv/ env/ .venv/ .env/ # lockfiles uv.lock bun.lock # Eclipse/pycharm settings files .idea .project .pydevproject # editor backup files *.swp *~ *.bak # patches *.patch *.diff # build/test .settings .coverage /build/ /dist/ /bumps.egg-info/ bumps.iss-include iss-version bumps/_version.py # doc targets /doc/_build/ /doc/api/ /doc/tutorial/ /doc/dream/ /doc/server/ # python droppings from running in place __pycache__/ *.pyc *.pyo *.so *.pyd *.dll *.dyld # run in place sets .mplconfig .mplconfig # session files *.h5 bumps-bumps-2b313fd/.pre-commit-config.yaml000066400000000000000000000042031521754644000206620ustar00rootroot00000000000000repos: - repo: https://github.com/pre-commit/mirrors-eslint rev: v9.15.0 # Matches your current version hooks: - id: eslint name: eslint # Use 'additional_dependencies' to ensure your Vue/TS plugins are present additional_dependencies: - "eslint@9.15.0" - "eslint-plugin-vue@10.5.1" - "@vue/eslint-config-typescript@14.6.0" - "eslint-plugin-vuejs-accessibility@2.4.1" # Point to your specific config file args: ["-c", "bumps/webview/client/eslint.config.js", "--fix"] files: \.(vue|js|ts|mjs|tsx|jsx)$ - repo: https://github.com/biomejs/pre-commit rev: v0.6.1 hooks: - id: biome-format name: biome-format # Point to your subdirectory config to ensure spaces vs tabs are correct files: \.(vue|js|ts|mjs|tsx|jsx|json)$ additional_dependencies: ["@biomejs/biome@2.4.13"] - repo: https://github.com/pre-commit/mirrors-prettier rev: v3.1.0 # Current stable mirror version hooks: - id: prettier name: prettier-docs # Limit strictly to non-code files files: \.(md|yaml|yml)$ # Recommended for Markdown to prevent weird line breaks in docs args: ["--prose-wrap", "always"] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 hooks: - id: check-added-large-files args: [--maxkb=8192] exclude: bumps/dream/random123 - id: check-merge-conflict - id: check-yaml args: [--allow-multiple-documents] - id: end-of-file-fixer exclude: | bumps/dream/random123 bumps/webview/client/src - id: trailing-whitespace exclude: | bumps/dream/random123 bumps/webview/client/src - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.6.1 hooks: # - id: ruff # args: [--fix, --exit-non-zero-on-fix] - id: ruff-format args: [--config=pyproject.toml] exclude: | bumps/dream/random123 bumps/webview/client/src extra/platform_scripts ci: autoupdate_schedule: monthly skip: [eslint] bumps-bumps-2b313fd/.readthedocs.yaml000066400000000000000000000011671521754644000176360ustar00rootroot00000000000000version: 2 build: os: ubuntu-22.04 tools: python: "3.11" jobs: post_checkout: # Unshallow the clone to ensure we have history - git fetch --unshallow || true # Specifically fetch all tags so versioningit can see them - git fetch --all --tags || true # Build documentation in the doc/ directory with Sphinx sphinx: configuration: doc/conf.py # If using Sphinx, optionally build your docs in additional formats such as PDF formats: - pdf - epub # Python environment required to build your docs python: install: - method: pip path: . extra_requirements: - dev bumps-bumps-2b313fd/CHANGES.rst000066400000000000000000000251061521754644000162100ustar00rootroot00000000000000************** Change History ************** v1.0.0 2025-06-24 ----------------- * a new web-based GUI (webview) has been added as the default GUI * a dataclass-based JSON serialization is available for bumps classes (and the serializer can be used for model classes as well, falling back to `dill`) * `FitProblem` class was refactored to support simultaneous fitting of multiple models, and `MultiFitProblem` is deprecated (currently an alias to `FitProblem`) * `Fitness` is a Protocol class defining the interface for models * `FitProblem.models` is now a generator that returns Fitness-compatible objects * `Parameter` was refactored and has an indirection attribute `slot`: * `Parameter.value` now returns `float(Parameter.slot)` * this allows `Parameter` to be a wrapper of other `Parameter` or `Expression` or `Calculation` or `Constant` objects * `Expression` is a new class that replaces `Operation` for combining `Parameter` objects in expressions (can be used in `Parameter` slots, and in a model) * Use `Parameter.equals(Expression|Parameter)` for equality constraints. This preserves bounds and limits on the parameter imposed by the model. Calling `Parameter.unlink()` clears the constraint. * `Calculation` is a new class to represent a calculable property of a class, which can be used in `Expression` instances (e.g. the total thickness of a stack of layers) * new `Constraint` class that can be created with e.g. `constraints = [par1 < par2, par2 < par3]` and passed directly to FitProblem * the `DE` stepper (also used by `dream`) is now fully accelerated with numba when numba is present (C DLL is still faster) * `setup.py` removed and the package is now defined in `pyproject.toml` * when using `trim` option in `dream`, the calculated stable portion is stored in the Dream state (`MCMCDraw.portion`), * this portion is used by default in `MCMCDraw.draw()` and plotting methods v0.9.3 2024-07-09 ----------------- * fixed issues with numpy > = 2.0 (see #140, `numpy.NaN` deprecated and removed) * fixed issues with refactor of scipy.stats (see #139) v0.9.2 2024-03-05 ----------------- * added testing for python 3.12 * fixed issue with matplotlib >= 3.8.0 (see #129) * added missing documents to dream manual (Convergence tests, Parallel coordinates plot) * added numba.njit-accelerated fallback bounds.apply methods to dream (still uses compiled C DLL if available) * provide MAX_CORR attribute on the CorrelationView; clear the figure if the number of variables exceeds MAX_CORR v0.9.1 2023-04-10 ----------------- * added support for python 3.11, scipy 1.10, numpy 1.24, wx 4.1.1 * fixed covariance calculation for n-D datasets * fixed batch mode I/O redirection cleanup * fixed issue with DREAM bounds checker when running in parallel * default to single precision derivatives with lm (fixes issue in SasView where OpenCL models failed with Levenberg-Marquardt) * improved support for repeat fitting within scripts and notebooks (*start_mapper* should now work after *stop_mapper*) v0.9.0 2022-03-15 ----------------- * use MPFit in place of scipy.leastsq for bounds-constrained Levenberg-Marquardt Breaking change: * simultaneous fit now scales individual nllfs by squared weight rather than weight v0.8.1 2021-11-18 ----------------- * "apply parameters" action added to GUI menu (does the same as --pars flag in CLI) * operators refactored (no more eval) * BoundedNormal keywords renamed (sigma, mu) -> (std, mean) * support for numba usage in models * fixed Parameters view jumping to top after toggling fit (linux, Mac) * fixed Summary view sliders disappearing in linux * fixed uncertainty plots regenerating at each parameter update * improved documentation of uncertainty analysis Breaking change: * python 2.7 support discontinued v0.8.0 2020-12-16 ----------------- * add stopping conditions to DREAM, using *--alpha=p-value* to reject convergence * require *--overwrite* or *--resume* when reusing a store directory * enable outlier trimming in DREAM with --outliers=iqr * add fitted slope and loglikelihood distribution to the loglikelihood plot * display seed value used for fit so it can be rerun with *--seed* * save MCMC files using gzip * remove R stat from saved state * restore *--pars* option, which was broken in 0.7.17 * terminate the MPI session when the fit is complete instead of waiting for the allocation to expire * allow a series of fits in the same MPI session * support newest matplotlib v0.7.18 2020-11-16 ------------------ * restore python 2.7 support v0.7.17 2020-11-06 ------------------ * restore DREAM fitter efficiency (it should now require fewer burn-in steps) * errplot.reload_errors allows full path to model file * clip values within bounds at start of fit so constraints aren't infinite * allow *--entropy=gmm|mvn|wnn|llf* to specify entropy estimation algorithm * allow duplicate parameter names in model on reload * expand tilde in path names * GUI: restore parallel processing * GUI: suppress uncertainty updates during fit to avoid memory leak * disable broken fitters: particle swarm, random lines, snobfit * minor doc changes v0.7.16 2020-06-11 ------------------ * improved handling of parameters for to_dict() json pickling v0.7.15 2020-06-09 ------------------ * parallel fitting suppressed in GUI for now---need to reuse thread pool * support *limits=(min, max)* for pm and pmp parameter ranges * cleaner handling of single/multiple fit specification * fix *--entropy* command line option * better support for pathlib with virtual file system v0.7.14 2020-01-03 ------------------ * support for *--checkpoint=n*, which updates the .mc files every n hours * fix bug for stuck fits on *--resume*: probabilities contain NaN * better error message for missing store directory * Python 3.8 support (time.clock no longer exists) v0.7.13 2019-10-15 ------------------ * fix pickle problem for parameterized functions * support multi-valued functions in Curve, shown with a coupled ODE example * update support for newer numpy and matplotlib v0.7.12 2019-07-30 ------------------ * --parallel defaults to using one process per CPU. * --pop=-k sets population size to k rather than k times num parameters * --resume=- resumes from --store=/path/to/store * use expanded canvas for parameter histograms to make plots more readable * use regular spaced tics for parameter histograms rather than 1- and 2-sigma * improve consistency between values of cov, stderr and chisq * fix handling of degenerate ranges on parameter output * add entropy calculator using gaussian mixture models (default is still Kramer) * vfs module allows loading of model and data from zip file (not yet enabled) * warn when model has no fitted parameters * update mpfit to support python 3 * support various versions of scipy and numpy v0.7.11 2018-09-24 ------------------ * add support for parameter serialization v0.7.10 2018-06-15 ------------------ * restructure parameter table in gui v0.7.9 2018-06-14 ----------------- * full support for python 3 in wx GUI * allow added or missing parameters in reloaded .par file * add dream state to return from fit() call v0.7.8 2018-05-18 ----------------- * fix source distribution (bin directory was missing) v0.7.7 2018-05-17 ----------------- * merge in amdahl branch for improved performance * update plot so that the displayed "chisq" is consistent with nllf * slight modification to the DREAM DE crossover ratio so that no crossover weight ever goes to zero. * par.dev(std) now uses the initial value of the parameter as the center of the distribution for a gaussian prior on par, as stated in the documentation. In older releases it was incorrectly defaulting to mean=0 if the mean was not specified. * save parameters and uncertainties as JSON as well as text * convert discrete variables to integer prior to computing DREAM statistics * allow relative imports from model files * support latest numpy/matplotlib stack * initial support for wxPhoenix/python 4 GUI (fit ranges can't yet be set) v0.7.6 2016-08-05 ----------------- * add --view option to command line which gets propagated to the model plotter * add support for probability p(x) for vector x using VectorPDF(f,x0) * rename DirectPDF to DirectProblem, and allow it to run in GUI * data reader supports multi-part files, with parts separated by blank lines * add gaussian mixture and laplace examples * bug fix: plots were failing if model name contains a '.' * miscellaneous code cleanup v0.7.5.10 2016-05-04 -------------------- * gui: undo code cleaning operation which broke the user interface v0.7.5.9 2016-04-22 ------------------- * population initializers allow indefinite bounds * use single precision criterion for levenberg-marquardt and bfgs * implement simple, faster, less accurate Hessian & Jacobian * compute uncertainty estimate from Jacobian if problem is sum of squares * gui: fit selection window acts like a dialog v0.7.5.8 2016-04-18 ------------------- * accept model.par output from a different model * show residuals with curve fit output * only show correlations for selected variables * show tics on correlations if small number * improve handling of uncertainty estimate from curvature * tweak dream algorithm -- maybe improve the acceptance ratio? * allow model to set visible variables in output * improve handling of arbitrary probability density functions * simplify loading of pymc models * update to numdifftools 0.9.14 * bug fix: improved handling of ill-conditioned fits * bug fix: avoid copying mcmc chain during run * bug fix: more robust handling of --time limit * bug fix: support newer versions of matplotlib and numpy * miscellaneous tweaks and fixes v0.7.5.7 2015-09-21 ------------------- * add entropy calculator (still unreliable for high dimensional problems) * adjust scaling of likelihood (the green line) to match histogram area * use --samples to specify the number of samples from the distribution * mark this and future releases with a DOI at zenodo.org v0.7.5.6 2015-06-03 ------------------- * tweak uncertainty calculations so they don't fail on bad models v0.7.5.5 2015-05-07 ------------------- * documentation updates v0.7.5.4 2014-12-05 ------------------- * use relative rather than absolute noise in dream, which lets us fit target values in the order of 1e-6 or less. * fix covariance population initializer v0.7.5.3 2014-11-21 ------------------- * use --time to stop after a given number of hours * Levenberg-Marquardt: fix "must be 1-d or 2-d" bug * improve curvefit interface v0.7.5.2 2014-09-26 ------------------- * pull numdifftools dependency into the repository v0.7.5.1 2014-09-25 ------------------- * improve the load_model interface v0.7.5 2014-09-10 ----------------- * Pure python release bumps-bumps-2b313fd/LICENSE.txt000077500000000000000000000152221521754644000162320ustar00rootroot00000000000000Bumps is in the public domain. Code in individual files has copyright and license set by individual authors. bumps.gui, bumps.quasinewton ---------------------------- Copyright (C) 2006-2011, University of Maryland Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/ or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. DREAM ----- Copyright (c) 2008, Los Alamos National Security, LLC All rights reserved. Copyright 2008. Los Alamos National Security, LLC. This software was produced under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National Laboratory (LANL), which is operated by Los Alamos National Security, LLC for the U.S. Department of Energy. The U.S. Government has rights to use, reproduce, and distribute this software. NEITHER THE GOVERNMENT NOR LOS ALAMOS NATIONAL SECURITY, LLC MAKES A NY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software is modified to produce derivative works, such modified software should be clearly marked, so as not to confuse it with the version available from LANL. Additionally, redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Los Alamos National Security, LLC, Los Alamos National Laboratory, LANL the U.S. Government, nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Random123 --------- Copyright 2010-2012, D. E. Shaw Research. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions, and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of D. E. Shaw Research nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. MPFit ----- The original version of this software, called LMFIT, was written in FORTRAN as part of the MINPACK-1 package by XXX. Craig Markwardt converted the FORTRAN code to IDL. The information for the IDL version is: Craig B. Markwardt, NASA/GSFC Code 662, Greenbelt, MD 20770 craigm@lheamail.gsfc.nasa.gov UPDATED VERSIONs can be found on my WEB PAGE: http://cow.physics.wisc.edu/~craigm/idl/idl.html Mark Rivers created this Python version from Craig's IDL version. Mark Rivers, University of Chicago Building 434A, Argonne National Laboratory 9700 South Cass Avenue, Argonne, IL 60439 rivers@cars.uchicago.edu Updated versions can be found at http://cars.uchicago.edu/software bumps.simplex ------------- :: # ******NOTICE*************** # From optimize.py module by Travis E. Oliphant # # You may copy and use this module as you see fit with no # guarantee implied provided you keep this notice in all copies. # *****END NOTICE************ bumps.cli.warn_with_traceback ----------------------------- From http://stackoverflow.com/questions/22373927/get-traceback-of-warnings answered by mgab (2014-03-13) edited by Gareth Rees (2015-11-28) bumps.dream.entropy.Timer ------------------------- Based on: Eli Bendersky https://stackoverflow.com/a/5849861 Extended with tic/toc by Paul Kienzle bumps.dream.entropy.MultivariateT.rvs ------------------------------------- From farhawa on stack overflow https://stackoverflow.com/questions/29798795/multivariate-student-t-distribution-with-python bumps.lsqerror.comb ------------------- From dheerosaur https://stackoverflow.com/questions/4941753/is-there-a-math-ncr-function-in-python/4941932#4941932 bumps-bumps-2b313fd/MANIFEST.in000077500000000000000000000024151521754644000161450ustar00rootroot00000000000000# The purpose of this file is to modify the list of files to include/exclude in # the source archive created by the 'python setup.py sdist' command. Executing # setup.py in the top level directory creates a default list (or manifest) and # the directives in this file add or subtract files from the resulting MANIFEST # file that drives the creation of the archive. # # Note: apparently due to a bug in setup, you cannot include a file whose name # starts with 'build' as in 'build_everything.py'. # Add files to the archive in addition to those that are installed by running # 'python setup.py install'. Typically these extra files are build related. include MANIFEST.in # this file include bin/* recursive-include bumps/webview/client/dist *.html *.js *.css *.svg *.png recursive-include bumps/webview/client/src *.html *.js *.mjs *.css *.svg *.png *.ts *.vue recursive-include bumps/dream/random123/include *.h recursive-include bumps/gui/resources *.ico *.png *.wav include bumps/dream/random123/LICENSE include bumps/dream/compiled.c include bumps/webview/client/*.html include bumps/webview/client/*.js include bumps/webview/client/*.json include bumps/webview/client/*.txt include extra/*.svg extra/*.png # Delete files # ex. prune this that prune bumps/webview/client/node_modules bumps-bumps-2b313fd/Makefile000066400000000000000000000044471521754644000160530ustar00rootroot00000000000000ROOTDIR = $(shell pwd) # Check for the presence of bun or npm ifneq ($(shell which bun),) FE_CMD=bun else ifneq ($(shell which npm),) FE_CMD=npm else echo "No frontend build tool found. Please install 'bun' or 'npm'." exit 1 endif # This nifty perl one-liner collects all comments headed by the double "#" symbols next to each target and recycles them as comments .PHONY: help help: ## Print this help message @perl -nle'print $& if m{^[/a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}' .PHONY: clean clean: ## Delete some cruft from builds/testing/etc. rm -f `find . -type f -name '*.py[co]'` rm -rf `find . -name __pycache__ -o -name "*.egg-info"` \ `find . -name 'output-*'` \ .coverage build dist \ doc/_build doc/api doc/tutorial \ .pytest_cache \ .ruff_cache .PHONY: test test: ## Run pytest and doc tests pytest -v python check_examples.py --chisq python check_fitters.py ####################### ### Dev environment ### ####################### .PHONY: dev-backend dev-backend: ## Start the backend server in headless mode bumps-webview --port 8080 --headless .PHONY: dev-frontend dev-frontend: ## Start the frontend server in development mode cd bumps/webview/client && \ $(FE_CMD) run dev ############################## ### Linting and formatting ### ############################## .PHONY: lint lint: lint-backend lint-frontend ## Run all linters .PHONY: lint-backend-check lint-backend-check: ## Run ruff linting on python code @ruff check bumps/ run.py test.py check_*.py .PHONY: lint-backend lint-backend: ## Run ruff linting fix on python code @ruff check --fix bumps/ run.py test.py check_*.py .PHONY: lint-frontend-check lint-frontend-check: ## Run bun linting check on javascript code cd bumps/webview/client && \ $(FE_CMD) run test:lint .PHONY: lint-frontend lint-frontend: ## Run bun linting fix on javascript code cd bumps/webview/client && \ $(FE_CMD) run lint .PHONY: format format: format-backend format-frontend ## Run all formatters .PHONY: format-backend format-backend: ## Run ruff formatting on python code @ruff format bumps/ run.py test.py check_*.py .PHONY: format-frontend format-frontend: ## Run bun formatting on javascript code cd bumps/webview/client && \ $(FE_CMD) run format bumps-bumps-2b313fd/README.amdahl000066400000000000000000000041101521754644000165030ustar00rootroot00000000000000For large fits across mutliple nodes you may find that the proposal step is a bottleneck. The compiled DE stepper can speed this up by a factor of 2 compared to the numba version that is usually used. This might help on large allocations, but not enough to support it in the automatic build infrastructure. Update: MSVC is 6x faster than numba on one machine. Need to check performance with and without compiled on HPC hardware to know if the compiled version is required. To use the compiled de stepper and bounds checks, first make sure the "random123" library submodule has been checked out git clone --branch v1.14.0 https://github.com/DEShawResearch/random123.git bumps/dream/random123 Then, to compile on unix use: (cd bumps/dream && cc compiled.c -I ./random123/include/ -O2 -fopenmp -shared -lm -o _compiled.so -fPIC -DMAX_THREADS=64) On OS/X clang doesn't support OpenMP: (cd bumps/dream && cc compiled.c -I ./random123/include/ -O2 -shared -lm -o _compiled.so -fPIC -DMAX_THREADS=64) MSVC on windows using Visual Studio build tools (2022): % set up compiler environment "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 cd bumps\dream cl compiled.c -I .\random123\include /O2 /openmp /LD /GL /Fe_compiled.so This only works when _compiled.so is in the bumps/dream directory. If running from a pip installed version, you will need to fetch the bumps repository: $ git clone https://github.com/bumps/bumps.git $ cd bumps Compile as above, then find the bumps install path using the following: $ python -c "import bumps.dream; print(bumps.dream.__file__)" #dream/path/__init__.py Copy the compiled module to the install, with the #dream/path printed above: $ cp bumps/dream/_compiled.so #dream/path There is no provision for using _compiled.so in a frozen application. Run with no more than 64 OMP threads. If the number of processors is more than 64, then use: OMP_NUM_THREADS=64 ./run.py ... I don't know how OMP_NUM_THREADS behaves if it is larger than the number of processors. bumps-bumps-2b313fd/README.rst000077500000000000000000000032701521754644000160760ustar00rootroot00000000000000============================================== Bumps: data fitting and uncertainty estimation ============================================== Bumps provides data fitting and Bayesian uncertainty modeling for inverse problems. It has a variety of optimization algorithms available for locating the most like value for function parameters given data, and for exploring the uncertainty around the minimum. Installation is with the usual python installation command:: pip install bumps Once the system is installed, you can verify that it is working with:: bumps doc/examples/peaks/model.py --chisq bumps -h To start the GUI use:: bumps Documentation is available at `readthedocs `_. See `CHANGES.rst `_ for details on recent changes. If a compiler is available, then significant speedup is possible for DREAM using:: python -m bumps.dream.build_compiled If you have installed from source, you must first check out the random123 library:: git clone --branch v1.14.0 https://github.com/DEShawResearch/random123.git bumps/dream/random123 python -m bumps.dream.build_compiled |CI| |RTD| |DOI| .. |CI| image:: https://github.com/bumps/bumps/actions/workflows/test-publish.yml/badge.svg :alt: Build status :target: https://github.com/bumps/bumps/actions/workflows/test-publish.yml .. |DOI| image:: https://zenodo.org/badge/DOI/10.5281/zenodo.594099.svg :alt: DOI tag :target: https://doi.org/10.5281/zenodo.594099 .. |RTD| image:: https://readthedocs.org/projects/bumps/badge/?version=latest :alt: Documentation status :target: https://bumps.readthedocs.io/en/latest/?badge=latest bumps-bumps-2b313fd/biome.json000066400000000000000000000005471521754644000163760ustar00rootroot00000000000000{ "$schema": "https://biomejs.dev/schemas/2.4.13/schema.json", "files": { "includes": ["**", "!**/package.json"] }, "formatter": { "enabled": true, "indentStyle": "space", "indentWidth": 2, "lineWidth": 120 }, "linter": { "enabled": false }, "javascript": { "formatter": { "quoteStyle": "double" } } } bumps-bumps-2b313fd/bumps/000077500000000000000000000000001521754644000155305ustar00rootroot00000000000000bumps-bumps-2b313fd/bumps/__init__.py000066400000000000000000000024421521754644000176430ustar00rootroot00000000000000# This program is in the public domain # Author: Paul Kienzle """ Bumps: curve fitter with uncertainty estimation This package provides tools for modeling parametric systems in a Bayesian context, with routines for finding the maximum likelihood and the posterior probability density function. A graphical interface allows direct manipulation of the model parameters. See https://bumps.readthedocs.io for online manuals. """ try: from ._version import __version__ # noqa: F401 except ImportError: __version__ = "unknown" __schema_version__ = "1" def data_files(): """ Return the data files associated with the package for setup_py2exe.py. The format is a list of (directory, [files...]) pairs which can be used directly in the py2exe setup script as:: setup(..., data_files=data_files(), ...) """ from .gui.utilities import data_files return data_files() def package_data(): """ Return the data files associated with the package for setup.py. The format is a dictionary of {'fully.qualified.module', [files...]} used directly in the setup script as:: setup(..., package_data=package_data(), ...) """ from .gui.utilities import package_data return package_data() bumps-bumps-2b313fd/bumps/__main__.py000066400000000000000000000003061521754644000176210ustar00rootroot00000000000000""" Bumps application. Run "bumps --help" for details, or "python -m bumps -help" if bumps isn't on your path. """ from bumps.webview.server.cli import main if __name__ == "__main__": main() bumps-bumps-2b313fd/bumps/bounds.py000066400000000000000000000774151521754644000174120ustar00rootroot00000000000000# This program is in the public domain # Author: Paul Kienzle """ Parameter bounds and prior probabilities. Parameter bounds encompass several features of our optimizers. First and most trivially they allow for bounded constraints on parameter values. Secondly, for parameter values known to follow some distribution, the bounds encodes a penalty function as the value strays from its nominal value. Using a negative log likelihood cost function on the fit, then this value naturally contributes to the overall likelihood measure. Predefined bounds are:: Unbounded range (-inf, inf) BoundedBelow range (base, inf) BoundedAbove range (-inf, base) Bounded range (low, high) Normal range (-inf, inf) with gaussian probability BoundedNormal range (low, high) with gaussian probability within SoftBounded range (low, high) with gaussian probability outside New bounds can be defined following the abstract base class interface defined in :class:`Bounds`, or using Distribution(rv) where rv is a scipy.stats continuous distribution. For generating bounds given a value, we provide a few helper functions:: v +/- d: pm(x,dx) or pm(x,-dm,+dp) or pm(x,+dp,-dm) return (x-dm,x+dm) limited to 2 significant digits v +/- p%: pmp(x,p) or pmp(x,-pm,+pp) or pmp(x,+pp,-pm) return (x-pm*x/100, x+pp*x/100) limited to 2 sig. digits pm_raw(x,dx) or raw_pm(x,-dm,+dp) or raw_pm(x,+dp,-dm) return (x-dm,x+dm) pmp_raw(x,p) or raw_pmp(x,-pm,+pp) or raw_pmp(x,+pp,-pm) return (x-pm*x/100, x+pp*x/100) nice_range(lo,hi) return (lo,hi) limited to 2 significant digits """ __all__ = [ "pm", "pmp", "pm_raw", "pmp_raw", "nice_range", "init_bounds", "DistProtocol", "Bounds", "Unbounded", "Bounded", "BoundedAbove", "BoundedBelow", "Distribution", "Normal", "BoundedNormal", "SoftBounded", ] import sys from dataclasses import dataclass, field import math from math import log, log10, sqrt, pi, ceil, floor from numpy import inf, isinf, isfinite, clip import numpy.random as RNG try: from scipy.stats import norm as normal_distribution except ImportError: # Normal distribution is an optional dependency. Leave it as a runtime # failure if it doesn't exist. pass from typing import Optional, Any, Dict, Union, Literal, Tuple, Protocol LimitValue = Union[float, Literal["-inf", "inf"]] LimitsType = Tuple[Union[float, Literal["-inf"]], Union[float, Literal["inf"]]] # TODO: should we use this in the bounds limits? # @dataclass(init=False) # class ExtendedFloat(float): # __root__: Union[float, Literal["inf", "-inf"]] # def __new__(cls, *args, **kw): # return super().__new__(cls, *args) # def __init__(self, __root__=None): # pass # def __repr__(self): # return float.__repr__(self) def pm(v, plus, minus=None, limits: Optional[LimitsType] = None): """ Return the tuple (~v-dv,~v+dv), where ~expr is a 'nice' number near to to the value of expr. For example:: >>> r = pm(0.78421, 0.0023145) >>> print("%g - %g"%r) 0.7818 - 0.7866 If called as pm(value, +dp, -dm) or pm(value, -dm, +dp), return (~v-dm, ~v+dp). """ return nice_range(limited_range(pm_raw(v, plus, minus), limits=limits)) def pmp(v, plus, minus=None, limits=None): """ Return the tuple (~v-%v,~v+%v), where ~expr is a 'nice' number near to the value of expr. For example:: >>> r = pmp(0.78421, 10) >>> print("%g - %g"%r) 0.7 - 0.87 >>> r = pmp(0.78421, 0.1) >>> print("%g - %g"%r) 0.7834 - 0.785 If called as pmp(value, +pp, -pm) or pmp(value, -pm, +pp), return (~v-pm%v, ~v+pp%v). """ return nice_range(limited_range(pmp_raw(v, plus, minus), limits=limits)) # Generate ranges using x +/- dx or x +/- p%*x def pm_raw(v, plus, minus=None): """ Return the tuple [v-dv,v+dv]. If called as pm_raw(value, +dp, -dm) or pm_raw(value, -dm, +dp), return (v-dm, v+dp). """ if minus is None: minus = -plus if plus < minus: plus, minus = minus, plus return v + minus, v + plus def pmp_raw(v, plus, minus=None): """ Return the tuple [v-%v,v+%v] If called as pmp_raw(value, +pp, -pm) or pmp_raw(value, -pm, +pp), return (v-pm%v, v+pp%v). """ if minus is None: minus = -plus if plus < minus: plus, minus = minus, plus b1, b2 = v * (1 + 0.01 * minus), v * (1 + 0.01 * plus) return (b1, b2) if v > 0 else (b2, b1) def limited_range(bounds, limits=None): """ Given a range and limits, fix the endpoints to lie within the range """ if limits is not None: return clip(bounds[0], *limits), clip(bounds[1], *limits) return bounds def nice_range(bounds): """ Given a range, return an enclosing range accurate to two digits. """ step = bounds[1] - bounds[0] if step > 0: d = 10 ** (floor(log10(step)) - 1) return floor(bounds[0] / d) * d, ceil(bounds[1] / d) * d else: return bounds def init_bounds(v) -> "Bounds": """ Returns a bounds object of the appropriate type given the arguments. This is a helper factory to simplify the user interface to parameter objects. """ # if it is none, then it is unbounded if v is None: return Unbounded() # if it isn't a tuple, assume it is a bounds type. try: lo, hi = v except TypeError: return v # if it is a tuple, then determine what kind of bounds we have if lo is None: lo = -inf if hi is None: hi = inf # TODO: consider issuing a warning instead of correcting reversed bounds if lo >= hi: lo, hi = hi, lo if isinf(lo) and isinf(hi): return Unbounded() elif isinf(lo): return BoundedAbove(hi) elif isinf(hi): return BoundedBelow(lo) else: return Bounded(lo, hi) class Bounds: """ Bounds abstract base class. A range is used for several purposes. One is that it transforms parameters between unbounded and bounded forms depending on the needs of the optimizer. Another is that it generates random values in the range for stochastic optimizers, and for initialization. A third is that it returns the likelihood of seeing that particular value for optimizers which use soft constraints. Assuming the cost function that is being optimized is also a probability, then this is an easy way to incorporate information from other sorts of measurements into the model. """ # TODO: need derivatives wrt bounds transforms @property def limits(self): return (-inf, inf) @property def dof(self): return 0 def get01(self, x): """ Convert value into [0,1] for optimizers which are bounds constrained. This can also be used as a scale bar to show approximately how close to the end of the range the value is. """ def put01(self, v): """ Convert [0,1] into value for optimizers which are bounds constrained. """ def getfull(self, x): """ Convert value into (-inf,inf) for optimizers which are unconstrained. """ def putfull(self, v): """ Convert (-inf,inf) into value for optimizers which are unconstrained. """ def random(self, n=1, target=1.0): """ Return a randomly generated valid value. *target* gives some scale independence to the random number generator, allowing the initial value of the parameter to influence the randomly generated value. Otherwise fits without bounds have too large a space to search through. """ def nllf(self, value): """ Return the negative log likelihood of seeing this value, with likelihood scaled so that the maximum probability is one. For uniform bounds, this either returns zero or inf. For bounds based on a probability distribution, this returns values between zero and inf. The scaling is necessary so that indefinite and semi-definite ranges return a sensible value. The scaling does not affect the likelihood maximization process, though the resulting likelihood is not easily interpreted. """ def residual(self, value): """ Return the parameter 'residual' in a way that is consistent with residuals in the normal distribution. The primary purpose is to graphically display exceptional values in a way that is familiar to the user. For fitting, the scaled likelihood should be used. To do this, we will match the cumulative density function value with that for N(0,1) and find the corresponding percent point function from the N(0,1) distribution. In this way, for example, a value to the right of 2.275% of the distribution would correspond to a residual of -2, or 2 standard deviations below the mean. For uniform distributions, with all values equally probable, we use a value of +/-4 for values outside the range, and 0 for values inside the range. """ def start_value(self): """ Return a default starting value if none given. """ return self.put01(0.5) def __contains__(self, v): return self.limits[0] <= v <= self.limits[1] def __str__(self): return f"({self.limits[0]},{self.limits[1]})" def satisfied(self, v) -> bool: lo, hi = self.limits return v >= lo and v <= hi def penalty(self, v) -> float: """ return a (differentiable) nonzero value when outside the bounds """ lo, hi = self.limits dlo = 0.0 if v >= lo else abs(v - lo) dhi = 0.0 if v <= hi else abs(v - hi) return dlo + dhi def to_dict(self): return dict( type=type(self).__name__, limits=self.limits, ) @dataclass(init=False) class Unbounded(Bounds): """ Unbounded parameter. The random initial condition is assumed to be between 0 and 1 The probability is uniformly 1/inf everywhere, which means the negative log likelihood of P is inf everywhere. A value inf will interfere with optimization routines, and so we instead choose P == 1 everywhere. """ type = "Unbounded" def __init__(self, *args, **kw): pass def random(self, n=1, target=1.0): scale = target + (target == 0.0) return RNG.randn(n) * scale def nllf(self, value): return 0 def residual(self, value): return 0 def get01(self, x): return _get01_inf(x) def put01(self, v): return _put01_inf(v) def getfull(self, x): return x def putfull(self, v): return v @dataclass(init=True) class BoundedBelow(Bounds): """ Semidefinite range bounded below. The random initial condition is assumed to be within 1 of the maximum. [base,inf] <-> (-inf,inf) is direct above base+1, -1/(x-base) below [base,inf] <-> [0,1] uses logarithmic compression. Logarithmic compression works by converting sign*m*2^e+base to sign*(e+1023+m), yielding a value in [0,2048]. This can then be converted to a value in [0,1]. Note that the likelihood function is problematic: the true probability of seeing any particular value in the range is infinitesimal, and that is indistinguishable from values outside the range. Instead we say that P = 1 in range, and 0 outside. """ base: float type = "BoundedBelow" @property def limits(self): return (self.base, inf) def start_value(self): return self.base + 1 def random(self, n=1, target: float = 1.0): target = max(abs(target), abs(self.base)) scale = target + float(target == 0.0) return self.base + abs(RNG.randn(n) * scale) def nllf(self, value): return 0 if value >= self.base else inf def residual(self, value): # TODO: penalty of 16 for being outside the bounds is too weak return 0 if value >= self.base else -4 def get01(self, x): # m in (-1, 1), e in [_E_MIN, _E_MAX] m, e = math.frexp(x - self.base) if m >= 0: v = (e - _E_MIN + m) / _E_RANGE return v else: # mantissa is negative; clip below return 0.0 def put01(self, v): v = v * _E_RANGE e = int(v) m = v - e x = math.ldexp(m, e + _E_MIN) + self.base return x def getfull(self, x): v = x - self.base return v if v >= 1 else 2 - 1.0 / v if v > 0 else -inf def putfull(self, v): x = v if v >= 1 else 1.0 / (2 - v) return x + self.base @dataclass(init=True) class BoundedAbove(Bounds): """ Semidefinite range bounded above. [-inf,base] <-> [0,1] uses logarithmic compression [-inf,base] <-> (-inf,inf) is direct below base-1, 1/(base-x) above Logarithmic compression works by converting sign*m*2^e+base to sign*(e+1023+m), yielding a value in [0,2048]. This can then be converted to a value in [0,1]. Note that the likelihood function is problematic: the true probability of seeing any particular value in the range is infinitesimal, and that is indistinguishable from values outside the range. Instead we say that P = 1 in range, and 0 outside. """ base: float @property def limits(self): return (-inf, self.base) def start_value(self): return self.base - 1 def random(self, n=1, target: float = 1.0): target = max(abs(self.base), abs(target)) scale = target + float(target == 0.0) return self.base - abs(RNG.randn(n) * scale) def nllf(self, value): return 0 if value <= self.base else inf def residual(self, value): # TODO: penalty of 16 for being outside the bounds is too weak return 0 if value <= self.base else 4 def get01(self, x): m, e = math.frexp(self.base - x) if m >= 0: v = (e - _E_MIN + m) / _E_RANGE return 1 - v else: return 1 if m < 0 else 0 def put01(self, v): v = (1 - v) * _E_RANGE e = int(v) m = v - e x = -(math.ldexp(m, e + _E_MIN) - self.base) return x def getfull(self, x): v = x - self.base return v if v <= -1 else -2 - 1.0 / v if v < 0 else inf def putfull(self, v): x = v if v <= -1 else -1.0 / (v + 2) return x + self.base @dataclass(init=True) class Bounded(Bounds): """ Bounded range. [lo,hi] <-> [0,1] scale is simple linear [lo,hi] <-> (-inf,inf) scale uses exponential expansion While technically the probability of seeing any value within the range is 1/range, for consistency with the semi-infinite ranges and for a more natural mapping between nllf and chisq, we instead set the probability to 0. This choice will not affect the fits. """ lo: float = field(metadata={"description": "lower end of bounds"}) hi: float = field(metadata={"description": "upper end of bounds"}) # @classmethod # def from_dict(cls, limits=None): # lo, hi = limits # return cls(lo, hi) # def __init__(self, lo, hi): # self.lo = lo # self.hi = hi # self._nllf_scale = log(hi - lo) @property def limits(self): return (self.lo, self.hi) def random(self, n=1, target=1.0): # print("= uniform",lo,hi) return RNG.uniform(self.lo, self.hi, size=n) def nllf(self, value): return 0 if self.lo <= value <= self.hi else inf # return self._nllf_scale if lo<=value<=hi else inf def residual(self, value): # TODO: penalty of 16 for being outside the bounds is too weak return -4 if self.lo > value else (4 if self.hi < value else 0) def get01(self, x): lo, hi = self.limits # TODO: check hi > lo during constructor return clip(float(x - lo) / (hi - lo), 0, 1) def put01(self, v): lo, hi = self.limits return (hi - lo) * v + lo def getfull(self, x): # TODO: value 0.2 in [0.1, 1.1] gives 4 digits of precision rather than 10 raise NotImplementedError("Bounded |getfull(putfull(x)) - x| is too large.") # v = self.get01(x) # Δ01 = self.put01(self.get01(x)) - x # Δfull = _get01_inf(_put01_inf(v)) - v # print(f"Bounded {x} in [{self.lo}, {self.hi}] => {v} {Δ01} {Δfull} {_put01_inf(v)}") return _put01_inf(self.get01(x)) def putfull(self, v): # print(f"Bounded {v} in [{self.lo}, {self.hi}] => {_get01_inf(v)}") return self.put01(_get01_inf(v)) class DistProtocol(Protocol): """ Protocol for a distribution object, implementing the scipy.stats interface. (also including args, kwds and name) """ name: str args: Tuple[float, ...] kwds: Dict[str, Any] def rvs(self, n: int) -> float: ... def nnlf(self, value: float) -> float: ... def cdf(self, value: float) -> float: ... def ppf(self, value: float) -> float: ... def pdf(self, value: float) -> float: ... class Distribution(Bounds): """ Parameter is pulled from a distribution. *dist* must implement the distribution interface from scipy.stats, described in the DistProtocol class. """ dist: DistProtocol = None def __init__(self, dist): object.__setattr__(self, "dist", dist) @property def dof(self): return 1 def random(self, n=1, target=1.0): return self.dist.rvs(n) def nllf(self, value): # TODO: This is inconsistent with normal below, which does not include normalizer # What is the chisq/2 equivalent for an arbitrary distribution? return -log(self.dist.pdf(value)) def residual(self, value): return normal_distribution.ppf(self.dist.cdf(value)) def get01(self, x): return self.dist.cdf(x) def put01(self, v): return self.dist.ppf(v) def getfull(self, x): return x def putfull(self, v): return v def __getstate__(self): # WARNING: does not preserve and restore seed return self.dist.__class__, self.dist.args, self.dist.kwds def __setstate__(self, state): cls, args, kwds = state self.dist = cls(*args, **kwds) def __str__(self): return "%s(%s)" % (self.dist.dist.name, ",".join(str(s) for s in self.dist.args)) def to_dict(self): return dict( type=type(self).__name__, limits=self.limits, # TODO: how to handle arbitrary distribution function in save/load? dist=type(self.dist).__name__, ) @dataclass(frozen=True) class Normal(Distribution): """ Parameter is pulled from a normal distribution. If you have measured a parameter value with some uncertainty (e.g., the film thickness is 35+/-5 according to TEM), then you can use this measurement to restrict the values given to the search, and to penalize choices of this fitting parameter which are different from this value. *mean* is the expected value of the parameter and *std* is the 1-sigma standard deviation. class is 'frozen' because a new object should be created if `mean` or `std` are changed. """ mean: float = 0.0 std: float = 1.0 _nllf_scale: float = field(init=False) def __post_init__(self): Distribution.__init__(self, normal_distribution(self.mean, self.std)) object.__setattr__(self, "_nllf_scale", log(2 * pi * self.std**2) / 2) def nllf(self, value): # P(v) = exp(-0.5*(v-mean)**2/std**2)/sqrt(2*pi*std**2) # -log(P(v)) = -(-0.5*(v-mean)**2/std**2 - log( (2*pi*std**2) ** 0.5)) # = 0.5*(v-mean)**2/std**2 + log(2*pi*std**2)/2 mean, std = self.dist.args return 0.5 * ((value - mean) / std) ** 2 # + self._nllf_scale def residual(self, value): mean, std = self.dist.args return (value - mean) / std def __getstate__(self): return self.dist.args # args is mean,std def __setstate__(self, state): mean, std = state self.__init__(mean=mean, std=std) @dataclass(init=False, frozen=True) class BoundedNormal(Bounds): """ truncated normal bounds """ mean: float = 0.0 std: float = 1.0 lo: Union[float, Literal["-inf"]] hi: Union[float, Literal["inf"]] _left: float = field(init=False) _delta: float = field(init=False) _nllf_scale: float = field(init=False) def __init__(self, mean: float = 0, std: float = 1, limits=(-inf, inf), hi="inf", lo="-inf"): if limits is not None: # for backward compatibility: lo, hi = limits limits = (-inf if lo is None else float(lo), inf if hi is None else float(hi)) # Note: Using object.__setattr__ because @dataclass(frozen) blocks setattr on BoundedNormal object.__setattr__(self, "lo", limits[0]) object.__setattr__(self, "hi", limits[1]) object.__setattr__(self, "mean", mean) object.__setattr__(self, "std", std) object.__setattr__(self, "_left", normal_distribution.cdf((limits[0] - mean) / std)) object.__setattr__(self, "_delta", normal_distribution.cdf((limits[1] - mean) / std) - self._left) object.__setattr__(self, "_nllf_scale", log(2 * pi * std**2) / 2 + log(self._delta)) @property def limits(self): return (self.lo, self.hi) @property def dof(self): return 1 def get01(self, x): """ Convert value into [0,1] for optimizers which are bounds constrained. This can also be used as a scale bar to show approximately how close to the end of the range the value is. """ v = (normal_distribution.cdf((x - self.mean) / self.std) - self._left) / self._delta return clip(v, 0, 1) def put01(self, v): """ Convert [0,1] into value for optimizers which are bounds constrained. """ x = v * self._delta + self._left return normal_distribution.ppf(x) * self.std + self.mean def getfull(self, x): """ Convert value into (-inf,inf) for optimizers which are unconstrained. """ raise NotImplementedError def putfull(self, v): """ Convert (-inf,inf) into value for optimizers which are unconstrained. """ raise NotImplementedError def random(self, n=1, target=1.0): """ Return a randomly generated valid value, or an array of values """ return self.get01(RNG.rand(n)) def nllf(self, value): """ Return the negative log likelihood of seeing this value, with likelihood scaled so that the maximum probability is one. """ if value in self: return 0.5 * ((value - self.mean) / self.std) ** 2 # + self._nllf_scale else: return inf def residual(self, value): """ Return the parameter 'residual' in a way that is consistent with residuals in the normal distribution. The primary purpose is to graphically display exceptional values in a way that is familiar to the user. For fitting, the scaled likelihood should be used. For the truncated normal distribution, we can just use the normal residuals. """ return (value - self.mean) / self.std def start_value(self): """ Return a default starting value if none given. """ return self.put01(0.5) def __contains__(self, v): return self.limits[0] <= v <= self.limits[1] def __str__(self): return f"({self.limits[0]},{self.limits[1]}), norm({self.mean},{self.std})" @dataclass(frozen=True) class SoftBounded(Bounds): """ Parameter is pulled from a stretched normal distribution. This is like a rectangular distribution, but with gaussian tails. The intent of this distribution is for soft constraints on the values. As such, the random generator will return values like the rectangular distribution, but the likelihood will return finite values based on the distance from the from the bounds rather than returning infinity. Note that for bounds constrained optimizers which force the value into the range [0,1] for each parameter we don't need to use soft constraints, and this acts just like the rectangular distribution. """ lo: float = 0.0 hi: float = 1.0 std: float = 1.0 def __post_init__(self): object.__setattr__(self, "_nllf_scale", log(self.hi - self.lo + sqrt(2 * pi * self.std))) @property def dof(self): # Treat as a uniform distribution, with no additional dof # associated with the parameter. return 0 def random(self, n=1, target=1.0): return RNG.uniform(self.lo, self.hi, size=n) def nllf(self, value): # To turn f(x) = 1 if x in [lo,hi] else G(tail) # into a probability p, we need to normalize by \int{f(x)dx}, # which is just hi-lo + sqrt(2*pi*std**2). if value < self.lo: z = self.lo - value elif value > self.hi: z = value - self.hi else: z = 0 return (z / self.std) ** 2 / 2 # + self._nllf_scale def residual(self, value): if value < self.lo: z = self.lo - value elif value > self.hi: z = value - self.hi else: z = 0 return z / self.std def get01(self, x): v = float(x - self.lo) / (self.hi - self.lo) # TODO: not clear that clipping is correct behaviour for soft bounded constraints # This turns the soft bounds into hard bounds for optimizers that use box constraint # optimizers that rely on get01/put01 return clip(v, 0, 1) def put01(self, v): return v * (self.hi - self.lo) + self.lo def getfull(self, x): return x def putfull(self, v): return v def __str__(self): return "box_norm(%g,%g,sigma=%g)" % (self.lo, self.hi, self.std) _E_MIN = -1023 _E_MAX = 1024 _E_RANGE = _E_MAX - _E_MIN + 1 def _get01_inf(x): """ Convert a floating point number to a value in [0,1]. The value sign*m*2^e to sign*(e+1023+m), yielding a value in [-2048,2048]. This can then be converted to a value in [0,1]. Sort order is preserved. Up to 14 bits of precision are lost from the 53 bit mantissa. """ # Arctan alternative # Arctan is approximately linear in (-0.5, 0.5), but the # transform is only useful up to (-10**15,10**15). # return atan(x)/pi + 0.5 if not isfinite(x): return 0.0 if x < 0 else 1.0 if x == 0: return 0.5 m, e = math.frexp(x) s = math.copysign(1.0, m) v = (e - _E_MIN + m * s) * s v = (v / _E_RANGE + 1) / 2 # print("> x,e,m,s,v",x,e,m,s,v) # print(">", type(x), type(e), type(v), type(m), type(s)) return v def _put01_inf(v): """ Convert a value in [0,1] to a full floating point number. Sort order is preserved. Reverses :func:`_get01_inf`, but with fewer bits of precision. """ # Arctan alternative # return tan(pi*(v-0.5)) if v <= 0: return -sys.float_info.max if v >= 1: return sys.float_info.max v = (2 * v - 1) * _E_RANGE s = math.copysign(1.0, v) v *= s e = int(v) m = v - e x = math.ldexp(s * m, e + _E_MIN) # print("< x,e,m,s,v",x,e,m,s,v) # print("<", type(x), type(e), type(v), type(m), type(s)) return x BoundsType = Union[Unbounded, Bounded, BoundedAbove, BoundedBelow, BoundedNormal, SoftBounded, Normal] def test_normal(): """ Test the normal distribution """ epsilon = 1e-10 n = Normal(mean=0.5, std=1.0) # assert abs(n.nllf(0.5) - 0.9189385332046727) < epsilon # root 2 pi sigma^2 norm assert abs(n.nllf(0.5) - 0.0) < epsilon assert abs(n.nllf(1.0) - n.nllf(0.0)) < epsilon assert abs(n.residual(0.5) - 0.0) < epsilon assert abs(n.residual(1.0) - 0.5) < epsilon def demo_01_inf(): import sys # TODO: doesn't handle inf/nan for v in [inf, sys.float_info.max, sys.float_info.max * 0.999999999, 1000, 1.1, 1.0, 0.9, sys.float_info.min, 0]: vp = _get01_inf(v) vpp = _put01_inf(vp) message = f"{v} => {vp} => {vpp}" # print(message) if isfinite(v) and v != 0.0: assert abs(v - vpp) < abs(v * 1e-12), message vp = _get01_inf(-1.0 * v) vpp = _put01_inf(vp) message = f"01_inf {-v} => {vp} => {vpp}" # print(message) if isfinite(v) and v != 0.0: assert abs(-v - vpp) < v * 1e-12, message print("All [0,1] => (-inf, inf) passed") def demo_01(): eps = 1e-10 for base in (-20, -0.001, 0.10, 50): for delta in (-35.5, -0.035, 0, 0.035, 1, 35.5, base): value = base + delta bounds = BoundedAbove(base=base) v01 = bounds.get01(value) vp = bounds.put01(v01) message = f"{value} => {v01} => {vp} in (-inf, {base}]" # print(message) target = value if delta < 0 else base assert abs(vp - target) < abs(value * eps), message bounds = BoundedBelow(base=base) v01 = bounds.get01(value) vp = bounds.put01(v01) message = f"{value} => {v01} => {vp} in [{base}, inf)" # print(message) target = base if delta < 0 else value assert abs(vp - target) < abs(value * eps), message bounds = Unbounded() v01 = bounds.get01(value) vp = bounds.put01(v01) message = f"{value} => {v01} => {vp} in (-inf, inf)" # print(message) target = value assert abs(vp - target) < abs(value * eps), f"{value} => {vp} in (-inf, inf)" bounds = Bounded(base, base + 1) v01 = bounds.get01(value) vp = bounds.put01(v01) message = f"{value} => {v01} => {vp} in [{base}, {base + 1}]" # print(message) target = clip(value, base, base + 1) assert abs(vp - target) < abs(value * eps), f"{value} => {vp} in [{base}, {base + 1}]" print("All 01 passed") def demo_full(): eps = 1e-10 for base in (-20, -0.001, 0.10, 50): for delta in (-35.5, -0.035, 0, 0.035, 0.5, 1.0, 35.5, base): value = base + delta bounds = BoundedAbove(base=base) vfull = bounds.getfull(value) vp = bounds.putfull(vfull) message = f"{value} => {vfull} => {vp} in (-inf, {base}]" # print(message) target = value if delta < 0 else base assert abs(vp - target) < abs(value * eps), message bounds = BoundedBelow(base=base) vfull = bounds.getfull(value) vp = bounds.putfull(vfull) message = f"{value} => {vfull} => {vp} in [{base}, inf)" # print(message) target = base if delta < 0 else value assert abs(vp - target) < abs(value * eps), message bounds = Unbounded() vfull = bounds.getfull(value) vp = bounds.putfull(vfull) message = f"{value} => {vfull} => {vp} in (-inf, inf)" # print(message) target = value assert abs(vp - target) < abs(value * eps), f"{value} => {vp} in (-inf, inf)" bounds = Bounded(base, base + 1) vfull = bounds.getfull(value) vp = bounds.putfull(vfull) message = f"{value} => {vfull} => {vp} in [{base}, {base + 1}]" # print(message) target = clip(value, base, base + 1) assert abs(vp - target) < abs(value * eps), f"{value} => {vp} in [{base}, {base + 1}]" print("All full passed") if __name__ == "__main__": demo_01() demo_01_inf() # demo_full() bumps-bumps-2b313fd/bumps/bspline.py000066400000000000000000000314111521754644000175360ustar00rootroot00000000000000# This program is public domain """ BSpline calculator. Given a set of knots, compute the cubic B-spline interpolation. """ __all__ = ["bspline", "pbs"] import numpy as np from numpy import maximum as max, minimum as min def pbs(x, y, t, clamp=True, parametric=True): """ Evaluate the parametric B-spline px(t),py(t). *x* and *y* are the control points, and *t* are the points in [0,1] at which they are evaluated. The *x* values are sorted so that the spline describes a function. The spline goes through the control points at the ends. If *clamp* is True, the derivative of the spline at both ends is zero. If *clamp* is False, the derivative at the ends is equal to the slope connecting the final pair of control points. If *parametric* is False, then parametric points t' are chosen such that x(t') = *t*. The B-spline knots are chosen to be equally spaced within [0,1]. """ x = list(sorted(x)) knot = np.hstack((0, 0, np.linspace(0, 1, len(y)), 1, 1)) cx = np.hstack((x[0], x[0], x[0], (2 * x[0] + x[1]) / 3, x[1:-1], (2 * x[-1] + x[-2]) / 3, x[-1])) if clamp: cy = np.hstack((y[0], y[0], y[0], y, y[-1])) else: cy = np.hstack((y[0], y[0], y[0], y[0] + (y[1] - y[0]) / 3, y[1:-1], y[-1] + (y[-2] - y[-1]) / 3, y[-1])) if parametric: return _bspline3(knot, cx, t), _bspline3(knot, cy, t) # Find parametric t values corresponding to given z values # First try a few newton steps xt = np.interp(t, x, np.linspace(0, 1, len(x))) with np.errstate(all="ignore"): for _ in range(6): pt, dpt = _bspline3(knot, cx, xt, nderiv=1) xt -= (pt - t) / dpt idx = np.isnan(xt) | (abs(_bspline3(knot, cx, xt) - t) > 1e-9) # Use bisection when newton fails if idx.any(): missing = t[idx] # print missing t_lo, t_hi = 0 * missing, 1 * missing for _ in range(30): # bisection with about 1e-9 tolerance trial = (t_lo + t_hi) / 2 ptrial = _bspline3(knot, cx, trial) tidx = ptrial < missing t_lo[tidx] = trial[tidx] t_hi[~tidx] = trial[~tidx] xt[idx] = (t_lo + t_hi) / 2 # print "err",np.max(abs(_bspline3(knot,cx,t)-xt)) # Return y evaluated at the interpolation points return _bspline3(knot, cx, xt), _bspline3(knot, cy, xt) def bspline(y, xt, clamp=True): """ Evaluate the B-spline with control points *y* at positions *xt* in [0,1]. The spline goes through the control points at the ends. If *clamp* is True, the derivative of the spline at both ends is zero. If *clamp* is False, the derivative at the ends is equal to the slope connecting the final pair of control points. B-spline knots are chosen to be equally spaced within [0,1]. """ knot = np.hstack((0, 0, np.linspace(0, 1, len(y)), 1, 1)) if clamp: cy = np.hstack(([y[0]] * 3, y, y[-1])) else: cy = np.hstack((y[0], y[0], y[0], y[0] + (y[1] - y[0]) / 3, y[1:-1], y[-1] + (y[-2] - y[-1]) / 3, y[-1])) return _bspline3(knot, cy, xt) def _bspline3(knot, control, t, nderiv=0): """ Evaluate the B-spline specified by the given *knot* sequence and *control* values at the parametric points *t*. *nderiv* selects the function or derivative to evaluate. """ knot, control, t = [np.asarray(v) for v in (knot, control, t)] # Deal with values outside the range valid = (t > knot[0]) & (t <= knot[-1]) tv = t[valid] f = np.zeros(t.shape) f[t <= knot[0]] = control[0] f[t >= knot[-1]] = control[-1] # Find B-Spline parameters for the individual segments end = len(knot) - 1 segment = knot.searchsorted(tv) - 1 tm2 = knot[max(segment - 2, 0)] tm1 = knot[max(segment - 1, 0)] tm0 = knot[max(segment - 0, 0)] tp1 = knot[min(segment + 1, end)] tp2 = knot[min(segment + 2, end)] tp3 = knot[min(segment + 3, end)] p4 = control[min(segment + 3, end)] p3 = control[min(segment + 2, end)] p2 = control[min(segment + 1, end)] p1 = control[min(segment + 0, end)] # Compute second and third derivatives. if nderiv > 1: # Normally we require a recursion for Q, R and S to compute # df, d2f and d3f respectively, however Q can be computed directly # from intermediate values of P, S has a recursion of depth 0, # which leaves only the R recursion of depth 1 in the calculation # below. q4 = (p4 - p3) * 3 / (tp3 - tm0) q3 = (p3 - p2) * 3 / (tp2 - tm1) q2 = (p2 - p1) * 3 / (tp1 - tm2) r4 = (q4 - q3) * 2 / (tp2 - tm0) r3 = (q3 - q2) * 2 / (tp1 - tm1) if nderiv > 2: s4 = (r4 - r3) / (tp1 - tm0) d3f = np.zeros(t.shape) d3f[valid] = s4 r4 = ((tv - tm0) * r4 + (tp1 - tv) * r3) / (tp1 - tm0) d2f = np.zeros(t.shape) d2f[valid] = r4 # Compute function value and first derivative p4 = ((tv - tm0) * p4 + (tp3 - tv) * p3) / (tp3 - tm0) p3 = ((tv - tm1) * p3 + (tp2 - tv) * p2) / (tp2 - tm1) p2 = ((tv - tm2) * p2 + (tp1 - tv) * p1) / (tp1 - tm2) p4 = ((tv - tm0) * p4 + (tp2 - tv) * p3) / (tp2 - tm0) p3 = ((tv - tm1) * p3 + (tp1 - tv) * p2) / (tp1 - tm1) if nderiv >= 1: df = np.zeros(t.shape) df[valid] = (p4 - p3) * 3 / (tp1 - tm0) p4 = ((tv - tm0) * p4 + (tp1 - tv) * p3) / (tp1 - tm0) f[valid] = p4 if nderiv == 0: return f elif nderiv == 1: return f, df elif nderiv == 2: return f, df, d2f else: return f, df, d2f, d3f """ def bspline_control(y, clamp=True): return _find_control(y, clamp=clamp) def pbs_control(x, y, clamp=True): return _find_control(x, clamp=clamp), _find_control(y, clamp=clamp) def _find_control(v, clamp=True): raise NotImplementedError("B-spline interpolation doesn't work yet") from scipy.linalg import solve_banded n = len(v) udiag = np.hstack([0, 0, 0, [1 / 6] * (n - 3), 0.25, 0.3]) ldiag = np.hstack([-0.3, 0.25, [1 / 6] * (n - 3), 0, 0, 0]) mdiag = np.hstack([1, 0.3, 7 / 12, [2 / 3] * (n - 4), 7 / 12, -0.3, 1]) A = np.vstack([ldiag, mdiag, udiag]) if clamp: # First derivative is zero at ends bl, br = 0, 0 else: # First derivative at ends follows line between final control points bl, br = (v[1] - v[0]) * n, (v[-1] - v[-2]) * n b = np.hstack([v[0], bl, v[1:n - 1], br, v[-1]]) x = solve_banded((1, 1), A, b) return x # x[1:-1] """ # =========================================================================== # test code def speed_check(): """ Print the time to evaluate 400 points on a 7 knot spline. """ import time x = np.linspace(0, 1, 7) x[1], x[-2] = x[2], x[-3] y = [9, 11, 2, 3, 8, 0, 2] t = np.linspace(0, 1, 400) t0 = time.time() for _ in range(1000): bspline(y, t, clamp=True) print("bspline (ms)", (time.time() - t0) / 1000) def _check(expected, got, tol): """ Check that value matches expected within tolerance. If *expected* is never zero, use relative error for tolerance. """ relative = (np.isscalar(expected) and expected != 0) or (not np.isscalar(expected) and all(expected != 0)) if relative: norm = np.linalg.norm((expected - got) / expected) else: norm = np.linalg.norm(expected - got) if norm >= tol: msg = [ "expected %s" % str(expected), "got %s" % str(got), "tol %s norm %s" % (tol, norm), ] raise ValueError("\n".join(msg)) def _derivs(x, y): """ Compute numerical derivative for a function evaluated on a fine grid. """ # difference formula return (y[1] - y[0]) / (x[1] - x[0]), (y[-1] - y[-2]) / (x[-1] - x[-2]) # 5-point difference formula # left = (y[0]-8*y[1]+8*y[3]-y[4]) / 12 / (x[1]-x[0]) # right = (y[-5]-8*y[-4]+8*y[-2]-y[-1]) / 12 / (x[-1]-x[-2]) # return left, right def test(): """bspline tests""" h = 1e-10 t = np.linspace(0, 1, 100) dt = np.array([0, h, 2 * h, 3 * h, 4 * h, 1 - 4 * h, 1 - 3 * h, 1 - 2 * h, 1 - h, 1]) y = [9, 11, 2, 3, 8, 0, 2] n = len(y) xeq = np.linspace(0, 1, n) x = xeq + 0 x[0], x[-1] = (x[0] + x[1]) / 2, (x[-2] + x[-1]) / 2 dx = np.array( [ x[0], x[0] + h, x[0] + 2 * h, x[0] + 3 * h, x[0] + 4 * h, x[-1] - 4 * h, x[-1] - 3 * h, x[-1] - 2 * h, x[-1] - h, x[-1], ] ) # ==== Check that bspline mat:w # ches pbs with equally spaced x yt = bspline(y, t, clamp=True) xtp, ytp = pbs(xeq, y, t, clamp=True, parametric=False) _check(t, xtp, 1e-8) _check(yt, ytp, 1e-8) xtp, ytp = pbs(xeq, y, t, clamp=True, parametric=True) _check(t, xtp, 1e-8) _check(yt, ytp, 1e-8) yt = bspline(y, t, clamp=False) xtp, ytp = pbs(xeq, y, t, clamp=False, parametric=False) _check(t, xtp, 1e-8) _check(yt, ytp, 1e-8) xtp, ytp = pbs(xeq, y, t, clamp=False, parametric=True) _check(t, xtp, 1e-8) _check(yt, ytp, 1e-8) # ==== Check bspline f at end points yt = bspline(y, t, clamp=True) _check(y[0], yt[0], 1e-12) _check(y[-1], yt[-1], 1e-12) yt = bspline(y, t, clamp=False) _check(y[0], yt[0], 1e-12) _check(y[-1], yt[-1], 1e-12) xt, yt = pbs(x, y, t, clamp=True, parametric=False) _check(x[0], xt[0], 1e-8) _check(x[-1], xt[-1], 1e-8) _check(y[0], yt[0], 1e-8) _check(y[-1], yt[-1], 1e-8) xt, yt = pbs(x, y, t, clamp=True, parametric=True) _check(x[0], xt[0], 1e-8) _check(x[-1], xt[-1], 1e-8) _check(y[0], yt[0], 1e-8) _check(y[-1], yt[-1], 1e-8) xt, yt = pbs(x, y, t, clamp=False, parametric=False) _check(x[0], xt[0], 1e-8) _check(x[-1], xt[-1], 1e-8) _check(y[0], yt[0], 1e-8) _check(y[-1], yt[-1], 1e-8) xt, yt = pbs(x, y, t, clamp=False, parametric=True) _check(x[0], xt[0], 1e-8) _check(x[-1], xt[-1], 1e-8) _check(y[0], yt[0], 1e-8) _check(y[-1], yt[-1], 1e-8) # ==== Check f' at end points yt = bspline(y, dt, clamp=True) left, right = _derivs(dt, yt) _check(0, left, 1e-8) _check(0, right, 1e-8) xt, yt = pbs(x, y, dx, clamp=True, parametric=False) left, right = _derivs(xt, yt) _check(0, left, 1e-8) _check(0, right, 1e-8) xt, yt = pbs(x, y, dt, clamp=True, parametric=True) left, right = _derivs(xt, yt) _check(0, left, 1e-8) _check(0, right, 1e-8) yt = bspline(y, dt, clamp=False) left, right = _derivs(dt, yt) _check((y[1] - y[0]) * (n - 1), left, 5e-4) _check((y[-1] - y[-2]) * (n - 1), right, 5e-4) xt, yt = pbs(x, y, dx, clamp=False, parametric=False) left, right = _derivs(xt, yt) _check((y[1] - y[0]) / (x[1] - x[0]), left, 5e-4) _check((y[-1] - y[-2]) / (x[-1] - x[-2]), right, 5e-4) xt, yt = pbs(x, y, dt, clamp=False, parametric=True) left, right = _derivs(xt, yt) _check((y[1] - y[0]) / (x[1] - x[0]), left, 5e-4) _check((y[-1] - y[-2]) / (x[-1] - x[-2]), right, 5e-4) # ==== Check interpolator # yc = bspline_control(y) # print("y",y) # print("p(yc)",bspline(yc,xeq)) def demo(): """Show bsline curve for a set of control points.""" import matplotlib.pyplot as plt # y = [9 ,6, 1, 3, 8, 4, 2] # y = [9, 11, 13, 3, -2, 0, 2] y = [9, 11, 2, 3, 8, 0] # y = [9, 9, 1, 3, 8, 2, 2] x = np.linspace(0, 1, len(y)) t = np.linspace(x[0], x[-1], 400) plt.subplot(211) plt.plot(t, bspline(y, t, clamp=False), "-.y", label="unclamped bspline") # bspline # bspline plt.plot(t, bspline(y, t, clamp=True), "-y", label="clamped bspline") plt.plot(sorted(x), y, ":oy", label="control points") plt.legend() # left, right = _derivs(t, bspline(y, t, clamp=False)) # print(left, (y[1] - y[0]) / (x[1] - x[0])) plt.subplot(212) xt, yt = pbs(x, y, t, clamp=False) plt.plot(xt, yt, "-.b", label="unclamped pbs") # pbs xt, yt = pbs(x, y, t, clamp=True) plt.plot(xt, yt, "-b", label="clamped pbs") # pbs # xt,yt = pbs(x,y,t,clamp=True, parametric=True) # plot(xt,yt,'-g') # pbs plt.plot(sorted(x), y, ":ob", label="control points") plt.legend() plt.show() # B-Spline control point inverse function is not yet implemented """ def demo_interp(): import matplotlib.pyplot as plt x = np.linspace(0, 1, 7) y = [9, 11, 2, 3, 8, 0, 2] t = np.linspace(0, 1, 400) yc = bspline_control(y, clamp=True) xc = np.linspace(x[0], x[-1], 9) plt.plot(xc, yc, ':oy', x, y, 'xg') #knot = np.hstack((0, np.linspace(0,1,len(y)), 1)) #fy = _bspline3(knot,yc,t) fy = bspline(yc, t, clamp=True) plt.plot(t, fy, '-.y') plt.show() """ if __name__ == "__main__": # test() demo() # demo_interp() # speed_check() bumps-bumps-2b313fd/bumps/cheby.py000066400000000000000000000146301521754644000172000ustar00rootroot00000000000000r""" Freeform modeling with Chebyshev polynomials. `Chebyshev polynomials `_ $T_k$ form a basis set for functions over $[-1,1]$. The truncated interpolating polynomial $P_n$ is a weighted sum of Chebyshev polynomials up to degree $n$: .. math:: f(x) \approx P_n(x) = \sum_{k=0}^n c_i T_k(x) The interpolating polynomial exactly matches $f(x)$ at the chebyshev nodes $z_k$ and is near the optimal polynomial approximation to $f$ of degree $n$ under the maximum norm. For well behaved functions, the coefficients $c_k$ decrease rapidly, and furthermore are independent of the degree $n$ of the polynomial. The models can either be defined directly in terms of the Chebyshev coefficients $c_k$ with *method* = 'direct', or in terms of control points $(z_k, f(z_k))$ at the Chebyshev nodes :func:`cheby_points` with *method* = 'interp'. Bounds on the parameters are easier to control using 'interp', but the function may oscillate wildly outside the bounds. Bounds on the oscillation are easier to control using 'direct', but the shape of the profile is difficult to control. """ # TODO: clipping volume fraction to [0,1] distorts parameter space # Option 0: clip to [0,1] # - Bayesian analysis: parameter values outside the domain will be equally # probable out to infinity # - Newton methods: the fit space is flat outside the domain, which leads # to a degenerate hessian. # - Direct methods: won't fail, but will be subject to random walk # performance outside the domain. # - trivial to implement! # Option 1: compress (-inf,0.001] and [0.999,inf) into (0,0.001], [0.999,1) # - won't address any of the problems of clipping # Option 2: have chisq return inf for points outside the domain # - Bayesian analysis: correctly assigns probability zero # - Newton methods: degenerate Hessian outside domain # - Direct methods: random walk outside domain # - easy to implement # Option 3: clip outside domain but add penalty based on amount of clipping # A profile based on clipping may have lower chisq than any profile that # can be described by a valid model (e.g., by having a sharper transition # than would be allowed by the model), leading to a minimum outside D. # Adding a penalty constant outside D would help, but there is no constant # that works everywhere. We could use a constant greater than the worst # chisq seen so far in D, which can guarantee an arbitrarily low P(x) and # a global minimum within D, but for Newton methods, the boundary may still # have spurious local minima and objective value now depends on history. # Linear compression of profile to fit within the domain would avoid # unreachable profile shapes (this is just a linear transform on chebyshev # coefficients), and the addition of the penalty value would reduce # parameter correlations that result from having transformed parameters # resulting in identical profiles. Returning T = ||A(x)|| from render, # with A being a transform that brings the profile within [0,1], the # objective function can return P'(x) = P(x)/(10*(1+sum(T_i)^4) for all # slabs i, or P(x) if no slabs return a penalty value. So long as T is # monotonic with increasing badness, with value of 0 within D, and so long # as no values of x outside D can generate models that cannot be # expressed for any x within D, then any optimizer should return a valid # result at the global minimum. There may still be local minima outside # the boundary, so information that the the value is outside the domain # still needs to pass through a local optimizer to the fitting program. # This approach could be used to transform a box constrained # problem to an unconstrained problem using clipping+penalty on the # parameter values and removing the need for constrained Newton optimizers. # - Bayesian analysis: parameters outside D have incorrect probability, but # with a sufficiently large penalty, P(x) ~ 0; if the penalty value is # too low, details of the correlations outside D may leak into D. # - Newton methods: Hessian should point back to domain # - Direct methods: random walk should be biased toward the domain # - moderately complicated __all__ = ["profile", "cheby_approx", "cheby_val", "cheby_points", "cheby_coeff"] import numpy as np from numpy import real, exp, pi, cos, arange, asarray from numpy.fft import fft def profile(c, t, method): r""" Evaluate the chebyshev approximation c at points x. If method is 'direct' then $c_i$ are the coefficients for the chebyshev polynomials $T_i$ yielding $P = \sum_i{c_i T_i(x)}$. If method is 'interp' then $c_i$ are the values of the interpolated function $f$ evaluated at the chebyshev points returned by :func:`cheby_points`. """ if method == "interp": c = cheby_coeff(c) return cheby_val(c, t) def cheby_approx(n, f, range=(0, 1)): """ Return the coefficients for the order n chebyshev approximation to function f evaluated over the range [low,high]. """ fx = f(cheby_points(n, range=range)) return cheby_coeff(fx) def cheby_val(c, x): r""" Evaluate the chebyshev approximation c at points x. The values $c_i$ are the coefficients for the chebyshev polynomials $T_i$ yielding $p(x) = \sum_i{c_i T_i(x)}$. """ c = np.asarray(c) if len(c) == 0: return 0 * x # Crenshaw recursion from numerical recipes sec. 5.8 y = 4 * x - 2 d = dd = 0 for c_j in c[:0:-1]: d, dd = y * d + (c_j - dd), d return y * (0.5 * d) + (0.5 * c[0] - dd) def cheby_points(n, range=(0, 1)): r""" Return the points in at which a function must be evaluated to generate the order $n$ Chebyshev approximation function. Over the range [-1,1], the points are $p_k = \cos(\pi(2 k + 1)/(2n))$. Adjusting the range to $[x_L,x_R]$, the points become $x_k = \frac{1}{2} (p_k - x_L + 1)/(x_R-x_L)$. """ return 0.5 * (cos(pi * (arange(n) + 0.5) / n) - range[0] + 1) / (range[1] - range[0]) def cheby_coeff(fx): """ Compute chebyshev coefficients for a polynomial of order n given the function evaluated at the chebyshev points for order n. This can be used as the basis of a direct interpolation method where the n control points are positioned at cheby_points(n). """ fx = asarray(fx) n = len(fx) w = exp((-0.5j * pi / n) * arange(n)) y = np.hstack((fx[0::2], fx[1::2][::-1])) c = (2.0 / n) * real(fft(y) * w) return c bumps-bumps-2b313fd/bumps/cli.py000066400000000000000000000405441521754644000166600ustar00rootroot00000000000000""" Bumps command line interface. The functions in this module are used by the bumps command to implement the command line interface. Bumps plugin models can use them to create stand alone applications with a similar interface. For example, the Refl1D application uses the following:: from . import fitplugin from bumps.plugin import install_plugin from bumps.plotutil import set_mplconfig from bumps.cli import main as bumps_main set_mplconfig(appdatadir='Refl1D') install_plugin(fitplugin) bumps_main() After completing a set of fits on related systems, a post-analysis script can use :func:`load_model` to load the problem definition and :func:`load_pars` to load the best value found in the fit. This can be used for example in experiment design, where you look at the expected parameter uncertainty when fitting simulated data from a range of experimental systems. """ __all__ = [ "main", "install_plugin", "set_mplconfig", "config_matplotlib", "load_model", "preview", "load_pars", "save_best", "resynth", ] import sys import os import re import warnings import shutil import traceback from pathlib import Path import numpy as np # np.seterr(all="raise") from .plotutil import config_matplotlib, set_mplconfig from .plugin import install_plugin from .fitproblem import load_pars from .fitters import save_best from .fitters import FitDriver, StepMonitor, ConsoleMonitor, CheckpointMonitor from .mapper import MPMapper, MPIMapper, SerialMapper from . import initpop from . import __version__ def load_model(path: Path | str, model_options: list[str] | None = None): """ *** DEPRECATED***. Use fitproblem.load_problem(path, [args=...]) instead. """ from .fitproblem import load_problem problem = load_problem(path, args=model_options) # CRUFT: support old 'problem.options' attribute problem.options = problem.script_args return problem def preview(problem, view=None): """ Show the problem plots and parameters. """ import matplotlib.pyplot as plt problem.show() problem.plot(view=view) plt.show() # CRUFT recall_best = load_best = load_pars def store_overwrite_query_gui(path): """ Ask if store path should be overwritten. Use this in a call to :func:`make_store` from a graphical user interface. """ import wx msg = path + " already exists. Press 'yes' to overwrite, or 'No' to abort and restart with newpath" msg_dlg = wx.MessageDialog(None, msg, "Overwrite Directory", wx.YES_NO | wx.ICON_QUESTION) retCode = msg_dlg.ShowModal() msg_dlg.Destroy() if retCode != wx.ID_YES: raise RuntimeError("Could not create path") def store_overwrite_query(path): """ Ask if store path should be overwritten. Use this in a call to :func:`make_store` from a command line interface. """ print(path, "already exists.") print("Press 'y' to overwrite, or 'n' to abort and restart with --overwrite, --resume, or --store=newpath") ans = input("Overwrite [y/n]? ") if ans not in ("y", "Y", "yes"): sys.exit(1) def sanitize_filename(name: str) -> str: return re.sub(r"[^A-Za-z0-9._-]", "_", name) def make_store(problem, opts, exists_handler): """ Create the store directory and populate it with the model definition file. """ # Determine if command line override if opts.store: problem.store = opts.store if getattr(problem, "store", None) is None: raise RuntimeError("Need to specify '--store=path' on command line or problem.store='path' in definition file.") stem = ( Path(problem.path).stem if hasattr(problem, "path") else sanitize_filename(problem.name) if problem.name else "problem" ) problem.output_path = os.path.join(problem.store, stem) # Check if already exists store_exists = os.path.exists(problem.output_path + ".par") if not opts.overwrite and opts.resume is None and store_exists: if opts.batch: print( problem.store + " already exists. Restart with --overwrite, --resume, or --store=newpath", file=sys.stderr, ) sys.exit(1) exists_handler(problem.output_path) # Create it and copy model if not os.path.exists(problem.store): os.mkdir(problem.store) shutil.copy2(problem.path, problem.store) def run_profiler(problem, steps): """ Model execution profiler. Run the program with "--profiler --steps=N" to generate a function profile chart breaking down the cost of evaluating N models. """ # Here is the findings from one profiling session:: # 23 ms total # 6 ms rendering model # 8 ms abeles # 4 ms convolution # 1 ms setting parameters and computing nllf from .util import profile p = initpop.random_init(int(steps), initial=None, bounds=None, use_point=False, problem=problem) # Note: map is an iterator in python 3 profile(lambda *args: list(map(*args)), problem.nllf, p) def run_timer(mapper, problem, steps): """ Model execution timer. Run the program with "--timer --steps=N" to determine the average run time of the model. If --parallel is included, then the model will be run in parallel on separate cores. """ import time T0 = time.time() steps = int(steps) p = initpop.generate(problem, init="random", pop=-steps, use_point=False) if p.shape == (0,): # No fitting parameters --- generate an empty population p = np.empty((steps, 0)) mapper(p) print("time per model eval: %g ms" % (1000 * (time.time() - T0) / steps,)) def start_remote_fit(problem, options, queue, notify): """ Queue remote fit. """ from jobqueue.client import connect from cloudpickle import dumps data = dict(package="bumps", version=__version__, problem=dumps(problem), options=dumps(options)) request = dict( service="fitter", version=__version__, # fitter service version notify=notify, name=problem.title, data=data, ) server = connect(queue) job = server.submit(request) return job # ==== Main ==== def initial_model(opts): """ Load and initialize the model. *opts* are the processed command line options. If --pars is in opts, then load the parameters from a .par file. If --simulate is in opts, then generate random data from the model. If --simrandom is in opts, then generate random data from a random model. If --shake is in opts, then use a random initial state for the fit. """ if opts.seed is not None: np.random.seed(opts.seed) if opts.args: problem = load_model(opts.args[0], opts.args[1:]) if opts.pars is not None: load_pars(problem, opts.pars) if opts.simrandom: problem.randomize() if opts.simulate or opts.simrandom: noise = None if opts.noise == "data" else float(opts.noise) problem.simulate_data(noise=noise) print("simulation parameters") print(problem.summarize()) print("chisq at simulation", problem.chisq_str()) if opts.shake: problem.randomize() else: problem = None return problem def resynth(fitdriver, problem, mapper, opts): """ Generate maximum likelihood fits to resynthesized data sets. *fitdriver* is a :class:`bumps.fitters.FitDriver` object with a fitter already chosen. *problem* is a :func:`bumps.fitproblem.FitProblem` object. It should be initialized with optimal values for the parameters. *mapper* is one of the available :mod:`bumps.mapper` classes. *opts* is a :class:`bumps.options.BumpsOpts` object representing the command line parameters. """ make_store(problem, opts, exists_handler=store_overwrite_query) fid = open(problem.output_path + ".rsy", "at") fitdriver.mapper = mapper.start_mapper(problem, opts.args) for i in range(opts.resynth): problem.resynth_data() best, fbest = fitdriver.fit() chisq = problem.chisq(nllf=fbest) print(f"step {i} chisq={chisq:.2f}") fid.write("%.15g " % chisq) fid.write(" ".join("%.15g" % v for v in best)) fid.write("\n") problem.restore_data() fid.close() def beep(): """ Audio signal that fit is complete. """ if sys.platform == "win32": try: import winsound winsound.MessageBeep(winsound.MB_OK) except Exception: pass else: print("\a", file=sys.__stdout__) def run_command(c): """ Run an arbitrary python command. """ exec(c, globals()) def setup_logging(): """Start logger""" import logging logging.basicConfig(level=logging.INFO) # From http://stackoverflow.com/questions/22373927/get-traceback-of-warnings # answered by mgab (2014-03-13) # edited by Gareth Rees (2015-11-28) def warn_with_traceback(message, category, filename, lineno, file=None, line=None): """ Alternate warning printer which shows a traceback with the warning. To use, set *warnings.showwarning = warn_with_traceback*. """ traceback.print_stack() log = file if hasattr(file, "write") else sys.stderr log.write(warnings.formatwarning(message, category, filename, lineno, line)) def main(): """ Run the bumps program with the command line interface. Input parameters are taken from sys.argv. """ from . import options # add full traceback to warnings # warnings.showwarning = warn_with_traceback if len(sys.argv) == 1: sys.argv.append("-?") # run command with bumps in the environment if sys.argv[1] == "-m": import runpy sys.argv = sys.argv[2:] runpy.run_module(sys.argv[0], run_name="__main__") sys.exit(0) elif sys.argv[1] == "-p": import runpy sys.argv = sys.argv[2:] runpy.run_path(sys.argv[0], run_name="__main__") sys.exit() elif sys.argv[1] == "-c": run_command(sys.argv[2]) sys.exit() elif sys.argv[1] == "-i": sys.argv = ["ipython", "--pylab"] from IPython import start_ipython sys.exit(start_ipython()) opts = options.getopts() setup_logging() if opts.edit: # TODO: circular imports from .gui.gui_app import main as gui gui() return # Set up the matplotlib backend to minimize the wx/gui dependency. # If no GUI specified and not editing, then use the default mpl # backend for the python version. if opts.batch or opts.remote or opts.noshow: # no interactivity config_matplotlib(backend="agg") else: # let preview use default graphs config_matplotlib() problem = initial_model(opts) if problem is None: print("\n!!! Model file missing from command line --- abort !!!.", file=sys.stderr) sys.exit(1) if opts.mpi: MPIMapper.start_worker(problem) mapper = MPIMapper elif opts.parallel != "" or opts.worker: if opts.transport == "mp": mapper = MPMapper else: raise ValueError("unknown mapper") else: mapper = SerialMapper if opts.worker: mapper.start_worker(problem) return if np.isfinite(float(opts.time)): import time start_time = time.time() stop_time = start_time + float(opts.time) * 3600 abort_test = lambda: time.time() >= stop_time else: abort_test = lambda: False fitdriver = FitDriver( opts.fit_config.selected_fitter, problem=problem, abort_test=abort_test, **opts.fit_config.selected_values ) # Start fitter within the domain so that constraints are valid clipped = fitdriver.clip() if clipped: print("Start value clipped to range for parameter", ", ".join(clipped)) if opts.time_model: run_timer(mapper.start_mapper(problem, opts.args), problem, steps=int(opts.steps)) elif opts.profile: run_profiler(problem, steps=int(opts.steps)) elif opts.chisq: if opts.cov: fitdriver.show_cov() print("chisq", problem.chisq_str()) # import pprint; pprint.pprint(problem.to_dict(), indent=2, width=272) elif opts.preview: if opts.cov: fitdriver.show_cov() preview(problem, view=opts.view) elif opts.resynth > 0: resynth(fitdriver, problem, mapper, opts) elif opts.remote: # Check that problem runs before submitting it remotely # TODO: this may fail if problem requires remote resources such as GPU print("initial chisq:", problem.chisq_str()) job = start_remote_fit(problem, opts, queue=opts.queue, notify=opts.notify) print("remote job:", job["id"]) else: # Show command line arguments and initial model print("#", " ".join(sys.argv), "--seed=%d" % opts.seed) problem.show() # Check that there are parameters to be fitted. if not len(problem.getp()): print("\n!!! No parameters selected for fitting---abort !!!\n", file=sys.stderr) sys.exit(1) # Run the fit if opts.resume == "-": opts.resume = opts.store if os.path.exists(opts.store) else None if opts.resume: resume_path = os.path.join(opts.resume, problem.name) else: resume_path = None make_store(problem, opts, exists_handler=store_overwrite_query) # Redirect sys.stdout to capture progress if opts.batch: sys.stdout = open(problem.output_path + ".mon", "w") # TODO: fix techical debt with checkpoint monitor implementation # * The current checkpoint implementation is self-referential: # checkpoint = lambda: save_best(fitdriver, ...) # fitdriver.monitors = [..., CheckpointMonitor(checkpoint), ...] # It is done this way because the checkpoint monitor needs the fitter # so it can ask it to save state, but the fitter needs the list of # monitors, including the checkpoint monitor, before it is run. # * Figures are cumulative, with each checkpoint adding a new set # * Figures are slow! Can they go into a separate thread? Can we # have the problem cache the best value? checkpoint_time = float(opts.checkpoint) * 3600 def checkpoint(history): problem = fitdriver.problem ## Use the following to save only the fitter state fitdriver.fitter.save(problem.output_path) ## Use the following to save the fitter state plus all other ## plots and other output files. This won't work yet since ## plots are generated sequentially, with each checkpoint producing ## a completely new set of plots. # best = history.point[0] # save_best(fitdriver, problem, best, view=opts.view) monitors = [ConsoleMonitor(problem)] if checkpoint_time > 0 and np.isfinite(checkpoint_time): mon = CheckpointMonitor(checkpoint, progress=checkpoint_time) monitors.append(mon) if opts.stepmon: fid = open(problem.output_path + ".log", "w") mon = StepMonitor(problem, fid, fields=["step", "value"]) monitors.append(mon) fitdriver.monitors = monitors # import time; t0=time.clock() cpus = int(opts.parallel) if opts.parallel != "" else 0 fitdriver.mapper = mapper.start_mapper(problem, opts.args, cpus=cpus) best, fbest = fitdriver.fit(resume=resume_path) # print("time=%g"%(time.clock()-t0),file=sys.__stdout__) # Note: keep this in sync with the checkpoint function above save_best(fitdriver, view=opts.view) fitdriver.show() if opts.err or opts.cov: fitdriver.show_err() if opts.cov: fitdriver.show_cov() if opts.entropy: fitdriver.show_entropy(opts.entropy) mapper.stop_mapper() # If in batch mode then explicitly close the monitor file on completion if opts.batch: sys.stdout.close() sys.stdout = sys.__stdout__ # Display the plots if not opts.batch and not opts.mpi and not opts.noshow: beep() import matplotlib.pyplot as plt plt.show() # Allow "$python -m bumps.cli args" calling pattern if __name__ == "__main__": main() bumps-bumps-2b313fd/bumps/curve.py000066400000000000000000000515141521754644000172340ustar00rootroot00000000000000""" Build a bumps model from a function and data. Example ------- Given a function *sin_model* which computes a sine wave at times *t*:: from numpy import sin def sin_model(t, freq, phase): return sin(2*pi*(freq*t + phase)) and given data *(y,dy)* measured at times *t*, we can define the fit problem as follows:: from bumps.names import * M = Curve(sin_model, t, y, dy, freq=20) The *freq* and *phase* keywords are optional initial values for the model parameters which otherwise default to zero. The model parameters can be accessed as attributes on the model to set fit range:: M.freq.range(2, 100) M.phase.range(0, 1) As usual, you can initialize or assign parameter expressions to the the parameters if you want to tie parameters together within or between models. Note: there is sometimes difficulty getting bumps to recognize the function during fits, which can be addressed by putting the definition in a separate file on the python path. With the windows binary distribution of bumps, this can be done in the problem definition file with the following code:: import os from bumps.names import * sys.path.insert(0, os.getcwd()) The model function can then be imported from the external module as usual:: from sin_model import sin_model """ __all__ = ["Curve", "PoissonCurve", "plot_err"] import inspect import warnings from typing import Callable, Literal, Union, Optional, Any, Dict, List, Callable from dataclasses import dataclass import numpy as np from numpy import log, pi, sqrt from numpy.typing import ArrayLike from .util import NDArray from .parameter import Parameter, ValueProtocol # import sys; sys.setrecursionlimit(60) # For debugging getattr/setattr def _parse_pars(fn, init=None, skip=0, name=""): """ Extract parameter names from function definition. *fn* is the function definition. This could be declared as *fn(p1, p2, p3, ...)* where *p1*, etc. are the fittable parameters. *init* is a dictionary of initial values for the parameters, overriding any default values. If called from a constructor with **kwargs representing unknown named arguments, use *init=kwargs*. *skip* is the number of parameters to skip. This will be *skip=0* for a function which defines the log likelihood directly or one that returns a set of residuals. For parameterized curves such as *fn(x, p1, p2, ...)* use *skip=1*. For surfaces with *fn(x, y, p1, p2, ...)* use *skip=2*. *name* is added to each parameter name to differentiate it from other parameters in the same fit. A default value in the function definition such as *pk=value* will be set as the default value for the parameter. If the default is *pk=None* then the parameter will be non-fittable, and instead set through *init*. """ sig = inspect.signature(fn) params = sig.parameters.values() pnames = [p.name for p in params] valid = [p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) for p in params] if not all(valid): raise TypeError(f"Only positional and keyword arguments allowed for {fn.__name__}") # TODO: need "self" handling for passed methods # Skip the first argument if it is x or maybe skip x, y. pnames = pnames[skip:] # Parameters default to zero defaults = dict((p, 0) for p in pnames) # If the function provides default values, use those. for param in list(params)[skip:]: if param.default is not inspect.Parameter.empty: defaults[param.name] = param.default # Non-fittable parameters need to be sent in as None state_vars = set(p for p, v in defaults.items() if v is None) # Regardless, use any values specified in the constructor, but first # check that they exist as function parameters. invalid = set(init.keys()) - set(pnames) if invalid: raise TypeError("Invalid initializers: %s" % ", ".join(sorted(invalid))) defaults.update(init) # Build parameters out of ranges and initial values # maybe: name=(p+name if name.startswith('_') else name+p) pars = dict((p, Parameter.default(defaults[p], name=name + p)) for p in pnames if p not in state_vars) state = dict((p, v) for p, v in defaults.items() if p in state_vars) # print("pars", pars) # print("state", state) return pars, state # TODO: separate the dataclass from the builder function # As currently implemented we need artificial parameters # "pars" and "state" to allow deserialization of the dataclass # but these are parameters that the user should never touch. # Instead, define a cleaner Curve object which can be dumped # and loaded independent of the calling parameters. @dataclass(init=False, eq=False) class Curve: r""" Model a measurement with a user defined function. The function *fn(x,p1,p2,...)* should return the expected value *y* for each point *x* given the parameters *p1*, *p2*, etc. *dy* is the uncertainty for each measured value *y*. If not specified, it defaults to 1. Multi-valued functions, which return multiple *y* values for each *x* value, should have *x* as a vector of length *n* and *y*, *dy* as arrays of size *[n, k]*. Initial values for the parameters can be set as *p=value* arguments to *Curve*. If no value is set, then the initial value will be taken from the default value given in the definition of *fn*, or set to 0 if the parameter is not defined with an initial value. Arbitrary non-fittable data can be passed to the function as parameters, but only if the parameter is given a default value of *None* in the function definition, and has the initial value set as an argument to *Curve*. Defining *state=dict(key=value, ...)* before *Curve*, and calling *Curve* as *Curve(..., \*\*state)* works pretty well. *Curve* takes the following special keyword arguments: * *name* is added to each parameter name when the parameter is defined. The filename for the data is a good choice, since this allows you to keep the parameters straight when fitting multiple datasets simultaneously. * *plot* is an alternative plotting function. The function should be defined as *plot(x,y,dy,fy,\*\*kw)*. The keyword arguments will be filled with the values of the parameters used to compute *fy*. It will be easiest to list the parameters you need to make your plot as arguments after *x,y,dy,fy* in the plot function declaration. For example, *plot(x,y,dy,fy,p3,\*\*kw)* will make the value of parameter *p3* available as a variable in your function. The special keyword *view* will be a string containing *linear*, *log*, *logx*, or *loglog*. If only showing the residuals, the string will be *residual*. * *plot_x* is an array giving the sample points to use when plotting the theory function, if different from the *x* values at which the function is sampled. Use this to draw a smooth curve between the fitted points. This value is ignored if you provide your own plot function. * *labels* are the axis labels for the plot. This should include units in parentheses. If the function is multi-valued then use *['x axis', 'y axis', 'line 1', 'line 2', ...]*. The data uncertainty is assumed to follow a gaussian distribution. If measurements draw from some other uncertainty distribution, then subclass Curve and replace nllf with the correct probability given the residuals. See the implementation of :class:`PoissonCurve` for an example. """ # TODO: Add resolution and uncertainty in x. Resolution will require oversampling. # TODO: Move data into its own class (pre 1.0?) # The data class can deal with x, y, dy, resolution, jitter, plotter, plot_x, labels fn: Callable x: NDArray y: NDArray dy: NDArray """Data uncertainty""" name: str """Name of the model""" pars: Dict[str, Parameter] = None # Needs to be None initially for getattr/setattr to work """Fittable parameters to the model""" state: Dict[str, Any] """Nonfittable parameters set during initialization. Values should be serializable.""" labels: List[str] plot_x: Optional[NDArray] plotter: Optional[Callable] def __init__( self, fn: Callable, x: ArrayLike, y: ArrayLike, dy: Optional[ArrayLike] = None, name: Optional[str] = "", labels: Optional[List[str]] = None, plotter: Optional[Callable[..., None]] = None, plot_x: Optional[NDArray] = None, plot: Optional[Callable] = None, pars: Optional[Dict[str, Parameter]] = None, state: Optional[Dict[str, Any]] = None, **kwargs, ): self.x, self.y = np.asarray(x), np.asarray(y) if dy is None: self.dy = np.ones_like(y) else: self.dy = np.asarray(dy) if (self.dy <= 0).any(): raise ValueError("measurement uncertainty must be positive") if len(self.x.shape) == 1 and len(self.y.shape) > 1: num_curves = self.y.shape[0] else: num_curves = 1 self._num_curves = num_curves # use same value everywhere # interpret labels parameter if labels is None: labels = ["x", "y"] elif len(labels) < 2 or len(labels) != num_curves + 2: if num_curves > 1: lines = "line1, ..., line%d" % num_curves else: lines = "line" raise TypeError("labels should be [x, y, %s]" % lines) if len(labels) == 2: if num_curves > 1: line_labels = ["y%d" % k for k in range(num_curves)] else: line_labels = [labels[1]] labels = list(labels) + line_labels self.labels = labels self.plot_x = plot_x # TODO: drop plot= option; likely nobody is using it, and this is a 1.0 release if plot is not None: warnings.warn("***DEPRECATED*** use Curve(..., plotter=...) instead of plot=...") self.plotter = plotter # _assign_pars(state, self) # ... and state variables as well self._cached_theory = None self._webview_plots = {} self.fn = fn self.name = name # if name else fn.__name__ + " " if pars is None: pars, state = _parse_pars(self.fn, init=kwargs, skip=1, name=name) # else: print("restoring", {k: float(v) for k, v in pars.items()}, state) # print("parsed", self.fn, pars, state) # Remember the function, parameters, and number of parameters # Note: we are remembering the parameter names and not the # parameters themselves so that the caller can tie parameters # together using model1.par = model2.par. Otherwise we would # need to override __setattr__ to intercept assignment to the # parameter attributes and redirect them to the a _pars dictionary. # ... and similarly for state if we decide to make them attributes. self.state = state # Need to do collision test before making parameters available as attributes # otherwise every parameter is a collision. Need to assign the pars object # last so that we know all the variables that could collide. collisions = [key for key in pars.keys() if hasattr(self, key)] if collisions: raise ValueError(f"parameter names shadowed by Curve object: {', '.join(collisions)}") self.pars = pars # Allow dot access to members of the parameter dictionary. Existing attributes # of the object take precedence. def __setattr__(self, key, value): # print(f"setting {key}") if self.pars and key in self.pars and key not in self.__dict__: if not isinstance(value, ValueProtocol): raise TypeError("Can only assign parameter or expression to a parameter slot") self.__dict__["pars"][key] = value else: super().__setattr__(key, value) def __getattr__(self, key): # print(f"getting {key}") if self.pars and key in self.pars: return self.pars[key] raise AttributeError(f"{type(self)!r} has no attribute {key!r}") def update(self): self._cached_theory = None def parameters(self): return self.pars def numpoints(self): return np.prod(self.y.shape) def theory(self, x=None): # Use cache if x is None, otherwise compute theory with x. if x is None: if self._cached_theory is None: self._cached_theory = self._compute_theory(self.x) return self._cached_theory return self._compute_theory(x) def _compute_theory(self, x): kw = self._fetch_pars() return self.fn(x, **kw) def _fetch_pars(self): kw = {k: float(v) for k, v in self.pars.items()} kw.update(**self.state) return kw def simulate_data(self, noise=None): theory = self.theory() if noise is not None: if noise == "data": pass elif noise < 0: self.dy = np.full_like(theory, -noise) else: self.dy = 0.01 * noise * abs(theory) self.y = theory + np.random.randn(*theory.shape) * self.dy def residuals(self): return (self.theory() - self.y) / self.dy def nllf(self): r = self.residuals() return 0.5 * np.sum(r**2) def save(self, basename): # TODO: need header line with state vars as json # TODO: need to support nD x,y,dy if len(self.x.shape) > 1: warnings.warn("Save not supported for nD x values") return theory = self.theory() if self._num_curves > 1: # Multivalued y, dy for single valued x. columns = [self.x] headers = ["x"] for k, (y, dy, fx) in enumerate(zip(self.y, self.dy, theory)): columns.extend((y, dy, fx)) headers.extend(("y[%d]" % (k + 1), "dy[%d]" % (k + 1), "fx[%d]" % (k + 1))) else: # Single-valued y, dy for single valued x. headers = ["x", "y", "dy", "fy"] columns = [self.x, self.y, self.dy, theory] data = np.vstack(columns) outfile = basename + ".dat" with open(outfile, "w") as fd: fd.write("# " + "\t ".join(headers) + "\n") np.savetxt(fd, data.T) def plot(self, view=None): if self.plotter is not None: kw = self._fetch_pars() self.plotter(self.x, self.y, self.dy, self.theory(), view=view, **kw) return import matplotlib.pyplot as plt from .plotutil import coordinated_colors x = self.x if self.plot_x is not None: theory_x, theory_y = self.plot_x, self.theory(self.plot_x) else: theory_x, theory_y = x, self.theory() resid = self.residuals() if self._num_curves > 1: y, dy, theory_y, resid = self.y.T, self.dy.T, theory_y.T, resid.T else: y, dy, theory_y, resid = (v[:, None] for v in (self.y, self.dy, theory_y, resid)) colors = tuple(coordinated_colors() for _ in range(self._num_curves)) # clear figure after calling coordinated_colors, which can create an axis object plt.clf() labels = self.labels # print "kw_plot",kw if view == "residual": _plot_resids(x, resid, colors, labels=labels, view=view) else: plot_ratio = 4 h = plt.subplot2grid((plot_ratio, 1), (0, 0), rowspan=plot_ratio - 1) for tick_label in h.get_xticklabels(): tick_label.set_visible(False) _plot_fits(data=(x, y, dy), theory=(theory_x, theory_y), colors=colors, labels=labels, view=view) # plt.gca().xaxis.set_visible(False) # plt.gca().spines['bottom'].set_visible(False) # plt.gca().set_xticks([]) plt.subplot2grid((plot_ratio, 1), (plot_ratio - 1, 0), sharex=h) _plot_resids(x, resid, colors=colors, labels=labels, view=view) def register_webview_plot( self, plot_title: str, plot_function: Callable, change_with: Literal["parameter", "uncertainty"] ): # Plot function syntax: f(model, problem, state) # change_with = 'parameter' or 'uncertainty' self._webview_plots[plot_title] = dict(func=plot_function, change_with=change_with) @property def webview_plots(self): return self._webview_plots def _plot_resids(x, resid, colors, labels, view): import matplotlib.pyplot as plt plt.axhline(y=1, ls="dotted", color="k") plt.axhline(y=0, ls="solid", color="k") plt.axhline(y=-1, ls="dotted", color="k") for k, color in enumerate(colors): plt.plot(x, resid[:, k], ".", color=color["base"]) plt.gca().locator_params(axis="y", tight=True, nbins=4) plt.xlabel(labels[0]) plt.ylabel("(f(x)-y)/dy") if view == "logx": plt.xscale("log") elif view == "loglog": plt.xscale("log") def _plot_fits(data, theory, colors, labels, view): import matplotlib.pyplot as plt x, y, dy = data theory_x, theory_y = theory for k, color in enumerate(colors): plt.errorbar(x, y[:, k], yerr=dy[:, k], fmt=".", color=color["base"], label="_") plt.plot(theory_x, theory_y[:, k], "-", color=color["dark"], label=labels[k + 2]) # Note: no xlabel since it is supplied by the residual plot below this plot plt.ylabel(labels[1]) if len(colors) > 1: plt.legend() if view == "log": plt.xscale("linear") plt.yscale("log") elif view == "logx": plt.xscale("log") plt.yscale("linear") elif view == "logy": plt.xscale("linear") plt.yscale("log") elif view == "loglog": plt.xscale("log") plt.yscale("log") else: # view == 'linear' plt.xscale("linear") plt.yscale("linear") def plot_resid(x, resid): """ **DEPRECATED** """ import matplotlib.pyplot as plt plt.axhline(y=1, ls="dotted", color="k") plt.axhline(y=0, ls="solid", color="k") plt.axhline(y=-1, ls="dotted", color="k") plt.plot(x, resid, ".") plt.gca().locator_params(axis="y", tight=True, nbins=4) plt.ylabel("Residuals") def plot_err(x, y, dy, fy, view=None, **kw): """ **DEPRECATED**: subclass Curve and override the plot function. Plot data *y* and error *dy* against *x*. *view* is one of linear, log, logx or loglog. """ import matplotlib.pyplot as plt plt.errorbar(x, y, yerr=dy, fmt=".") plt.plot(x, fy, "-") if view == "log": plt.xscale("linear") plt.yscale("log") elif view == "logx": plt.xscale("log") plt.yscale("linear") elif view == "loglog": plt.xscale("log") plt.yscale("log") else: # view == 'linear' plt.xscale("linear") plt.yscale("linear") _LOGFACTORIAL = np.array([log(np.prod(np.arange(1.0, k + 1))) for k in range(21)]) def logfactorial(n): """Compute the log factorial for each element of an array""" result = np.empty(n.shape, dtype="double") idx = n <= 20 result[idx] = _LOGFACTORIAL[np.asarray(n[idx], "int32")] n = n[~idx] result[~idx] = n * log(n) - n + log(n * (1 + 4 * n * (1 + 2 * n))) / 6 + log(pi) / 2 return result @dataclass(init=False, eq=False) class PoissonCurve(Curve): r""" Model a measurement with Poisson uncertainty. The nllf is calculated using Poisson probabilities, but the curve itself is displayed using the approximation that $\sigma_y \approx \sqrt(y)$. Note that the *dy* argument is ignored. It is only present for deserialization of the Curve base class. See :class:`Curve` for details. """ def __init__(self, fn, x, y, dy=None, name="", **fnkw): # dy ignored. It is needed for dataclass deserialization dy = sqrt(y) + (y == 0) if y is not None else None Curve.__init__(self, fn, x, y, dy, name=name, **fnkw) self._logfacty = logfactorial(y) if y is not None else None self._logfactysum = np.sum(self._logfacty) ## Assume gaussian residuals for now # def residuals(self): # # TODO: provide individual probabilities as residuals # # or perhaps the square roots --- whatever gives a better feel for # # which points are driving the fit # theory = self.theory() # return np.sqrt(self.y * log(theory) - theory - self._logfacty) def nllf(self): theory = self.theory() if (theory <= 0).any(): return 1e308 return -sum(self.y * log(theory) - theory) + self._logfactysum def simulate_data(self, noise=None): theory = self.theory() self.y = np.random.poisson(theory) self.dy = sqrt(self.y) + (self.y == 0) self._logfacty = logfactorial(self.y) self._logfactysum = np.sum(self._logfacty) bumps-bumps-2b313fd/bumps/data.py000066400000000000000000000210541521754644000170150ustar00rootroot00000000000000""" Data handling utilities. """ import os import gzip from contextlib import contextmanager import numpy as np from numpy import inf, nan __all__ = ["indfloat", "parse_file"] def parse_multi(file, keysep=None, sep=None, comment="#"): """ Parse a multi-part file. Return a list of (header, data) pairs, where header is a key: value dictionary and data is a numpy array. The header section is list of key value pairs, with the *comment* character at the start of each line. Key and value will be separated by *keysep*, or by spaces if *keysep = None*. The data section is a sequence of floating point numbers separated by *sep*, or by spaces if *sep* is None. inf and nan are parsed as inf and nan. Comments at the end of the data line will be ignored. Data points can be commented out by including a comment character at the start of the data line, assuming the next character is a digit, plus, or decimal separator. Quotes around keys are removed, but not around values. Use :func:`strip_quotes` to remove them if they are present. This is different from the :func:`parse_file` interface, which strips quotes around values. The new interface allows *json.loads()* calls on values if values are stored as *key: json.dumps(value)*. Special hack for binned data: if the first column contains bin edges, then the last row will only have the bin edge. To make the array square, we replace the bin edges with bin centers. The original bins can be found in the header using the 'bins' key (unless that key already exists in the header, in which case the key will be ignored). """ parts = [] with maybe_open(file) as fh: while True: header, data, bins = _read_part(fh, comment=comment, multi_part=True, col_sep=sep, key_sep=keysep) if header is None: break if bins is not None: header.setdefault("bins", bins) parts.append((header, data)) return parts def parse_file(file, keysep=None, sep=None, comment="#"): """ Parse a file into a header and data. Return a (header, data) pair, where header is a key: value dictionary and data is a numpy array. The header section is list of key value pairs, with the *comment* character at the start of each line. Key and value will be separated by *keysep*, or by spaces if *keysep = None*. The data section is a sequence of floating point numbers separated by *sep*, or by spaces if *sep* is None. inf and nan are parsed as inf and nan. Comments at the end of the data line will be ignored. Data points can be commented out by including a comment character at the start of the data line, assuming the next character is a digit, plus, or decimal separator. Quotes around keys are removed. For compatibility with the old interface, quotes around values are removed as well. Special hack for binned data: if the first column contains bin edges, then the last row will only have the bin edge. To make the array square, we replace the bin edges with bin centers. The original bins can be found in the header using the 'bins' key (unless that key already exists in the header, in which case the key will be ignored). """ with maybe_open(file) as fh: header, data, bins = _read_part(fh, comment=comment, multi_part=False, col_sep=sep, key_sep=keysep) if header is None: raise IOError("data file is empty") # compatibility: strip quotes from values in key-value pairs header = dict((k, strip_quotes(v)) for k, v in header.items()) if bins is not None: header.setdefault("bins", bins) return header, data def _read_part(fh, key_sep=None, col_sep=None, comment="#", multi_part=False): header = {} data = [] iseof = True for line in fh: # Blank lines indicate a section break. if not line.strip(): # Skip blank lines if we are parsing the data as a single part file # or if the "section" has no data due to blank lines in the header. if not multi_part or not data: continue # If we are at the beginning of a section, then iseof is True and # continuing to the next loop iteration will skip them. If we have # already consumed some non-blank lines, then iseof will be false, # and we need to break this section of the data. If we have blank # lines at the end of the file, we will never set iseof to False # and they will be ignored. if iseof: continue break # Line is not blank, so process it. columns, key, value = _parse_line(line, comment=comment, col_sep=col_sep, key_sep=key_sep) if columns: data.append([indfloat(v) for v in columns]) if key is not None: if key in header: header[key] = "\n".join((header[key], value)) else: header[key] = value # We have processed some data, so iseof = False if iseof: return None, None, None # print data # print "\n".join(k+":"+v for k,v in header.items()) if len(data) and len(data[-1]) == 1: # For TOF data, the first column is the bin edge, which has one # more row than the remaining columns; fill those columns with # bin centers instead last_edge = data[-1][0] data = np.array(data[:-1]).T edges = np.hstack((data[0], last_edge)) data[0] = 0.5 * (edges[:-1] + edges[1:]) bins = edges else: data = np.array(data).T bins = None return header, data, bins @contextmanager def maybe_open(file_or_path): """ A context manager for file opening, given as a file path or an open handle. If *file_or_path* is a string ending in ".gz" then open with gzip. """ if hasattr(file_or_path, "readline"): # If it is a file handle, yield it and return without closing. fh = file_or_path yield fh else: # Otherwise it should be a path. Make sure it is at least a string. if not string_like(file_or_path): raise ValueError("file must be a name or a file handle") # Open file; if name ends in .gz then assume it is compressed. path = os.path.expanduser(file_or_path) fh = gzip.open(path) if path.endswith(".gz") else open(path) try: yield fh finally: fh.close() def string_like(s): """ Return True if s operates like a string. """ try: s + "" except Exception: return False return True def _parse_line(line, key_sep=None, col_sep=None, comment="#"): # Find location of the comment character on the line idx = line.find(comment) # If the line does not contain a comment character or if the comment # character is not in the first column, then this is a data line which # should be returned as a sequence of text columns separated by spaces. # The caller can turn the columns into numbers or leave them as strings. # Data on the line after the comment character is ignored. # TODO: allow quoted strings or backslash escaped spaces for text columns if idx != 0: if idx > 0: return line[:idx].split(col_sep), None, "" else: return line.split(col_sep), None, "" # Split line on key separator parts = [p.strip() for p in line[1:].split(key_sep, 1)] key, value = parts if len(parts) > 1 else (parts[0], "") key = strip_quotes(key) # If key is a number assume it is simply a commented out data point if len(key) and (key[0] in ".-+0123456789" or key == "inf" or key == "nan"): return [], None, None return [], key, value def strip_quotes(s): return s[1:-1] if len(s) and s[0] in "'\"" and s[0] == s[-1] else s INF_VALUES = set(("inf", "1/0", "1.#inf", "infinity")) NAN_VALUES = set(("nan", "0/0", "1.#qnan", "na", "n/a")) def indfloat(s): """ Convert string to float, with support for inf and nan. Example:: >>> from numpy import isinf, isnan >>> print(isinf(indfloat('inf'))) True >>> print(isinf(indfloat('-inf'))) True >>> print(isnan(indfloat('nan'))) True """ try: return float(s) except Exception: s = s.lower() if s in INF_VALUES: return inf elif s and s[0] == "-" and s[1:] in INF_VALUES: return -inf elif s in NAN_VALUES: return nan raise bumps-bumps-2b313fd/bumps/dream/000077500000000000000000000000001521754644000166205ustar00rootroot00000000000000bumps-bumps-2b313fd/bumps/dream/__init__.py000066400000000000000000000021771521754644000207400ustar00rootroot00000000000000""" Notes on random numbers ======================= Uses dream.util.rng as the random number generator. You can set the seed using:: dream.util.rng = numpy.random.RandomState(seed) """ # TODO: move the rng control to the Dream class rather than a shared global # This interface doesn't feel right, since one instance of DREAM may # influence another if they are running within one another. Putting # the rng on the dream class may be a better option. # TODO: Maybe reenable these three exports; they might be used by third parties # Suppress these imports because I was getting some circular import problems # from .model import MCMCModel # from .core import Dream, Model, dream # from .state import MCMCDraw # from .initpop import cov_init, lhs_init # from .model import MCMCModel, Density, LogDensity, Simulation, MVNormal, Mixture # from .state import MCMCDraw, Draw, dream_load, h5load, h5dump #, load_state save_state # from .views import plot_all, plot_corrmatrix, plot_corr, plot_traces, plot_logp, plot_acceptance_rate # from .util import console, draw # from .stats import var_stats, format_num, format_vars, save_vars, parse_var bumps-bumps-2b313fd/bumps/dream/acr.py000066400000000000000000000055001521754644000177370ustar00rootroot00000000000000""" ACR upper percentiles critical value for test of single multivariate normal outlier. From the method given by Wilks (1963) and approaching to a F distribution function by the Yang and Lee (1987) formulation, we compute the critical value of the maximum squared Mahalanobis distance to detect outliers from a normal multivariate sample. We can generate all the critical values of the maximum squared Mahalanobis distance presented on the Table XXXII of by Barnett and Lewis (1978) and Table A.6 of Rencher (2002). Also with any given significance level (alpha). Example:: >>> print("%.4f"%ACR(3, 25, 0.01)) 13.1753 Created by:: A. Trujillo-Ortiz, R. Hernandez-Walls, A. Castro-Perez and K. Barba-Rojo Facultad de Ciencias Marinas Universidad Autonoma de Baja California Apdo. Postal 453 Ensenada, Baja California Mexico. atrujo@uabc.mx Copyright. August 20, 2006. To cite this file, this would be an appropriate format:: Trujillo-Ortiz, A., R. Hernandez-Walls, A. Castro-Perez and K. Barba-Rojo. (2006). *ACR:Upper percentiles critical value for test of single multivariate normal outlier.* A MATLAB file. [WWW document]. URL http://www.mathworks.com/matlabcentral/fileexchange/loadFile.do?objectId=12161 The function's name is given in honour of Dr. Alvin C. Rencher for his invaluable contribution to multivariate statistics with his text 'Methods of Multivariate Analysis'. References: [1] Barnett, V. and Lewis, T. (1978), Outliers on Statistical Data. New-York:John Wiley & Sons. [2] Rencher, A. C. (2002), Methods of Multivariate Analysis. 2nd. ed. New-Jersey:John Wiley & Sons. Chapter 13 (pp. 408-450). [3] Wilks, S. S. (1963), Multivariate Statistical Outliers. Sankhya, Series A, 25: 407-426. [4] Yang, S. S. and Lee, Y. (1987), Identification of a Multivariate Outlier. Presented at the Annual Meeting of the American Statistical Association, San Francisco, August 1987. """ __all__ = ["ACR"] from scipy.stats import f finv = f.ppf def ACR(p, n, alpha=0.05): """ Return critical value for test of single multivariate normal outlier using the Mahalanobis distance metric. *p* is the number of independent variables, *n* is the number of samples, and *alpha* is the significance level cutoff (default=0.05). """ if alpha <= 0 or alpha >= 1: raise ValueError("significance level must be between 0 and 1") a = alpha # F distribution critical value with p and n-p-1 degrees of freedom # using the Bonferroni correction. fc = finv(1 - a / n, p, n - p - 1) result = (p * (n - 1) ** 2 * fc) / (n * (n - p - 1) + (n * p * fc)) # = ((-1*((1/(1+(fc*p/(n-p-1))))-1))*((n-1)^2))/n; return result def test(): assert abs(ACR(3, 25, 0.01) - 13.1753251622586) < 1e-13 if __name__ == "__main__": test() bumps-bumps-2b313fd/bumps/dream/bounds.py000066400000000000000000000171451521754644000204740ustar00rootroot00000000000000""" Bounds handling. Use bounds(low, high, style) to create a bounds handling object. This function operates on a point x, transforming it so that all dimensions are within the bounds. Options are available, including reflecting, wrapping, clipping or randomizing the point, or ignoring the bounds. The returned bounds object should have an apply(x) method which transforms the point *x*. """ __all__ = ["make_bounds_handler", "Bounds", "ReflectBounds", "ClipBounds", "FoldBounds", "RandomBounds", "IgnoreBounds"] import numpy as np from numpy import inf, isinf from numpy.typing import NDArray from . import util from .compiled import dll try: from numba import njit except ImportError: def njit(*args, **kw): return lambda f: f def make_bounds_handler(bounds, style="reflect"): """ Return a bounds object which can update the bounds. Bounds handling *style* name is one of:: reflect: reflect off the boundary clip: stop at the boundary fold: wrap values to the other side of the boundary randomize: move to a random point in the bounds none: ignore the bounds With semi-infinite intervals folding and randomizing aren't well defined, and reflection is used instead. With finite intervals the the reflected or folded point may still be outside the bounds (which can happen if the step size is too large), and a random uniform value is used instead. """ if bounds is None: return IgnoreBounds() low, high = bounds # Do boundary handling -- what to do when points fall outside bound style = style.lower() if style == "reflect": f = ReflectBounds(low, high) elif style == "clip": f = ClipBounds(low, high) elif style == "fold": f = FoldBounds(low, high) elif style == "randomize": f = RandomBounds(low, high) elif style == "none" or style is None: f = IgnoreBounds() else: raise ValueError("bounds style %s is not valid" % style) return f class Bounds(object): """ Base class for all times of bounds objects. """ c_interface = None # type: Callable[[int, int, Any, Any, Any], None] low: NDArray = None high: NDArray = None @staticmethod def apply(minn, maxn, pop): """Force pop (population) values within bounds""" raise NotImplementedError def __call__(self, population): """ Force parameter values within bounds for each member of the population (population is expected to be a 2d array of shape (M, N) where - M is the size of the population - N is then number of parameters Returns population for convenience. E.g., y = bounds(x+0) """ if self.c_interface is not None: self.c_interface(len(population), len(self.low), population.ctypes, self.low.ctypes, self.high.ctypes) else: self.apply(self.low, self.high, population) return population class ReflectBounds(Bounds): """ Reflect parameter values into bounded region """ c_interface = dll.bounds_reflect if dll else None def __init__(self, low, high): self.low, self.high = [np.ascontiguousarray(v, "d") for v in (low, high)] @staticmethod @njit(cache=True) def apply(minn, maxn, pop): """ Update pop so all values lie within bounds """ for y in pop: # Reflect points which are out of bounds idx = y < minn y[idx] = 2 * minn[idx] - y[idx] idx = y > maxn y[idx] = 2 * maxn[idx] - y[idx] # Randomize points which are still out of bounds idx = (y < minn) | (y > maxn) y[idx] = minn[idx] + util.rng.rand(sum(idx)) * (maxn[idx] - minn[idx]) class ClipBounds(Bounds): """ Clip values to bounded region """ c_interface = dll.bounds_clip if dll else None def __init__(self, low, high): self.low, self.high = [np.ascontiguousarray(v, "d") for v in (low, high)] @staticmethod @njit(cache=True) def apply(minn, maxn, pop): for y in pop: idx = y < minn y[idx] = minn[idx] idx = y > maxn y[idx] = maxn[idx] class FoldBounds(Bounds): """ Wrap values into the bounded region """ c_interface = dll.bounds_fold if dll else None def __init__(self, low, high): self.low, self.high = [np.ascontiguousarray(v, "d") for v in (low, high)] @staticmethod @njit(cache=True) def apply(minn, maxn, pop): for y in pop: # Deal with semi-infinite cases using reflection idx = (y < minn) & isinf(maxn) y[idx] = 2 * minn[idx] - y[idx] idx = (y > maxn) & isinf(minn) y[idx] = 2 * maxn[idx] - y[idx] # Wrap points which are out of bounds idx = y < minn y[idx] = maxn[idx] - (minn[idx] - y[idx]) idx = y > maxn y[idx] = minn[idx] + (y[idx] - maxn[idx]) # Randomize points which are still out of bounds idx = (y < minn) | (y > maxn) y[idx] = minn[idx] + util.rng.rand(sum(idx)) * (maxn[idx] - minn[idx]) class RandomBounds(Bounds): """ Randomize values into the bounded region """ c_interface = dll.bounds_random if dll else None def __init__(self, low, high): self.low, self.high = [np.ascontiguousarray(v, "d") for v in (low, high)] @staticmethod @njit(cache=True) def apply(minn, maxn, pop): for y in pop: # Deal with semi-infinite cases using reflection idx = (y < minn) & isinf(maxn) y[idx] = 2 * minn[idx] - y[idx] idx = (y > maxn) & isinf(minn) y[idx] = 2 * maxn[idx] - y[idx] # The remainder are selected uniformly from the bounded region idx = (y < minn) | (y > maxn) y[idx] = minn[idx] + util.rng.rand(sum(idx)) * (maxn[idx] - minn[idx]) class IgnoreBounds(Bounds): """ Leave values outside the bounded region """ c_interface = dll.bounds_ignore if dll else None def __init__(self, low=None, high=None): self.low, self.high = [np.ascontiguousarray(v, "d") for v in (low, high)] @staticmethod def apply(minn, maxn, pop): pass def test(): """bounds handlers test""" from numpy.linalg import norm from numpy import array bounds = list(zip([5, 10], [-inf, -10], [-5, inf], [-inf, inf])) v = np.ascontiguousarray([[6, -12, 6, -12]], "d") for t in "none", "reflect", "clip", "fold", "randomize": w = make_bounds_handler(bounds, t) assert norm(w(v + 0) - v) == 0 v = np.ascontiguousarray([[12, 12, -12, -12]], "d") for t in "none", "reflect", "clip", "fold": w = make_bounds_handler(bounds, t) assert norm(w(v.repeat(3, axis=0)) - w(v + 0)) == 0 assert norm(make_bounds_handler(bounds, "none")(v + 0) - v) == 0 assert norm(make_bounds_handler(bounds, "reflect")(v + 0) - [8, -32, 2, -12]) == 0 assert norm(make_bounds_handler(bounds, "clip")(v + 0) - [10, -10, -5, -12]) == 0 assert norm(make_bounds_handler(bounds, "fold")(v + 0) - [7, -32, 2, -12]) == 0 w = make_bounds_handler(bounds, "randomize")(v + 0) assert 5 <= w[0, 0] <= 10 and w[0, 1] == -32 and w[0, 2] == 2 and w[0, 3] == -12 v = np.ascontiguousarray([[20, 1, 1, 1]], "d") w = make_bounds_handler(bounds, "reflect")(v + 0) assert 5 <= w[0, 0] <= 10 v = np.ascontiguousarray([[20, 1, 1, 1]], "d") w = make_bounds_handler(bounds, "fold")(v + 0) assert 5 <= w[0, 0] <= 10 if __name__ == "__main__": test() bumps-bumps-2b313fd/bumps/dream/build_compiled.py000066400000000000000000000041571521754644000221540ustar00rootroot00000000000000import argparse import multiprocessing import os from pathlib import Path import sysconfig CC = sysconfig.get_config_vars().get("CC", os.environ.get("CC", "gcc")) def compile_dll(nthreads=None, use_openmp=True, dry_run=False): num_cores = multiprocessing.cpu_count() if nthreads == 0: nthreads = num_cores os.chdir(Path(__file__).parent) openmp_flag = "-fopenmp" if use_openmp else "" flags = f"-I ./random123/include/ -O2 {openmp_flag} -shared -lm -o _compiled.so -fPIC -DMAX_THREADS={nthreads}" compile_command = f"{CC} compiled.c {flags}" if dry_run: print(f"Would compile: {compile_command}") return False if not (Path(__file__).parent / "random123" / "include" / "Random123").exists(): print("""\ Random123 not found in the expected location: clone it with the following command: git clone --branch v1.14.0 https://github.com/DEShawResearch/random123 bumps/dream/random123 """) print(f"Compiling: {compile_command}") os.system(compile_command) return os.path.exists("_compiled.so") def remove_dll(): dll_path = Path(__file__).parent / "_compiled.so" if dll_path.exists(): print(f"Removing {dll_path}") dll_path.unlink() else: print(f"{dll_path} does not exist") def main(): parser = argparse.ArgumentParser() parser.add_argument( "--nthreads", type=int, default=64, help="Number of threads to compile for, use 0 for all available cores, default is 64", ) parser.add_argument( "--no-openmp", action="store_true", help="Disable OpenMP (e.g. for clang, where it is not supported)", ) parser.add_argument("--remove", action="store_true", help="Remove the compiled dll") parser.add_argument( "--dry-run", action="store_true", help="Print the compile command without running it", ) args = parser.parse_args() if args.remove: remove_dll() else: compile_dll(nthreads=args.nthreads, use_openmp=not args.no_openmp, dry_run=args.dry_run) if __name__ == "__main__": main() bumps-bumps-2b313fd/bumps/dream/compiled.c000066400000000000000000000400071521754644000205610ustar00rootroot00000000000000/* Differential evolution MCMC stepper. */ #define _GNU_SOURCE // sincos isn't standard? #include #include #include // for debugging #ifdef _MSC_VER #define DLL_EXPORT __declspec(dllexport) #else #define DLL_EXPORT #endif // Random library with a separate generator for each thread of // an OpenMP threaded program. Assumes mafx 64 threads. If OpenMP is // not available, then operates single threaded. #ifdef _OPENMP #include #else #define omp_get_thread_num() 0 #endif // M_PI missing from MSVC math.h #ifndef M_PI # define M_PI 3.141592653589793 #endif // Limit to the number of threads so static thread-local data can be // pre-allocated with the right size. #ifndef MAX_THREADS #define MAX_THREADS 64 #endif // ==== Generator definition ==== // Uses: // Salmon, J. K.; Moraes, M. A.; Dror, R. O.; Shaw, D. E. (2011) // Parallel random numbers: as easy as 1, 2, 3. In Proceedings of 2011 // International Conference for High Performance Computing, Networking, // Storage and Analysis; SC '11; ACM: New York, NY; p 16:1016:12. // doi: 10.1145/2063384.2063405 // https://www.deshawresearch.com/resources_random123.html v1.09 // may want to swap it for a different generator, and update the following #include typedef threefry4x64_ctr_t r123_ctr_t; typedef threefry4x64_key_t r123_key_t; typedef threefry4x64_ukey_t r123_ukey_t; #define r123_init threefry4x64keyinit #define r123_next threefry4x64 #define R123_SIZE 4 // the 4 in 4x64 typedef uint64_t randint_t; // the 64 in 4x64 const randint_t R123_MAX = 18446744073709551615UL; const double R123_TO_01 = 1.0/18446744073709551616.0; const double R123_TO_M11 = 2.0/18446744073709551616.0; // ==== end generator definition ==== typedef struct { r123_ctr_t counter; // position in sequence r123_key_t key; // seed r123_ctr_t values; // cached values not yet used int have_normal; // Have a precomputer random normal double normal; // the precomputed random normal } Random; Random streams[MAX_THREADS]; // Max of 64 different threads in OpenMP double u_01_open(randint_t v) { return (((double)v) + 0.5)*R123_TO_01; } double u_m11_closed(randint_t v) { return ((double)((int64_t)v) + 0.5)*R123_TO_M11; } void _rand_init(randint_t seed) { int thread_id = omp_get_thread_num(); Random *rng = streams + thread_id; r123_ukey_t user_key; r123_key_t counter; int k; if (thread_id >= MAX_THREADS) { printf("Too many threads for random number generator. Set OMP_NUM_THREADS=%d\n", MAX_THREADS); exit(1); } for (k = 0; k < R123_SIZE; k++) user_key.v[k] = counter.v[k] = 0; user_key.v[0] = seed; //user_key.v[1] = omp_get_thread_num(); rng->key = r123_init(user_key); rng->counter = counter; //printf("%d initializing %p with seed %llu and counter %llu\n", omp_get_thread_num(), rng, rng->key.v[0], rng->counter.v[0]); rng->have_normal = 0; } void rand_init(randint_t seed) { #ifdef _OPENMP #pragma omp parallel #endif _rand_init(seed); } randint_t rand_next(void) { Random *rng = streams+omp_get_thread_num(); //printf("retrieving from %p with key %ld and counter %ld\n",rng, rng->key.v[0], rng->counter.v[0]); if (rng->counter.v[0]%R123_SIZE == 0) { rng->values = r123_next(rng->counter, rng->key); } return rng->values.v[(rng->counter.v[0]++)%R123_SIZE]; } double randn(void) { Random *rng = &streams[omp_get_thread_num()]; if (rng->have_normal) { rng->have_normal = 0; return rng->normal; } else { // Box-Muller transform converts two ints into two normals // Return one now and save the other for later. double x, y, r, arg; arg = M_PI*u_m11_closed(rand_next()); x = sin(arg); y = cos(arg); r = sqrt(-2. * log(u_01_open(rand_next()))); rng->have_normal = 1; rng->normal = y*r; return x*r; } } randint_t randint(randint_t range) { while (1) { randint_t value = rand_next(); // TODO: correct for very tiny bias against higher numbers. // Something like the following? // if (value > R123_MAX-(R123_MAX%range)) continue; return value%range; } } double randu(void) { return u_01_open(rand_next()); } /* draw k unique from n objects not equal to q */ // Specialized for k << n. If n is large and k -> n then argsort on // a random uniform draw is a better bet. If you don't want to exclude // any numbers, set not_matching to total_num. // TODO: raise an error instead of silently using replacement. // The current behaviour is good enough for this code base, so not fixing here. void rand_draw(int num_to_draw, int total_num, randint_t not_matching, randint_t p[]) { int i, j; // Handle the case where num_to_draw is too big if (num_to_draw > total_num - 1) { for (i = 0; i < total_num; i++) { p[i] = i; } num_to_draw -= total_num; p += total_num; } //printf("draw %d from %d != %llu\n", num_to_draw, total_num, not_matching); for (i=0; i < num_to_draw; i++) { while (1) { int proposed = randint(total_num); int unique = (proposed != not_matching); for (j=0; j < i && unique; j++) unique = (proposed != p[j]); if (unique) { p[i] = proposed; break; } } } } #if 0 #include #include #include randint_t random_seed() { randint_t seed; FILE* urandom = fopen("/dev/urandom", "r"); fread(&seed, sizeof(seed), 1, urandom); fclose(urandom); return seed; } void main(int argc, char *argv[]) { int j, k; randint_t seed, draw[10]; seed = (argc == 1 ? random_seed() : atoi(argv[1])); printf("seed: %ld\n", seed); rand_init(seed); printf("i randint(10):\n"); #pragma omp parallel for for (k=0; k < 10; k++) printf("i %d %ld\n", omp_get_thread_num(), randint(10)); printf("u randu:\n"); #pragma omp parallel for for (k=0; k < 10; k++) printf("u %d %g\n", omp_get_thread_num(), randu()); printf("n randn:\n"); #pragma omp parallel for for (k=0; k < 10; k++) printf("n %d %g\n", omp_get_thread_num(), randn()); printf("d rand_draw(10,52,!5):\n"); #pragma omp parallel for private(draw, j) for (k=0; k < 10; k++) { char buf[200]; rand_draw(10, 52, 5, draw); sprintf(buf, "d %d ", omp_get_thread_num()); for (j=0; j < 10; j++) sprintf(buf+strlen(buf), "%ld ", draw[j]); printf("%s\n", buf); } } #endif #define _SNOOKER 0 #define _DE 1 #define _DIRECT 2 #define EPS 1e-6 #define MAX_CHAINS 20 /* Generates offspring using METROPOLIS HASTINGS monte-carlo markov chain The number of chains may be smaller than the population size if the population is selected from both the current generation and the ancestors. */ void _perform_step(int qq, int Nchain, int Nvar, int NCR, double pop[], double CR[][2], int max_pairs, double eps, double snooker_rate, double de_rate, double noise, double scale, double x_new[], double step_alpha[], double CR_used[]) { randint_t chains[2*MAX_CHAINS]; double u = randu(); int alg = (u < snooker_rate ? _SNOOKER : u < de_rate ? _DE : _DIRECT); double *xin = &pop[qq*Nvar]; int k; //for (k=0; k < NCR; k++) printf("CR %d: %g %g\n", k, CR[k][0], CR[k][1]); //printf("pop in c: "); //for (k=0; k < Nvar; k++) printf("%g ", pop[qq*Nvar+k]); //printf("\n"); //printf("alg: %d\n", alg); switch (alg) { case _DE: { // Use DE with cross-over ratio int var, num_crossover, active; double crossover_ratio, CR_cdf, distance, jiggle; // Select to number of vector pair differences to use in update // using k ~ discrete U[1, max pairs] int num_pairs = randint(max_pairs)+1; // [PAK: same as k = DEversion[qq, 1] in matlab version] // Weight the size of the jump inversely proportional to the // number of contributions, both from the parameters being // updated and from the population defining the step direction. double gamma_scale = 2.38/sqrt(2 * Nvar * num_pairs); // [PAK: same as F=Table_JumpRate[len(vars), k] in matlab version] // Select 2*k members at random different from the current member rand_draw(2*num_pairs, Nchain, qq, chains); // Select crossover ratio u = randu(); CR_cdf = 0.; for (k=0; k < NCR-1; k++) { CR_cdf += CR[k][1]; if (u <= CR_cdf) break; } crossover_ratio = CR[k][0]; CR_used[qq] = crossover_ratio; // Select the dims to update based on the crossover ratio, making // sure at least one dim is selected num_crossover = 0; for (var=0; var < Nvar || num_crossover == 0; var++) { if (var == Nvar) { active = randint(Nvar); } else if (randu() <= crossover_ratio) { active = var; } else { x_new[var] = 0.; continue; } num_crossover++; // Find and average step from the selected pairs distance = 0.; for (k=0; k < num_pairs; k++) { distance += pop[chains[2*k]*Nvar + active] - pop[chains[2*k+1]*Nvar + active]; } // Apply that step with F scaling and noise jiggle = 1 + eps * (2 * randu() - 1); x_new[active] = jiggle*gamma_scale*distance; } step_alpha[qq] = 1.; break; } case _SNOOKER: { // Use snooker update double num, denom, gamma_scale; // Select current and three others rand_draw(3, Nchain, qq, chains); double *z = &pop[chains[0]*Nvar]; double *R1 = &pop[chains[1]*Nvar]; double *R2 = &pop[chains[2]*Nvar]; // Find the step direction and scale it to the length of the // projection of R1-R2 onto the step direction. // TODO: population sometimes not unique! for (k=0; k < Nvar; k++) x_new[k] = xin[k] - z[k]; while (1) { denom = 0.; for (k=0; k < Nvar; k++) denom += x_new[k]*x_new[k]; if (denom != 0.) break; for (k=0; k < Nvar; k++) x_new[k] = EPS*randn(); } num = 0.; for (k=0; k < Nvar; k++) num += ((R1[k]-R2[k])*x_new[k]); // Step using gamma of 2.38/sqrt(2) + U(-0.5, 0.5) gamma_scale = (1.2 + randu())*num/denom; for (k=0; k < Nvar; k++) x_new[k] *= gamma_scale; // Scale Metropolis probability by (||xi* - z||/||xi - z||)^(d-1) num = 0.; for (k=0; k < Nvar; k++) num += (xin[k]+x_new[k]-z[k])*(xin[k]+x_new[k]-z[k]); step_alpha[qq] = pow(num/denom, (Nvar-1)/2); CR_used[qq] = 0.; break; } case _DIRECT: { // Use one pair and all dimensions // Note that there is no F scaling, dimension selection or noise int p[2]; rand_draw(2, Nchain, qq, chains); double *R1 = &pop[chains[0]*Nvar]; double *R2 = &pop[chains[1]*Nvar]; for (k=0; k < Nvar; k++) x_new[k] = R1[k] - R2[k]; step_alpha[qq] = 1.; CR_used[qq] = 0.; break; } } //printf("alg %d -> ", alg); //for (k=0; k < Nvar; k++) printf("%g ", x_new[k]); //printf("\n"); // Update x_old with delta_x and noise for (k=0; k < Nvar; k++) x_new[k] *= scale; // [PAK] The noise term needs to depend on the fitting range // of the parameter rather than using a fixed noise value for all // parameters. The current parameter value is a pretty good proxy // in most cases (i.e., relative noise), but it breaks down if the // parameter is zero, or if the range is something like 1 +/- eps. // absolute noise //for (k=0; k < Nvar; k++) x_new[k] += xin[k] + scale*noise*randn(); // relative noise for (k=0; k < Nvar; k++) x_new[k] += xin[k]*(1.+scale*noise*randn()); //printf("alg %d -> ", alg); //for (k=0; k < Nvar; k++) printf("%g ", x_new[k]); //printf("\n"); // no noise //for (k=0; k < Nvar; k++) x_new[k] += xin[k]; } DLL_EXPORT void de_step(int Nchain, int Nvar, int NCR, double pop[], double CR[][2], int max_pairs, double eps, double snooker_rate, double noise, double scale, double x_new[], double step_alpha[], double CR_used[]) { int qq; double de_rate = snooker_rate + 0.8 * (1-snooker_rate); //Choose snooker, de or direct according to snooker_rate, and 80:20 // ratio of de to direct. //printf("in de_step with (%d,%d,%d) pairs=%d eps=%g snooker=%g noise=%g scale=%g\n", //Nchain, Nvar, NCR, max_pairs, eps, snooker_rate, noise, scale); //printf("points pop=%p CR=%p x_new=%p step_alpha=%p CR_used=%p\n", pop, CR, x_new, step_alpha, CR_used); // Chains evolve using information from other chains to create offspring #ifdef _OPENMP #pragma omp parallel for #endif for (qq = 0; qq < Nchain; qq++) { _perform_step(qq, Nchain, Nvar, NCR, pop, CR, max_pairs, eps, snooker_rate, de_rate, noise, scale, &x_new[qq*Nvar], step_alpha, CR_used); } } DLL_EXPORT void bounds_reflect(int Nchain, int Nvar, double pop[], double low[], double high[]) { int k, p, idx; #ifdef _OPENMP #pragma omp parallel for private(idx, k) #endif for (p=0; p < Nchain; p++) { for (k=0; k < Nvar; k++) { idx = p*Nvar+k; if (pop[idx] < low[k]) { pop[idx] = 2*low[k] - pop[idx]; } else if (pop[idx] > high[k]) { pop[idx] = 2*high[k] - pop[idx]; } if (pop[idx] < low[k] || pop[idx] > high[k]) { pop[idx] = low[k] + randu()*(high[k]-low[k]); } } } } DLL_EXPORT void bounds_clip(int Nchain, int Nvar, double pop[], double low[], double high[]) { int k, p, idx; #ifdef _OPENMP #pragma omp parallel for private(idx, k) #endif for (p=0; p < Nchain; p++) { for (k=0; k < Nvar; k++) { idx = p*Nvar+k; if (pop[idx] < low[k]) { pop[idx] = low[k]; } else if (pop[idx] > high[k]) { pop[idx] = high[k]; } } } } DLL_EXPORT void bounds_fold(int Nchain, int Nvar, double pop[], double low[], double high[]) { int k, p, idx; #ifdef _OPENMP #pragma omp parallel for private(idx, k) #endif for (p=0; p < Nchain; p++) { for (k=0; k < Nvar; k++) { idx = p*Nvar+k; if (pop[idx] < low[k]) { if (isinf(high[k])) { pop[idx] = 2*low[k] - pop[idx]; } else { pop[idx] = high[k] - (low[k] - pop[idx]); } } else if (pop[idx] > high[k]) { if (isinf(low[k])) { pop[idx] = 2*high[k] - pop[idx]; } else { pop[idx] = low[k] - (high[k] - pop[idx]); } } if (pop[idx] < low[k] || pop[idx] > high[k]) { pop[idx] = low[k] + randu()*(high[k]-low[k]); } } } } DLL_EXPORT void bounds_random(int Nchain, int Nvar, double pop[], double low[], double high[]) { int k, p, idx; #ifdef _OPENMP #pragma omp parallel for private(idx, k) #endif for (p=0; p < Nchain; p++) { for (k=0; k < Nvar; k++) { idx = p*Nvar+k; if (pop[idx] < low[k]) { if (isinf(high[k])) { pop[idx] = 2*low[k] - pop[idx]; } else { pop[idx] = low[k] + randu()*(high[k]-low[k]); } } else if (pop[idx] > high[k]) { if (isinf(low[k])) { pop[idx] = 2*high[k] - pop[idx]; } else { pop[idx] = low[k] + randu()*(high[k]-low[k]); } } } } } DLL_EXPORT void bounds_ignore(int Nchain, int Nvar, double pop[], double low[], double high[]) { } bumps-bumps-2b313fd/bumps/dream/compiled.py000066400000000000000000000003201521754644000207610ustar00rootroot00000000000000from os.path import join as joinpath, realpath, dirname, exists from ctypes import CDLL _dll_path = joinpath(realpath(dirname(__file__)), "_compiled.so") dll = CDLL(_dll_path) if exists(_dll_path) else None bumps-bumps-2b313fd/bumps/dream/convergence.py000066400000000000000000000253501521754644000214750ustar00rootroot00000000000000r""" Convergence diagnostics ======================= The function :func:`burn_point` returns the point within the MCMC chain at which the chain can be said to have converged, or -1 if the log probabilities are still improving throughout the chain. """ __all__ = ["burn_point", "ks_converged"] from typing import TYPE_CHECKING import warnings import numpy as np from numpy.random import choice from scipy.stats import chi2, ks_2samp, kstest if TYPE_CHECKING: from .state import MCMCDraw # TODO is cramer von mises better than a KS test? # defaults should match for ks_converged and burn_point trimming DENSITY = 0.6 ALPHA = 0.01 SAMPLES = 1000 TRIALS = 5 MIN_WINDOW = 100 def ks_converged( state: "MCMCDraw", trials: int = TRIALS, density: float = DENSITY, alpha: float = ALPHA, samples: int = SAMPLES ) -> bool: """ Return True if the MCMC has converged according to the K-S window test. Since we are only looking at the distribution of logp values, and not the individual points, we should be relatively stable regardless of the properties of the sampler. The main reason for failure will be "stuck" fits which have not had a chance to jump to a lower minimum. *state* contains the MCMC chain information. *trials* is the number of times to run the K-S test. *density* is the proportion of samples to select from the window. Prefer lower density from larger number of samples so the sets chosen for the K-S test have fewer duplicates. For density=0.1 about 5% of samples will be duplicates. For density=0.6 about 25% will be duplicates. *alpha* is the significance level for the test. With smaller alpha values the K-S test is less likely to reject the current window when testing against the tail of the distribution, and so the fit will end earlier, with more samples after the burn point. *samples* is the size of the sample window. If the window is too big the test will falsly end burn when the start of the window is still converging. If the window is too small the test will take a long time, and will start to show effects of autocorrelation (efficient MCMC samplers move slowly across the posterior probability space, showing short term autocorrelation between samples.) A minimum of 10 generations and a maximum of 1/2 the generations will be used. There is a strong interaction between density, alpha, samples and trials. If the K-S test has too many points (=density*samples), it will often reject simply because the different portions of the Markov chain are not identical (Markov chains can have short range correlations yet still have the full chain as a representative draw from the posterior distribution) unless alpha is reduced. With fewer points, the estimated K-S statistic will have more variance, and so more trials will be needed to avoid spurious accept/reject decisions. """ # Make sure we are testing for convergence if alpha == 0.0: return False # Make sure we have a full buffer. if state.generation < state.Ngen: # print(f"convergence {state.generation}: need {state.Ngen} for a full cycle") return False # Quick fail if best occurred within current buffer. We want to make sure we have # time to explore a new minimum before concluding that it has converged. if not state.stable_best(): # print(f"convergence {state.generation}: best {state._best_gen} still in pop {state.total_generations} - {state.Ngen} = {state.total_generations - state.Ngen}") return False # Grab a window at the start and the end window_size = min(max(samples // state.Npop + 1, MIN_WINDOW), state.Ngen // 2) head = state.logp_slice(window_size).flatten() tail = state.logp_slice(-window_size).flatten() # # Note: best not in buffer should cover this case. # # Quick fail if logp head is worse than logp tail # if np.min(head) > np.min(tail): # # print(f"convergence {state.generation}: tail {np.min(tail)} better than head {np.min(head)}") # return False n_draw = int(density * samples) p = _robust_ks_2samp(head, tail, n_draw, trials) # print(f"convergence {state.generation}: pval={p} < {alpha}?") # comparison returns np.bool type which is not exactly the same as bool return bool(p < alpha) def check_nllf_distribution(state: "MCMCDraw") -> bool: """ Check if the nllf distribution looks like chisq. Note: test is not used. It is only true for gaussian. It fails pretty badly for doc/examples/test_functions.py griewank 2. It fails even worse with pure integer models such as the OpenBugs asia example, which has discrete levels in the posterior pdf corresponding to the various binary configurations of the model values. """ # Check that likelihood distribution looks like chi2 # Note: cheating, and looking at the stored logp without unrolling reject = _check_nllf_distribution(data=-state._gen_logp.flatten(), df=state.Nvar, n_draw=10, trials=5, alpha=0.01) return not reject def _check_nllf_distribution(data, df, n_draw, trials, alpha): # fit the best chisq to the data given df float_df, loc, scale = chi2.fit(data, f0=df) df = int(float_df + 0.5) cdf = lambda x: chi2.cdf(x, df, loc, scale) # check the quality of the fit (i.e., does the set of nllfs look vaguely # like the fitted chisq distribution). Repeat the test a few times on # small data sets for consistency. p_vals = [] for _ in range(trials): f_samp = choice(data, n_draw, replace=True) p_vals.append(kstest(data, cdf)[1]) print("llf dist", p_vals, df, loc, scale) return alpha > np.mean(p_vals) def burn_point( state: "MCMCDraw", trials: int = TRIALS, density: float = DENSITY, alpha: float = ALPHA, samples: int = SAMPLES ) -> int: r""" Determines the point at which the MCMC chain seems to have converged, using a Kolmogorov-Smirnov sliding window test. *state* contains the MCMC chain information. *trials* is the number of times to run the K-S test. *density* is the proportion of samples to select from the window. Prefer lower density from larger number of samples so the sets chosen for the K-S test have fewer duplicates. For density=0.1 about 5% of samples will be duplicates. For density=0.6 about 25% will be duplicates. *alpha* is the significance level for the test. With smaller alpha values the K-S test is less likely to reject the current window when testing against the tail of the distribution, and so the fit will end earlier, with more samples after the burn point. *samples* is the size of the sample window. If the window is too big the test will falsly end burn when the start of the window is still converging. If the window is too small the test will take a long time, and will start to show effects of autocorrelation (efficient MCMC samplers move slowly across the posterior probability space, showing short term autocorrelation between samples.) A minimum of 10 generations and a maximum of 1/2 the generations will be used. Returns the index of the burn points, or -1 if no good burn point is found. **Kolmogorov-Smirnov sliding window** Detects convergence by comparing the distribution of $\log(p)$ values in a small window at the start of the chains and the values in a section at the end of the chain using a Kolmogorov-Smirnov test. """ index = _ks_sliding_window(state=state, trials=trials, density=density, alpha=alpha, samples=samples) # TODO: need a better way to report convergence failure if index < 0: warnings.warn("Did not converge!") return index def _ks_sliding_window(*, state: "MCMCDraw", trials: int, density: float, alpha: float, samples: int) -> int: _, logp = state.logp() window_size = min(max(samples // state.Npop + 1, MIN_WINDOW), state.Ngen // 2) tiny_window = window_size // 11 + 1 half = state.Ngen // 2 max_index = len(logp) - half - window_size # print("max index", max_index, "window size", window_size, "half", half, "Ngen", state.Ngen, "len(logp)", len(logp)) if max_index < 0: return -1 tail = logp[-half:].flatten() min_tail = np.min(tail) # Check in large bunches n_draw = int(density * samples) for index in range(0, max_index + 1, window_size): # [PAK] make sure the worst point is not in the first window. # Stastically this will introduce some bias (by chance the max could # happen to occur in the first window) but it will be small when the # window is small relative to the full pool. A better test would # count the number of samples worse than the all the tail, compute # the probability, and reject according to a comparison with a uniform # number in [0,1]. To much work for so little bias. window = logp[index : index + window_size].flatten() if np.min(window) < min_tail: # print("step llf", index, window_size, len(window), np.min(window), min_tail) continue # if head and tail are different, slide to the next window # p_val represents the probability that the two samples are from the same distribution. p_val = _robust_ks_2samp(window, tail, n_draw, trials) # print("ks", index, window_size, len(window), p_val, alpha) if p_val >= alpha: # two samples look the same, so we do not reject the null hypothesis. break if index >= max_index: return -1 # check in smaller steps for fine tuned stopping for index in range(index, index + window_size, tiny_window): window = logp[index : index + tiny_window].flatten() if np.min(window) < min_tail: # print("tiny llf", index, tiny_window, len(window), np.min(window), min_tail) continue p_val = _robust_ks_2samp(window, tail, n_draw, trials) # print("tiny ks", index, tiny_window, len(window), p_val, alpha) if p_val >= alpha: # head and tail are not significantly different, so stop break return index def _robust_ks_2samp(f_data, r_data, n_draw, trials): """ Repeat ks test n times for a more robust statistic. Returns True if f_data is significantly different from r_data. """ p_vals = [] for _ in range(trials): # [PAK] Using replace=True since it is more efficient and practically # indistinguishable for a small sampling portion such as 10% or less. # When drawing 60% of the sample size, 25% of the samples are repeated. f_samp = choice(f_data, n_draw, replace=True) r_samp = choice(r_data, n_draw, replace=True) p_vals.append(ks_2samp(f_samp, r_samp)[1]) return np.median(p_vals) bumps-bumps-2b313fd/bumps/dream/core.py000066400000000000000000000514741521754644000201350ustar00rootroot00000000000000""" DiffeRential Evolution Adaptive Metropolis algorithm DREAM runs multiple different chains simultaneously for global exploration, and automatically tunes the scale and orientation of the proposal distribution using differential evolution. The algorithm maintains detailed balance and ergodicity and works well and efficient for a large range of problems, especially in the presence of high-dimensionality and multimodality. DREAM developed by Jasper A. Vrugt and Cajo ter Braak This algorithm has been described in: Vrugt, J.A., C.J.F. ter Braak, M.P. Clark, J.M. Hyman, and B.A. Robinson, *Treatment of input uncertainty in hydrologic modeling: Doing hydrology backward with Markov chain Monte Carlo simulation*, Water Resources Research, 44, W00B09, 2008. `doi:10.1029/2007WR006720 `_ Vrugt, J.A., C.J.F. ter Braak, C.G.H. Diks, D. Higdon, B.A. Robinson, and J.M. Hyman, *Accelerating Markov chain Monte Carlo simulation by differential evolution with self-adaptive randomized subspace sampling*, International Journal of Nonlinear Sciences and Numerical Simulation, 10(3), 271-288, 2009. Vrugt, J.A., C.J.F. ter Braak, H.V. Gupta, and B.A. Robinson, *Equifinality of formal (DREAM) and informal (GLUE) Bayesian approaches in hydrologic modeling*, Stochastic Environmental Research and Risk Assessment, 1-16, 2009, In Press. `doi:10.1007/s00477-008-0274-y `_ For more information please read: Ter Braak, C.J.F., *A Markov Chain Monte Carlo version of the genetic algorithm Differential Evolution: easy Bayesian computing for real parameter spaces*, Stat. Comput., 16, 239 - 249, 2006. `doi:10.1007/s11222-006-8769-1 `_ Vrugt, J.A., H.V. Gupta, W. Bouten and S. Sorooshian, *A Shuffled Complex Evolution Metropolis algorithm for optimization and uncertainty assessment of hydrologic model parameters*, Water Resour. Res., 39 (8), 1201, 2003. `doi:10.1029/2002WR001642 `_ Ter Braak, C.J.F., and J.A. Vrugt, *Differential Evolution Markov Chain with snooker updater and fewer chains*, Statistics and Computing, 2008. `doi:10.1007/s11222-008-9104-9 `_ Vrugt, J.A., C.J.F. ter Braak, and J.M. Hyman, *Differential evolution adaptive Metropolis with snooker update and sampling from past states*, SIAM journal on Optimization, 2009. Vrugt, J.A., C.J.F. ter Braak, and J.M. Hyman, *Parallel Markov chain Monte Carlo simulation on distributed computing networks using multi-try Metropolis with sampling from past states*, SIAM journal on Scientific Computing, 2009. G. Schoups, and J.A. Vrugt, *A formal likelihood function for Bayesian inference of hydrologic models with correlated, heteroscedastic and non-Gaussian errors*, Water Resources Research, 2010, In Press. G. Schoups, J.A. Vrugt, F. Fenicia, and N.C. van de Giesen, *Inaccurate numerical solution of hydrologic models corrupts efficiency and robustness of MCMC simulation*, Water Resources Research, 2010, In Press. Copyright (c) 2008, Los Alamos National Security, LLC All rights reserved. Copyright 2008. Los Alamos National Security, LLC. This software was produced under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National Laboratory (LANL), which is operated by Los Alamos National Security, LLC for the U.S. Department of Energy. The U.S. Government has rights to use, reproduce, and distribute this software. NEITHER THE GOVERNMENT NOR LOS ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software is modified to produce derivative works, such modified software should be clearly marked, so as not to confuse it with the version available from LANL. Additionally, redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Los Alamos National Security, LLC, Los Alamos National Laboratory, LANL the U.S. Government, nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. MATLAB code written by Jasper A. Vrugt, Center for NonLinear Studies (CNLS) Written by Jasper A. Vrugt: vrugt@lanl.gov Version 0.5: June 2008 Version 1.0: October 2008 Adaption updated and generalized CR implementation 2010-04-20 Paul Kienzle * Convert to python Complete changelog on github.com/bumps/bumps """ __all__ = ["Dream", "Model"] import sys import time from ctypes import c_double from typing import Protocol, List, Sequence, Optional import numpy as np from numpy.typing import NDArray from .state import MCMCDraw from .metropolis import metropolis, metropolis_dr, dr_step from .crossover import AdaptiveCrossover, LogAdaptiveCrossover, Crossover from .diffev import de_step from .bounds import make_bounds_handler from .compiled import dll from .util import rng from .convergence import ks_converged # Everything should be available in state, but lets be lazy for now _LAST_TIME = 0 """Used by console_monitor to count time until the next update""" def console_monitor(state, pop, logp): """ Print progress of fit on the console. This is the default update monitor when running DREAM standalone. It is not used by the bumps package. """ global _LAST_TIME if state.generation == 1: print("#gen", "logp(x)", " ".join("<%s>" % par for par in state.labels)) _LAST_TIME = time.perf_counter() current_time = time.perf_counter() if current_time >= _LAST_TIME + 1: _LAST_TIME = current_time print(state.generation, state._best_logp, " ".join("%.15g" % v for v in state._best_x)) sys.stdout.flush() class Model(Protocol): """ Dream model interface definition. """ labels: List[str] """Labels for all the parameters""" bounds: Sequence[Sequence[float]] """Bounds for each parameter as a pair of sequences of the same length as the labels.""" def map(self, pop: NDArray) -> NDArray: """ Function which takes an array of [k x n] and returns an array of n where k is the number of parameters. TODO: check if """ ... class Dream: """ Data structure containing the details of the running DREAM analysis code. """ model: Model = None # Sampling parameters burn: int = 0 draws: int = 100000 thinning: int = 1 # TODO: change the default outlier test to IQR and control with options outlier_test = "none" population: NDArray = None #: convergence criteria alpha: float = 0.01 # DE parameters DE_steps: int = 10 DE_pairs: int = 3 DE_eps: float = 0.05 DE_snooker_rate: float = 0.1 DE_noise: float = 1e-6 bounds_style: str = "reflect" # reflect, clip, fold, randomize, none # Crossover parameters CR: Crossover = None CR_spacing: str = "linear" # 'log' or 'linear' # Delay rejection parameters use_delayed_rejection: bool = False DR_scale: float = 1 # 1-sigma step size using cov of population # Local optimizer best fit injection The optimizer has # the following interface: # x, fx = goalseek(mapper, bounds_handler, pop, fpop) # where: # x, fx are the local optimum point and its value # pop is the starting population # fpop is the nllf for each point in pop # mapper is a function which takes pop and returns fpop # bounds_handler takes pop and forces all points into the range goalseek_optimizer = None goalseek_interval: int = 2e9 # close enough to never goalseek_minburn: int = 1000 state: MCMCDraw = None def __init__(self, **kw): self.monitor = console_monitor for k, v in kw.items(): if hasattr(self, k): setattr(self, k, v) else: raise TypeError("Unknown attribute " + k) self._initialized = False def sample(self, state: Optional[MCMCDraw] = None, abort_test=lambda: False): """ Pull the requisite number of samples from the distribution """ if not self._initialized: self._initialized = True self.state = state try: _run_dream(self, abort_test=abort_test) except KeyboardInterrupt: pass return self.state def _run_dream(dream: Dream, abort_test=lambda: False): """ Collect posterior distribution samples using DREAM sampler. """ if dll: dll.rand_init(rng.randint(1e9)) # Step 1: Sample s points in the parameter space # [PAK] I moved this out of dream so that the user can use whatever # complicated sampling scheme they want. Unfortunately, this means # the user needs to know some complex sampling scheme. if dream.population is None: raise ValueError("initial population not defined") # Remember the problem dimensions n_gen, n_chain, n_var = dream.population.shape n_pop = n_gen * n_chain if dream.CR is None: if dream.CR_spacing == "log": dream.CR = LogAdaptiveCrossover(n_var) else: # linear dream.CR = AdaptiveCrossover(3) # Step 2: Calculate posterior density associated with each value in x apply_bounds = make_bounds_handler(dream.model.bounds, style=dream.bounds_style) # Record initial state allocate_state(dream) state = dream.state # Reset plot portion to 100% during fitting so that the intermediate plots show the # full set of samples. state.portion = 1.0 state.labels = dream.model.labels previous_draws = state.draws if previous_draws: x, logp = state._last_gen() else: # No initial state, so evaluate initial population for x in dream.population: apply_bounds(x) # ********************** MAP ***************************** logp = dream.model.map(x) state._generation(new_draws=n_chain, x=x, logp=logp, accept=n_chain, force_keep=True) dream.monitor(state, x, logp) # print(f"Initial pop has draws={state.draws} gen={state.generation} pop={dream.population.shape}") state._update(CR_weight=dream.CR.weight) # Now start drawing samples # print "previous draws", previous_draws, "new draws", dream.draws+dream.burn last_goalseek = (dream.draws + dream.burn) / n_pop - dream.goalseek_minburn next_goalseek = state.generation + dream.goalseek_interval if dream.goalseek_optimizer else 1e100 if dll: xtry = np.empty((n_chain, n_var), "d") step_alpha = np.empty(n_chain, "d") CR_used = np.empty(n_chain, "d") scale = 1.0 # serial_time = parallel_time = 0. # last_time = time.time() # Make sure that the pop we are drawing doesn't need to be copied before # sending it to the compiled C code. pop = state._draw_pop() assert pop.ctypes.data == np.ascontiguousarray(pop).ctypes.data # frame = 0 # For debugging... next_outlier_test = max(state.Ngen, 2 * state.Ngen - 10) next_convergence_test = state.Ngen final_gen = dream.draws + dream.burn # print(f"running dream with {dream.draws} draws and {dream.burn} burn; final gen={final_gen} draws={state.draws} gen={state.generation}") while state.draws < final_gen: # Age the population using differential evolution dream.CR.reset() CR = np.ascontiguousarray(np.vstack((dream.CR.CR, dream.CR.weight)).T, "d") for gen in range(dream.DE_steps): # Define the current locations and associated posterior densities xold, logp_old = x, logp pop = state._draw_pop() # print(pop.ctypes.data, np.ascontiguousarray(pop).ctypes.data) # print(pop) # print("gen", gen, pop.shape) # Generate candidates for each sequence if dll is None: xtry, step_alpha, CR_used = de_step( n_chain, pop, CR, max_pairs=dream.DE_pairs, eps=dream.DE_eps, snooker_rate=dream.DE_snooker_rate, noise=dream.DE_noise, scale=scale, ) else: dll.de_step( n_chain, n_var, len(CR), pop.ctypes, CR.ctypes, dream.DE_pairs, c_double(dream.DE_eps), c_double(dream.DE_snooker_rate), c_double(dream.DE_noise), c_double(scale), xtry.ctypes, step_alpha.ctypes, CR_used.ctypes, ) # print("try", xtry) # PAK: Try a local optimizer every N generations if next_goalseek <= state.generation <= last_goalseek: best, logp_best = dream.goalseek_optimizer(dream.model.map, apply_bounds, xold, logp_old) xtry[0] = best # Note: it is slightly inefficient to throw away logp_best, # but it makes the the code cleaner if we do next_goalseek += dream.goalseek_interval # Compute the likelihood of the candidates apply_bounds(xtry) # ********************** MAP ***************************** # next_time = time.time() # serial_time += next_time - last_time # last_time = next_time logp_try = dream.model.map(xtry) # next_time = time.time() # parallel_time += next_time - last_time # last_time = next_time draws = len(logp_try) # Apply the metropolis acceptance/rejection rule x, logp, alpha, accept = metropolis(xtry, logp_try, xold, logp_old, step_alpha) # Process delayed rejection # PAK NOTE: this updates according to the covariance matrix of the # current sample, which may be useful on unimodal systems, but # doesn't seem to be of any value in general; the DREAM papers # found that the acceptance rate did indeed improve with delayed # rejection, but the overall performance did not. Worse, this # requires a linear system solution O(nPop^3) which can be near # singular for complex posterior distributions. if dream.use_delayed_rejection and not accept.all(): # Generate alternate candidates using the covariance of xold xdr, r = dr_step(x=xold, scale=dream.DR_scale) # Compute the likelihood of the new candidates reject = ~accept apply_bounds(xdr) # ********************** MAP ***************************** logp_xdr = dream.model.map(xdr[reject]) draws += len(logp_xdr) # Apply the metropolis delayed rejection rule. x[reject], logp[reject], alpha[reject], accept[reject] = metropolis_dr( xtry[reject], logp_try[reject], x[reject], logp[reject], xold[reject], logp_old[reject], alpha[reject], r, ) # els = zip(logp_old, logp_try, logp, x[:, -2], x[:, -1], accept) # print "pop", "\n ".join((("%12.3e "*(len(el)-1))%el[:-1]) # +("T " if el[-3]<=el[-2] else " ") # +("accept" if el[-1] else "") # for el in els) # Update Sequences with the new population. state._generation(draws, x, logp, accept) # ********************** NOTIFY ************************** dream.monitor(state, x, logp) # print state.generation, ":", state._best_logp # Keep track of which CR ratios were successful if state.draws <= dream.burn: dream.CR.update(xold, x, CR_used) if abort_test(): break # print("serial¶llel",serial_time,parallel_time) # End of differential evolution aging # --------------------------------------------------------------------- # Save update information state._update(CR_weight=dream.CR.weight) if abort_test(): # print("<<< DREAM abort >>>") break # if state.draws <= 0.1 * dream.draws: if state.draws <= dream.burn: # Adapt the crossover ratio, but only during burn-in. dream.CR.adapt() if False: # TODO: Suppress scale update until we have a chance to verify that it # doesn't skew the resulting statistics. # TODO: scale needs to be stored in state in order for resume to work correctly. _, r = state.acceptance_rate() ravg = np.mean(r[-dream.DE_steps :]) if ravg > 0.4: scale *= 1.01 elif ravg < 0.2: scale /= 1.01 if state.generation >= next_convergence_test: converged = ks_converged(state, alpha=dream.alpha) # print(f"ks convergence for α={dream.alpha} is {converged}") else: converged = False # See whether there are any outlier chains, and remove them # Only do this once per frame, and only if there is some time # left to adapt the distribution. Also do it if we believe # that we have converged. if (converged or state.generation >= next_outlier_test) and state.generation + state.Ngen < final_gen: outliers = state.remove_outliers(x, logp, dream.outlier_test) next_outlier_test = state.generation + 2 * state.Ngen if len(outliers): # TODO: use monitors to report arbitrary information print("step %d trimmed %d outliers from %d chains" % (state.generation, len(outliers), len(logp))) next_convergence_test = state.generation + state.Ngen converged = False if converged: # print("<<< DREAM converged >>>") # _show_logp_frame(dream, state, frame+1) break # Draw the next frame (for debugging...) # next_frame = state.generation // state.Ngen # if frame != next_frame: _show_logp_frame(dream, state, next_frame) # frame = next_frame def _show_logp_frame(dream, state, frame): import matplotlib.pyplot as plt from . import views plt.clf() views.plot_logp(state) plt.savefig("logp%03d.png" % frame) plt.clf() def allocate_state(dream): """ Estimate the size of the output """ # Determine problem dimensions from the initial population n_pop, n_chain, n_var = dream.population.shape steps = dream.DE_steps thinning = dream.thinning n_cr = len(dream.CR.CR) draws = dream.draws n_update = int(np.ceil(draws / (steps * n_chain))) n_gen = n_update * steps n_thin = int(np.ceil(n_gen / thinning)) # print("new state", n_var, n_chain, n_cr, n_gen, n_thin, n_update, draws, steps) if dream.state is not None: # print("existing", dream.state.Nvar, dream.state.Npop, dream.state.Ncr) dream.state.resize(n_gen, n_thin, n_update, n_var, n_chain, n_cr, thinning) else: dream.state = MCMCDraw(n_gen, n_thin, n_update, n_var, n_chain, n_cr, thinning) bumps-bumps-2b313fd/bumps/dream/corrplot.py000066400000000000000000000127741521754644000210510ustar00rootroot00000000000000# This program is public domain # Author Paul Kienzle """ 2-D correlation histograms Generate 2-D correlation histograms and display them in a figure. Uses false color plots of density. """ __all__ = ["Corr2d"] import numpy as np from numpy import inf from matplotlib import cm, colors, image, artist from matplotlib.font_manager import FontProperties from matplotlib.ticker import MaxNLocator try: COLORMAP = colors.LinearSegmentedColormap.from_list("density", ("w", "y", "g", "b", "r")) except Exception: COLORMAP = cm.gist_earth_r class Corr2d(object): """ Generate and manage 2D correlation histograms. """ def __init__(self, data, labels=None, **kw): if labels is None: labels = ["P" + str(i + 1) for i, _ in enumerate(data)] self.N = len(data) self.labels = labels self.data = data self.hists = _hists(data, **kw) # for k, v in self.hists.items(): # print k, (v[1][0], v[1][-1]), (v[2][0], v[2][-1]) self.ax = None # will be set on plot def R(self): return np.corrcoef(self.data) def __getitem__(self, key): """ Retrieve correlation histogram for data[i] X data[j]. Returns bin i edges, bin j edges, and histogram """ i, j = key return self.hists[i, j] def plot(self, title=None, fig=None): """ Plot the correlation histograms on the specified figure """ import matplotlib.pyplot as plt if fig is None: plt.clf() fig = plt.gcf() if title is not None: fig.text(0.5, 0.95, title, horizontalalignment="center", fontproperties=FontProperties(size=16)) self.ax = _plot(fig, self.hists, self.labels, self.N) def _hists(data, ranges=None, **kw): """ Generate pair-wise correlation histograms """ n = len(data) if ranges is None: low, high = np.min(data, axis=1), np.max(data, axis=1) ranges = [(l, h) for l, h in zip(low, high)] return dict( ((i, j), np.histogram2d(data[i], data[j], range=[ranges[i], ranges[j]], **kw)) for i in range(0, n) for j in range(i + 1, n) ) def _plot(fig, hists, labels, n, show_ticks=None): """ Plot pair-wise correlation histograms """ if n <= 1: fig.text(0.5, 0.5, "No correlation plots when only one variable", ha="center", va="center") return vmin, vmax = inf, -inf for data, _, _ in hists.values(): positive = data[data > 0] if len(positive) > 0: vmin = min(vmin, np.amin(positive)) vmax = max(vmax, np.amax(positive)) norm = colors.LogNorm(vmin=vmin, vmax=vmax, clip=False) # norm = colors.Normalize(vmin=vmin, vmax=vmax) mapper = image.FigureImage(fig) mapper.set_array(np.zeros((1, 1))) mapper.set_cmap(cmap=COLORMAP) mapper.set_norm(norm) if show_ticks is None: show_ticks = n < 3 ax = {} Nr = Nc = n - 1 for i in range(0, n - 1): for j in range(i + 1, n): sharex = ax.get((0, j), None) sharey = ax.get((i, i + 1), None) a = fig.add_subplot(Nr, Nc, (Nr - i - 1) * Nc + j, sharex=sharex, sharey=sharey) ax[(i, j)] = a a.xaxis.set_major_locator(MaxNLocator(4, steps=[1, 2, 4, 5, 10])) a.yaxis.set_major_locator(MaxNLocator(4, steps=[1, 2, 4, 5, 10])) data, x, y = hists[(i, j)] data = np.clip(data, vmin, vmax) a.pcolorfast(y, x, data, cmap=COLORMAP, norm=norm) # Show labels or hide ticks if i != 0: artist.setp(a.get_xticklabels(), visible=False) if i == n - 2 and j == n - 1: a.text(0.5, 1.02, labels[j], va="bottom", ha="center", transform=a.transAxes) # a.set_xlabel(labels[j]) # a.xaxis.set_label_position("top") # a.xaxis.set_offset_position("top") if not show_ticks: a.xaxis.set_ticks([]) if j == i + 1: a.set_ylabel(labels[i]) else: artist.setp(a.get_yticklabels(), visible=False) if not show_ticks: a.yaxis.set_ticks([]) a.zoomable = True # Adjust subplots and add the colorbar fig.subplots_adjust(left=0.07, bottom=0.07, top=0.9, right=0.85, wspace=0.0, hspace=0.0) cax = fig.add_axes([0.88, 0.2, 0.04, 0.6]) fig.colorbar(mapper, cax=cax, orientation="vertical") return ax def zoom(event, step): ax = event.inaxes if not hasattr(ax, "zoomable"): return # TODO: test logscale step *= 3 if ax.zoomable is not True and "mapper" in ax.zoomable: mapper = ax.zoomable["mapper"] if event.ydata is not None: lo, hi = mapper.get_clim() pt = event.ydata * (hi - lo) + lo lo, hi = _rescale(lo, hi, pt, step) mapper.set_clim((lo, hi)) if ax.zoomable is True and event.xdata is not None: lo, hi = ax.get_xlim() lo, hi = _rescale(lo, hi, event.xdata, step) ax.set_xlim((lo, hi)) if ax.zoomable is True and event.ydata is not None: lo, hi = ax.get_ylim() lo, hi = _rescale(lo, hi, event.ydata, step) ax.set_ylim((lo, hi)) ax.figure.canvas.draw_idle() def _rescale(lo, hi, pt, step): scale = float(hi - lo) * step / (100 if step > 0 else 100 - step) bal = float(pt - lo) / (hi - lo) new_lo = lo - bal * scale new_hi = hi + (1 - bal) * scale return new_lo, new_hi bumps-bumps-2b313fd/bumps/dream/crossover.py000066400000000000000000000147551521754644000212330ustar00rootroot00000000000000""" Crossover ratios The crossover ratio (CR) determines what percentage of parameters in the target vector are updated with difference vector selected from the population. In traditional differential evolution a CR value is chosen somewhere in [0, 1] at the start of the search and stays constant throughout. DREAM extends this by allowing multiple CRs at the same time with different probabilities. Adaptive crossover adjusts the relative weights of the CRs based on the average distance of the steps taken when that CR was used. This distance will be zero for unsuccessful metropolis steps, and so the relative weights on those CRs which generate many unsuccessful steps will be reduced. Usage ----- 1. Traditional differential evolution:: crossover = Crossover(CR=CR) 2. Weighted crossover ratios:: crossover = Crossover(CR=[CR1, CR2, ...], weight=[weight1, weight2, ...]) The weights are normalized to one, and default to equally weighted CRs. 3. Adaptive weighted crossover ratios:: crossover = AdaptiveCrossover(N) The CRs are set to *[1/N, 2/N, ... 1]*, and start out equally weighted. The weights are adapted during burn-in (10% of the runs) and fixed for the remainder of the analysis. Compatibility Notes ------------------- For *Extra.pCR == 'Update'* in the matlab interface use:: CR = AdaptiveCrossover(Ncr=MCMCPar.nCR) For *Extra.pCR != 'Update'* in the matlab interface use:: CR = Crossover(CR=[1./Ncr], pCR=[1]) """ __all__ = [ "Crossover", "BaseAdaptiveCrossover", "AdaptiveCrossover", "LogAdaptiveCrossover", ] import numpy as np from numpy import ones, zeros, arange, isscalar, std, trunc, log10, logspace from numpy.typing import NDArray class Crossover(object): """ Fixed weight crossover ratios. *CR* is a scalar if there is a single crossover ratio, or a vector of numbers in (0, 1]. *weight* is the relative weighting of each CR, or None for equal weights. """ CR: NDArray weight: NDArray def __init__(self, CR: NDArray, weight: NDArray): if isscalar(CR): CR, weight = [CR], [1] CR, weight = [np.asarray(v, "d") for v in (CR, weight)] self.CR, self.weight = CR, weight / np.sum(weight) def reset(self): pass def update(self, xold, xnew, used): """ Gather adaptation data on *xold*, *xnew* for each CR that was *used* in step *N*. """ pass def adapt(self): """ Update CR weights based on the available adaptation data. """ pass class BaseAdaptiveCrossover(object): """ Adapted weight crossover ratios. """ weight: NDArray = None _count: NDArray = None _distance: NDArray = None _generations: int = 0 _Nchains: int = 0 def _set_CRs(self, CR): self.CR = CR # Start with all CRs equally probable self.weight = ones(len(self.CR)) / len(self.CR) # No initial statistics for adaptation self._count = zeros(len(self.CR)) self._distance = zeros(len(self.CR)) self._generations = 0 self._Nchains = 0 def reset(self): # TODO: do we reset count and distance? pass def update(self, xold, xnew, used): """ Gather adaptation data on *xold*, *xnew* for each CR that was *used* in step *N*. """ # Calculate the standard deviation of each dimension of X r = std(xnew, ddof=1, axis=0) # Compute the Euclidean distance between new X and old X d = np.sum(((xold - xnew) / r) ** 2, axis=1) # Use this information to update sum_p2 to update N_CR count, total = distance_per_CR(self.CR, d, used) self._count += count self._distance += total self._generations += 1 # [PAK] Accumulate number of steps so counts can be normalized self._Nchains += len(used) def adapt(self): """ Update CR weights based on the available adaptation data. """ # [PAK] Protect against completely stuck fits (no accepted updates # [PAK] and therefore a total distance of zero) by reusing the # [PAK] the existing weights. total_distance = np.sum(self._distance) if total_distance > 0 and self._Nchains > 0: # [PAK] Make sure no count is zero by adding one to all counts self.weight = self._distance / (self._count + 1) self.weight *= self._Nchains / total_distance # [PAK] Adjust weights toward uniform; this guarantees nonzero weights. self.weight += 0.1 * np.sum(self.weight) self.weight /= np.sum(self.weight) class AdaptiveCrossover(BaseAdaptiveCrossover): """ Adapted weight crossover ratios. *N* is the number of CRs to use. CR is set to [1/N, 2/N, ..., 1], with initial weights [1/N, 1/N, ..., 1/N]. """ def __init__(self, N): if N < 2: raise ValueError("Need more than one CR for AdaptiveCrossover") self._set_CRs((arange(N) + 1) / N) # Equally spaced CRs # [PAK] Add log spaced adaptive cross-over for high dimensional tightly # constrained problems. class LogAdaptiveCrossover(BaseAdaptiveCrossover): """ Adapted weight crossover ratios, log-spaced. *dim* is the number of dimensions in the problem. *N* is the number of CRs to use per decade. CR is set to [k/dim] where k is log-spaced from 1 to dim. The CRs start equally weighted as [1, ..., 1]/len(CR). *N* should be around 4.5. This gives good low end density, with 1, 2, 3, and 5 parameters changed at a time, and proceeds up to 60% and 100% of parameters each time. Lower values of *N* give too few high density CRs, and higher values give too many low density CRs. """ def __init__(self, dim, N=4.5): # Log spaced CR from 1/dim to dim/dim self._set_CRs(logspace(0, log10(dim), trunc(N * log10(dim) + 1)) / dim) def distance_per_CR(available_CRs: NDArray, distances: NDArray, used: NDArray): """ Accumulate normalized Euclidean distance for each crossover value Returns the number of times each available CR was used and the total distance for that CR. """ # TODO: could use sparse array trick to evaluate totals by CR # Set distance[k] to coordinate (k, used[k]), then sum by columns # Note: currently setting unused CRs to -1, so this won't work total = np.asarray([np.sum(distances[used == p]) for p in available_CRs]) count = np.asarray([np.sum(used == p) for p in available_CRs]) return count, total bumps-bumps-2b313fd/bumps/dream/diffev.py000066400000000000000000000245151521754644000204440ustar00rootroot00000000000000""" Differential evolution MCMC stepper. """ __all__ = ["de_step"] from numpy import zeros, ones, empty, sqrt, sum from numpy import where, array from numpy.linalg import norm from .util import draw, rng try: # raise ImportError("skip numpy") from numba import njit, prange @njit(cache=True) def pchoice(choices, size=0, replace=True, p=None): if p is None: return rng.choice(choices, size=size, replace=replace) # TODO: if choices is an array, then shape should be an array result = empty(size, dtype=choices.dtype) num_choices = choices.shape[0] for index in prange(size): u = rng.rand() cdf = 0.0 for k in range(num_choices): cdf += p[k] if u <= cdf: result[index] = choices[k] # else: should never get here if choices sum to 1 return result except ImportError: def njit(*args, **kw): return lambda f: f prange = range pchoice = rng.choice EPS = 1e-6 _DE, _SNOOKER, _DIRECT = 0, 1, 2 @njit(cache=True) def de_step(Nchain, pop, CR, max_pairs=2, eps=0.05, snooker_rate=0.1, noise=1e-6, scale=1.0): """ Generates offspring using METROPOLIS HASTINGS monte-carlo markov chain *Nchain* is the number of simultaneous changes that are running. *pop* is an array of shape [Npop x Nvar] providing the active points used to generate the next proposal for each chain. This may be larger than the Nchains if the caller is using ancestor generations for active population. The current population is assumed to be the first *Nchain* rows of *pip*.. *CR* is an array of [Ncrossover x 2] crossover ratios with weights. The crossover ratio is the probability of selecting a particular dimension when generating the difference vector. The weights are used to select the crossover ratio. The weights are adjusted dynamically during the fit based on the acceptance rate of points generated with each crossover ratio. *max_pairs* determines the maximum number of pairs which contribute to the differential evolution step. The number of pairs is chosen at random, with the difference vectors between the pairs averaged when creating the DE step. *eps* determines the jitter added to the DE step. *snooker_rate* determines the probability of using the snooker stepper. Otherwise use DE stepper 80% of the time, or apply the difference between pairs the other 20% of the time. *scale=1* scales the difference vector (constant, not stochastic) *noise=1e-6* adds random noise to the non-zero components of the difference vector. This noise is relative rather than absolute to allow for parameter values far from 1.0. Noise is also scaled by *scale*. """ Npop, Nvar = pop.shape # Initialize the delta update to zero delta_x = zeros((Nchain, Nvar)) step_alpha = ones(Nchain) # Choose snooker, de or direct according` to snooker_rate, and 80:20 # ratio of de to direct. u = rng.rand(Nchain) de_rate = 0.8 * (1 - snooker_rate) alg = zeros(Nchain, dtype="int") # _DE = 0 alg[u >= de_rate] = _SNOOKER alg[u >= de_rate + snooker_rate] = _DIRECT # alg = select([u < snooker_rate, u < snooker_rate+de_rate], # [_SNOOKER, _DE], default=_DIRECT) # [PAK] CR selection moved from crossover into DE step CR_used = pchoice(CR[:, 0], size=Nchain, replace=True, p=CR[:, 1]) # Chains evolve using information from other chains to create offspring for qq in prange(Nchain): if alg[qq] == _DE: # Use DE with cross-over ratio # Select to number of vector pair differences to use in update # using k ~ discrete U[1, max pairs] k = rng.randint(max_pairs) + 1 # [PAK: same as k = DEversion[qq, 1] in matlab version] # TODO: make sure we don't fail if too few chains. # Select 2*k members at random different from the current member perm = draw(2 * k, Npop - 1) # TODO: rewrite draw so that it accepts a not_matching int perm[perm >= qq] += 1 r1, r2 = perm[:k], perm[k : 2 * k] # Select the dims to update based on the crossover ratio, making # sure at least one dim is selected vars = where(rng.rand(Nvar) <= CR_used[qq])[0] if len(vars) == 0: vars = array([rng.randint(Nvar)]) # print("for chain", qq, CR_used[qq], "% update", vars) # Weight the size of the jump inversely proportional to the # number of contributions, both from the parameters being # updated and from the population defining the step direction. gamma = 2.38 / sqrt(2 * len(vars) * k) # [PAK: same as F=Table_JumpRate[len(vars), k] in matlab version] # Find and average step from the selected pairs step = sum(pop[r1] - pop[r2], axis=0) # Apply that step with F scaling and noise jiggle = 1 + eps * (2 * rng.rand(*step.shape) - 1) delta_x[qq, vars] = (jiggle * gamma * step)[vars] elif alg[qq] == _SNOOKER: # Use snooker update # Select current and three others perm = draw(3, Npop - 1) perm[perm >= qq] += 1 xi = pop[qq] z, R1, R2 = [pop[i] for i in perm] # Find the step direction and scale it to the length of the # projection of R1-R2 onto the step direction. # TODO: population sometimes not unique! step = xi - z denom = sum(step**2) if denom == 0: # identical points; should be extremely rare step = EPS * rng.randn(*step.shape) denom = sum(step**2) step_scale = sum((R1 - R2) * step) / denom # Step using gamma of 2.38/sqrt(2) + U(-0.5, 0.5) gamma = 1.2 + rng.rand() delta_x[qq] = gamma * step_scale * step # Scale Metropolis probability by (||xi* - z||/||xi - z||)^(d-1) step_alpha[qq] = (norm(delta_x[qq] + step) / norm(step)) ** ((Nvar - 1) / 2) CR_used[qq] = 0.0 elif alg[qq] == _DIRECT: # Use one pair and all dimensions # Note that there is no F scaling, dimension selection or noise perm = draw(2, Npop - 1) perm[perm >= qq] += 1 delta_x[qq, :] = pop[perm[0], :] - pop[perm[1], :] CR_used[qq] = 0.0 else: raise RuntimeError("Select failed...should never happen") # Didn't implement this in the compiled version (yet) ## If no step was specified (exceedingly unlikely!), then ## select a delta at random from a gaussian approximation to the ## current population # if all(delta_x[qq] == 0): # from numpy.linalg import cholesky, LinAlgError # try: # #print "No step" # # Compute the Cholesky Decomposition of x_old # R = (2.38/sqrt(Nvar)) * cholesky(cov(pop.T) + EPS*eye(Nvar)) # # Generate jump using multinormal distribution # delta_x[qq] = dot(rng.randn(*(1, Nvar)), R) # except LinAlgError: # print("Bad cholesky") # delta_x[qq] = rng.randn(Nvar) # Update x_old with delta_x and noise delta_x *= scale # print("alg", alg) # print("CR_used", CR_used) # print("delta_x", delta_x) # [PAK] The noise term needs to depend on the fitting range # of the parameter rather than using a fixed noise value for all # parameters. The current parameter value is a pretty good proxy # in most cases (i.e., relative noise), but it breaks down if the # parameter is zero, or if the range is something like 1 +/- eps. # absolute noise # x_new = pop[:Nchain] + delta_x + scale*noise*rng.randn(Nchain, Nvar) # relative noise x_new = pop[:Nchain] * (1 + scale * noise * rng.randn(Nchain, Nvar)) + delta_x # no noise # x_new = pop[:Nchain] + delta_x return x_new, step_alpha, CR_used def _check(): from numpy import arange, vstack, ascontiguousarray max_pairs, snooker_rate, eps, noise, scale = 2, 0.1, 0.05, 1e-6, 1.0 nchain, npop, nvar, ncr = 4, 10, 7, 4 pop = 100 * arange(npop * nvar, dtype="d").reshape((npop, nvar)) pop = pop * (1 + rng.rand(*pop.shape) * 0.1) ratios = 1.0 / (rng.randint(4, size=ncr) + 1) # 4 => ratios in [0.2, 0.25, 0.333, 0.5, 1.0] weights = [1 / ncr] * ncr # equal-weight for each CR # print(ratios, weights) CR = ascontiguousarray(vstack((ratios, weights)).T, dtype="d") work = lambda: de_step(nchain, pop, CR, max_pairs=max_pairs, eps=eps) x_new, _step_alpha, used = work() # print(f"{pop=}") # print(f"{x_new=}") # print(f"{_step_alpha=}") # print(f"{used=}") print("""\ The following table shows the expected portion of the dimensions that are changed and the rounded value of the change for each point in the population. """) for k, u in enumerate(used): rstr = f"{int(u * 100):4d}% " if u else " full " vstr = " ".join(f"{v:.2f}" for v in x_new[k] - pop[k]) print(rstr + vstr) if 1: # timing check from timeit import timeit from ctypes import c_double from .compiled import dll dll_work = lambda: dll.de_step( nchain, nvar, len(CR), pop.ctypes, CR.ctypes, max_pairs, c_double(eps), c_double(snooker_rate), c_double(noise), c_double(scale), x_new.ctypes, _step_alpha.ctypes, used.ctypes, ) print("small pop time (ms)", timeit(work, number=10000) / 10) if dll: print("small pop time compiled (ms)", timeit(dll_work, number=10000) / 10) else: print("no dlls") nchain, nvar = 1000, 50 npop = nchain pop = 100 * arange(npop * nvar, dtype="d").reshape((npop, nvar)) pop = pop * (1 + rng.rand(*pop.shape) * 0.1) print("large pop time (ms)", timeit(work, number=100)) if dll: x_new, _step_alpha, used = work() # need this line to define new return vectors for dll print("large pop time compiled (ms)", timeit(dll_work, number=100)) if __name__ == "__main__": _check() bumps-bumps-2b313fd/bumps/dream/digits.py000066400000000000000000000150231521754644000204560ustar00rootroot00000000000000""" Significant digits ------------------ Calculate the number significant digits in a statistic using boot strapping/jackknife. The 'gum' method from the GUM supplement[1] splits the sequence into n independent sequences, computes the statistic for each sequence, and returns the mean and variance of the statistic as the estimated value with uncertainty. Each block should have at least 10000 samples, or 100/(1-ci) samples if the desired credible interval ci > 0.99. The 'jack' method uses notes: - the gum method gives a slight underestimate of the sd, while jackknife gives an overestimate. - Jackknife performs better than gum on small samples - in current implementation, gum is much faster - in picking the number of cuts for jackknife, lower is usually better (but not too low...) [1] JCGM, 2008. Evaluation of measurement data - Supplement 1 to the "Guide to the expression of uncertainty in measurement" - Propagation of distributions using a Monte Carlo method (No. JCGM 101:2008). Geneva, Switzerland. """ """ THINGS TO DO: - actually useful display of information in output - an effective, fast, online version to use as a stoping crit - use as a method to guess how many more samples: - provides estimate of sd of sampling distribution of statistic - sd of stat for n samples = sd for k samples / sqrt(n/k - 1) - not actually well tested... - provide better user accessible control hooks - do better deciding between jack and gum - these don't deal with lists of CIs (as the actual ci functions do) - very slow, needs a restructure """ # TODO: needs a refactor before going into production import numpy as np import numpy.ma as ma from .stats import credible_interval def credible_interval_sd(data, ci, fn=None, method=None, cuts=None): if fn is None: # ugly hack here fn = lambda *a, **kw: credible_interval(*a, **kw)[0] if method is None: # do something clever to choose gum or jack method = "gum" # Determine the number of cuts if cuts is None or cuts <= 0: # Use groups of 10000 unless looking for ci > 0.99, in which case # use groups of 100/(1-ci). M = max(int(100 / (1 - ci)), 10000) cuts = len(data) // M if cuts < 3: cuts = 3 if method == "gum": return gum_sd(data, fn, ci) elif method == "jack": return jack_sd(data, fn, ci) else: raise ValueError("Unknown sd method" + method) # TODO does not handle either CI function natively # requires a 1d ndarray return value def gum_sd(data, f, ci, cuts=10): """ adaptation of method in section 7.9.4 of gum suppliment 1 """ # reshape data into blocks, skipping the first partial block M = len(data) // cuts data = data[-M * cuts :].reshape((cuts, M)) # compute the variance of the statistic over the sub-blocks stat = np.apply_along_axis(f, 1, data, ci) var = np.var(stat, axis=0, ddof=1) # estimate standard deviation of the statistic for the full data set return np.sqrt(var / cuts) def jack_sd(data, f, ci, cuts=10): """ does an average of sd on smaller blocks to counteract skewed distribution of jackknife """ # reshape data into blocks, skipping the first partial block M = len(data) // cuts data = data[-M * cuts :].reshape((cuts, M)) # compute the standard deviation of the statistic over the sub-blocks std = np.apply_along_axis(fast_jack, 1, data, f, ci) # estimate standard deviation of the statistic for the full data set return np.mean(std, axis=0) / np.sqrt(cuts) # TODO: does not deal well with a list of CIs # returns the sd of a statistic, f def fast_jack(data, f, *a, **kw): jack = FastJackknife(data=data, fn=f) jack.set_args(*a, **kw) return jack.std() # TODO: this is coded badly and so is very slow class FastJackknife(object): def __init__(self, **kw): self.fn = None self._x = None for k, v in kw.items(): try: setattr(self, k, v) except AttributeError: pass @property def data(self): return self._x @data.setter def data(self, value): self._x = np.sort(value) self.n = len(value) @property def fn(self): if self._fn is not None: return lambda x: self._fn(x, *self._args, **self._kwargs) else: return None @fn.setter def fn(self, f): self._fn = f def set_args(self, *a, **kw): self._args = a self._kwargs = kw def _f_drop(self, i): from numpy import zeros_like mask = zeros_like(self.data) mask[i] = 1 mx = ma.array(self.data, mask=mask) return np.array(self.fn(mx.compressed())) def pooled(self): assert self.data is not None and self.fn is not None return self.fn(self.data) def std(self): from numpy import sqrt, var, empty, searchsorted true_stat = self.fn(self.data) drop_stats = empty((self.n, len(true_stat))) # this part is specific for CI idx_lo, idx_hi = searchsorted(self.data, true_stat) self._fill_interval(drop_stats, (0, idx_lo), true_stat) self._fill_interval(drop_stats, (idx_lo, idx_hi), true_stat) self._fill_interval(drop_stats, (idx_hi, self.n - 1), true_stat) return sqrt((self.n - 1) * var(drop_stats, axis=0, ddof=1)) def _fill_interval(self, drop_stats, interval, ev, splits=None): lo, hi = interval v_hi = self._f_drop(hi) v_lo = self._f_drop(lo) a = np.array([v_lo, v_hi, ev]).T g = lambda ar: (ar[0] == ar).all() if np.apply_along_axis(g, 1, a).all(): for i in range(lo, hi + 1): drop_stats[i] = v_hi elif hi == lo: drop_stats[hi] = v_hi elif hi == lo + 1: drop_stats[hi] = v_hi drop_stats[lo] = v_lo else: splits = max(int(np.ceil(np.log10(hi - lo))), 2) dx = (hi - lo) // splits intervals = [[lo + dx * i, lo + dx * (i + 1)] for i in range(splits)] intervals[-1][1] = hi for inter in intervals: self._fill_interval(drop_stats, inter, np.mean((v_lo, v_hi), axis=0)) def n_dig(sx, sd): """ Converts standard deviation to # of significand digits. *sx* is standard deviation of the statistic *sd* is standard deviation of the sample Adapted from section 7.9.2 of gum suppliment 1 """ l = np.log10(4 * sx) return np.floor(np.log10(sd)) - l + 1 # return as float or round now? bumps-bumps-2b313fd/bumps/dream/entropy.py000066400000000000000000001324071521754644000207010ustar00rootroot00000000000000r""" Estimate entropy after a fit. The :func:`gmm_entropy` function computes the entropy from a Gaussian mixture model. This provides a reasonable estimate even for non-Gaussian distributions. This is the recommended method for estimating the entropy of a sample. The :func:`cov_entropy` method computes the entropy associated with the covariance matrix. This covariance matrix can be estimated during the fitting procedure (BFGS updates an estimate of the Hessian matrix for example), or computed by estimating derivatives when the fit is complete. The :class:`MVNEntropy` class estimates the covariance from an MCMC sample and uses this covariance to estimate the entropy. This gives a better estimate of the entropy than the equivalent direct calculation, which requires many more samples for a good kernel density estimate. The *reject_normal* attribute is *True* if the MCMC sample is significantly different from normal. Unfortunately, this almost always the case for any reasonable sample size that isn't strictly gaussian. The :func:`entropy` function computes the entropy directly from a set of MCMC samples, normalized by a scale factor computed from the kernel density estimate at a subset of the points.\ [#Kramer]_ There are many other entropy calculations implemented within this file,\ [#Turjillo-Ortiz]_\ [#Mardia1970]_\ [#Mardia1974]_\ [#Stevens]_ as well as a number of sampling distributions for which the true entropy is known. Furthermore, entropy was computed against dream output and checked for consistency. None of the methods is truly excellent in terms of minimum sample size, maximum dimensions and speed, but many of them are pretty good. The following is an informal summary of the results from different algorithms applied to DREAM output:: from .entropy import Timer as T # Try MVN ... only good for normal distributions, but very fast with T(): M = entropy.MVNEntropy(drawn.points) print("Entropy from MVN: %s"%str(M)) # Try wnn ... no good. with T(): S_wnn, Serr_wnn = entropy.wnn_entropy(drawn.points, n_est=20000) print("Entropy from wnn: %s"%str(S_wnn)) # Try wnn with bootstrap ... still no good. with T(): S_wnn, Serr_wnn = entropy.wnn_bootstrap(drawn.points) print("Entropy from wnn bootstrap: %s"%str(S_wnn)) # Try wnn entropy with thinning ... still no good. #drawn = self.draw(portion=portion, vars=vars, # selection=selection, thin=10) with T(): S_wnn, Serr_wnn = entropy.wnn_entropy(points) print("Entropy from wnn: %s"%str(S_wnn)) # Try wnn with gmm ... still no good with T(): S_wnn, Serr_wnn = entropy.wnn_entropy(drawn.points, n_est=20000, gmm=20) print("Entropy from wnn with gmm: %s"%str(S_wnn)) # Try pure gmm ... pretty good with T(): S_gmm, Serr_gmm = entropy.gmm_entropy(drawn.points, n_est=10000) print("Entropy from gmm: %s"%str(S_gmm)) # Try kde from statsmodels ... pretty good with T(): S_kde_stats = entropy.kde_entropy_statsmodels(drawn.points, n_est=10000) print("Entropy from kde statsmodels: %s"%str(S_kde_stats)) # Try kde from sklearn ... pretty good with T(): S_kde = entropy.kde_entropy_sklearn(drawn.points, n_est=10000) print("Entropy from kde sklearn: %s"%str(S_kde)) # Try kde from sklearn at points from gmm ... pretty good with T(): S_kde_gmm = entropy.kde_entropy_sklearn_gmm(drawn.points, n_est=10000) print("Entropy from kde+gmm: %s"%str(S_kde_gmm)) # Try Kramer ... pretty good, but doesn't support marginal entropy with T(): S, Serr = entropy.entropy(drawn.points, drawn.logp, N_entropy=n_est) print("Entropy from Kramer: %s"%str(S)) .. [#Kramer] Kramer, A., Hasenauer, J., Allgower, F., Radde, N., 2010. Computation of the posterior entropy in a Bayesian framework for parameter estimation in biological networks, in: 2010 IEEE International Conference on Control Applications (CCA). Presented at the 2010 IEEE International Conference on Control Applications (CCA), pp. 493-498. doi:10.1109/CCA.2010.5611198 .. [#Turjillo-Ortiz] Trujillo-Ortiz, A. and R. Hernandez-Walls. (2003). Mskekur: Mardia's multivariate skewness and kurtosis coefficients and its hypotheses testing. A MATLAB file. [WWW document]. ``_ .. [#Mardia1970] Mardia, K. V. (1970), Measures of multivariate skewnees and kurtosis with applications. Biometrika, 57(3):519-530. .. [#Mardia1974] Mardia, K. V. (1974), Applications of some measures of multivariate skewness and kurtosis for testing normality and robustness studies. Sankhy A, 36:115-128 .. [#Stevens] Stevens, J. (1992), Applied Multivariate Statistics for Social Sciences. 2nd. ed. New-Jersey:Lawrance Erlbaum Associates Publishers. pp. 247-248. """ __all__ = ["entropy", "gmm_entropy", "cov_entropy", "wnn_entropy", "MVNEntropy"] import numpy as np from numpy import e, exp, log, log2, mean, nan, pi, sqrt, std from numpy.random import choice, permutation from scipy import stats from scipy.special import digamma, gammaln from scipy.stats import chi2, norm LN2 = log(2) def standardize(x): """ Standardize the points by removing the mean and scaling by the standard deviation. """ # TODO: check if it is better to multiply by inverse covariance # That would serve to unrotate and unscale the dimensions together, # but squishing them down individually might be just as good. # compute zscores for the each variable independently mu, sigma = mean(x, axis=0), std(x, axis=0, ddof=1) # Protect against NaN when sigma is zero. If sigma is zero # then all points are equal, so x == mu and z-score is zero. return (x - mu) / (sigma + (sigma == 0.0)), mu, sigma def kde_entropy_statsmodels(points, n_est=None): """ Use statsmodels KDEMultivariate pdf to estimate entropy. Density evaluated at sample points. Slow and fails for bimodal, dirichlet; poor for high dimensional MVN. """ from statsmodels.nonparametric.kernel_density import KDEMultivariate n, d = points.shape # Default to the full set if n_est is None: n_est = n # reduce size of draw to n_est if n_est >= n: x = points else: x = points[permutation(n)[:n_est]] n = n_est predictor = KDEMultivariate(data=x, var_type="c" * d) p = predictor.pdf() H = -np.mean(log(p)) return H / LN2 def kde_entropy_sklearn(points, n_est=None): """ Use sklearn.neigbors.KernelDensity pdf to estimate entropy. Data is standardized before analysis. Sample points drawn from the kernel density estimate. Fails for bimodal and dirichlet, similar to statsmodels kde. """ n, d = points.shape # Default to the full set if n_est is None: n_est = n # reduce size of draw to n_est if n_est >= n: x = points else: x = points[permutation(n)[:n_est]] n = n_est # logp = sklearn_log_density(points, evaluation_points=n_est) logp = sklearn_log_density(x, evaluation_points=x) H = -np.mean(logp) return H / LN2 def kde_entropy_sklearn_gmm(points, n_est=None, n_components=None): """ Use sklearn.neigbors.KernelDensity pdf to estimate entropy. Data is standardized before kde. Sample points drawn from gaussian mixture model from original points. Fails for bimodal and dirichlet, similar to statsmodels kde. """ from sklearn.mixture import BayesianGaussianMixture as GMM n, d = points.shape # Default to the full set if n_est is None: n_est = n # reduce size of draw to n_est if n_est >= n: x = points else: x = points[permutation(n)[:n_est]] n = n_est if n_components is None: n_components = int(5 * sqrt(d)) predictor = GMM( n_components=n_components, covariance_type="full", # verbose=True, max_iter=1000, ) predictor.fit(x) evaluation_points, _ = predictor.sample(n_est) logp = sklearn_log_density(x, evaluation_points=evaluation_points) H = -np.mean(logp) return H / LN2 def gmm_entropy(points, n_est=None, n_components=None): r""" Use sklearn.mixture.BayesianGaussianMixture to estimate entropy. *points* are the data points in the sample. *n_est* are the number of points to use in the estimation; default is 10,000 points, or 0 for all the points. *n_components* are the number of Gaussians in the mixture. Default is $5 \sqrt{d}$ where $d$ is the number of dimensions. Returns estimated entropy and uncertainty in the estimate. This method uses BayesianGaussianMixture from scikit-learn to build a model of the point distribution, then uses Monte Carlo sampling to determine the entropy of that distribution. The entropy uncertainty is computed from the variance in the MC sample scaled by the number of samples. This does not incorporate any uncertainty in the sampling that generated the point distribution or the uncertainty in the GMM used to model that distribution. """ # from sklearn.mixture import GaussianMixture as GMM from sklearn.mixture import BayesianGaussianMixture as GMM n, d = points.shape # Default to the full set if n_est is None: n_est = 10000 elif n_est == 0: n_est = n # reduce size of draw to n_est if n_est >= n: x = points n_est = n else: x = points[permutation(n)[:n_est]] n = n_est if n_components is None: n_components = int(5 * sqrt(d)) ## Standardization doesn't seem to help ## Note: sigma may be zero # x, mu, sigma = standardize(x) # if standardized predictor = GMM( n_components=n_components, covariance_type="full", # verbose=True, max_iter=1000, ) predictor.fit(x) eval_x, _ = predictor.sample(n_est) weight_x = predictor.score_samples(eval_x) H = -np.mean(weight_x) # with np.errstate(divide='ignore'): H = H + np.sum(np.log(sigma)) # if standardized dH = np.std(weight_x, ddof=1) / sqrt(n) ## cross-check against own calcs # alt = GaussianMixture(predictor.weights_, mu=predictor.means_, sigma=predictor.covariances_) # print("alt", H, alt.entropy()) # print(np.vstack((weight_x[:10], alt.logpdf(eval_x[:10]))).T) return H / LN2, dH / LN2 def wnn_bootstrap(points, k=None, weights=True, n_est=None, reps=10, parts=10): # raise NotImplementedError("deprecated; bootstrap doesn't help.") n, d = points.shape if n_est is None: n_est = n // parts results = [wnn_entropy(points, k=k, weights=weights, n_est=n_est) for _ in range(reps)] # print(results) S, Serr = list(zip(*results)) return np.mean(S), np.std(S) def wnn_entropy(points, k=None, weights=True, n_est=None, gmm=None): r""" Weighted Kozachenko-Leonenko nearest-neighbour entropy calculation. *k* is the number of neighbours to consider, with default $k=n^{1/3}$ *n_est* is the number of points to use for estimating the entropy, with default $n_\rm{est} = n$ *weights* is True for default weights, False for unweighted (using the distance to the kth neighbour only), or a vector of weights of length *k*. *gmm* is the number of gaussians to use to model the distribution using a gaussian mixture model. Default is 0, and the points represent an empirical distribution. Returns entropy H in bits and its uncertainty. Berrett, T. B., Samworth, R.J., Yuan, M., 2016. Efficient multivariate entropy estimation via k-nearest neighbour distances. DOI:10.1214/18-AOS1688 https://arxiv.org/abs/1606.00304 """ from sklearn.neighbors import NearestNeighbors n, d = points.shape # Default to the full set if n_est is None: n_est = 10000 elif n_est == 0: n_est = n # reduce size of draw to n_est if n_est >= n: x = points n_est = n else: x = points[permutation(n)[:n_est]] n = n_est # Default k based on n if k is None: # Private communication: cube root of n is a good choice for k # Personal observation: k should be much bigger than d k = max(int(n ** (1 / 3)), 3 * d) # If weights are given then use them (setting the appropriate k), # otherwise use the default weights. if isinstance(weights, bool): weights = _wnn_weights(k, d, weights) else: k = len(weights) # print("weights", weights, sum(weights)) # select knn algorithm algorithm = "auto" # algorithm = 'kd_tree' # algorithm = 'ball_tree' # algorithm = 'brute' n_components = 0 if gmm is None else gmm # H = 1/n sum_i=1^n sum_j=1^k w_j log E_{j,i} # E_{j,i} = e^-Psi(j) V_d (n-1) z_{j,i}^d = C z^d # logC = -Psi(j) + log(V_d) + log(n-1) # H = 1/n sum sum w_j logC + d/n sum sum w_j log(z) # = sum w_j logC + d/n sum sum w_j log(z) # = A + d/n B # H^2 = 1/n sum Psi = digamma(np.arange(1, k + 1)) logVd = d / 2 * log(pi) - gammaln(1 + d / 2) logC = -Psi + logVd + log(n - 1) # TODO: standardizing points doesn't work. # Standardize the data so that distances conform. This is equivalent to # a u-substitution u = sigma x + mu, so the integral needs to be corrected # for dU = det(sigma) dx. Since the standardization squishes the dimensions # independently, sigma is a diagonal matrix, with the determinant equal to # the product of the diagonal elements. # x, mu, sigma = standardize(x) # Note: sigma may be zero # detDU = np.prod(sigma) detDU = 1.0 if n_components > 0: # Use Gaussian mixture to model the distribution from sklearn.mixture import GaussianMixture as GMM predictor = GMM(n_components=gmm, covariance_type="full") predictor.fit(x) eval_x, _ = predictor.sample(n_est) # weight_x = predictor.score_samples(eval_x) skip = 0 else: # Empirical distribution # TODO: should we use the full draw for kNN and a subset for eval points? # Choose a subset for evaluating the entropy estimate, if desired # print(n_est, n) # eval_x = x if n_est >= n else x[permutation(n)[:n_est]] eval_x = x # weight_x = 1 skip = 1 tree = NearestNeighbors(algorithm=algorithm, n_neighbors=k + skip) tree.fit(x) dist, _ind = tree.kneighbors(eval_x, n_neighbors=k + skip, return_distance=True) # Remove first column. Since test points are in x, the first column will # be a point from x with distance 0, and can be ignored. if skip: dist = dist[:, skip:] # Find log distances. This can be problematic for MCMC runs where a # step is rejected, and therefore identical points are in the distribution. # Ignore them by replacing these points with nan and using nanmean. # TODO: need proper analysis of duplicated points in MCMC chain dist[dist == 0] = nan logdist = log(dist) H_unweighted = logC + d * np.nanmean(logdist, axis=0) H = np.dot(H_unweighted, weights)[0] Hsq_k = np.nanmean((logC[-1] + d * logdist[:, -1]) ** 2) # TODO: abs shouldn't be needed? if Hsq_k < H**2: print("warning: avg(H^2) < avg(H)^2") dH = sqrt(abs(Hsq_k - H**2) / n_est) # print("unweighted", H_unweighted) # print("weighted", H, Hsq_k, H**2, dH, detDU, LN2) return H * detDU / LN2, dH * detDU / LN2 def _wnn_weights(k, d, weighted=True): # Private communication: ignore w_j = 0 constraints (they are in the # paper for mathematical nicety), and find the L2 norm of the # remaining underdeterimined system described in Eq 2. # Personal observation: k should be some small multiple of d # otherwise the weights blow up. if d < 4 or not weighted: # with few dimensions go unweighted with the kth nearest neighbour. return np.array([[0.0] * (k - 1) + [1.0]]).T j = np.arange(1, k + 1) sum_zero = [exp(gammaln(j + 2 * i / d) - gammaln(j)) for i in range(1, d // 4 + 1)] sum_one = [[1.0] * k] A = np.array(sum_zero + sum_one) b = np.array([[0.0] * (d // 4) + [1.0]]).T return np.dot(np.linalg.pinv(A), b) def scipy_stats_density(sample_points, evaluation_points): # pragma: no cover """ Estimate the probability density function from which a set of sample points was drawn and return the estimated density at the evaluation points. """ ## standardize data so that we can use uniform bandwidth ## Note: this didn't help with singular matrix ## Note: if re-enable, protect against sigma=0 in some dimensions # mu, sigma = mean(data, axis=0), std(data, axis=0) # data,points = (data - mu)/sigma, (points - mu)/sigma kde = stats.gaussian_kde(sample_points) return kde(evaluation_points) def sklearn_log_density(sample_points, evaluation_points): """ Estimate the log probability density function from which a set of sample points was drawn and return the estimated density at the evaluation points. *sample_points* is an [n x m] matrix. *evaluation_points* is the set of points at which to evaluate the kde. Note: if any dimension has all points equal then the entire distribution is treated as a dirac distribution with infinite density at each point. This makes the entropy calculation better behaved (narrowing the distribution increases the entropy) but is not so useful in other contexts. Other packages will (correctly) ignore dimensions of width zero. """ # Ugly hack warning: if *evaluation_points* is an integer, then sample # that many points from the kde and return the log density at each # sampled point. Since the code that uses this is looking only at # the mean log density, it doesn't need the sample points themselves. # This interface should be considered internal to the entropy module # and not used by outside functions. If you need it externally, then # restructure the api so that the function always returns both the # points and the density, as well as any other function (such as the # denisty function and the sister function scipy_stats_density) so # that all share the new interface. from sklearn.neighbors import KernelDensity # Standardize data so we can use spherical kernels and uniform bandwidth data, mu, sigma = standardize(sample_points) # Note that sigma will be zero for dimensions w_o where all points are equal. # With P(w) = P(w, w_o) / P(w_o | w) and P(w_o) = 1 for all points in # the set, then P(w) = P(w, w_o) and we can ignore the zero dimensions. # However, as another ugly hack, we want the differential entropy to go # to -inf as the distribution narrows, so pretend that P = 0 everywhere. # Uncomment the following line to return the sample probability instead. ## sigma[sigma == 0.] = 1. # Silverman bandwidth estimator n, d = sample_points.shape bandwidth = (n * (d + 2) / 4.0) ** (-1.0 / (d + 4)) # print("starting grid search for bandwidth over %d points"%n) # from sklearn.grid_search import GridSearchCV # from numpy import logspace # params = {'bandwidth': logspace(-1, 1, 20)} # fitter = GridSearchCV(KernelDensity(), params) # fitter.fit(data) # kde = fitter.best_estimator_ # print("best bandwidth: {0}".format(kde.bandwidth)) # import time; T0 = time.time() kde = KernelDensity(kernel="gaussian", bandwidth=bandwidth, rtol=1e-6, atol=1e-6) kde.fit(data) if isinstance(evaluation_points, int): # For generated points, they already follow the distribution points = kde.sample(n) elif evaluation_points is not None: # Standardized evaluation points to match sample distribution # Note: for dimensions where all sample points are equal, sigma # has been artificially set equal to one. This means that the # evaluation points which do not match the sample value will # use the simple differences for the z-score rather than # pushing them out to plus/minus infinity. points = (evaluation_points - mu) / (sigma + (sigma == 0.0)) else: points = sample_points # Evaluate pdf, scaling the resulting density by sigma to correct the area. # If sigma is zero, return entropy as -inf; this seems to not be the # case for discrete distributions (consider Bernoulli with p=1, q=0, # => H = -p log p - q log q = 0), so need to do something else, both # for the kde and for the entropy calculation. with np.errstate(divide="ignore"): log_pdf = kde.score_samples(points) - np.sum(np.log(sigma)) return log_pdf def sklearn_density(sample_points, evaluation_points): """ Estimate the probability density function from which a set of sample points was drawn and return the estimated density at the evaluation points. """ return exp(sklearn_log_density(sample_points, evaluation_points)) # scipy kde fails with singular matrix, so we will use scikit.learn # density = scipy_stats_density density = sklearn_density def entropy(points, logp, N_entropy=10000, N_norm=2500): r""" Return entropy estimate and uncertainty from a random sample. *points* is a set of draws from an underlying distribution, as returned by a Markov chain Monte Carlo process for example. *logp* is the log-likelihood for each draw. *N_norm* is the number of points $k$ to use to estimate the posterior density normalization factor $P(D) = \hat N$, converting from $\log( P(D|M) P(M) )$ to $\log( P(D|M)P(M)/P(D) )$. The relative uncertainty $\Delta\hat S/\hat S$ scales with $\sqrt{k}$, with the default *N_norm=2500* corresponding to 2% relative uncertainty. Computation cost is $O(nk)$ where $n$ is number of points in the draw. *N_entropy* is the number of points used to estimate the entropy $\hat S = - \int P(M|D) \log P(M|D)$ from the normalized log likelihood values. """ # Use a random subset to estimate density if N_norm >= len(logp): norm_points = points else: idx = permutation(len(points))[:N_entropy] norm_points = points[idx] # Use a different subset to estimate the scale factor between density # and logp. if N_entropy is None: N_entropy = 10000 if N_entropy >= len(logp): entropy_points, eval_logp = points, logp else: idx = permutation(len(points))[:N_entropy] entropy_points, eval_logp = points[idx], logp[idx] """ # Try again, just using the points from the high probability regions # to determine the scale factor N_norm = min(len(logp), 5000) N_entropy = int(0.8*N_norm) idx = np.argsort(logp) norm_points = points[idx[-N_norm:]] entropy_points = points[idx[-N_entropy:]] eval_logp = logp[idx[-N_entropy:]] """ # Normalize p to a peak probability of 1 so that exp() doesn't underflow. # # This should be okay since for the normalizing constant C: # # u' = e^(ln u + ln C) = e^(ln u)e^(ln C) = C u # # Using eq. 11 of Kramer with u' substituted for u: # # N_est = < u'/p > = < C u/p > = C < u/p > # # S_est = - < ln q > # = - < ln (u'/N_est) > # = - < ln C + ln u - ln (C ) > # = - < ln u + ln C - ln C - ln > # = - < ln u - ln > # = - < ln u > + ln # # Uncertainty comes from eq. 13: # # N_err^2 = 1/(k-1) sum( (u'/p - )^2 ) # = 1/(k-1) sum( (C u/p - )^2 ) # = C^2 std(u/p)^2 # S_err = std(u'/p) / = (C std(u/p))/(C ) = std(u/p)/ # # So even though the constant C shows up in N_est, N_err, it cancels # again when S_est, S_err is formed. log_scale = np.max(eval_logp) # print("max log sample: %g"%log_scale) eval_logp -= log_scale # Compute entropy and uncertainty in nats # Note: if all values are the same in any dimension then we have a dirac # functional with infinite probability at every sample point, and the # differential entropy estimate will yield H = -inf. rho = density(norm_points, entropy_points) # print(rho.min(), rho.max(), eval_logp.min(), eval_logp.max()) frac = exp(eval_logp) / rho n_est, n_err = mean(frac), std(frac) if n_est == 0.0: s_est, s_err = -np.inf, 0.0 else: s_est = log(n_est) - mean(eval_logp) s_err = n_err / n_est # print(n_est, n_err, s_est/LN2, s_err/LN2) ##print(np.median(frac), log(np.median(frac))/LN2, log(n_est)/LN2) if False: import matplotlib.pyplot as plt idx = plt.argsort(entropy_points[:, 0]) plt.figure() plt.subplot(221) plt.hist(points[:, 0], bins=50, density=True, log=True) plt.plot(entropy_points[idx, 0], rho[idx], label="density") plt.plot(entropy_points[idx, 0], exp(eval_logp + log_scale)[idx], label="p") plt.ylabel("p(x)") plt.legend() plt.subplot(222) plt.hist(points[:, 0], bins=50, density=True, log=False) plt.plot(entropy_points[idx, 0], rho[idx], label="density") plt.plot(entropy_points[idx, 0], exp(eval_logp + log_scale)[idx], label="p") plt.ylabel("p(x)") plt.legend() plt.subplot(212) plt.plot(entropy_points[idx, 0], frac[idx], ".") plt.xlabel("P[0] value") plt.ylabel("p(x)/kernel density") # return entropy and uncertainty in bits return s_est / LN2, s_err / LN2 class MVNEntropy(object): """ Multivariate normal entropy approximation. Uses Mardia's multivariate skewness and kurtosis test to estimate normality. *x* is a set of points *alpha* is the cutoff for the normality test. *max_points* is the maximum number of points to use when checking normality. Since the normality test is $O(n^2)$ in memory and time, where $n$ is the number of points, *max_points* defaults to 1000. The entropy is computed from the full dataset. The returned object has the following attributes: *p_kurtosis* is the p-value for the kurtosis normality test *p_skewness* is the p-value for the skewness normality test *reject_normal* is True if either the the kurtosis or the skew test fails *entropy* is the estimated entropy of the best normal approximation to the distribution """ # TODO: use robust covariance estimator for mean and covariance # FastMSD is available in sklearn.covariance.MinDetCov. There are methods # such as (Zhahg, 2012), which may be faster if performance is an issue. # [1] Zhang (2012) DOI: 10.5539/ijsp.v1n2p119 def __init__(self, x, alpha=0.05, max_points=1000): # compute Mardia test coefficient n, p = x.shape # num points, num dimensions mu = np.mean(x, axis=0) C = np.cov(x.T, bias=1) if p > 1 else np.array([[np.var(x.T, ddof=1)]]) # squared Mahalanobis distance matrix # Note: this forms a full n x n matrix of distances, so will # fail for a large number of points. Kurtosis only requires # the diagonal elements so can be computed cheaply. If there # is no order to the points, skew could be estimated using only # the block diagonal dx = (x - mu[None, :])[:max_points] D = np.dot(dx, np.linalg.solve(C, dx.T)) kurtosis = np.sum(np.diag(D) ** 2) / n skewness = np.sum(D**3) / n**2 kurtosis_stat = (kurtosis - p * (p + 2)) / sqrt(8 * p * (p + 2) / n) raw_skewness_stat = n * skewness / 6 # Small sample correction converges to 1 as n increases, so it is # always safe to apply it small_sample_correction = (p + 1) * (n + 1) * (n + 3) / ((p + 1) * (n + 1) * n - n * 6) skewness_stat = raw_skewness_stat * small_sample_correction dof = (p * (p + 1) * (p + 2)) / 6 # degrees of freedom for chisq test self.p_kurtosis = 2 * (1 - norm.cdf(abs(kurtosis_stat))) self.p_skewness = 1 - chi2.cdf(skewness_stat, dof) self.reject_normal = self.p_kurtosis < alpha or self.p_skewness < alpha # print("kurtosis", kurtosis, kurtosis_stat, self.p_kurtosis) # print("skewness", skewness, skewness_stat, self.p_skewness) # compute entropy self.entropy = cov_entropy(C) def __str__(self): return "H=%.1f bits%s" % (self.entropy, " (not normal)" if self.reject_normal else "") def cov_entropy(C): """ Entropy estimate from covariance matrix C """ return 0.5 * (len(C) * log2(2 * pi * e) + log2(abs(np.linalg.det(C)))) def mvn_entropy_bootstrap(points, samples=50): """ Use bootstrap method to estimate entropy and its uncertainty """ n, d = points.shape results = [] for _ in range(samples): # sample n points with replacement in 0 ... n-1. x = points[choice(n, size=n)] C = np.cov(x.T, bias=1) if d > 1 else np.array([[np.var(x.T, ddof=1)]]) # print(f"cov {samples}, {x.shape}, {C.shape}") results.append(cov_entropy(C)) return np.mean(results), np.std(results) # ====================================================================== # Testing code # ====================================================================== # Based on: Eli Bendersky https://stackoverflow.com/a/5849861 # Extended with tic/toc by Paul Kienzle import time class Timer(object): @staticmethod def tic(name=None): return Timer(name).toc def __init__(self, name=None): self.name = name self.step_number = 0 self.tlast = self.tstart = time.time() def toc(self, step=None): self.step_number += 1 if step is None: step = str(self.step_number) label = self.name + "-" + step if self.name else step tnext = time.time() total = tnext - self.tstart delta = tnext - self.tlast print("[%s] Elapsed: %s, Delta: %s" % (label, total, delta)) self.tlast = tnext def __enter__(self): self.tlast = self.tstart = time.time() def __exit__(self, type, value, traceback): if self.name: print("[%s]" % self.name, end="") print("Elapsed: %s" % (time.time() - self.tstart)) def entropy_mc(D, N=1000000): logp = D.logpdf(D.rvs(N)) return -np.mean(logp) # return -np.mean(logp[np.isfinite(logp)]) # CRUFT: dirichlet needs transpose of theta for logpdf class Dirichlet: def __init__(self, alpha): self.alpha = alpha self._dist = stats.dirichlet(alpha) self.dim = len(alpha) def logpdf(self, theta): return self._dist.logpdf(theta.T) def rvs(self, *args, **kw): x = self._dist.rvs(*args, **kw) # Dirichlet logpdf is failing if x=0 for any x when alpha<1. # The simplex check allows fudge of 1e-10. x[x == 0] = 1e-100 return x def entropy(self, *args, **kw): return self._dist.entropy(*args, **kw) class Box: def __init__(self, width=None, center=None): if width is None: width = np.ones(len(center), dtype="d") if center is None: center = np.zeros(len(width), dtype="d") self.center = center self.width = width self.dim = len(width) self._logpdf = -np.sum(np.log(self.width)) def rvs(self, size=1): x = np.random.rand(size, len(self.width)) x = (x - 0.5) * self.width + self.center return x def logpdf(self, theta): y = (theta - self.center) / self.width + 0.5 logp = np.ones(len(theta)) * self._logpdf logp[np.any(y < 0, axis=1)] = -np.inf logp[np.any(y > 1, axis=1)] = -np.inf return logp def entropy(self): return -self._logpdf # CRUFT: scipy MVN gives wrong entropy for singular (and near singular) matrices # This solution gives wrong results for near-singular rvs(), but for the simple # case of a diagonal Sigma with one zero it does what I need for the test. class MVNSingular: def __init__(self, *args, **kw): kw["allow_singular"] = True self.dist = stats.multivariate_normal(*args, **kw) @property def dim(self): return self.dist.dim def pdf(self, theta): return self.dist.pdf(theta) def logpdf(self, theta): return self.dist.logpdf(theta) def rvs(self, size=1): return self.dist.rvs(size=size) def entropy(self, N=10000): # CRUFT scipy==1.10.0: scipy.stats briefly removed the dist.cov attribute. if hasattr(self.dist, "cov"): cov = self.dist.cov else: cov = self.dist.cov_object.covariance with np.errstate(divide="ignore"): return 0.5 * log(np.linalg.det((2 * pi * np.e) * cov)) class GaussianMixture: def __init__(self, w, mu=None, sigma=None): mu = np.asarray(mu) dim = mu.shape[1] if sigma is None: sigma = [None] * len(mu) sigma = [(np.ones(dim) if s is None else np.asarray(s)) for s in sigma] sigma = [(np.diag(s) if len(s.shape) == 1 else s) for s in sigma] self.dim = dim self.weight = np.asarray(w, "d") / np.sum(w) self.dist = [stats.multivariate_normal(mean=m, cov=s) for m, s in zip(mu, sigma)] def pdf(self, theta): return sum(w * D.pdf(theta) for w, D in zip(self.weight, self.dist)) def logpdf(self, theta): return log(self.pdf(theta)) def rvs(self, size=1): # TODO: should randomize the output sizes = partition(size, self.weight) draws = [D.rvs(size=n) for n, D in zip(sizes, self.dist)] return np.random.permutation(np.vstack(draws)) def entropy(self, N=10000): # No analytic expression, so estimate entropy using MC integration return entropy_mc(self, N=N) class MultivariateT: def __init__(self, mu=None, sigma=None, df=None): if sigma is not None: sigma = np.asarray(sigma) self.mu = np.zeros(sigma.shape[0]) if mu is None else np.asarray(mu) if sigma is None: sigma = np.ones(len(mu)) if len(sigma.shape) == 1: sigma = np.diag(sigma) self.dim = len(self.mu) self.sigma = sigma self.df = df # Use scipy stats to compute |Sigma| and (x-mu)^T Sigma^{-1} (x - mu), # and to estimate dimension p from rank. Formula for pdf from wikipedia # https://en.wikipedia.org/wiki/Multivariate_t-distribution from scipy.stats._multivariate import _PSD self._psd = _PSD(self.sigma) nu, p = self.df, self._psd.rank self._log_norm = gammaln((nu + p) / 2) - gammaln(nu / 2) - p / 2 * log(pi * nu) - self._psd.log_pdet / 2 def logpdf(self, theta): dev = theta - self.mu maha = np.sum(np.square(np.dot(dev, self._psd.U)), axis=-1) nu, p = self.df, self._psd.rank return self._log_norm - (nu + p) / 2 * np.log1p(maha / nu) def pdf(self, theta): return exp(self.logpdf(theta)) def rvs(self, size=1): # From farhawa on stack overflow # https://stackoverflow.com/questions/29798795/multivariate-student-t-distribution-with-python nu, p = self.df, len(self.mu) g = np.tile(np.random.gamma(nu / 2, 2 / nu, size=size), (p, 1)).T Z = np.random.multivariate_normal(np.zeros(p), self.sigma, size=size) return self.mu + Z / np.sqrt(g) def entropy(self, N=100000): # No analytic expression, so estimate entropy using MC integration return entropy_mc(self, N=N) def MultivariateCauchy(mu=None, sigma=None): return MultivariateT(mu=mu, sigma=sigma, df=1) class Joint: def __init__(self, distributions): # Note: list(x) converts any sequence, including generators, into a list self.distributions = list(distributions) self.dim = len(self.distributions) def rvs(self, size=1): return np.stack([D.rvs(size=size) for D in self.distributions], axis=-1) def pdf(self, theta): return exp(self.logpdf(theta)) def logpdf(self, theta): return sum(D.logpdf(theta[..., k]) for k, D in enumerate(self.distributions)) def cdf(self, theta): return exp(self.logcdf(theta)) def logcdf(self, theta): return sum(D.logcdf(theta[..., k]) for k, D in enumerate(self.distributions)) def sf(self, theta): return -np.expm1(self.logcdf(theta)) def logsf(self, theta): return log(self.sf(theta)) def entropy(self): return sum(D.entropy() for D in self.distributions) def partition(n, w): # TODO: build an efficient algorithm for splitting n things into k buckets indices = np.arange(len(w), dtype="i") choices = np.random.choice(indices, size=n, replace=True, p=w) bins = np.arange(len(w) + 1, dtype="f") - 0.5 sizes, _ = np.histogram(choices, bins=bins) return sizes def _check_entropy(name, D, seed=1, N=10000, N_entropy=None, N_norm=2500, demo=False): """ Check if entropy from a random draw matches analytic entropy. """ use_kramer = use_mvn = use_wnn = use_gmm = use_kde = False if demo: # use_kramer = True # use_wnn = True use_mvn = True use_gmm = True use_kde = True else: use_kramer = True state = np.random.get_state() np.random.seed(seed) try: theta = D.rvs(size=N) if getattr(D, "dim", 1) == 1: theta = theta.reshape(N, 1) if use_kramer: logp_theta = D.logpdf(theta) logp_theta += 27 # result should be independent of scale factor S, Serr = entropy(theta, logp_theta, N_entropy=N_entropy, N_norm=N_norm) if use_wnn: S_wnn, Serr_wnn = wnn_entropy(theta, n_est=N_entropy) if use_gmm: S_gmm, Serr_gmm = gmm_entropy(theta, n_est=N_entropy) if use_mvn: M = MVNEntropy(theta) S_mvn = M.entropy if use_kde: S_kde = kde_entropy_statsmodels(theta, n_est=N_entropy) # S_kde = kde_entropy_sklearn(theta, n_est=N_entropy) # S_kde = kde_entropy_sklearn_gmm(theta, n_est=N_entropy) finally: np.random.set_state(state) H = D.entropy() / LN2 if demo: print("entropy", N, "~", name, H, end="") if use_kramer: print(" Kramer", S, Serr, end="") if use_wnn: print(" wnn", S_wnn, Serr_wnn, end="") if use_gmm: print(" gmm", S_gmm, Serr_gmm, end="") if use_mvn: print(" MVN", S_mvn, end="") if use_kde: print(" KDE", S_kde, end="") print() else: if use_kramer: # assert Serr < 0.05*S, "incorrect error est. for Kramer" if np.isfinite(H): assert abs(S - H) < 3 * Serr, "incorrect est. for Kramer" else: assert not np.isfinite(S), "incorrect est. for Kramer" if use_wnn: if np.isfinite(H): assert Serr_wnn < 0.05 * S_wnn, "incorrect error est. for wnn" assert abs(S_wnn - H) < 3 * Serr_wnn, "incorrect est. for wnn" else: assert not np.isfinite(S_wnn), "incorrect est. for wnn" def _show_entropy(name, D, **kw): with Timer(): return _check_entropy(name, D, seed=None, demo=True, **kw) def _check_smoke(D): theta = D.rvs(size=1000) if getattr(D, "dim", 1) == 1: theta = theta.reshape(-1, 1) logp_theta = D.logpdf(theta) entropy(theta, logp_theta) wnn_entropy(theta) MVNEntropy(theta).entropy def test(): """check entropy estimates from known distributions""" # entropy test is optional: don't test if sklearn is not installed try: import sklearn except ImportError: return # Smoke test - do all the methods run in 1-D and 10-D? _check_smoke(stats.norm(10, 8)) if hasattr(stats, "multivariate_normal"): _check_smoke(stats.multivariate_normal(cov=np.diag([1] * 10))) D = stats.norm(10, 8) _check_entropy("N[100,8]", D, N=2000) _check_entropy("N[100,8]", D, N=12000) if hasattr(stats, "multivariate_normal"): D = stats.multivariate_normal(cov=np.diag([1, 12**2, 0.2**2])) _check_entropy("MVN[1,12,0.2]", D) D = stats.multivariate_normal(cov=np.diag([1] * 10)) _check_entropy("MVN[1]*10", D, N=10000) # Make sure zero-width dimensions return H = -inf D = MVNSingular(cov=np.diag([1, 1, 0])) _check_entropy("MVN[1,1,0]", D, N=10000) # raise TestFailure("make bumps testing fail so we know that test harness works") def mvn_entropy_test(): """ Test against results from the R MVN pacakge (using the web version) and the matlab Mskekur program (using Octave), both of which produce the same value. Note that MVNEntropy uses the small sample correction for the skewness stat since it converges to the large sample value for large n. """ x = np.array( [ [2.4, 2.1, 2.4], [4.5, 4.9, 5.7], [3.5, 1.8, 3.9], [3.9, 4.7, 4.7], [6.7, 3.6, 5.9], [4.0, 3.6, 2.9], [5.3, 3.3, 6.1], [5.7, 5.5, 6.2], [5.2, 4.1, 6.4], [2.4, 2.9, 3.2], [3.2, 2.7, 4.0], [2.7, 2.6, 4.1], ] ) M = MVNEntropy(x) # print(M) # print("%.15g %.15g %.15g"%(M.p_kurtosis, M.p_skewness, M.entropy)) assert abs(M.p_kurtosis - 0.265317890462476) <= 1e-10 assert abs(M.p_skewness - 0.773508066109368) <= 1e-10 assert abs(M.entropy - 5.7920040570988) <= 1e-10 ## wnn_entropy doesn't work for small sample sizes (no surprise there!) # S_wnn, Serr_wnn = wnn_entropy(x) # assert abs(S_wnn - 5.7920040570988) <= 1e-10 # print("wnn %.15g, target %g"%(S_wnn, 5.7920040570988)) def demo(): # hide module load time from Timer from sklearn.neighbors import NearestNeighbors ## Bootstrap didn't help, but leave the test code in place for now # D = Dirichlet(alpha=[0.02]*20) # theta = D.rvs(size=1000) # S, Serr = wnn_bootstrap(D.rvs(size=200000)) # print("bootstrap", S, D.entropy()) # return if False: # Multivariate T distribution D = stats.t(df=4) _show_entropy("T;df=4", D, N=20000) D = MultivariateT(sigma=np.diag([1]), df=4) _show_entropy("MT[1];df=4", D, N=20000) D = MultivariateT(sigma=np.diag([1, 12, 0.2]) ** 2, df=4) _show_entropy("MT[1,12,0.2];df=4", D, N=10000) D = MultivariateT(sigma=np.diag([1] * 10), df=4) _show_entropy("MT[1]*10;df=4", D, N=10000) D = MultivariateT(sigma=np.diag([1, 12, 0.2, 1e2, 1e-2, 1]) ** 2, df=4) _show_entropy("MT[1,12,0.2,1e3,1e-3,1];df=4", D, N=10000) return if False: # Multivariate skew normal distribution D = stats.skewnorm(5) _show_entropy("skew=5 N[1]", D, N=20000) D = Joint(stats.skewnorm(5, 0, s) for s in [1, 12, 0.2]) _show_entropy("skew=5 N[1,12,0.2]", D, N=10000) D = Joint(stats.skewnorm(5, 0, s) for s in [1] * 10) _show_entropy("skew=5 N[1]*10", D, N=10000) D = Joint(stats.skewnorm(5, 0, s) for s in [1, 12, 0.2, 1e2, 1e-2, 1]) _show_entropy("skew=5 N[1,12,0.2,1e3,1e-3,1]", D, N=10000) # print("double check entropy", D.entropy()/LN2, entropy_mc(D)/LN2) return D = Box(center=[100] * 10, width=np.linspace(1, 10, 10)) _show_entropy("Box 10!", D, N=10000) D = stats.norm(10, 8) # _show_entropy("N[100,8]", D, N=100) # _show_entropy("N[100,8]", D, N=200) # _show_entropy("N[100,8]", D, N=500) # _show_entropy("N[100,8]", D, N=1000) # _show_entropy("N[100,8]", D, N=2000) # _show_entropy("N[100,8]", D, N=5000) _show_entropy("N[100,8]", D, N=10000) # _show_entropy("N[100,8]", D, N=20000) # _show_entropy("N[100,8]", D, N=50000) # _show_entropy("N[100,8]", D, N=100000) D = stats.multivariate_normal(cov=np.diag([1, 12, 0.2]) ** 2) # _show_entropy("MVN[1,12,0.2]", D) D = stats.multivariate_normal(cov=np.diag([1] * 10) ** 2) # _show_entropy("MVN[1]*10", D, N=1000) _show_entropy("MVN[1]*10", D, N=10000) # _show_entropy("MVN[1]*10", D, N=100000) # _show_entropy("MVN[1]*10", D, N=200000, N_entropy=20000) D = stats.multivariate_normal(cov=np.diag([1, 12, 0.2, 1, 1, 1]) ** 2) # _show_entropy("MVN[1,12,0.2,1,1,1]", D, N=100) # _show_entropy("MVN[1,12,0.2,1,1,1]", D, N=1000) _show_entropy("MVN[1,12,0.2,1,1,1]", D, N=10000) # _show_entropy("MVN[1,12,0.2,1,1,1]", D, N=100000) D = stats.multivariate_normal(cov=np.diag([1, 12, 0.2, 1e2, 1e-2, 1]) ** 2) # _show_entropy("MVN[1,12,0.2,1e3,1e-3,1]", D, N=100) # _show_entropy("MVN[1,12,0.2,1e3,1e-3,1]", D, N=1000) _show_entropy("MVN[1,12,0.2,1e3,1e-3,1]", D, N=10000) # _show_entropy("MVN[1,12,0.2,1e3,1e-3,1]", D, N=100000) D = GaussianMixture([1, 10], mu=[[0] * 10, [100] * 10], sigma=[[10] * 10, [0.1] * 10]) _show_entropy("bimodal mixture", D) D = Dirichlet(alpha=[0.02] * 20) # _show_entropy("Dirichlet[0.02]*20", D, N=1000) # _show_entropy("Dirichlet[0.02]*20", D, N=2000) # _show_entropy("Dirichlet[0.02]*20", D, N=5000) # _show_entropy("Dirichlet[0.02]*20", D, N=10000) _show_entropy("Dirichlet[0.02]*20", D, N=20000) # _show_entropy("Dirichlet[0.02]*20", D, N=50000) # _show_entropy("Dirichlet[0.02]*20", D, N=200000, N_entropy=20000) if __name__ == "__main__": # pragma: no cover demo() bumps-bumps-2b313fd/bumps/dream/exppow.py000066400000000000000000000023431521754644000205160ustar00rootroot00000000000000""" Exponential power density parameter calculator. """ __all__ = ["exppow_pars"] from math import sqrt from scipy.special import gamma def exppow_pars(B): r""" Return w(B) and c(B) for the exponential power density: .. math:: p(v|S,B) = \frac{w(B)}{S} \exp\left(-c(B) |v/S|^{2/(1+B)}\right) *B* in (-1,1] is a measure of kurtosis:: B = 1: double exponential B = 0: normal B -> -1: uniform [1] Thiemann, M., M. Trosser, H. Gupta, and S. Sorooshian (2001). *Bayesian recursive parameter estimation for hydrologic models*, Water Resour. Res. 37(10) 2521-2535. """ # First calculate some dummy variables A1 = gamma(3 * (1 + B) / 2) A2 = gamma((1 + B) / 2) # And use these to derive Cb and Wb cB = (A1 / A2) ** (1 / (1 + B)) wB = sqrt(A1 / A2**3) / (1 + B) return cB, wB def test(): import math cB, wB = exppow_pars(13) assert abs(cB - 12.8587702619708) < 1e-13 assert abs(wB - 5766.80847609837) < 1e-11 # Check that beta=0 yields a normal distribution cB, wB = exppow_pars(0) assert abs(2 * math.pi * wB**2 - 1) < 1e-14 assert abs(cB - 0.5) < 1e-14 if __name__ == "__main__": # print calc_CbWb(13) test() bumps-bumps-2b313fd/bumps/dream/gelman.py000066400000000000000000000053131521754644000204370ustar00rootroot00000000000000""" Convergence test statistic from Gelman and Rubin, 1992.[1] [1] Gelman, Andrew, and Donald B. Rubin. "Inference from Iterative Simulation Using Multiple Sequences." Statistical Science 7, no. 4 (November 1, 1992): 457-72. https://doi.org/10.2307/2246093. """ __all__ = ["gelman"] from numpy import var, mean, ones, sqrt def gelman(sequences, portion=0.5): """ Calculates the R-statistic convergence diagnostic For more information please refer to: Gelman, A. and D.R. Rubin, 1992. Inference from Iterative Simulation Using Multiple Sequences, Statistical Science, Volume 7, Issue 4, 457-472. doi:10.1214/ss/1177011136 """ # Find the size of the sample chain_len, nchains, nvar = sequences.shape # print sequences[:20, 0, 0] # Only use the last portion of the sample chain_len = int(chain_len * portion) sequences = sequences[-chain_len:] if chain_len < 2: # Set the R-statistic to a large value r_stat = -2 * ones(nvar) else: # Step 1: Determine the sequence means mean_seq = mean(sequences, axis=0) # Step 1: Determine the variance between the sequence means b = chain_len * var(mean_seq, axis=0, ddof=1) # Step 2: Compute the variance of the various sequences var_seq = var(sequences, axis=0, ddof=1) # Step 2: Calculate the average of the within sequence variances w = mean(var_seq, axis=0) # Step 3: Estimate the target mean # mu = mean(mean_seq) # Step 4: Estimate the target variance (Eq. 3) sigma2 = ((chain_len - 1) / chain_len) * w + (1 / chain_len) * b # TODO: the second term, -(N-1)/(K N), doesn't appear in [1] # Step 5: Compute the R-statistic r_stat = sqrt((nchains + 1) / nchains * sigma2 / w - (chain_len - 1) / nchains / chain_len) # par=2 # print chain_len,b[par],var_seq[...,par],w[par],r_stat[par] return r_stat def test(): from numpy import reshape, arange, transpose from numpy.linalg import norm # Targe values computed from octave: # format long # s = reshape([1:15*6*7],[15,6,7]); # r = gelman(s,struct('n',6,'seq',7)) s = reshape(arange(1.0, 15 * 6 * 7 + 1) ** -2, (15, 6, 7), order="F") s = transpose(s, [0, 2, 1]) target = [ 1.06169861367116, 2.75325774624905, 4.46256647696399, 6.12792266170178, 7.74538715553575, 9.31276519155232, ] r = gelman(s, portion=1) # print r # print "target", array(target), "\nactual", r assert norm(r - target) < 1e-14 r = gelman(s, portion=0.1) assert norm(r - [-2, -2, -2, -2, -2, -2]) == 0 if __name__ == "__main__": test() bumps-bumps-2b313fd/bumps/dream/geweke.py000066400000000000000000000041061521754644000204420ustar00rootroot00000000000000""" Convergence test statistic from Gelman and Rubin, 1992. """ __all__ = ["geweke"] from numpy import var, mean, ones, sqrt, reshape, log10, abs def geweke(sequences, portion=0.25): """ Calculates the Geweke convergence diagnostic Refer to: pymc-devs.github.com/pymc/modelchecking.html#informal-methods support.sas.com/documentation/cdl/en/statug/63033/HTML/default/viewer.htm#statug_introbayes_sect008.html """ # Find the size of the sample chain_len, nchains, nvar = sequences.shape z_stat = -2 * ones(nvar) if chain_len >= 2: # Only use the last portion of the sample try: front_portion, back_portion = portion except TypeError: front_portion = back_portion = portion front_len = int(chain_len * front_portion) back_len = int(chain_len * back_portion) # print "STARTING SHAPE", sequences.shape seq1 = reshape(sequences[:front_len, :, :], (front_len * nchains, nvar)) seq2 = reshape(sequences[-back_len:, :, :], (back_len * nchains, nvar)) # print "SEQ1", seq1.shape, 'SEQ2', seq2.shape # Step 1: Determine the sequence means meanseq1 = mean(seq1, axis=0) meanseq2 = mean(seq2, axis=0) # print "SHAPEs", meanseq1.shape, meanseq2.shape var1 = var(seq1, axis=0) var2 = var(seq2, axis=0) denom = sqrt(var1 + var2) index = denom > 0 z_stat[index] = (meanseq1 - meanseq2)[index] / denom[index] # z_stat is now the Z score for every chain and parameter # in that with shape (chains, vars) # To make it easier to look at, return the average for the vars. if 0: avg_z = mean(z_stat, axis=0) lavg_z = log10(abs(avg_z)) return lavg_z.tolist() if 0: avg_z = z_stat lavg_z = log10(abs(avg_z)) return lavg_z.flatten().tolist() else: return z_stat.flatten().tolist() # TODO: code is wrong if chain length is 1, since lavg_z is not defined return lavg_z.tolist() bumps-bumps-2b313fd/bumps/dream/initpop.py000066400000000000000000000055511521754644000206620ustar00rootroot00000000000000""" Population initialization routines. To start the analysis an initial population is required. This will be an array of size M x N, where M is the number of dimensions in the fitting problem and N is the number of Markov chains. Two functions are provided: 1. lhs_init(N, bounds) returns a latin hypercube sampling, which tests every parameter at each of N levels. 2. cov_init(N, x, cov) returns a Gaussian sample along the ellipse defined by the covariance matrix, cov. Covariance defaults to diag(dx) if dx is provided as a parameter, or to I if it is not. Additional options are random box: rand(M, N) or random scatter: randn(M, N). """ __all__ = ["lhs_init", "cov_init"] from numpy import asarray, diag, empty, eye from . import util def lhs_init(N, bounds): """ Latin Hypercube Sampling Returns an array whose columns each have *N* samples from equally spaced bins between *bounds=(xmin, xmax)* for the column. DREAM bounds objects, with bounds.low and bounds.high can be used as well. Note: Indefinite ranges are not supported. """ try: xmin, xmax = bounds.low, bounds.high except AttributeError: xmin, xmax = bounds # Define the size of xmin nvar = len(xmin) # Initialize array ran with random numbers ran = util.rng.rand(N, nvar) # Initialize array s with zeros s = empty((N, nvar)) # Now fill s for j in range(nvar): # Random permutation idx = util.rng.permutation(N) + 1 p = (idx - ran[:, j]) / N s[:, j] = xmin[j] + p * (xmax[j] - xmin[j]) return s def cov_init(N, x, cov=None, dx=None): """ Initialize *N* sets of random variables from a gaussian model. The center is at *x* with an uncertainty ellipse specified by the 1-sigma independent uncertainty values *dx* or the full covariance matrix uncertainty *cov*. For example, create an initial population for 20 sequences for a model with local minimum x with covariance matrix C:: pop = cov_init(cov=C, x=x, N=20) """ # return mean + dot(util.rng.randn(N, len(mean)), chol(cov)) if cov is None and dx is None: cov = eye(len(x)) elif cov is None: cov = diag(asarray(dx) ** 2) return util.rng.multivariate_normal(mean=x, cov=cov, size=N) def demo(): from numpy import arange print("Three ways of calling cov_init:") print("with cov", cov_init(N=4, x=[5, 6], cov=diag([0.1, 0.001]))) print("with dx", cov_init(N=4, x=[5, 6], dx=[0.1, 0.001])) print("with nothing", cov_init(N=4, x=[5, 6])) print(""" The following array should have four columns. Column 1 should have the numbers from 10 to 19, column 2 from 20 to 29, etc. The columns are in random order with a random fractional part. """) pop = lhs_init(N=10, bounds=(arange(1, 5), arange(2, 6))) * 10 print(pop) if __name__ == "__main__": demo() bumps-bumps-2b313fd/bumps/dream/ksmirnov.py000066400000000000000000000021631521754644000210440ustar00rootroot00000000000000""" Kolmogorov-Smirnov test for MCMC convergence. Use the K-S tests to compare the distribution of values at the front of the chain to that at the end of the chain. If the distributions are significantly different, then the MCMC chain has not converged. """ __all__ = ["ksmirnov"] from numpy import reshape, apply_along_axis from scipy.stats import ks_2samp def ksmirnov(seq, portion=0.25, filter_order=15): """ Kolmogorov-Smirnov test of similarity between the empirical distribution at the start and at the end of the chain. Apply a median filter (filter=15) on neighbouring K-S values to reduce variation in the test statistic value. """ chlen, nchains, nvars = seq.shape count = portion * chlen * nchains n = filter_order ks, p = apply_along_axis(lambda chain: _ksm(chain, n, count), 0, reshape(seq, (chlen * nchains, nvars))) return ks, p def _ksm(chain, n, count): # return ks_2samp(chain[:count], chain[-count:]) ks, p = zip(*[ks_2samp(chain[i : count + i], chain[-count - n + i : -n + i]) for i in range(n)]) return sorted(ks)[(n - 1) // 2], sorted(p)[(n - 1) // 2] bumps-bumps-2b313fd/bumps/dream/mahal.py000066400000000000000000000031151521754644000202540ustar00rootroot00000000000000# This program is public domain # Author: Paul Kienzle, April 2010 """ Mahalanobis distance calculator Compute the `Mahalanobis distance `_ between observations and a reference set. The principle components of the reference set define the basis of the space for the observations. The simple Euclidean distance is used within this space. """ __all__ = ["mahalanobis"] from numpy import dot, mean, sum from numpy.linalg import svd def mahalanobis(Y, X): """ Returns the distances of the observations from a reference set. Observations are stored in rows *Y* and the reference set in *X*. """ M = mean(X, axis=0) # mean Xc = X - mean(X, axis=0) # center the reference W = dot(Xc.T, Xc) / (Xc.shape[0] - 1) # covariance of reference Yc = Y - M # center the observations # Distance is diag(Yc * inv(W) * Yc.H) # solve Wb = Yc.H using singular value decomposition because it is # the most accurate with numpy; QR decomposition does not use pivoting, # and is less accurate. The built-in solve routine is between the two. u, s, vh = svd(W, 0) SVinv = vh.T.conj() / s Uy = dot(u.T.conj(), Yc.T.conj()) b = dot(SVinv, Uy) D = sum(Yc.T * b, axis=0) # compute distance return D def test(): from numpy import array from numpy.linalg import norm d = mahalanobis(array([[2, 3, 4], [2, 3, 4]]), array([[1, 0, 0], [2, 1, 0], [1, 1, 0], [2, 0, 1]])) assert norm(d - [290.25, 290.25]) < 1e-12, "diff=%s" % str(d - [290.25, 290.25]) if __name__ == "__main__": test() bumps-bumps-2b313fd/bumps/dream/matlab.py000066400000000000000000000174721521754644000204450ustar00rootroot00000000000000""" Interface compatible with matlab dream. Usage ----- This interface is identical to dream in matlab:: [Sequences, Reduced_Seq, X, output, hist_logp] = \ dream(MCMCPar, ParRange, Measurement, ModelName, Extra, option) With care, you will be able to use the same model definition file on both platforms, with only minor edits required to switch between them. Clearly, you won't be able to use if statements and loops since the syntax for python and matlab are incompatible. Similarly, you will have to be careful about indentation and line breaks. And create your model without comments. Python requires that structures be defined before you assign values to their fields, so you will also need the following lines. Note however, that they are safe to use in matlab as well:: MCMCPar = struct() Extra = struct() Measurement = struct() ParRange = struct() Another challenge is *ModelName*. In matlab this is the name of the m-file that contains the model definition. Following that convention, we will try to using "from import ", and if this fails, assume that *ModelName* is actually the function itself. For this to work you will need to translate the function in ModelName.m to the equivalent function in ModelName.py. *option* is the same option number as before IPython usage ------------- Within ipython you can interact with your models something like you do in matlab. For example:: $ ipython -pylab In [1]: from dream.matlab import * In [2]: from dream import * In [3]: %run example.m You can now use various dream visualization tools or use the matlab-like plotting functions from matplotlib:: In [4]: out.state.save('modeloutput') In [5]: out.state.plot_state() Command line usage ------------------ You can also run a suitable m-file example from the command line. This will place you at an ipython command line with all the variables from your m-file available. For example:: python -m dream.matlab example.m In [1]: out.state.save('modeloutput') In [2]: out.state.plot_state() Script usage ------------ You can create a driver script which calls the m-file example and uses pylab commands to plot the results. For example:: -- example.py -- #!/usr/bin/env python from pylab import * from dream.matlab import * execfile('example.m') from dream import * out.state.save('modeloutput') plot_state(out.state) This can be run from the command prompt:: $ python example.py """ __all__ = ["struct", "dream", "setup", "convert_output"] import numpy as np from .core import Dream from .model import Density, LogDensity, Simulation from .initpop import cov_init, lhs_init from .crossover import Crossover, AdaptiveCrossover class struct(object): """ Matlab compatible structure creation. """ def __init__(self, *pairs, **kw): for k, v in zip(pairs[::2], pairs[1::2]): setattr(self, k, v) for k, v in kw.items(): setattr(self, k, v) def __getattr__(self, k): return None def dream(MCMCPar, ParRange, Measurement, ModelName, Extra, option): """ Emulate the matlab dream call. """ dreamer = setup(MCMCPar, ParRange, Measurement, ModelName, Extra, option) dreamer.sample() return convert_state(dreamer.state) def setup(MCMCPar, ParRange, Measurement, ModelName, Extra, option): """ Convert matlab dream models to a python Dream object. """ dreamer = Dream() # Problem specification bounds = ParRange.minn, ParRange.maxn dreamer.bounds_style = Extra.BoundHandling if ModelName == "Banshp": # specific properties of the banana function # Extra.imat is computed from cmat # MCMCPar.n is implicit in Extra.cmat f = Banana(mu=Extra.mu.flatten(), bpar=Extra.bpar, cmat=Extra.cmat) option = 4 else: try: f = None # keep lint happy # Try matlab style of having the function in the same named file. exec("from " + ModelName + " import " + ModelName + " as f") except ImportError: # The import failed; hope the caller supplied a function instead. f = ModelName if option == 1: model = Density(f, bounds=bounds) elif option == 4: model = LogDensity(f, bounds=bounds) elif option in [2, 3, 5]: # Measurement.N is implicit in Measurement.MeasData model = Simulation(f, data=Measurement.MeasData, bounds=bounds, sigma=Measurement.Sigma, gamma=MCMCPar.Gamma) else: raise ValueError("option should be in 1 to 5") dreamer.model = model # Sampling parameters if Extra.save_in_memory == "Yes": thinning = 1 elif Extra.reduced_sample_collection == "Yes": thinning = Extra.T else: thinning = 1 dreamer.thinning = thinning dreamer.draws = MCMCPar.ndraw # Outlier detection T = MCMCPar.outlierTest if T.endswith("_test"): T = T[:-5] dreamer.outlier_test = T # DE parameters dreamer.DE_steps = MCMCPar.steps dreamer.DE_pairs = MCMCPar.DEpairs dreamer.DE_eps = MCMCPar.eps # Initial population if Extra.InitPopulation == "COV_BASED": pop = cov_init(N=MCMCPar.seq, x=Extra.muX.flatten(), cov=Extra.qcov) elif Extra.InitPopulation == "LHS_BASED": pop = lhs_init(N=MCMCPar.seq, bounds=(ParRange.minn, ParRange.maxn)) else: raise ValueError("Extra.InitPopulation must be COV_BASED or LHS_BASED") dreamer.population = pop # Crossover parameters if Extra.pCR == "Update": CR = AdaptiveCrossover(MCMCPar.nCR) else: CR = Crossover(1.0 / MCMCPar.nCR) dreamer.CR = CR # Delayed rejection parameters dreamer.use_delayed_rejection = Extra.DR == "Yes" dreamer.DR_scale = Extra.DRscale return dreamer def convert_state(state): """ Convert a completed dream run into a form compatible with the matlab dream interface:: Sequences, Reduced_Seq, X, out, hist_logp The original state is stored in out.state """ _, points, logp = state.sample() logp = logp[:, :, None] Sequences = np.concatenate((points, np.exp(logp), logp), axis=2) X = Sequences[-1, :, :] draws, logp = state.logp(full=True) hist_logp = np.concatenate((draws[:, None], logp), axis=1) out = struct() _, AR = state.acceptance_rate() out.AR = np.concatenate((draws[:, None], AR[:, None]), axis=1) draws, w = state.CR_weight() out.CR = np.concatenate((draws[:, None], w), axis=1) out.outlier = state.outliers()[:, :2] Nupdate, Nvar = len(draws), points.shape[2] R = np.zeros((Nupdate, Nvar), "d") # R is no longer calculated out.R_stat = np.concatenate((draws[:, None], R), axis=1) # save the dreamer state data structure as well out.state = state return Sequences, Sequences, X, out, hist_logp def run_script(filename): exec(compile(open(filename).read(), filename, "exec")) class Banana(object): """ Banana shaped function. Note that this is not one of the N dimensional Rosenbrock variants documented on wikipedia as it only operates "banana-like" in the x0-x1 plane. """ def __init__(self, mu, bpar, cmat): self.mu, self.bpar, self.cmat = mu, bpar, cmat self.imat = np.linalg.inv(cmat) def __call__(self, x): x = x + 0 # make a copy x[1] += self.bpar * (x[0] ** 2 - 100) ret = -0.5 * np.dot(np.dot(x[None, :], self.imat), x[:, None]) return ret[0, 0] def main(): import sys if len(sys.argv) == 2: import pylab run_script(sys.argv[1]) user_ns = pylab.__dict__.copy().update(locals()) import IPython IPython.Shell.IPShell(user_ns=user_ns).mainloop() else: print("usage: python -m dream.matlab model.m") if __name__ == "__main__": main() bumps-bumps-2b313fd/bumps/dream/metropolis.py000066400000000000000000000051421521754644000213710ustar00rootroot00000000000000""" MCMC step acceptance test. """ __all__ = ["metropolis", "metropolis_dr"] import os from numpy import array, cov, dot, errstate, exp, eye, minimum, sqrt, where from numpy.linalg import cholesky, inv, norm from . import util BUMPS_TEMPERATURE = float(os.environ.get("BUMPS_TEMPERATURE", "1")) def paccept(logp_old, logp_try): """ Returns the probability of taking a metropolis step given two log density values. """ return exp(minimum(logp_try - logp_old, 0) / BUMPS_TEMPERATURE) def metropolis(xtry, logp_try, xold, logp_old, step_alpha): """ Metropolis rule for acceptance or rejection Generates the next generation, *newgen* from:: x_new[k] = x[k] if U > alpha = x_old[k] if U <= alpha where alpha is p/p_old and accept is U > alpha. Returns x_new, logp_new, alpha, accept """ with errstate(under="ignore"): alpha = paccept(logp_try=logp_try, logp_old=logp_old) alpha *= step_alpha accept = alpha > util.rng.rand(*alpha.shape) logp_new = where(accept, logp_try, logp_old) ## The following only works for vectors: # xnew = where(accept, xtry, xold) xnew = xtry + 0 for i, a in enumerate(accept): if not a: xnew[i] = xold[i] return xnew, logp_new, alpha, accept def dr_step(x, scale): """ Delayed rejection step. """ # Compute the Cholesky Decomposition of X nchains, npars = x.shape r = (2.38 / sqrt(npars)) * cholesky(cov(x.T) + 1e-5 * eye(npars)) # Now do a delayed rejection step for each chain delta_x = dot(util.rng.randn(*x.shape), r) / scale # Generate ergodicity term eps = 1e-6 * util.rng.randn(*x.shape) # Update x_old with delta_x and eps; return x + delta_x + eps, r def metropolis_dr(xtry, logp_try, x, logp, xold, logp_old, alpha12, R): """ Delayed rejection metropolis """ # Compute alpha32 (note we turned x and xtry around!) alpha32 = paccept(logp_try=logp, logp_old=logp_try) # Calculate alpha for each chain l2 = paccept(logp_try=logp_try, logp_old=logp_old) iR = inv(R) q1 = array( [exp(-0.5 * (norm(dot(x2 - x1, iR)) ** 2 - norm(dot(x1 - x0, iR)) ** 2)) for x0, x1, x2 in zip(xold, x, xtry)] ) alpha13 = l2 * q1 * (1 - alpha32) / (1 - alpha12) accept = alpha13 > util.rng.rand(*alpha13.shape) logp_new = where(accept, logp_try, logp) ## The following only works for vectors: # xnew = where(accept, xtry, x) xnew = xtry + 0 for i, a in enumerate(accept): if not a: xnew[i] = x[i] return xnew, logp_new, alpha13, accept bumps-bumps-2b313fd/bumps/dream/model.py000066400000000000000000000157751521754644000203110ustar00rootroot00000000000000""" MCMC model types Usage ----- First create a :class:`bumps.dream.bounds.Bounds` object. This stores the ranges available on the parameters, and controls how values outside the range are handled:: M_bounds = bounds(minx, maxx, style='reflect|clip|fold|randomize|none') For simple functions you can use one of the existing models. If your model *f* computes the probability density, use :class:`Density`:: M = Density(f, bounds=M_bounds) If your model *f* computes the log probability density, use :class:`LogDensity`:: M = LogDensity(f, bounds=M_bounds) If your model *f* computes a simulation which returns a vector, and you have *data* associated with the simulation, use :class:`Simulation`:: M = Simulation(f, data=data, bounds=M_bounds) The measurement *data* can have a 1-sigma uncertainty associated with it, as well as a *gamma* factor if the uncertainty distribution has non-Gaussian kurtosis associated with it. Multivariate normal distribution:: M = MVNormal(mu, sigma) Mixture models:: M = Mixture(M1, w1, M2, w2, ...) For more complex functions, you can subclass MCMCModel:: class Model(MCMCModel): def __init__(self, ..., bounds=None, ...): ... self.bounds = bounds ... def nnlf(self, x): "Return the negative log likelihood of seeing x" p = probability of seeing x return -log(p) M = Model(..., bounds=M_bounds, ...) The MCMC program uses only two methods from the model:: apply_bounds(pop) log_density(pop) If your model provides these methods, you will not need to subclass MCMCModel in order to interact with DREAM. Compatibility with matlab DREAM ------------------------------- First generate a bounds handling function:: M_bounds = bounds(ParRange.minn, ParRange.maxn) Then generate a model, depending on what kind of function you have. Option 1. Model directly computes posterior density:: model = Density(f, bounds=M_bounds) Option 2. Model computes simulation, data has known 1-sigma uncertainty:: model = Simulation(f, data=Measurement.MeasData, bounds=M_bounds, sigma=Measurement.Sigma, gamma = MCMCPar.Gamma) Option 3. Model computes simulation, data has unknown 1-sigma uncertainty:: model = Simulation(f, data=Measurement.MeasData, bounds=M_bounds, gamma = MCMCPar.Gamma) Option 4. Model directly computes log posterior density:: model = LogDensity(f, bounds=M_bounds) Option 5 is like option 2 but the reported likelihoods do not take the 1-sigma uncertainty into account. The metropolis steps are still based on the 1-sigma uncertainty, so use the style given in option 2 for this case. """ __all__ = ["MCMCModel", "Density", "LogDensity", "Simulation", "MVNormal", "Mixture"] from abc import abstractmethod from typing import List import numpy as np from numpy import diag, log, exp, pi from numpy.linalg import cholesky, inv from numpy.typing import NDArray from . import exppow class MCMCModel: """ MCMC model abstract base class. Each model must have a negative log likelihood function which operates on a point x, returning the negative log likelihood, or inf if the point is outside the domain. """ labels: List[str] = None bounds: NDArray = None @abstractmethod def nllf(self, x): raise NotImplementedError def plot(self, x): pass def map(self, pop): return np.array([self.nllf(x) for x in pop]) class Density(MCMCModel): """ Construct an MCMC model from a probablility density function. *f* is the density function """ def __init__(self, f, bounds=None, labels=None): self.f, self.bounds, self.labels = f, bounds, labels def nllf(self, x): return -log(self.f(x)) class LogDensity(MCMCModel): """ Construct an MCMC model from a log probablility density function. *f* is the log density function """ def __init__(self, f, bounds=None, labels=None): self.f, self.bounds, self.labels = f, bounds, labels def nllf(self, x): return -self.f(x) class Simulation(MCMCModel): """ Construct an MCMC model from a simulation function. *f* is the function which simulates the data *data* is the measurement(s) to compare it to *sigma* is the 1-sigma uncertainty of the measurement(s). *gamma* in (-1, 1] represents kurtosis on the data measurement uncertainty. Data is assumed to come from an exponential power density:: p(v|S, G) = w(G)/S exp(-c(G) |v/S|^(2/(1+G))) where S is *sigma* and G is *gamma*. The values of *sigma* and *gamma* can be uniform or can vary with the individual measurement points. Certain values of *gamma* select particular distributions:: G = 0: normal G = 1: double exponential G -> -1: uniform """ def __init__(self, f=None, bounds=None, data=None, sigma=1, gamma=0, labels=None): self.f, self.bounds, self.labels = f, bounds, labels self.data, self.sigma, self.gamma = data, sigma, gamma cb, wb = exppow.exppow_pars(gamma) self._offset = np.sum(log(wb / sigma * np.ones_like(data))) self._cb = cb self._pow = 2 / (1 + gamma) # print "cb", cb, "sqrt(2pi)*wb", sqrt(2*pi)*wb # print "offset", self._offset def nllf(self, x): err = self.f(x) - self.data log_p = self._offset - np.sum(self._cb * abs(err / self.sigma) ** self._pow) return log_p def plot(self, x): import matplotlib.pyplot as plt v = np.arange(len(self.data)) plt.plot(v, self.data, "x", v, self.f(x), "-") class MVNormal(MCMCModel): """ multivariate normal negative log likelihood function """ def __init__(self, mu, sigma): self.mu, self.sigma = np.asarray(mu), np.asarray(sigma) # Precompute sigma contributions r = cholesky(sigma) self._rinv = inv(r) self._c = 0.5 * len(mu) * log(2 * pi) + 0.5 * np.sum(diag(r)) def nllf(self, x): mu, c, rinv = self.mu, self._c, self._rinv y = c + 0.5 * np.sum(np.dot(x - mu, rinv) ** 2) return y class Mixture(MCMCModel): """ Create a mixture model from a list of weighted density models. MixtureModel( M1, w1, M2, w2, ...) Models M1, M2, ... are MCMC models with M.nllf(x) returning the negative log likelihood of x. Weights w1, w2, ... are arbitrary scalars. """ def __init__(self, *args): models = args[::2] weights = args[1::2] if ( len(args) % 2 != 0 or not all(hasattr(M, "nllf") for M in models) or not all(np.isscalar(w) for w in weights) ): raise TypeError("Expected MixtureModel(M1, w1, M2, w2, ...)") self.pairs = tuple(zip(models, weights)) self.weight = np.sum(w for w in weights) def nllf(self, x): p = [w * exp(-M.nllf(x)) for M, w in self.pairs] total = np.sum(p) / self.weight return -log(total) if total > 0 else 1e100 bumps-bumps-2b313fd/bumps/dream/outliers.py000066400000000000000000000144661521754644000210530ustar00rootroot00000000000000""" Chain outlier tests. """ __all__ = ["identify_outliers"] import os from numpy import mean, std, sqrt, where, argmin, arange, array from numpy import sort from scipy.stats import t as student_t from scipy.stats import scoreatpercentile from .mahal import mahalanobis from .acr import ACR tinv = student_t.ppf # Hack to adjust the aggressiveness of the interquartile range test. # TODO: Document/remove BUMPS_INTERQUARTILES or replace with command option. BUMPS_INTERQUARTILES = float(os.environ.get("BUMPS_INTERQUARTILES", "2.0")) # CRUFT: scoreatpercentile not accepting array arguments in older scipy def prctile(v, Q): v = sort(v) return [scoreatpercentile(v, Qi) for Qi in Q] def identify_outliers(test, llf, x=None): """ Determine which chains have converged on a local maximum much lower than the maximum likelihood. *test* is the name of the test to use (one of IQR, Grubbs, Mahal or none). IQR rejects any chains with mean log likelihood more than than twice the inter-quartile range below the value of the 25% quartile. The Grubbs method uses a t-test to determine which chains have a mean log likelihood extremely far below the mean across all the chains. The Mahal test looks at the head of the chain with the worst mean log likelihood and marks it as an outlier if it is far from the centroid of the population. This assumes that the posterior is approximately gaussian, which is not true in general. *llf* is a set of log likelihood values for all chains, which is an array of shape (chain len, num chains) *x* is the current population with one point for each each, which is an array of shape (num chains, num vars). This is only used for the Mahal test. Returns an integer array of outlier indices. """ # Check whether any of these active chains are outlier chains test = test.lower() if test == "iqr": # Determine the mean log density of the active chains v = mean(llf, axis=0) # Derive the upper and lower quartile of the chain averages q1, q3 = prctile(v, [25.0, 75.0]) # Derive the Inter Quartile Range (IQR) iqr = q3 - q1 # See whether there are any outlier chains # 2017-10-06 [PAK] test against chain max rather than chain mean. # Chains wandering inside the active region should not be punished. # Since removing outliers will delay convergence tests until the # outlier removal effects have disappeared (i.e., an entire frame), # a less aggressive test will speed completion. vmax = llf.max(axis=0) outliers = where(vmax < q1 - BUMPS_INTERQUARTILES * iqr)[0] elif test == "grubbs": # Determine the mean log density of the active chains v = mean(llf, axis=0) # Compute zscore for chain averages zscore = (mean(v) - v) / std(v, ddof=1) # Determine t-value of one-sided interval n = len(v) t2 = tinv(1 - 0.01 / n, n - 2) ** 2 # 95% interval # Determine the critical value gcrit = ((n - 1) / sqrt(n)) * sqrt(t2 / (n - 2 + t2)) # Then check against this outliers = where(zscore > gcrit)[0] elif test == "mahal": # Determine the mean log density of the active chains v = mean(llf, axis=0) # Find which chain has minimum log_density minidx = argmin(v) # Use the Mahalanobis distance to find outliers in the population alpha = 0.01 npop, nvar = x.shape gcrit = ACR(nvar, npop - 1, alpha) # print "alpha", alpha, "nvar", nvar, "npop", npop, "gcrit", gcrit # check the Mahalanobis distance of the current point to other chains d1 = mahalanobis(x[minidx, :], x[minidx != arange(npop), :]) # print "d1", d1, "minidx", minidx # and see if it is an outlier outliers = array([minidx]) if d1 > gcrit else array([]) elif test == "none": outliers = array([]) else: raise ValueError("Unknown outlier test " + test) return outliers def test_outliers(): from .walk import walk from numpy.random import multivariate_normal, seed from numpy import vstack, ones, eye seed(2) # Remove uncertainty on tests # Set a number of good and bad chains ngood, nbad = 25, 2 # Make chains mean-reverting chains with widely separated values for # bad and good; put bad chains first. chains = walk(1000, mu=[1] * nbad + [5] * ngood, sigma=0.45, alpha=0.1) # Check IQR and Grubbs assert (identify_outliers("IQR", chains, None) == arange(nbad)).all() assert (identify_outliers("Grubbs", chains, None) == arange(nbad)).all() # Put points for 'bad' chains at [-1,...,-1] and 'good' chains at [1,...,1] x = vstack( (multivariate_normal(-ones(4), 0.1 * eye(4), size=nbad), multivariate_normal(ones(4), 0.1 * eye(4), size=ngood)) ) assert identify_outliers("Mahal", chains, x)[0] in range(nbad) # Put points for _all_ chains at [1,...,1] and check that mahal return [] xsame = multivariate_normal(ones(4), 0.2 * eye(4), size=ngood + nbad) assert len(identify_outliers("Mahal", chains, xsame)) == 0 # Check again with large variance x = vstack( (multivariate_normal(-3 * ones(4), eye(4), size=nbad), multivariate_normal(ones(4), 10 * eye(4), size=ngood)) ) assert len(identify_outliers("Mahal", chains, x)) == 0 # ===================================================================== # Test replacement # Construct a state object from numpy.linalg import norm from .state import MCMCDraw ngen, npop = chains.shape npop, nvar = x.shape state = MCMCDraw(Ngen=ngen, Nthin=ngen, Nupdate=0, Nvar=nvar, Npop=npop, Ncr=0, thinning=0) # Fill it with chains for i in range(ngen): state._generation(new_draws=npop, x=x, logp=chains[i], accept=npop) # Make a copy of the current state so we can check it was updated nx, nlogp = x + 0, chains[-1] + 0 # Remove outliers state.remove_outliers(nx, nlogp, test="IQR") # Check that the outliers were removed outliers = state.outliers() assert outliers.shape[0] == nbad for i in range(nbad): assert nlogp[outliers[i, 1]] == chains[-1][outliers[i, 2]] assert nx[outliers[i, 1], -1] == x[outliers[i, 2], -1] if __name__ == "__main__": test_outliers() bumps-bumps-2b313fd/bumps/dream/parcoord.py000066400000000000000000000145511521754644000210110ustar00rootroot00000000000000import numpy as np def plot(draw, nlines=150, control_var=None): """ Plot parallel coordinates from a draw from the distribution. *draw* is the draw from the sample *nlines* is the number of lines to draw If *control_var* is provided, then the best value from each bin in the histogram for that variable will be chosen as a line to draw on the coordination plot. This will will produce a roughly equally spaced set of coordination points for that variable. If *control_var* is None, then lines will be picked at random. """ data = draw.points labels = draw.labels logp = draw.logp # TODO: confirm that plot is called with 95% interval draw # Note: the 95% interval draw is based on the range of the data? # Drop points outside the 95% interval of chisq # index = np.argsort(logp) # npoints = data.shape[0] # maxn = int(npoints*0.95) # index = index[:maxn] # logp = logp[index] # data = data[index] # TODO: order data by strength of correlation # Some options: # Pearson: only works for linear # Kendal, Spearman: only works for monotonic # distance (Szekely, 2005) doi:10.1214/009053607000000505 # HGG (Heller, Heller, Gorfine, 2013) doi:10.1093/biomet/ass070 # estimate correlations # corr = matrix_corr(data) # order = list(range(data.shape[1])) # data = data[order] if control_var is None: index = np.random.choice(data.shape[0], size=nlines, replace=False) else: index = best_in_bin(data[:, control_var], -logp, bins=nlines) # color_value, color_label = data[index, var], labels[var] color_value, color_label = -logp[index], "-log(P)" parallel_coordinates(data[index], labels=labels, value=color_value, value_label=color_label) def best_in_bin(x, value, bins=50, range=None, keep_empty=False): """ Find the index of the minimum *value* in each bin for the data in *x*. *x* are the coordinates to be binned. *value* are the objective to be minimized in each bin, such as chisq. There is one value for each x coordinate. *bins* are the number of bins within *range* or an array of bin edges if the bins are not uniform. *range* is the data range for the bins. The default is (x.min(), x.max()). *keep_empty* is True if empty bins will return an index of -1. When False (the default), empty bins are removed. Returns indices of the elements which are the minimum within each bin. This list may be shorter than the number of bins if *keep_empty* is False. The algorithm works by assigning a bin number to each point then adding an offset in [0, 1) according to the scaled *value*. These point values are then sorted, and searched by bin number. The returned index will correspond to the first value in each bin, and therefore, the best value in that bin. From this the index into the original list can be returned. """ # Find bin number for each value. if isinstance(bins, int): # Find intervals for x and use integer division to assign bin number. # Don't worry that this might create bin numbers such as -3 or nbins+5 # since these will be ignored during bin lookup. xmin, xmax = range if range is not None else (np.min(x), np.max(x)) dx = (xmax - xmin) / bins value_in_bin = (x - xmin) // dx if dx != 0.0 else np.full_like(x, bins // 2) nbins = bins else: # Lookup x in bin edges. searchsorted returns index 0 for elements # before the first edge so we need to put an extra start edge and # subtract one from the index so that bin number is -1 before first. value_in_bin = np.searchsorted(np.hstack((-np.inf, bins)), x) - 1 nbins = len(bins) - 1 # Increment bin number offset using scaled value. Limit the scaled # value to 0.99 rather 1.0 so that the worst value isn't 1.0, which # will be the best value in the next bin. value_in_bin += scale(value) * 0.99 # Sort values, preserving original indices sort_index = np.argsort(value_in_bin) sorted_value_in_bin = value_in_bin[sort_index] # Lookup first point in each bin, which will now by the min chisq in bin. bin_numbers = np.arange(nbins) min_index = np.searchsorted(sorted_value_in_bin, bin_numbers) # Find the point number for the best points in the original list. index = sort_index[min_index] # Indentify empty bins. This will occur if base value returned by # search sorted does not match the bin number. empty = np.floor(sorted_value_in_bin[min_index]) != bin_numbers # Process empty bins. if keep_empty: index[empty] = -1 else: index = index[~empty] return index def parallel_coordinates(data, labels=None, value=None, value_label=""): """ Produce a parallel coordinates plot. *data* is a set of points to draw, one row per point. *labels* is the label to assign to the dimensions, one per column in *data*. *value* is an optional value to use to color the different lines. This could be the value of the control variable, or some additional dimension such as overall quality for the individual points. *value_label* is the label to put on the colorbar for the line colors. """ import matplotlib.pyplot as plt from matplotlib.collections import LineCollection ndim = data.shape[1] if labels is None: labels = ["p%d" % k for k in range(ndim)] x = np.arange(ndim) data = scale(data, axis=0) # We need to set the plot limits, they will not autoscale fig, ax = plt.gcf(), plt.gca() ax.set_xlim(0, ndim - 1) ax.set_ylim(0, 1) plt.xticks(x, labels, rotation=30) lines = LineCollection([np.column_stack([x, y]) for y in data], linewidths=1, linestyles="solid") ax.add_collection(lines) ax.set_yticklabels([]) if value is not None: lines.set_array(value) cbar = fig.colorbar(lines) cbar.set_label(value_label) plt.sci(lines) def scale(x, axis=None): """ Returns *x* with values scaled to [0, 1]. If *axis* is not None, then scale values within each axis independently, otherwise use the min/max value across all dimensions. """ low = x.min(axis=axis, keepdims=True) high = x.max(axis=axis, keepdims=True) dx = high - low dx[dx == 0.0] = 1.0 scaled = (x - low) / dx return scaled bumps-bumps-2b313fd/bumps/dream/state.py000066400000000000000000002233171521754644000203220ustar00rootroot00000000000000""" Sampling history for MCMC. MCMC keeps track of a number of things during sampling. The results may be queried as follows:: draws, generation, thinning, total_generations sample(condition) returns draws, points, logp gen_logp() returns genid, logp acceptance_rate() returns draws, AR traces() returns genid, chains CR_weight() returns draws, CR_weight best() returns best_x, best_logp outliers() returns outliers show()/save(file)/load(file) Data is stored in circular arrays, which keeps the last N generations and throws the rest away. draws is the total number of draws from the sampler. generation is the total number of generations. Due to tests for partially full circular buffers throughout the code (state.generation < state.Ngen) we are resetting generation to the size of the stored history on resume, and setting the generation_offset to the start of the history. If you need the number of generations across resume then use total_generations. thinning is the number of generations per stored sample. draws[i] is the number of draws including those required to produce the information in the corresponding return vector. Note that draw numbers need not be linearly spaced, since techniques like delayed rejection will result in a varying number of samples per generation. logp[i] is the set of log likelihoods, one for each member of the population. The logp() method returns the complete set, and the sample() method returns a thinned set, with on element of logp[i] for each vector point[i, :]. AR[i] is the acceptance rate at generation i, showing the proportion of proposed points which are accepted into the population. chains[i, :, :] is the set of points in the differential evolution population at thinned generation i. Ideally, the thinning rate of the MCMC process is chosen so that thinned generations i and i+1 are independent samples from the posterior distribution, though there is a chance that this may not be the case, and indeed, some points in generation i+1 may be identical to those in generation i. Actual generation number is i*thinning. points[i, :] is the ith point in a returned sample. The i is just a place holder; there is no inherent ordering to the sample once they have been extracted from the chains. Note that the sample may be from a marginal distribution. R[i] is the Gelman R statistic measuring convergence of the Markov chain. CR_weight[i] is the set of weights used for selecting between the crossover ratios available to the candidate generation process of differential evolution. These will be fixed early in the sampling, even when adaptive differential evolution is selected. outliers[i] is a vector containing the thinned generation number at which an outlier chain was removed, the id of the chain that was removed and the id of the chain that replaced it. We leave it to the reader to decide if the cloned samples, point[:generation, :, removed_id], should be included in further analysis. best_logp is the highest log likelihood observed during the analysis and best_x is the corresponding point at which it was observed. generation is the last generation number """ # TODO: state should be collected in files as we go __all__ = ["MCMCDraw", "Draw", "dream_load", "h5load", "h5dump"] import os.path import re import gzip from typing import List, Dict, TextIO, Tuple, Union, Optional, Callable from pathlib import Path import warnings from fnmatch import fnmatch import numpy as np from numpy import empty, sum, asarray, inf, argmax, hstack, dstack from numpy import loadtxt, savetxt, reshape from numpy.typing import NDArray from .convergence import burn_point from .outliers import identify_outliers from .util import rng from .gelman import gelman # Don't load hdf5 if you don't need it from typing import TYPE_CHECKING if TYPE_CHECKING: from h5py import Group EXT = ".mc.gz" CREATE = gzip.open # EXT = ".mc" # CREATE = open SelectionType = Optional[Dict[Union[int, str], Tuple[float, float]]] UNCERTAINTY_DTYPE = "d" MAX_LABEL_LENGTH = 1024 LABEL_DTYPE = f"|S{MAX_LABEL_LENGTH}" H5_COMPRESSION = 5 # TODO: Unused code. Using webview.server.cli.reload_export instead def dream_load(store): """ Load the saved DREAM state. *store* is the path to the stored state, either as an HDF5 history file or as a bumps export directory. If the directory contains multiple exports then use the path to the .par file within the directory. See also: h5load, load_state """ from pathlib import Path import h5py store = Path(store).expanduser().resolve() modelname = None if not store.exists(): raise RuntimeError(f"Path {store} does not exist.") # First try the HDF reader since the user can save the session to any file. if store.is_file(): h5 = None try: with h5py.File(store) as h5: if "fitting/fit_state" in h5: return h5load(h5["fitting/fit_state"]) else: raise RuntimeError(f"fitting/fit_state not found in {store}") except Exception: if h5 is not None: raise # Fall through to the non-hdf5 loader if store.is_dir(): # It's a directory: look for model name in store/*.par pars = list(store.glob("*.par")) if not pars: raise RuntimeError("No .par file found in %s" % store) if len(pars) > 1: print(pars) raise RuntimeError("Multiple models found in %s: %s" % (store, ", ".join(p.name for p in pars))) modelname = pars[0].stem else: # It's a filename: parse store=path/model.par if store.suffix != ".par": raise RuntimeError(f"Expected model.par file, got {store}") modelname = store.stem store = store.parent # Reload the MCMC data basename = str(store / modelname) state = load_state(basename) return state def _h5_write_field(group: "Group", field: str, data: Union[NDArray, str]): import h5py # print(f"h5 writing {field} in {group} with {data}") if isinstance(data, str): dtype = h5py.string_dtype(encoding="utf-8", length=len(data)) return group.create_dataset(field, data=data, dtype=dtype) elif isinstance(data, (int, np.integer)): return group.create_dataset(field, data=data, dtype=np.int64) elif isinstance(data, (float, np.floating)): return group.create_dataset(field, data=data, dtype=np.double) elif np.isscalar(data): return group.create_dataset(field, data=data, dtype=data.dtype) else: return group.create_dataset(field, data=data, dtype=data.dtype, compression=H5_COMPRESSION) def _h5_read_field(group: "Group", field: str): raw_data = group[field][()] size = raw_data.size if isinstance(raw_data, np.integer): return int(raw_data) elif isinstance(raw_data, np.floating): return float(raw_data) elif size is not None and size > 0: return raw_data else: return None def h5dump(group: "Group", state: "MCMCDraw"): class Fields: total_generations = state.total_generations thinning = state.thinning gen_draws, gen_logp = state.logp(full=True) _, AR = state.acceptance_rate() thin_draws, thin_point, thin_logp = state.chains() update_draws, update_CR_weight = state.CR_weight() labels = np.array(state.labels, dtype=LABEL_DTYPE) # These are marked outliers. They should still be marked when reloading the # state, but they need to be cleared when updates are given. Make sure the # request for the population from dream includes the outliers. good_chains = None if isinstance(state._good_chains, slice) else state._good_chains # In case the MCMC chains are stored as 32-bit, save the current population # as 64-bit so that resume is consistent. current_population, current_logp = state._last_gen() best_x, best_logp = state.best() best_gen = state._best_gen portion = state.portion # print(f"wrote {Fields.total_generations} generations") # print(f"{state._gen_offset=} {state.Ngen=}") # print("wrote", Fields.__dict__) for field, value in Fields.__dict__.items(): if field[0] != "_" and value is not None: try: _h5_write_field(group, field, value) except Exception as exc: exc.args = (f"{exc.args[0]} while writing {field}",) + exc.args[1:] raise def h5load(group: "Group"): # print(f"dream.state.h5load {group}") class Fields: ... for field in group: setattr(Fields, field, _h5_read_field(group, field)) # print("read", Fields.__dict__) # Guess dimensions Ngen = Fields.gen_draws.shape[0] Nthin, Npop, Nvar = Fields.thin_point.shape Nupdate, Ncr = Fields.update_CR_weight.shape # Nthin -= skip good_chains = getattr(Fields, "good_chains", None) total_generations = getattr(Fields, "total_generations", Ngen) thinning = getattr(Fields, "thinning", 1) current_population = getattr(Fields, "current_population", None) if current_population is None: current_population = Fields.thin_point[-1].astype("d") current_logp = getattr(Fields, "current_logp", None) if current_logp is None: current_logp = Fields.thin_logp[-1].astype("d") best_x = getattr(Fields, "best_x", None) best_logp = getattr(Fields, "best_logp", 0.0) best_gen = getattr(Fields, "best_gen", 0) portion = getattr(Fields, "portion", 1.0) # Create empty draw and fill it with loaded data state = MCMCDraw(0, 0, 0, 0, 0, 0, thinning) state.portion = portion state.draws = Ngen * Npop state.labels = [label.decode() for label in Fields.labels] state.generation = Ngen # print(f"loading {total_generations=} {Ngen=}") state._gen_offset = total_generations - Ngen state._gen_index = 0 state._gen_draws = Fields.gen_draws.astype(np.int64) state._gen_acceptance_rate = Fields.AR state._gen_logp = Fields.gen_logp state._thin_count = Nthin state._thin_index = 0 state._thin_draws = Fields.thin_draws.astype(np.int64) state._thin_logp = Fields.thin_logp state._thin_point = Fields.thin_point state._gen_current = current_population state._gen_current_logp = current_logp state._update_count = Nupdate state._update_index = 0 state._update_draws = Fields.update_draws.astype(np.int64) state._update_CR_weight = Fields.update_CR_weight state._outliers = [] if best_x is not None: state._best_x = best_x state._best_logp = best_logp state._best_gen = best_gen else: bestidx = np.unravel_index(np.argmax(Fields.thin_logp), Fields.thin_logp.shape) state._best_logp = Fields.thin_logp[bestidx] state._best_x = Fields.thin_point[bestidx] # We are not multiplying the following by thinning because thinning and best_gen # were added at the same time, so the reloaded thinning=1. Even if it were not # one this could still be wrong since thinning may have changed on resume. state._best_gen = bestidx[0] state._good_chains = slice(None, None) if good_chains is None else good_chains.astype(int) return state def save_state(state: "MCMCDraw", filename: str): # import sys; trace = sys.stdout # trace = open(filename+"-trace.mc", "wt") # trace.write("starting trace\n") # Build 2-D data structures # trace.write("extracting draws, logp\n") draws, logp = state.logp(full=True) # trace.write("extracting acceptance rate\n") _, AR = state.acceptance_rate(portion=1) # trace.write("building chain from draws, AR and logp\n") chain = hstack((draws[:, None], AR[:, None], logp)) # trace.write("extracting point, logp\n") _, point, logp = state.chains() Nthin, Npop, Nvar = point.shape # trace.write("shape is %d,%d,%d\n" % (Nthin, Npop, Nvar)) # trace.write("adding logp to point\n") point = dstack((logp[:, :, None], point)) # trace.write("collapsing to draws x point\n") point = reshape(point, (point.shape[0] * point.shape[1], point.shape[2])) # trace.write("extracting CR_weight\n") draws, CR_weight = state.CR_weight() Nupdate, Ncr = CR_weight.shape # trace.write("building stats\n") stats = hstack((draws[:, None], CR_weight)) # TODO: missing _outliers from save_state # Write convergence info # trace.write("writing chain\n") fid = CREATE(filename + "-chain" + EXT, "wb") fid.write(f"# draws acceptance_rate {Npop}*logp\n".encode()) savetxt(fid, chain) fid.close() # Write point info # trace.write("writing point\n") fid = CREATE(filename + "-point" + EXT, "wb") fid.write(f"# logp point (Nthin x Npop x Nvar = [{Nthin},{Npop},{Nvar}])\n".encode()) savetxt(fid, point) fid.close() # Write stats # trace.write("writing stats\n") fid = CREATE(filename + "-stats" + EXT, "wb") fid.write(f"# draws {Ncr}*CR_weight\n".encode()) savetxt(fid, stats) fid.close() # trace.write("done state save\n") # trace.close() IND_PAT = re.compile("-1#IND") INF_PAT = re.compile("1#INF") def loadtxt_with_fallback(file: TextIO, report=0): """ Try to load the file with numpy.loadtxt, but if it fails then fall back to loadtxt_MSVC, which is adapted for windows non-finite numbers. """ # record current position in file so that we can reset it if loadtxt fails pos = file.tell() try: return loadtxt(file) except Exception: file.seek(pos) return loadtxt_MSVC(file, report=report) def loadtxt_MSVC(fh: TextIO, report=0): """ Like numpy loadtxt, but adapted for windows non-finite numbers. """ res = [] section = 0 lineno = 0 for line in fh: lineno += 1 if report and lineno % report == 0: print("read", section * report) section += 1 IND_PAT.sub("nan", line) INF_PAT.sub("inf", line) line = line.split("#")[0].strip() values = line.split() if len(values) > 0: try: res.append([float(v) for v in values]) except ValueError: print("Parse error:", values) return asarray(res) def path_contains_saved_state(filename: str) -> bool: chain_file = filename + "-chain" + EXT return os.path.exists(chain_file) def openmc(filename: str) -> TextIO: # If filename ends in .mc.gz, also check for a .mc file. # If filename ends in .mc, also check for a .mc.gz file. if filename.endswith(".gz"): if os.path.exists(filename): # print("opening with gzip") fh = gzip.open(filename, "rt") elif os.path.exists(filename[:-3]): fh = open(filename[:-3], "rt") else: raise RuntimeError("file %s does not exist" % filename) else: if os.path.exists(filename): fh = open(filename, "rt") elif os.path.exists(filename + ".gz"): # print("opening with gzip") fh = gzip.open(filename + ".gz", "rt") else: raise RuntimeError("file %s does not exist" % filename) return fh def load_state(filename: str, skip=0, report=0, derived_vars=0): """ *filename* is the path to the saved MCMC state up to the final -chain.mc, etc. Any extension will be removed before using. *derived_vars* is the number of columns added to each point, derived from other columns in that point. The newer set_derived_vars interface generates the derived variables on demand rather than storing them in the state object and so it will be zero always. """ # Trim extension from filename filename = str(Path(filename).with_suffix("")) # Read chain file with openmc(filename + "-chain" + EXT) as fid: chain = loadtxt_with_fallback(fid) # Read point file with openmc(filename + "-point" + EXT) as fid: line = fid.readline() point_dims = line[line.find("[") + 1 : line.find("]")] Nthin, Npop, Nvar = eval(point_dims) for _ in range(skip * Npop): fid.readline() point = loadtxt_with_fallback(fid, report=report * Npop) # Read stats file with openmc(filename + "-stats" + EXT) as fd: stats_header = fd.readline() stats = loadtxt_with_fallback(fd) # Determine number of R-stat stored in the stats file if "R-stat" in stats_header: # Old header looks like: # # draws {Nvar}*R-stat {Ncr}*CR_weight # however, number of R-stat stored in stats file is the number of # variables stored each generation, not including the derived variables # calculated after the MCMC has completed. num_r = int(stats_header.split("*")[0].split()[-1]) - derived_vars else: num_r = 0 # Guess dimensions Ngen = chain.shape[0] thinning = 1 Nthin -= skip Nupdate = stats.shape[0] # Create empty draw and fill it with loaded data state = MCMCDraw(0, 0, 0, 0, 0, 0, thinning) # print("gen, var, pop", Ngen, Nvar, Npop) state.portion = 1.0 state.draws = Ngen * Npop state.generation = Ngen state._gen_offset = 0 state._gen_index = 0 state._gen_draws = chain[:, 0].astype(np.int64) state._gen_acceptance_rate = chain[:, 1] state._gen_logp = chain[:, 2:] state._thin_count = Ngen // thinning state._thin_index = 0 state._thin_draws = state._gen_draws[(skip + 1) * thinning - 1 :: thinning] state._thin_logp = point[:, 0].reshape((Nthin, Npop)) state._thin_point = reshape(point[:, 1 : Nvar + 1 - derived_vars], (Nthin, Npop, -1)) state._gen_current = state._thin_point[-1].copy() state._gen_current_logp = state._thin_logp[-1].copy() state._update_count = Nupdate state._update_index = 0 state._update_draws = stats[:, 0].astype(np.int64) state._update_CR_weight = stats[:, 1 + num_r :] state._outliers = [] bestidx = np.argmax(point[:, 0]) state._best_logp = point[bestidx, 0] state._best_x = point[bestidx, 1 : Nvar + 1 - derived_vars] state._best_gen = 0 # Clean up _thin dimensions. # It seems that some 0.x versions of bumps are yielding thin draws one shorter # than logp and points, so throw away the first logp and points. if state._thin_draws.shape[0] == state._thin_logp.shape[0] - 1: # print(f"shapes differ {state._gen_draws.shape=}, {state._thin_draws.shape=}, {state._thin_logp.shape=}") state._thin_logp = state._thin_logp[1:] state._thin_point = state._thin_point[1:] # Load labels from the .par file if it exists. parfile = Path(f"{filename}.par") if parfile.exists(): try: text = parfile.read_text().strip() lines = text.split("\n") labels = [line.rsplit(None, 1)[0].strip() for line in lines] state.labels = labels except Exception as exc: print(f"Couldn't load labels from {parfile}: {exc}") return state # TODO: Unused code. Using webview.server.cli.reload_export instead def reload_state_from_export(parfile): """ For the simplified fit with an export path we can reload the bumps dream state from the exported MCMC files. *parfile* is the name of the model.par file saved in the export directory. Use *state.show()* for the returned state to produce the plots. """ assert parfile.endswith(".par"), "need model.par file" path = str(parfile)[:-4] state = load_state(path) state.mark_outliers() state.portion = state.trim_portion() return state class MCMCDraw(object): """ Initialization parameters: *Ngen* is the number of generations to store. These are retrievable through *gen_logp()* and *acceptance_rate()* methods. Note that this is not the same as the length of the saved chains. *Nthin* is the number of thinned generations to store. These are retrievable through the *draw()* and *traces()* methods. *Nupdate* is the number of crossover ratio sets to save. The DREAM algorithm runs in batches of steps, with various checks such as convergence and crossover ratio updates after each batch. The results are retrievable through the *CR_weight()* method. *Nvar* is the number of parameters stored in each point. *Npop* is the number of running chains. Note that unlike the --pop command line option, this is not the number of chains per parameter. *Ncr* is the number of crossover ratios. DREAM maintains a fixed set of crossover ratios, choosing amongst them at each DE step. Depending on the success rate of the steps, it will adapt the weights after every batch of updates. *thinning* is the number of update generations to skip between saves to the MC chains. Additional thinning can be done when drawing samples from the saved points. Attributes and properties: *Ngen*, *Nthin*, *Nupdate*, *Nvar*, *Npop*, *Ncr* gives the size of the buffers stored in state. If the buffers are not yet full, the returned sizes for the *traces()*, *gen_logp()*, *acceptance_rate()*, *CR_weight()* and *draw()* methods will be smaller. *generation* is the number of generations in the current fit run. This is reset to zero each time the fit is resumed. *draws* is the number of draws in the current fit run. This is reset to zero each time the fit is resumed. Use *total_generations* time *Npop* if you want the number of draws including all previous runs. *total_generations* is the total number of generations for the fit across all fit runs. *title* a title for the plot *labels* a list of names for each parameter *thinning* amount of thinning between the gen_logp buffer and the traces buffer. DREAM stores every nth generation starting at generation n (1-origin). It is possible to change thinning on resume, which will change the short range correlation in the traces, however these changes are not recorded. Generation number on the trace plot will be incorrect until DREAM burns through the entire buffer. *portion* gives the fraction of each chain to use for statistical plots. A value can be automatically determined by calling *state.portion = state.trim_portion()*, or supplied by the user to various functions that work with the chains. The value should be between 0.0 and 1.0, where 1.0 means the entire chain is used for statistical plots. """ _derived_fn: Callable[[NDArray], NDArray] = None _derived_labels: List[str] = None _labels = None _integer_vars = None # boolean array of integer variables, or None title = None def __init__( self, Ngen: int, Nthin: int, Nupdate: int, Nvar: int, Npop: int, Ncr: int, thinning: int, ): # self.generation and self.draws are used to control the number of iterations in # the dream loop, and must therefore be set to the current size of the state on # resume rather than the cumulative across all runs. To retrieve the totals use # generations=self.total_generations and draws=self.total_generations*self.Npop. # Total number of draws so far self.draws = 0 self.portion = 1.0 # Maximum observed likelihood # Note: with thinning and burn the best may not be in the set of samples self._best_x = None self._best_logp = -inf self._best_gen = 0 # TODO: Scrap the per-generation data. It takes memory and disk, but doesn't add value. # Per generation iteration self.generation = 0 self._gen_offset = 0 self._gen_index = 0 self._gen_draws = empty(Ngen, np.int64) self._gen_logp = empty((Ngen, Npop), dtype=UNCERTAINTY_DTYPE) self._gen_acceptance_rate = empty(Ngen, dtype=UNCERTAINTY_DTYPE) # TODO: should current generation be single precision if the model is single precision? # The current generation, stored as double precision so that nothing # is lost on resume. self._gen_current = empty((Npop, Nvar), dtype="d") self._gen_current_logp = empty((Npop,), dtype="d") # Per thinned generation iteration self.thinning = thinning self._thin_index = 0 self._thin_count = 0 self._thin_timer = 0 self._thin_draws = empty(Nthin, np.int64) self._thin_point = empty((Nthin, Npop, Nvar), dtype=UNCERTAINTY_DTYPE) self._thin_logp = empty((Nthin, Npop), dtype=UNCERTAINTY_DTYPE) # Per update iteration self._update_index = 0 self._update_count = 0 self._update_draws = empty(Nupdate, np.int64) self._update_CR_weight = empty((Nupdate, Ncr), dtype=UNCERTAINTY_DTYPE) self._outliers = [] # Query functions will not return outlier chains; initially, all # chains are marked as good. Call mark_outliers to remove # outlier chains from the set. self._good_chains = slice(None, None) @property def Ngen(self): return self._gen_draws.shape[0] @property def total_generations(self): return self._gen_offset + self.generation @property def Nsamples(self): return self._gen_logp.size @property def Nthin(self): return self._thin_draws.shape[0] @property def Nupdate(self): return self._update_draws.shape[0] @property def Nvar(self): """Number of parameters in the fit""" return self._thin_point.shape[2] @property def Npop(self): return self._gen_logp.shape[1] @property def Ncr(self): return self._update_CR_weight.shape[1] def resize(self, Ngen: int, Nthin: int, Nupdate: int, Nvar: int, Npop: int, Ncr: int, thinning: int): if self.Nvar != Nvar or self.Npop != Npop or self.Ncr != Ncr: raise ValueError("Cannot change Nvar, Npop or Ncr on resize") # print("resize") # print(self.generation, self.Ngen, Ngen) # print(self._update_count, self.Nupdate, Nupdate) # print(self._thin_count, self.Nthin, Nthin) # When resuming from live state, make sure we unroll the state before resizing. # This is probably a no-op since plotting will already have forced an unroll. self._unroll() self.thinning = thinning def buf_resize(v, new_size, position): """ Resize a circular buffer to *new_size*. *position* is the number of entries that have been added to the buffer. This may be less than the existing size if the buffer is not yet filled with one complete cycle. Assume the buffer has been unrolled so that the next location index is zero. Don't assume that the next location index is *position%size*. When extending, we add space to the end of the buffer and set the next index after the previous end of the buffer. When contracting, we keep the last N entries in the buffer. If there are fewer than N entries, keep all of them and include enough empty entries to make the buffer size N. Need to set the next index to min(position, old_size, new_size)%new_size """ if v.shape[0] < new_size: # expand v = np.resize(v, (new_size, *v.shape[1:])) elif position < new_size: # contract when not enough v = v[:new_size] else: # contract when have enough v = v[-new_size:] return v self._gen_offset += max(self.generation - Ngen, 0) self.generation = min(self.generation, Ngen, self.Ngen) self.draws = self.generation * self.Npop self._gen_index = 0 # Reset sample size to current size on resume. self.generation = min(self.generation, self.Ngen, Ngen) self.draws = self.generation * self.Npop self._gen_index = self.generation % Ngen self._gen_draws = buf_resize(self._gen_draws, Ngen, self.generation) self._gen_logp = buf_resize(self._gen_logp, Ngen, self.generation) self._gen_acceptance_rate = buf_resize(self._gen_acceptance_rate, Ngen, self.generation) self._thin_count = min(self._thin_count, self.Nthin, Nthin) self._thin_index = self._thin_count % Nthin self._thin_draws = buf_resize(self._thin_draws, Nthin, self._thin_count) self._thin_point = buf_resize(self._thin_point, Nthin, self._thin_count) self._thin_logp = buf_resize(self._thin_logp, Nthin, self._thin_count) self._update_count = min(self._update_count, self.Nupdate, Nupdate) self._update_index = self._update_count % Nupdate self._update_draws = buf_resize(self._update_draws, Nupdate, self._update_count) self._update_CR_weight = buf_resize(self._update_CR_weight, Nupdate, self._update_count) def save(self, filename: Union[Path, str]): save_state(self, filename) def trim_portion(self): """ Estimate the point at which the Markov chains have reached equilibrium, returning it as a portion of the currently stored length. If not converged, then trim the first half of the samples. """ index = burn_point(self) saved_gens = min(self.Ngen, self.generation) portion = 1 - (index / saved_gens) if index >= 0 else 0.5 return portion def trim_index(self, portion: Optional[float] = None, generation: Optional[int] = None): """ Returns the generation index corresponding to the trim portion. The returned generation is relative to the start of the sampling, even if resume has been called multiple times to extend the burn or the sample size. The optional *generation* parameter is needed in case we have a state object that is out of date with respect to the number of generations seen so far. This can happen when the state is transferred to the user interface thread much less frequently than the step monitor. """ # Note: self.total_generations = self.generation + self._gen_offset # If generation is provided, it represents self.total_generations, so # we can reconstruct the number of saved generations we would see if the # state object were up to date. if generation is None: total_gens = self.total_generations saved_gens = min(self.Ngen, self.generation) else: total_gens = generation saved_gens = min(self.Ngen, generation - self._gen_offset) portion = self.portion if portion is None else portion return total_gens - int(portion * saved_gens) def show(self, portion: Optional[float] = None, figfile: Union[str, Path, None] = None): from .views import plot_all plot_all(self, portion=portion, figfile=figfile) # TODO: _last_gen and _draw_pop do similar things. Can they be merged? def _last_gen(self): """ Returns x, logp for most recently saved generation. This may be several generations ago if thinning is enabled. """ return self._gen_current, self._gen_current_logp def _generation(self, new_draws: int, x: NDArray, logp: NDArray, accept: NDArray, force_keep: bool = False): """ Called from dream.py after each generation is completed with a set of accepted points and their values. """ # Keep track of the total number of draws # Note: this is first so that we tag the record with the number of # draws taken so far, including the current draw. self.draws += new_draws self.generation += 1 # Record if this is the best so far maxid = argmax(logp) if logp[maxid] > self._best_logp: self._best_logp = logp[maxid] self._best_x = x[maxid, :] + 0 # Force a copy self._best_gen = self.total_generations # print("new best", logp[maxid], self.generation) # Record acceptance rate and cost i = self._gen_index # print("generation", i, self.draws, "\n x", x, "\n logp", logp, "\n accept", accept) self._gen_draws[i] = self.draws self._gen_acceptance_rate[i] = 100 * sum(accept) / new_draws self._gen_logp[i] = logp i = i + 1 if i == len(self._gen_draws): i = 0 self._gen_index = i # Keep every nth iteration self._thin_timer += 1 if self._thin_timer == self.thinning or force_keep: self._thin_timer = 0 self._thin_count += 1 i = self._thin_index self._thin_draws[i] = self.draws self._thin_point[i] = x self._thin_logp[i] = logp i = i + 1 if i == len(self._thin_draws): i = 0 self._thin_index = i self._gen_current[:] = x self._gen_current_logp[:] = logp def _update(self, CR_weight: NDArray): """ Called from dream.py when a series of DE steps is completed and summary statistics/adaptations are ready to be stored. """ self._update_count += 1 i = self._update_index # print("update", i, self.draws, "\n CR weight", CR_weight) self._update_draws[i] = self.draws self._update_CR_weight[i] = CR_weight i = i + 1 if i == len(self._update_draws): i = 0 self._update_index = i @property def labels(self): """ Labels associated with each parameter in a point. This doesn't include the labels for derived quantities. For these you will need to query state.draw().labels. """ if self._labels is None: result = ["P%d" % i for i in range(self._thin_point.shape[2])] else: result = self._labels return result @labels.setter def labels(self, v: List[str]): self._labels = v def _draw_pop(self): """ Return the current population. """ return self._gen_current def _unroll(self): """ Unroll the circular queue so that data access can be done inplace. Call this when done stepping, and before plotting. Calls to logp, sample, etc. assume the data is already unrolled. """ if self.generation > self._gen_index > 0: self._gen_draws[:] = np.roll(self._gen_draws, -self._gen_index, axis=0) self._gen_logp[:] = np.roll(self._gen_logp, -self._gen_index, axis=0) self._gen_acceptance_rate[:] = np.roll(self._gen_acceptance_rate, -self._gen_index, axis=0) self._gen_index = 0 if self._thin_count > self._thin_index > 0: self._thin_draws[:] = np.roll(self._thin_draws, -self._thin_index, axis=0) self._thin_point[:] = np.roll(self._thin_point, -self._thin_index, axis=0) self._thin_logp[:] = np.roll(self._thin_logp, -self._thin_index, axis=0) self._thin_index = 0 if self._update_count > self._update_index > 0: self._update_draws[:] = np.roll(self._update_draws, -self._update_index, axis=0) self._update_CR_weight[:] = np.roll(self._update_CR_weight, -self._update_index, axis=0) self._update_index = 0 def remove_outliers(self, x: NDArray, logp: NDArray, test: str = "IQR"): """ Replace outlier chains with clones of good ones. This should happen early in the sampling processes so the clones have an opportunity to evolve their own identity. Only the head of the chain is modified. *state* contains the chains, with log likelihood for each point. *x*, *logp* are the current population and the corresponding log likelihoods; these are updated with cloned chain values. *test* is the name of the test to use (one of IQR, Grubbs, Mahal or none). See :func:`.outliers.identify_outliers` for details. Updates *state*, *x* and *logp* to reflect the changes. Returns a list of the outliers that were removed. """ # Grab the last part of the chain histories _, chains = self.logp() chain_len, Nchains = chains.shape outliers = identify_outliers(test, chains, x) # if len(outliers): print("old llf", logp[outliers]) # Loop over each outlier chain, replacing each with another for old in outliers: # Draw another chain at random, with replacement # TODO: consider using relative likelihood as a weight factor while True: new = rng.randint(Nchains) if new not in outliers: break # Update the saved state and current population self._replace_outlier(old=old, new=new) x[old, :] = x[new, :] logp[old] = logp[new] # if len(outliers): print("new llf", logp[outliers]) return outliers def _replace_outlier(self, old: int, new: int): """ Called from outliers.py when a chain is replaced by the clone of another. """ self._outliers.append((self._thin_index, old, new)) # 2017-10-06 [PAK] only replace the head, not the full chain index = self._gen_index self._gen_current[old] = self._gen_current[new] self._gen_logp[index, old] = self._gen_logp[index, new] self._thin_logp[index, old] = self._thin_logp[index, new] self._thin_point[index, old, :] = self._thin_point[index, new, :] def mark_outliers(self, test: str = "IQR", portion: Optional[float] = None): """ Mark some chains as outliers but don't remove them. This can happen after drawing is complete, so that chains that did not converge are not included in the statistics. *test* is 'IQR', 'mahal', 'grubbs', or 'none'. *portion* indicates what portion of the samples should be included in the outlier test. If None, then the stored portion is used. """ _, chains, logp = self.chains() if test == "none": self._good_chains = slice(None, None) else: thin_saved_gens = chains.shape[0] portion = self.portion if portion is None else portion start = int(thin_saved_gens * (1 - portion)) outliers = identify_outliers(test, logp[start:], chains[-1]) # print("outliers", outliers, logp.shape, chains.shape) if len(outliers) > 0: self._good_chains = np.array([i for i in range(logp.shape[1]) if i not in outliers]) else: self._good_chains = slice(None, None) # print(self._good_chains) def logp(self, full: bool = False): """ Return the cumulative number of draws and the log likelihoods for each chain. Note that draw[i] represents the total number of samples taken, including those for the samples in logp[i]. If full is True, then return all chains, not just good chains. ** Deprecated ** Use state.gen_logp() instead. """ # self._unroll() # draws, logp = self._gen_draws, self._gen_logp # if self.generation == self._gen_index: # draws, logp = [v[:self.generation] for v in (draws, logp)] # Don't do a full unroll here if self.generation == self._gen_index: draws = self._gen_draws[: self._gen_index] logp = self._gen_logp[: self._gen_index] elif self._gen_index > 0: draws = np.roll(self._gen_draws, -self._gen_index, axis=0) logp = np.roll(self._gen_logp, -self._gen_index, axis=0) else: draws = self._gen_draws logp = self._gen_logp return draws, (logp if full else logp[:, self._good_chains]) def gen_logp(self, portion: Optional[float] = None, outliers: bool = False): """ Return the iteration number and the log likelihood for each point in the individual sequences in that iteration. For example, to plot the convergence of each sequence:: genid, logp = state.gen_logp() plot(genid, logp) *portion* is the amount to trim from the head of the chain, or None to use the stored burn point. If *outliers* is True, then return all chains, not just good chains. """ # Don't do a full unroll here if self.generation == self._gen_index: logp = self._gen_logp[: self._gen_index] elif self._gen_index > 0: logp = np.roll(self._gen_logp, -self._gen_index, axis=0) else: logp = self._gen_logp stop, step = self.total_generations, 1 # No thinning on _gen_logp, _gen_draws start = stop - (len(logp) - 1) * step genid = np.arange(start, stop + 1, step) # 1-origin, end-inclusive assert len(logp) == len(genid) portion = self.portion if portion is None else portion start = int((1 - portion) * len(logp)) gen_index = slice(start, None) chain_index = slice(None) if outliers else self._good_chains return genid[gen_index], logp[gen_index, chain_index] def logp_slice(self, n: int): """ Return a slice of the logp chains, either the first n if n > 0 or the last n if n < 0. Avoids unrolling the circular buffer if possible. """ if n < 0: # tail if self._gen_index >= -n: return self._gen_logp[self._gen_index + n : self._gen_index] elif self._gen_index == 0: return self._gen_logp[n:] else: # unroll across boundary return np.vstack((self._gen_logp[n + self._gen_index :], self._gen_logp[: self._gen_index])) else: # head if self.generation < self.Ngen: return self._gen_logp[:n] elif self._gen_index + n <= self.Ngen: return self._gen_logp[self._gen_index : self._gen_index + n] else: return np.vstack((self._gen_logp[self._gen_index :], self._gen_logp[-n + self._gen_index :])) def min_slice(self, n: int): """ Return the minimum logp for n slices, from the head if positive or the tail if negative. This is a specialized function so it can be fast. Convergence can be quickly rejected if the min in a short head is smaller than the min in a long tail. Unfortunately, if the data is wrapped, then the max function will cost extra. """ # Copy the logic of slice if n < 0: # tail if self._gen_index >= -n: return np.min(self._gen_logp[self._gen_index + n : self._gen_index]) elif self._gen_index == 0: return np.min(self._gen_logp[n:]) else: # max across boundary return min(np.min(self._gen_logp[n + self._gen_index :]), np.min(self._gen_logp[: self._gen_index])) else: # head if self.generation < self.Ngen: return np.min(self._gen_logp[:n]) elif self._gen_index + n <= self.Ngen: return np.min(self._gen_logp[self._gen_index : self._gen_index + n]) else: return min(np.min(self._gen_logp[self._gen_index :]), np.min(self._gen_logp[-n + self._gen_index :])) def acceptance_rate(self, portion: Optional[float] = None): """ Return the iteration number and the acceptance rate for that iteration. For example, to plot the acceptance rate over time:: genid, AR = state.acceptance_rate() plot(genid, AR) V1.0.2: Now returns (genid, AR) rather than (num_draws, AR) """ AR = self._gen_acceptance_rate if self.generation == self._gen_index: AR = AR[: self.generation] elif self._gen_index > 0: AR = np.roll(AR, -self._gen_index, axis=0) # Generation numbers for the buffered acceptance rates. The acceptance # rate is stored for each generation regardless of thinning, so this is # a simple increment. last, step = self.total_generations, 1 first = last - (len(AR) - 1) * step generation = np.arange(first, last + 1, step) # 1-origin, end-inclusive portion = self.portion if portion is None else portion index = int((1 - portion) * len(AR)) return generation[index:], AR[index:] def chains(self): """ Returns the observed Markov chains and the corresponding likelihoods. The return value is a tuple (*draws*, *chains*, *logp*). *draws* is the number of samples taken up to and including the samples for the current generation. *chains* is a three dimensional array of generations X chains X vars giving the set of points observed for each chain in every generation. Only the thinned samples are returned. *logp* is a two dimensional array of generation X population giving the log likelihood of observing the set of variable values given in chains. """ self._unroll() retval = self._thin_draws, self._thin_point, self._thin_logp if self._thin_count == self._thin_index: retval = [v[: self._thin_count] for v in retval] return retval def traces(self, portion: Optional[float] = None, thin: int = 1, outliers: bool = False): """ Grab traces for trace plot. *portion* gives the starting point of the trace, skipping over the initial frames that may not have converged. *thin* is the amount of additional thinning to apply on the traces. *outliers* (default False) is True if bad chains should be included in the trace. Returns *generation* [Ngen] and *chains* [Ngen x Nchain x Nvar] """ portion = self.portion if portion is None else portion # TODO: Chains code is duplicated in Draw # Draw samples from the state according to portion, thinning and outliers. # If we also want derived variables and value range limits then we should # switch to the code in Draw. _, chains, _ = self.chains() start = int((1 - portion) * len(chains)) gen_index = slice(start, None, thin) chain_index = slice(None) if outliers else self._good_chains chains = chains[gen_index, chain_index, :] num_steps = chains.shape[0] last, step = self.total_generations, self.thinning * thin first = last - (num_steps - 1) * step generation = np.arange(first + 1, last + 2, step) # 1-origin inclusive return generation, chains def gelman(self, portion: Optional[float] = None): """ Compute the Gelman and Rubin R-statistic for the Markov chains. """ # Note: Not allowing a portion parameter here because that would # require unrolling the chains or otherwise making the logic in # gelman() too complicated. Keep it simple so that we can show # a simple R statistic over time without thrashing memory. if self.generation < self.Ngen: return gelman(self._thin_point[: self.generation], portion=1.0) else: return gelman(self._thin_point, portion=1.0) def CR_weight(self): """ Return the crossover ratio weights to be used in the next generation. For example, to see if the adaptive CR is stable use:: draw, weight = state.CR_weight() plot(draw, weight) See :mod:`.crossover` for details. """ self._unroll() retval = self._update_draws, self._update_CR_weight if self._update_count == self._update_index: retval = [v[: self._update_count] for v in retval] return retval def outliers(self): """ Return a list of outlier removal operations. Each outlier operation is a tuple giving the thinned generation in which it occurred, the old chain id and the new chain id. The chains themselves have already been updated to reflect the removal. Curiously, it is possible for the maximum likelihood seen so far to be removed by this operation. """ return asarray(self._outliers, "i") def best(self): """ Return the best point seen and its log likelihood. """ return self._best_x, self._best_logp def stable_best(self): """ Return True if the at least one full cycle of the circular buffer has passed since the best logp was first observed. """ # print(f"stable_best {self._best_gen} + {self.Ngen} <= {self.total_generations}") return self._best_gen + self.Ngen <= self.total_generations def keep_best(self): """ Place the best point at the end of the last good chain. Good chains are defined by mark_outliers. Because the Markov chain is designed to wander the parameter space, the best individual seen during the random walk may have been observed during the burn-in period, and may no longer be present in the chain. If this is the case, replace the final point with the best, otherwise swap the positions of the final and the best. """ # Get state as a 1D array _, chains, logp = self.chains() Ngen, Npop, Nvar = chains.shape points = reshape(chains, (Ngen * Npop, Nvar)) logp = reshape(logp, Ngen * Npop) # Set the final position to the end of the last good chain. If # mark_outliers has not been called, then _good_chains will # just be slice(None, None) if isinstance(self._good_chains, slice): final = -1 else: final = self._good_chains[-1] - Npop # Find the location of the best point if it exists and swap with # the final position, or overwrite the final position if it does # not exist. idx = np.where(logp == self._best_logp)[0] if len(idx) == 0: logp[final] = self._best_logp points[final, :] = self._best_x else: # If there are multiple best values, arbitrarily choose one of them. # The chances are very high that it is the same point that was kept # after a proposed step was rejected. idx = idx[0] logp[final], logp[idx] = logp[idx], logp[final] points[final, :], points[idx, :] = points[idx, :], points[final, :] def sample(self, **kw): """ Return a sample from the posterior distribution. **Deprecated** use :meth:`draw` instead. """ drawn = self.draw(**kw) return drawn.points, drawn.logp def entropy( self, vars: Optional[List[int]] = None, portion: Optional[float] = None, selection: SelectionType = None, n_est: int = 10000, thin: Optional[int] = None, method: Optional[str] = None, ): r""" Return entropy estimate and uncertainty from an MCMC draw. *portion* is the portion of each chain to use (uses self.portion if None). *vars* is the set of variables to marginalize over. It is None for the visible variables, or a list of variables. *vars* is the list of variables to use for marginalization. *selection* sets the range each parameter in the returned distribution, using {variable: (low, high)}. Missing variables use the full range. *n_est* is the number of points to use from the draw when estimating the entropy (default=10000). *thin* is the amount of thinning to use when selecting points from the draw. *method* determines which entropy calculation to use: * gmm: fit sample to a gaussian mixture model (GMM) with $5 \sqrt{d}$ components where $d$ is the number fitted parameters and estimate entropy by sampling from the GMM. * llf: estimates likelihood scale factor from ratio of density estimate to model likelihood, then computes Monte Carlo entropy from sample; this does not work for marginal likelihood estimates. DOI:10.1109/CCA.2010.5611198 * mvn: fit sample to a multi-variate Gaussian and return the entropy of the best fit gaussian; uses bootstrap to estimate uncertainty. * wnn: estimate entropy from nearest-neighbor distances in sample. DOI:10.1214/18-AOS1688 """ from . import entropy # Get the sample from the state. # set default thinning to max((steps * samples/step) // n_est, 1) if thin is None: Nsteps = min(self.Nthin, self._thin_count) thin = max(Nsteps * self.Npop // n_est, 1) # print("thin", thin, Nsteps, self.Npop, self.Nthin, self._thin_count) drawn = self.draw(portion=portion, vars=vars, selection=selection, thin=thin) # TODO: don't print within a library function! M = entropy.MVNEntropy(drawn.points) print("Entropy from MVN: %s" % str(M)) if method is None: # TODO: change default to gmm method = "llf" if method == "llf": S, Serr = entropy.entropy(drawn.points, drawn.logp, N_entropy=n_est) # print("Entropy from llf (Kramer): %s"%str(S)) elif method == "gmm": # Try pure gmm ... pretty good S, Serr = entropy.gmm_entropy(drawn.points, n_est=n_est) # print("Entropy from gmm: %g +/- %g"% (S, Serr)) elif method == "wnn": # Try pure wnn ... no good S, Serr = entropy.wnn_entropy(drawn.points, n_est=n_est) # print("Entropy from wnn: %s"%str(S)) elif method == "mvn": S, Serr = entropy.mvn_entropy_bootstrap(drawn.points) # print("Entropy from mvn: %s"%str(S)) else: raise ValueError("unknown method %r" % method) # Always return entropy estimate from draw, even if it is normal return S, Serr def show_labels(self): """ List of parameters in the state, with the parameter number for each. Use this to help with the inputs to state.draw(). """ print("\n".join(f"p[{k}] = {v}" for k, v in enumerate(self.labels))) def draw( self, portion: Optional[float] = None, vars: Optional[List[int]] = None, exclude: Optional[List[int]] = None, selection: SelectionType = None, thin: int = 1, outliers: bool = False, derived: Optional[Callable] = None, ): """ Return a sample from the posterior distribution. *portion* is the portion of each chain to use *vars* is the list of vars that are active. Variables can be 0-origin integers or strings matching a variable label. Match uses standard glob rules, with `*` matching any sequence, `?` matching any character, `[abc]` matching one of `abc`, and `[!abc]` not matching one of `abc`. *selection* sets the range each parameter in the returned distribution, using {var: (low, high)}. If *var* matches multiple labels, then use the first restriction only. Missing variables use the full range. *thin* takes every nth item. *outliers* is True if outlier chains should be included (default False). To plot the distribution for parameter p1:: draw = state.draw() hist(draw.points[:, 0]) To plot the interdependence of p1 and p2:: draw = state.sample() plot(draw.points[:, 0], draw.points[:, 1], '.') """ # Fill in defaults vars = vars if vars is not None else getattr(self, "_shown", None) portion = self.portion if portion is None else portion return Draw( state=self, portion=portion, vars=vars, exclude=exclude, selection=selection, thin=thin, outliers=outliers, derived=derived, ) # TODO: Move processing of visible/integer/derived out of state def set_visible_vars(self, labels: List[str]): self._shown = [self.labels.index(v) for v in labels] # print("\n".join(str(pair) for pair in enumerate(self.labels))) # print(labels) # print(self._shown) def set_integer_vars(self, labels: List[str]): """ Indicate tha variables should be considered integer variables when computing statistics. """ self._integer_vars = np.array([var in labels for var in self.labels]) def set_derived_vars(self, fn: Callable[[NDArray], NDArray], labels: List[str]): """ Define derived variables from the sample. When calling draw() it will add columns for the derived variables to each sample. *fn* is a function taking points p[:, k] for k in 0 ... samples and returning a set of derived variables pj[k] for each sample k. The variables can be returned as any kind of sequence including an array or a tuple with one entry per variable. The caller uses asarray to convert the returned variables into a vars X samples array. For convenience, a single variable can be returned by itself. *labels* are the labels to use for the derived variables. The following example adds the new variable x+y = P[0] + P[1]:: state.derive_vars(lambda p: p[0]+p[1], labels=["x+y"]) """ self._derived_fn = fn self._derived_labels = labels def derive_vars(self, fn: Callable[[NDArray], NDArray], labels: Optional[List[str]] = None): """ *** DEPRECATED *** Like set_derived_vars but operating in place, modifying the points in the history. """ warnings.warn("Use state.set_derived_vars() instead of state.derive_vars()", DeprecationWarning, stacklevel=2) # Grab all samples as a set of points _, chains, _ = self.chains() Ngen, Npop, Nvar = chains.shape points = reshape(chains, (Ngen * Npop, Nvar)) # Compute new variables from the points newvars = asarray(fn(points.T)).T Nnew = newvars.shape[1] if len(newvars.shape) == 2 else 1 newvars.reshape((Ngen, Npop, Nnew)) # Extend new variables to be the same length as the stored selection Nthin = self._thin_point.shape[0] newvars = np.resize(newvars, (Nthin, Npop, Nnew)) # Add labels for the new variables, if available. This must # occur before updating self._thin_point, otherwise a default # label may be generated for the new column. if labels is not None: self.labels = self.labels + labels elif self._labels is not None: labels = ["P%d" % i for i in range(Nvar, Nvar + Nnew)] self.labels = self.labels + labels else: # no labels specified, old or new pass # Add new variables to the points self._thin_point = dstack((self._thin_point, newvars)) class Draw: """ A set of sample points from an MCMC draw with restrictions applied. **Inputs** *state* is the :class:`MCMCDraw` containing all points. *portion* is the fraction of points to use for the sample, starting from the ends of the Markov chains. *vars* is the list of vars that are active. Variables can be 0-origin integers or strings matching a variable label. Match uses standard glob rules, with `*` matching any sequence, `?` matching any character, `[abc]` matching one of `abc`, and `[!abc]` not matching one of `abc`. *exclude* all labels that match a pattern in the list. Matching rules are the same as for *vars*. If a variable matches both *vars* and *exclude* then it is included. *selection* restricts the points returned to a specific region of the parameter space using {var: (low, high)}. The var can be a 0-origin integer or label as for *vars*. It can also be *logp* if restricting by log likelihood. If a label matches multiple patterns then choose the first restriction (this may change to all that match in future). *thin* is the amount of thinning to apply, using every nth point in the chain. This reduces autocorrelation in the chains and reduces artificial spikes in the marginal distributions. (The distributions will still be spiky, but the spikiness will be appropriate for the number of points in the bins.) **Attributes** *points[n,k]* are the set of sampled points, where n is the number of points and k is the number of parameters. The earlier points come from earlier generations, but this is an accident of the implementation and should not be relied on. n is usually the number of chains times the number of saved generations, but this will change depending on *portion*, *selection*, *thin* and *outliers* parameters. k is usually the number of fitted parameters, but the derived and active parameter lists will adjust this. *Nvar* is the number of dimensions per point in the draw after including derived parameters and excluding nuisance parameters. *weights* are the weights associated with each point. Since the chains are run at a single temperature the weights will all be 1. *logp[n]* is the negative log likelihuud for each sampled point. *labels* are the labels for the variables remaining after applying derived variables and restricting to the requested *vars* list. *integers* is an array of flags, one per parameter* indicating whether the parameter is float (0) or int (1=floor). The values in the chain are managed as floats, and it is the responsibility of the likelihood function to transform them to integers. Models which use round, trunc or ceil rather than float are not yet supported.[1] *state* is the original MCMCDraw. [1] Note that discrete parameter handling is likely to change in the future. The derivative based optimizers in particular will need to know that the minimum step size for discrete parameters is one when computing the partial derivative. If bumps converts the parameters to integers before calling the likelihood function then we will only need a discrete flag since we will always use the *floor()* function. """ state: MCMCDraw vars: Optional[List[Union[int, str]]] portion: float selection: SelectionType thin: int Nvar: int title: Optional[str] labels: List[str] integers: NDArray # boolean logp: NDArray points: NDArray weights: NDArray # # MC chains, with mask for included points. # generation: NDArray # chains: NDArray # chains_logp: NDArray # mask: Optional[NDArray] def __init__( self, state: MCMCDraw, portion: float = 1.0, vars: Optional[List[Union[int, str]]] = None, exclude: Optional[List[Union[int, str]]] = None, selection: SelectionType = None, thin: int = 1, outliers: bool = False, derived: Optional[Callable] = None, ): self.state = state self.portion = portion self.thin = thin self.selection = selection self.labels = state.labels[:] # copy of labels self.integers = ( state._integer_vars if state._integer_vars is not None else np.zeros(len(self.labels), dtype=bool) ) self.title = state.title # TODO: Make a copy? # Draw samples from the state according to portion and thinning. _, chains, logp = state.chains() start = int((1 - portion) * len(chains)) gen_index = slice(start, None, thin) chain_index = slice(None) if outliers else state._good_chains chains = chains[gen_index, chain_index, :] chains_logp = logp[gen_index, chain_index] Ngen, Npop, Nvar = chains.shape # Derived variables are created during the draw so that existing data isn't altered. # This allows resume to work without extra effort. The code is also much simpler. if derived is None: derived = state._derived_fn if derived is not None: newvars = derived(chains.reshape(-1, Nvar).T) # If derived returns a dict, then use its keys as parameter labels if isinstance(newvars, dict): newlabels, newvars = zip(*newvars.items()) else: # Backward compatibility: old interface had derived returning a list newlabels = state._derived_labels if not newlabels: newlabels = [f"F{k+1}" for k in range(len(derived))] newvars = asarray(newvars).T newvars = newvars.reshape(Ngen, Npop, -1) chains = np.dstack((chains, newvars)) Nvar = chains.shape[2] # Update the number of variables self.integers = np.append(self.integers, [False] * newvars.shape[2]) self.labels.extend(newlabels) # Select points within range limits given by selection, if any. # This happens before variable subsetting so that numerical indices # are independent of the displayed parameters. It happens after # calculating derived variables so that we can apply selection to derived values. mask = True selection = self._selection_pattern_to_index(selection) for var, limits in selection.items(): if var == "logp": mask = mask & (chains_logp >= limits[0]) & (chains_logp <= limits[1]) else: mask = mask & (chains[..., var] >= limits[0]) & (chains[..., var] <= limits[1]) # TODO: with pattern matching we've lost the ability to reorder parameters # Restrict draw to a subset of the variables. Labels matching include take # precedence over those matching exclude. include = self._var_pattern_to_index(vars) exclude = self._var_pattern_to_index(exclude) exclude = [k for k in exclude if k not in include] include = [k for k in include if k not in exclude] # if vars and not include: # # Note: could be empty because everything is excluded # logging.warning(f"No parameters match {vars} in {self.labels}") if include: # if nothing matches then match everything chains = chains[:, :, include] # Mask doesn't include the final dimension, so doesn't need to be trimmed. Nvar = chains.shape[-1] self.labels = [self.labels[v] for v in include] self.integers = self.integers[include] self.vars = include # indices of selected items else: self.vars = None # Convert to a vector of selected points. if mask is True: # All points selected, so simply adjust the view self.points = chains.reshape(-1, Nvar) self.logp = chains_logp.flatten() else: # Only some points selected, so grab those under the mask self.points = chains[mask, :] self.logp = chains_logp[mask] # # Keep the thinned chains and the mask for trace plots # self.chains = chains # self.chains_logp = chains_logp # self.mask = None if mask is True else mask # stop, step = state.total_generations, thin*state.thinning # self.generation = np.arange(stop - (Ngen-1)*step + 1, stop + 2, step) # 1-origin inclusive self.weights = None self.Nvar = Nvar # Cache for the sorted argument indices. self._argsort_indices = {} def get_argsort_indices(self, var: int): """ Sort var by value. Unlike argsort this caches the results for reuse. """ if var not in self._argsort_indices: self._argsort_indices[var] = np.argsort(self.points[:, var].flatten()) return self._argsort_indices[var] def _selection_pattern_to_index(self, selection: dict[int | str, tuple[float, float]] | None): """ For the selection dictionary {pattern: data}, return a dictionary of {index: data} where data comes from the first pattern to matched by the label. Patterns are the standard unix glob patterns, or integers for specific numbered variables (see :meth:`State.show_labels`). """ if selection is None: return {} result = {} if data := selection.get("logp", None): result["logp"] = data for index, label in enumerate(self.labels): for var, data in selection.items(): if index == var or (isinstance(var, str) and fnmatch(label, var)): result[index] = data break return result def _var_pattern_to_index(self, vars: list[int | str] | None): """ Convert the include/exclude list of var patterns into a list of integer parameter indices. Patterns are the standard unix glob patterns, or integers for specific numbered variables (see :meth:`State.show_labels`) """ if vars is None: return [] def var_match(var, label, index): return index == var or (isinstance(var, str) and fnmatch(label, var)) return [k for k, p in enumerate(self.labels) if any(var_match(v, p, k) for v in vars)] def test(): from numpy.linalg import norm from numpy.random import rand from numpy import arange # Make some fake data Nupdate, Nstep = 3, 5 Ngen = Nupdate * Nstep Nvar, Npop, Ncr = 3, 6, 2 xin = rand(Ngen, Npop, Nvar) xin[..., 2] *= 9 # make one parameter range large enough for integer tests xin += 0.1 # force above 0.1 so that arrays print nicer pin = rand(Ngen, Npop) accept = rand(Ngen, Npop) < 0.8 CRin = rand(Nupdate, Ncr) # thinning = 2 # Nthin = int(Ngen/thinning) # Put it into a state thinning = 2 Nthin = int(Ngen / thinning) thinned = xin[thinning - 1 :: thinning, :, :] state = MCMCDraw(Ngen=Ngen, Nthin=Nthin, Nupdate=Nupdate, Nvar=Nvar, Npop=Npop, Ncr=Ncr, thinning=thinning) for i in range(Nupdate): state._update(CR_weight=CRin[i]) for j in range(Nstep): gen = i * Nstep + j state._generation(new_draws=Npop, x=xin[gen], logp=pin[gen], accept=accept[gen]) # Check that it got there assert (thinned == state.chains()[1]).all() draws, logp = state.logp() assert norm(draws - Npop * arange(1, Ngen + 1)) == 0 # Data is generated as double but may be stored as single, # so there could be a loss of precision. assert norm(logp - pin.astype(UNCERTAINTY_DTYPE)) == 0 genid, logp = state.gen_logp() assert norm(genid - arange(1, Ngen + 1)) == 0 assert norm(logp - pin) == 0 genid, AR = state.acceptance_rate() assert norm(genid - arange(1, Ngen + 1)) == 0 assert norm(AR - (100 * sum(accept, axis=1) / Npop).astype(UNCERTAINTY_DTYPE)) == 0 draws, logp = state.sample() # assert norm(draws - thinning*Npop*arange(1, Nthin+1)) == 0 # assert norm(sample - xin[thinning-1::thinning]) == 0 # assert norm(logp - pin[thinning-1::thinning]) == 0 draws, CR = state.CR_weight() assert norm(draws - Npop * Nstep * arange(Nupdate)) == 0 assert norm(CR - CRin.astype(UNCERTAINTY_DTYPE)) == 0 x, p = state.best() bestid = argmax(pin) i, j = bestid // Npop, bestid % Npop assert pin[i, j] == p assert norm(xin[i, j, :] - x) == 0 draw = state.draw(selection={"P1": (0.4, 1.1)}, portion=1, outliers=True) assert ((draw.points[:, 1] >= 0.4) & (draw.points[:, 1] <= 1.1)).all() # print(thinned[..., 1]) # print(thinned[..., 1] >= 0.4, thinned.shape) # print(len(draw.points[:, 1]), np.sum(thinned[..., 1] >= 0.4)) # print("thinned", thinned[thinned[..., 1] >= 0.4][:, 1]) # print("points", draw.points[:, 1]) mask = (thinned[..., 1] >= 0.4) & (thinned[..., 1] <= 1.1) assert len(draw.points[:, 1]) == np.sum(mask) if 0: # test of deprecated interface # Test derived variables (inplace update! ick!) state.derive_vars(lambda p: p[0] + p[1], labels=["inplace x+y"]) assert state.traces()[1].shape[2] == Nvar + 1 draw = state.draw() assert draw.labels[-1] == "inplace x+y" assert (draw.points[:, -1] == draw.points[:, 0] + draw.points[:, 1]).all() assert draw.points.shape[1] == Nvar + 1 assert len(draw.labels) == Nvar + 1 # Test derived variables (ephemeral) state.set_derived_vars(lambda p: p[0] + p[1], labels=["x+y"]) draw = state.draw() assert draw.labels[-1] == "x+y" assert (draw.points[:, -1] == draw.points[:, 0] + draw.points[:, 1]).all() assert draw.points.shape[1] == Nvar + 1 assert len(draw.labels) == Nvar + 1 from .stats import var_stats, format_vars vstats = var_stats(state.draw(vars=[2, "x+y"])) print(format_vars(vstats)) # Test integer vars state.set_integer_vars(labels=["P2"]) draw = state.draw() vstats_int = var_stats(draw) # print("integers", draw.integers) # print(format_vars(vstats_int)) assert vstats_int[2].mean == np.mean(np.floor(draw.points[:, 2])) draw = state.draw(selection={"P1": (0.4, 1.1)}) # print(format_vars(var_stats(draw))) assert (draw.points[:, 1] >= 0.4).all() # print(thinned[..., 1]) # print(thinned[..., 1] >= 0.4, thinned.shape) # print(len(draw.points[:, 1]), np.sum(thinned[..., 1] >= 0.4)) # print("thinned", thinned[thinned[..., 1] >= 0.4][:, 1]) # print("points", draw.points[:, 1]) assert len(draw.points[:, 1]) == np.sum(thinned[..., 1] >= 0.4) # Note: check mods to the state data structures last # Check that outlier updates properly state._replace_outlier(1, 2) outliers = state.outliers() draws, logp = state.sample() assert norm(outliers - asarray([[state._thin_index, 1, 2]])) == 0 # assert norm(sample[:, 1, :] - xin[thinning-1::thinning, 2, :]) == 0 # assert norm(sample[:, 2, :] - xin[thinning-1::thinning, 2, :]) == 0 # assert norm(logp[:, 1] - pin[thinning-1::thinning, 2]) == 0 # assert norm(logp[:, 2] - pin[thinning-1::thinning, 2]) == 0 if __name__ == "__main__": test() bumps-bumps-2b313fd/bumps/dream/stats.py000066400000000000000000000322321521754644000203320ustar00rootroot00000000000000""" Statistics helper functions. """ __all__ = [ "VarStats", "var_stats", "format_uncertainty", "format_num", "format_vars", "parse_var", "stats", "credible_interval", "shortest_credible_interval", ] from dataclasses import dataclass, field import re import json import numpy as np from uncertainties import ufloat, ufloat_fromstr def clean_exponent(s: str) -> str: """Remove leading + and leading zeros from exponent in a scientific notation number.""" head, sep, tail = s.partition("e") # If there is no 'e', return the original string if not sep: return s # Reassemble: head + 'e' + cleaned integer exponent return f"{head}e{int(tail)}" def format_uncertainty(mean, std): """Opinionated formatting of mean and standard deviation.""" # Handle indefinite value if np.isinf(mean): return "inf" if mean > 0 else "-inf" if np.isnan(mean): return "NaN" if std is None or std <= 0 or np.isnan(std): return str(mean) if np.isinf(std): # Format val(inf) using two digits of precision two_digits = f"{ufloat(mean, abs(mean)):.2uS}" return clean_exponent(re.sub(r"\(.*\)", "(inf)", two_digits)) return clean_exponent(f"{ufloat(mean, std):.2uS}") def test_format_uncertainty(): from math import inf, nan def check_val(val, unc, expected): res = format_uncertainty(val, unc) # if res != expected: # print(f"{val} ± {unc} => {res} != {expected}") assert res == expected, f"{val} ± {unc} => {res} != {expected}" # non-finite values check_val(-inf, None, "-inf") check_val(inf, None, "inf") check_val(nan, None, "NaN") # bad or missing uncertainty check_val(-1.23567, nan, "-1.23567") check_val(-1.23567, -inf, "-1.23567") check_val(-1.23567, -0.1, "-1.23567") check_val(-1.23567, 0, "-1.23567") check_val(-1.23567, None, "-1.23567") check_val(-1.23567, inf, "-1.2(inf)") check_val(-123.567, inf, "-1.2(inf)e2") check_val(-0.00123567, inf, "-0.0012(inf)") check_val(-0.0000123567, inf, "-1.2(inf)e-5") @dataclass class VarStats(object): label: str index: int p95: tuple[float, float] p68: tuple[float, float] # p95sd: float # p68sd: float median: float mean: float std: float best: float integer: bool = False p95_range: tuple[float, float] = field(init=False) p68_range: tuple[float, float] = field(init=False) def __post_init__(self): self.p95_range = (self.p95[0], self.p95[1] + self.integer * 0.9999999999) self.p68_range = (self.p68[0], self.p68[1] + self.integer * 0.9999999999) def __str__(self): return f"{self.label}={format_uncertainty(self.mean, self.std)}" # CRUFT: sasview is looking for name instead of label @property def name(self): return self.label def var_stats(draw, vars=None): if vars is None: vars = range(draw.points.shape[1]) return [_var_stats_one(draw, v) for v in vars] ONE_SIGMA = 1 - 2 * 0.15865525393145705 def _var_stats_one(draw, var): weights, values = draw.weights, draw.points[:, var].flatten() integer = draw.integers is not None and draw.integers[var] if integer: values = np.floor(values) best_idx = np.argmax(draw.logp) best = values[best_idx] x_sort_index = draw.get_argsort_indices(var) # Choose the interval for the histogram # credible_interval = shortest_credible_interval p95, p68, p0 = credible_interval(x=values, weights=weights, ci=[0.95, ONE_SIGMA, 0.0], x_sort_index=x_sort_index) ## reporting uncertainty on credible intervals? ## might be nice to pair sd on credible intervals ## with the actual CIs, rather than use a separate param # from .digits import credible_inderval_sd # p95sd = credible_interval_sd(values, 0.95) # p68sd = credible_interval_sd(values, ONE_SIGMA) # open('/tmp/out','a').write( # "in vstats: p68=%s, p95=%s, p0=%s, value range=%s\n" # % (p68,p95,p0,(min(values),max(values)))) # if p0[0] != p0[1]: raise RuntimeError("wrong median %s"%(str(p0),)) mean, std = stats(x=values, weights=weights, x_sort_index=x_sort_index) vstats = VarStats( label=draw.labels[var], index=var + 1, p95=p95, p68=p68, # p95sd=p95sd, p68sd=p68sd, median=p0[0], mean=mean, std=std, best=best, integer=integer, ) return vstats def format_num(x, place): precision = 10**place digits_after_decimal = abs(place) if place < 0 else 0 return "%.*f" % (digits_after_decimal, np.round(x / precision) * precision) def format_vars(all_vstats): v = dict( parameter="Parameter", mean="mean", median="median", best="best", interval68="68% interval", interval95="95% interval", ) s = [" %(parameter)20s %(mean)10s %(median)7s %(best)7s [%(interval68)15s] [%(interval95)15s]" % v] for v in all_vstats: # Make sure numbers are formatted with the appropriate precision place = ( int(np.log10(v.p95[1] - v.p95[0])) - 2 if v.p95[1] > v.p95[0] else int(np.log10(abs(v.p95[0]))) - 3 if v.p95[0] != 0 else 0 ) summary = dict( mean=format_uncertainty(v.mean, v.std), median=format_num(v.median, place - 1), best=format_num(v.best, place - 1), lo68=format_num(v.p68[0], place), hi68=format_num(v.p68[1], place), loci=format_num(v.p95[0], place), hici=format_num(v.p95[1], place), parameter=v.label, index=v.index, ) s.append( "%(index)2d %(parameter)20s %(mean)10s %(median)7s %(best)7s " "[%(lo68)7s %(hi68)7s] [%(loci)7s %(hici)7s]" % summary ) return "\n".join(s) def save_vars(all_vstats, filename): with open(filename, "w") as fid: json.dump( dict((v.label, v.__dict__) for v in all_vstats), fid, default=numpy_json, sort_keys=True, indent=2, ) def numpy_json(o): """ JSON encoder for numpy data. To automatically convert numpy data to lists when writing a datastream use json.dumps(object, default=numpy_json). """ if isinstance(o, np.integer): return int(o) elif isinstance(o, np.floating): return float(o) elif isinstance(o, np.bool_): return bool(o) elif isinstance(o, np.ndarray): return o.tolist() else: raise TypeError(f"unexpected type found: {type(o)}, {o}") VAR_PATTERN = re.compile( r""" ^\ * (?P[0-9]+)\ + (?P.+?)\ + (?P[0-9.-]+?\([0-9.]+\)(e[+-]?[0-9]+)?)\ + (?P[0-9.eE+-]+?)\ + (?P[0-9.eE+-]+?)\ + \[\ *(?P[0-9.eE+-]+?)\ + (?P[0-9.eE+-]+?)\]\ + \[\ *(?P[0-9.eE+-]+?)\ + (?P[0-9.eE+-]+?)\] \ *$ """, re.VERBOSE, ) def parse_var(line): """ Parse a line returned by format_vars back into the statistics for the variable on that line. """ m = VAR_PATTERN.match(line) if m: # Parse the full ufloat string (e.g., "1.23(8)" or "1.2346(8)e+06") parsed_value = ufloat_fromstr(m.group("mean_with_std")) mean = parsed_value.nominal_value std = parsed_value.std_dev return VarStats( label=m.group("parname"), index=int(m.group("parnum")), p95=(float(m.group("lo95")), float(m.group("hi95"))), p68=(float(m.group("lo68")), float(m.group("hi68"))), median=float(m.group("median")), mean=mean, std=std, best=float(m.group("best")), # TODO: The format_vars line does not show whether the var is integer. # integer=False, ) else: return None def stats(x, weights=None, x_sort_index=None): """ Find mean and standard deviation of a set of weighted samples. Note that the median is not strictly correct (we choose an endpoint of the sample for the case where the median falls between two values in the sample), but this is good enough when the sample size is large. """ if weights is None: if x_sort_index is None: x_sort_index = np.argsort(x) x = x[x_sort_index] mean, std = np.mean(x), np.std(x, ddof=1) else: mean = np.mean(x * weights) / np.sum(weights) # TODO: this is biased by selection of mean; need an unbiased formula var = np.sum((weights * (x - mean)) ** 2) / np.sum(weights) std = np.sqrt(var) return mean, std def credible_interval(x, ci, weights=None, x_sort_index=None): r""" Find the credible interval covering the portion *ci* of the data. *x* are samples from the posterior distribution. *ci* is a set of intervals in [0,1]. For a $1-\sigma$ interval use *ci=erf(1/sqrt(2))*, or 0.68. About 1e5 samples are needed for 2 digits of precision on a $1-\sigma$ credible interval. For a 95% interval, about 1e6 samples are needed for 2 digits of precision. At least 1000 points are needed for an unbiased result, otherwise the resulting interval will be shorter than expected (tested on a variety of distributions including exponential, cauchy, gaussian, beta and gamma). *weights* is a vector of weights for each x, or None for unweighted. One could weight points according to temperature in a parallel tempering dataset. Returns an array *[[x1_low, x1_high], [l2_low, x2_high], ...]* where *[xi_low, xi_high]* are the starting and ending values for credible interval *i*. This function is faster if the inputs are already sorted. """ n = x.size ci = np.asarray(ci, "d") target = (1 + np.vstack((-ci, +ci))).T / 2 if x_sort_index is None: x_sort_index = np.argsort(x) if weights is None: cdf = np.linspace(0.5 / n, 1 - 0.5 / n, n) # cdf = np.linspace(1, n, n)/(n+1) result = np.interp(target, cdf, x[x_sort_index]) else: x, weights = x[x_sort_index], weights[x_sort_index] # convert weights to cdf cdf = np.cumsum(weights) cdf /= cdf[-1] cdf -= 0.5 * cdf[0] # cdf *= n/(cdf[-1]*(n+1)) result = np.interp(target, cdf, x) return result if ci.shape else result[0] def shortest_credible_interval(x, ci=0.95, weights=None): """ Find the credible interval covering the portion *ci* of the data. *x* are samples from the posterior distribution. *ci* is the interval size in (0,1], and defaults to 0.95. For a 1-sigma interval use *ci=erf(1/sqrt(2))*. *weights* is a vector of weights for each x, or None for unweighted. Returns the minimum and maximum values of the interval. If *ci* is a vector, return a vector of intervals. This function is faster if the inputs are already sorted. About 1e6 samples are needed for 2 digits of precision on a 95% credible interval, or 1e5 for 2 digits on a 1-sigma credible interval. To remove bias towards toward smaller intervals, the midpoints between the surrounding intervals are used as the end points. """ if weights is None: x = np.sort(x) # Simple solution: ci*N is the number of points in the interval, so # find the width of every interval of that size and return the smallest. if np.isscalar(ci): return _unweighted_hpd(x, ci) else: return [_unweighted_hpd(x, ci_k) for ci_k in ci] else: index = np.argsort(x) x, weights = x[index], weights[index] # Work from the empirical cdf, finding the corresponding right # interval for each possible left interval and choosing that with # the shortest distance. cdf = np.cumsum(weights) cdf /= cdf[-1] # jcdf -= 0.5*cdf[0] if np.isscalar(ci): return _weighted_hpd(x, cdf, ci) else: return [_weighted_hpd(x, cdf, ci_k) for ci_k in ci] def _unweighted_hpd(x, ci): """ Find shortest credible interval ci in sorted, unweighted x """ n = len(x) size = int(ci * n) if size >= n: return x[0], x[-1] else: width = x[size:] - x[:-size] index = np.argmin(width) # left, right = x[idx], x[idx+size] left = x[0] if index == 0 else (x[index - 1] + x[index]) / 2 right = x[-1] if index + size == n - 1 else (x[index + size] + x[index + size + 1]) / 2 return left, right def _weighted_hpd(z, cdf, ci): # extra one-half interval """ Find shortest credible interval ci in sorted, weighted x """ size = np.searchsorted(cdf, 1 - ci) if size == 0: return z[0], z[-1] p_left = cdf[:size] z_left = z[:size] # avoid spurious floating point bugs, e.g., where .1+0.9 > 1.0 i_right = np.searchsorted(cdf[:-1], p_left + ci) z_right = z[i_right] index = np.argmin(z_right - z_left) left = z_left[0] if index == 0 else (z_left[index - 1] + z_left[index]) / 2 right = z_right[-1] if index + 1 == len(z_right) else (z_right[index] + z_right[index + 1]) / 2 return left, right bumps-bumps-2b313fd/bumps/dream/tile.py000066400000000000000000000027451521754644000201370ustar00rootroot00000000000000""" Split a rectangle into n panes. """ __all__ = ["max_tile_size"] import math def max_tile_size(tile_count, rect_size): """ Determine the maximum sized tile possible. Keyword arguments: tile_count -- Number of tiles to fit rect_size -- 2-tuple of rectangle size as (width, height) """ # If the rectangle is taller than it is wide, reverse its dimensions if rect_size[0] < rect_size[1]: rect_size = rect_size[1], rect_size[0] # Rectangle aspect ratio rect_ar = rect_size[0] / rect_size[1] # tiles_max_height is the square root of tile_count, rounded up tiles_max_height = int(math.ceil(math.sqrt(tile_count))) best_tile_size = 0 # i in the range [1, tile_max_height], inclusive for i in range(1, tiles_max_height + 1): # tiles_used is the arrangement of tiles (width, height) tiles_used = math.ceil(tile_count / i), i # tiles_ar is the aspect ratio of this arrangement tile_ar = tiles_used[0] / tiles_used[1] # Calculate the size of each tile # Tile pattern is flatter than rectangle if tile_ar > rect_ar: tile_size = rect_size[0] / tiles_used[0] # Tile pattern is skinnier than rectangle else: tile_size = rect_size[1] / tiles_used[1] # Check if this is the best answer so far if tile_size > best_tile_size: best_tile_size = tile_size return best_tile_size # print(max_tile_size(6, (100, 100))) bumps-bumps-2b313fd/bumps/dream/util.py000066400000000000000000000056541521754644000201610ustar00rootroot00000000000000""" Miscellaneous utilities. """ __all__ = ["draw", "console"] import numpy as np import numpy.random as rng try: from numba import njit except ImportError: def njit(*args, **kw): return lambda f: f @njit(cache=True) def draw(k, n): """ Select k things from a pool of n without replacement. """ # At k == n/4, an extra 0.15*k draws are needed to get k unique draws # TODO: silently returns too few values if k > n if k > n / 4: result = rng.permutation(n)[:k] else: result = np.empty(k, np.int64) for i in range(k): # select an item not already selected while True: p = rng.randint(n) for j in range(i): # if there is a match then break out of the for-loop and # go to the next while iteration, generating a new proposal if j == p: break # unusual syntax: if you make it to the end of the for loop # then there were no matches, so break out of the while loop else: break result[i] = p return result def _check_uniform_draw(): """ Draws from history should """ import matplotlib.pyplot as plt k, n = 50, 400 counts = np.zeros(n * k) idx = np.arange(k) for _ in range(100000): t = draw(k, n) counts[k * t + idx] += 1 plt.subplot(211) plt.pcolormesh(np.reshape(counts, (n, k))) plt.colorbar() plt.title("drawn number vs. draw position") plt.subplot(212) plt.hist(counts) plt.title("number of draws per (number,position) bin") plt.show() def console(): """ Start the python console with the local variables available. console() should be the last thing in the file, after sampling and showing the default plots. """ import os import sys # Hack for eclipse console: can't actually run ipython in the eclipse # console and get it to plot, so instead guess whether we are in a # console by checking if we are attached to a proper tty through stdin. # For eclipse, just show the plots. try: tty = os.isatty(sys.stdin.fileno()) except Exception: tty = False if tty: # Display outstanding plots and turn interactive on from matplotlib import interactive from matplotlib._pylab_helpers import Gcf for fig in Gcf.get_all_fig_managers(): try: # CRUFT fig.show() except AttributeError: fig.frame.Show() interactive(True) # Start an ipython shell with the caller's local variables import IPython symbols = sys._getframe(1).f_locals ip = IPython.Shell.IPShell(user_ns=symbols) ip.mainloop() else: # Not a tty; try doing show() anyway import matplotlib.pyplot as plt plt.show() bumps-bumps-2b313fd/bumps/dream/varplot.py000066400000000000000000000322521521754644000206650ustar00rootroot00000000000000""" Build layout for histogram plots """ __all__ = ["var_plot_size", "plot_vars", "plot_var"] from math import ceil, sqrt import numpy as np from matplotlib import pyplot as plt # Set space between plots in horiz and vert. H_SPACE = 0.2 V_SPACE = 0.3 # Set top, bottom, left margins. T_MARGIN = 0.2 B_MARGIN = 0.1 L_MARGIN = 0.2 R_MARGIN = 0.2 # Set desired plot sizes. TILE_W = 3.0 TILE_H = 2.0 CBAR_WIDTH = 0.75 def var_plot_size(n): ncol, nrow = tile_axes_square(n) # Calculate total width and figure size plots_width = (TILE_W + H_SPACE) * ncol figwidth = plots_width + CBAR_WIDTH + L_MARGIN + R_MARGIN figheight = (TILE_H + V_SPACE) * nrow + T_MARGIN + B_MARGIN return figwidth, figheight def _make_var_axes(n, fig=None): """ Build a figure with one axis per parameter, and one axis (the last one) to contain the colorbar. Use to make the vars histogram figure. """ if fig is None: fig = plt.gcf() fig.clf() total_width, total_height = fig.get_size_inches() ncol, nrow = tile_axes_square(n) # Calculate dimensions as a faction of figure size. v_space_f = V_SPACE / total_height h_space_f = H_SPACE / total_width t_margin_f = T_MARGIN / total_height b_margin_f = B_MARGIN / total_height l_margin_f = L_MARGIN / total_width top = 1 - t_margin_f + v_space_f left = l_margin_f tile_h = (total_height - T_MARGIN - B_MARGIN) / nrow - V_SPACE tile_w = (total_width - L_MARGIN - R_MARGIN - CBAR_WIDTH) / ncol - H_SPACE tile_h_f = tile_h / total_height tile_w_f = tile_w / total_width # Calculate colorbar location (left, bottom) and colorbar height. l_cbar_f = l_margin_f + ncol * (tile_w_f + h_space_f) b_cbar_f = b_margin_f + v_space_f cbar_w_f = CBAR_WIDTH / total_width cbar_h_f = 1 - t_margin_f - b_margin_f - v_space_f cbar_box = [l_cbar_f, b_cbar_f, cbar_w_f, cbar_h_f] k = 0 for j in range(1, nrow + 1): for i in range(0, ncol): if k >= n: break dims = [left + i * (tile_w_f + h_space_f), top - j * (tile_h_f + v_space_f), tile_w_f, tile_h_f] ax = fig.add_axes(dims) ax.set_facecolor("none") k += 1 fig.add_axes(cbar_box) # fig.set_size_inches(total_width, total_height) return fig def tile_axes_square(n): """ Determine number of columns by finding the next greatest square, then determine number of rows needed. """ cols = int(ceil(sqrt(n))) rows = int(ceil(n / float(cols))) return cols, rows def plot_vars(draw, all_vstats, fig=None, nbins: int = 30, full: bool = False): """ Plot parameter histograms in a grid on the figure. Each bar on the histogram is shaded according the nllf of points in that bar, sorted to make a color gradient. The colorbar showing the nllf range is shared across all plots. If *fig* is not provided and new figure will be created. *nbins* controls the number of bars on each histogram. Use *full=True* to plot the histogram across the full range of sample values. By default *full=False* and only the central 95% range is histogrammed. The plots can be unreadable when labels overwrite each other. You can adjust the layout by setting H_SPACE, V_SPACE, T_MARGIN, B_MARGIN, L_MARGIN, R_MARGIN (see pyplot.subplots_adjust), TILE_W, TILE_H (for aspect ratio of the subplots) and CBAR_WIDTH (it's complicated). This will affect every subsequent plot. """ n = len(all_vstats) fig = _make_var_axes(n, fig=fig) cbar = _make_fig_colorbar(draw.logp, fig=fig) for k, vstats in enumerate(all_vstats): axes = fig.axes[k] plot_var(draw, vstats, k, cbar, axes=axes, nbins=nbins, full=full) fig.canvas.draw() def plot_var(draw, vstats, var, cbar, axes=None, nbins=30, full=False): import matplotlib.pyplot as plt values = draw.points[:, var].flatten() if full: bin_range = np.min(values), np.max(values) else: bin_range = vstats.p95_range if axes is None: axes = plt.gca() make_logp_histogram(values, draw.logp, nbins, bin_range, draw.weights, cbar, axes) decorate_histogram(vstats, axes) def decorate_histogram(vstats, axes): from matplotlib.transforms import blended_transform_factory as blend l95, h95 = vstats.p95_range l68, h68 = vstats.p68_range # Shade things inside 1-sigma axes.axvspan(l68, h68, color="gold", alpha=0.5, zorder=-2) # Mark the median with a vertical line axes.axvline(x=vstats.median, color="g", ls=":", alpha=0.7, zorder=-1) # build transform with x=data, y=axes(0,1) transform = blend(axes.transData, axes.transAxes) # Mark the mean and best with symbols def marker(symbol: str, position: float) -> None: if position < l95: text, x, ha = f"{symbol}←", l95, "left" elif position > h95: text, x, ha = f"→{symbol}", h95, "right" else: text, x, ha = symbol, position, "center" y, va = -0.01, "top" axes.text(x, 1 + y, text, va=va, ha=ha, transform=transform, zorder=3, color="g") # axes.axvline(v) marker("▽", vstats.mean) marker("∗", vstats.best) # Put the parameter label on the line with mean and best markers. Use the side without # the mean/best marker so that they don't overwrite each other. if (vstats.mean - l95) / (h95 - l95) > 0.4 or (vstats.best - l95) / (h95 - l95) > 0.4: x, ha = 0.02, "left" else: x, ha = 0.98, "right" axes.text( x, 0.99, vstats.label, zorder=2, # backgroundcolor=(0.6, 1.0, 0.6, 0.6), verticalalignment="top", horizontalalignment=ha, transform=axes.transAxes, ) axes.set_yticklabels([]) def _make_fig_colorbar(logp, fig=None): import matplotlib as mpl import matplotlib.pyplot as plt # Option 1: min to min + 4 # vmin=-max(logp); vmax=vmin+4 # Option 1b: min to min log10(num samples) # vmin=-max(logp); vmax=vmin+log10(len(logp)) # Option 2: full range of best 98% snllf = np.sort(-logp) vmin, vmax = snllf[0], snllf[int(0.98 * (len(snllf) - 1))] # robust range # Option 3: full range # vmin,vmax = -max(logp),-min(logp) if fig is None: fig = plt.gcf() ax = fig.axes[-1] cmap = mpl.cm.copper # Set the colormap and norm to correspond to the data for which # the colorbar will be used. norm = mpl.colors.Normalize(vmin=vmin, vmax=vmax) # ColorbarBase derives from ScalarMappable and puts a colorbar # in a specified axes, so it has everything needed for a # standalone colorbar. There are many more kwargs, but the # following gives a basic continuous colorbar with ticks # and labels. class MinDigitsFormatter(mpl.ticker.Formatter): def __init__(self, low, high): self.delta = high - low def __call__(self, x, pos=None): # TODO: where did format_value come from? # it does not exist anywhere in the project. # return format_value(x, self.delta) return "{:.3G}".format(x) ticks = () # (vmin, vmax) formatter = MinDigitsFormatter(vmin, vmax) cbar = mpl.colorbar.ColorbarBase(ax, cmap=cmap, norm=norm, ticks=ticks, format=formatter, orientation="vertical") # cb.set_ticks(ticks) # cb.set_ticklabels(labels) # cb.set_label('negative log likelihood') cbar_box = ax.get_position().bounds fig.text(cbar_box[0], cbar_box[1], "{:.3G}".format(vmin), va="top") fig.text(cbar_box[0], cbar_box[1] + cbar_box[3], "{:.3G}".format(vmax), va="bottom") return cbar def make_logp_histogram(values, logp, nbins, ci, weights, cbar, ax): from numpy import ones_like, searchsorted, linspace, cumsum, unique, argsort, array, hstack, exp if weights is None: weights = ones_like(logp) # TODO: values are being sorted to collect stats and again to plot idx = argsort(values) values, weights, logp = values[idx], weights[idx], logp[idx] # open('/tmp/out','a').write("ci=%s, range=%s\n" # % (ci,(min(values),max(values)))) edges = linspace(ci[0], ci[1], nbins + 1) idx = searchsorted(values[1:-1], edges) # weightsum = cumsum(weights) # heights = diff(weightsum[idx])/weightsum[-1] # normalized weights edgecolors = None cmap = cbar.cmap cmap_edges = linspace(0, 1, cmap.N + 1)[1:-1] bins = [] # marginalized maximum likelihood for s, e, xlo, xhi in zip(idx[:-1], idx[1:], edges[:-1], edges[1:]): if s == e: continue # parameter interval endpoints x = array([xlo, xhi], "d") # -logp values within interval, with sort index from low to high pv = -logp[s:e] pidx = argsort(pv) pv = pv[pidx] # weights for samples within interval, sorted pw = weights[s:e][pidx] # vertical colorbar top edges is the cumulative sum of the weights y_top = cumsum(pw) # For debugging compare with one rectangle per sample if False: import matplotlib as mpl import matplotlib.pyplot as plt cmap = mpl.cm.flag edgecolors = "k" xmid = (xlo + xhi) / 2 x = [xlo, xmid] y = hstack((0, y_top)) z = pv[:, None] plt.pcolormesh(x, y, z, norm=cbar.norm, cmap=cmap) x = [xmid, xhi] # Possibly millions of samples, so group those which have the # same colour instead of drawing each as its own rectangle. # # Norm the values then look up the colormap edges in the sorted # normed negative log probabilities. Drop duplicates, which # represent zero-width bars. Assign the value for each interval # according to the value at the change point. # # The indexing logic is very ugly. The searchsorted() function # returns 0 if before the first or N if after the last, so the # end points of the range are dropped so that there is and implicit # [-inf, ... interior points ..., inf] range. Similarly, colours # below vmin go to vmin and above vmax go to vmax, so drop those # end points as well. Then put the end points back on in the # found indices [0, ... interior edges ..., N-1]. Use the value # at the end of the boundary to colour the section. # Something is not quite right: with this algorithm the first # block appears to always be one element long and is often the # same colour as the next block. This is only visible if edges # are drawn so ignore it for now. change_point = searchsorted(cbar.norm(pv[1:-1]), cmap_edges) tops = unique(hstack((change_point, len(pv) - 1))) y = hstack((0, y_top[tops])) z = pv[tops][:, None] ax.pcolormesh(x, y, z, norm=cbar.norm, cmap=cmap, edgecolors=edgecolors) # centerpoint, histogram height, maximum likelihood for each bin bins.append(((xlo + xhi) / 2, y_top[-1], exp(cbar.norm.vmin - pv[0]))) # Check for broken distribution if not bins: return centers, height, maxlikelihood = array(bins).T # Normalize maximum likelihood plot so it contains the same area as the # histogram, unless it is really spikey, in which case make sure it has # about the same height as the histogram. maxlikelihood *= np.sum(height) / np.sum(maxlikelihood) hist_peak = np.max(height) ml_peak = np.max(maxlikelihood) if ml_peak > hist_peak * 1.3: maxlikelihood *= hist_peak * 1.3 / ml_peak ml_peak = hist_peak * 1.3 ax.plot(centers, maxlikelihood, "-b") # Leave space for parameter label and statistics markers at the top of the plot ax.autoscale(enable=True, axis="x", tight=True) ax.set_ylim(0, 1.1 * max(hist_peak, ml_peak)) # ax.autoscale(enable=True, axis='y', tight=False) ## plot marginal gaussian approximation along with histogram # def G(x, mean, std): # return np.exp(-((x-mean)/std)**2/2)/np.sqrt(2*np.pi*std**2) ## TODO: use weighted average for standard deviation # mean, std = np.average(values, weights=weights), np.std(values, ddof=1) # pdf = G(centers, mean, std) # plt.plot(centers, pdf*np.sum(height)/np.sum(pdf), '-b') def make_var_histogram(values, logp, nbins, ci, weights): # Produce a histogram hist, bins = np.histogram( values, bins=nbins, range=ci, # new=True, density=True, weights=weights, ) # Find the max likelihood for values in each bin edges = np.searchsorted(values, bins) histbest = [np.max(logp[edges[i] : edges[i + 1]]) if edges[i] < edges[i + 1] else -np.inf for i in range(nbins)] # scale to marginalized probability with peak the same height as hist histbest = np.exp(np.asarray(histbest) - max(logp)) * np.max(hist) import matplotlib.pyplot as plt # Plot the histogram plt.bar(bins[:-1], hist, width=bins[1] - bins[0]) # Plot the kernel density estimate # density = KDE1D(values) # x = linspace(bins[0],bins[-1],100) # plt.plot(x, density(x), '-k') # Plot the marginal maximum likelihood centers = (bins[:-1] + bins[1:]) / 2 plt.plot(centers, histbest, "-g") bumps-bumps-2b313fd/bumps/dream/views.py000066400000000000000000000252501521754644000203330ustar00rootroot00000000000000""" MCMC plotting methods. """ __all__ = ["plot_all", "plot_corr", "plot_corrmatrix", "plot_trace", "plot_logp", "format_vars"] import math from typing import Optional, List import numpy as np from scipy.stats import gaussian_kde from . import corrplot, varplot from .stats import format_vars, format_uncertainty, save_vars, var_stats from .state import MCMCDraw, Draw CORRPLOT_MAXVAR = 25 # maximum number of variables to plot in correlation matrix TRACE_WSPACE = 0.15 TRACE_HSPACE = 0.1 # TODO: plot_all does not allow us to specify variables or their ranges # TODO: plot_all not tested with revised handling of derived parameters # TODO: remove hack allowing plot_all to accept a state.draw as well as a state object def plot_all(state: MCMCDraw | Draw, portion: Optional[float] = None, figfile=None): # Print/save uncertainty report before loading matplotlib or creating plots if isinstance(state, MCMCDraw): draw = state.draw(portion=portion) else: draw = state state = draw.state all_vstats = var_stats(draw) print(format_vars(all_vstats)) print( f"\nStatistics and plots based on {len(draw.points)} samples ({int(100 * draw.portion)}% of total samples drawn)" ) if figfile is not None: save_vars(all_vstats, figfile + "-err.json") import matplotlib.pyplot as plt figext = "." + plt.rcParams.get("savefig.format", "png") # Use finer binning with more samples. For 1% bin variation p, # points per bin k = (100/p)**2 = 10000, and nbins = N // k. nbins = max(min(draw.points.shape[0] // 10000, 400), 30) # histograms plt.figure(figsize=varplot.var_plot_size(len(all_vstats))) varplot.plot_vars(draw, all_vstats, nbins=nbins) if draw.title: plt.suptitle(draw.title, x=0, y=1, va="top", ha="left") if figfile is not None: plt.savefig(figfile + "-vars" + figext) # parameter traces plt.figure() plot_traces(draw.state, portion=draw.portion, vars=draw.vars) plt.suptitle("Parameter history" + (" for " + draw.title if draw.title else "")) if figfile is not None: plt.savefig(figfile + "-trace" + figext) # Acceptance rate if False: plt.figure() plot_acceptance_rate(draw.state, portion=draw.portion) if figfile is not None: plt.savefig(figfile + "-acceptance" + figext) # convergence plot plt.figure() plot_logp(draw.state, portion=draw.portion) if draw.title: plt.suptitle(draw.title) if figfile is not None: plt.savefig(figfile + "-logp" + figext) # correlation plot if draw.Nvar <= CORRPLOT_MAXVAR: plt.figure() plot_corrmatrix(draw, nbins=nbins) if draw.title: plt.suptitle(draw.title) if figfile is not None: plt.savefig(figfile + "-corr" + figext) # parallel coordinates plot if draw.Nvar > 1: from . import parcoord plt.figure() parcoord.plot(draw, control_var=0) if draw.title: plt.suptitle(draw.title) if figfile is not None: plt.savefig(figfile + "-parcor" + figext) def plot_corrmatrix(draw, nbins=50, vstats=None, full=False, fig=None): ranges = None if vstats is None or full else [v.p95_range for v in vstats] c = corrplot.Corr2d(draw.points.T, bins=nbins, ranges=ranges, labels=draw.labels) c.plot(fig=fig) # print "Correlation matrix\n",c.R() class KDE1D(gaussian_kde): covariance_factor = lambda self: 2 * self.silverman_factor() class KDE2D(gaussian_kde): covariance_factor = gaussian_kde.silverman_factor def __init__(self, dataset): gaussian_kde.__init__(self, dataset.T) def evalxy(self, x, y): grid_x, grid_y = np.meshgrid(x, y) dxy = self.evaluate(np.vstack([grid_x.flatten(), grid_y.flatten()])) return dxy.reshape(grid_x.shape) __call__ = evalxy def plot_corr(draw, vars=(0, 1)): """ Plot kernel density estimate of the parameter correlation. *vars* is the pair of parameters to plot (from *draw.labels*, 0-origin). """ import matplotlib.pyplot as plt _, _ = vars # Make sure vars is length 2 labels = [draw.labels[v] for v in vars] values = [draw.points[:, v] for v in vars] # Form kernel density estimates of the parameters xmin, xmax = min(values[0]), max(values[0]) density_x = KDE1D(values[0]) x = np.linspace(xmin, xmax, 100) px = density_x(x) density_y = KDE1D(values[1]) ymin, ymax = min(values[1]), max(values[1]) y = np.linspace(ymin, ymax, 100) py = density_y(y) nbins = 50 ax_data = plt.axes([0.1, 0.1, 0.63, 0.63]) # x,y,w,h density_xy = KDE2D(values[vars]) dxy = density_xy(x, y) * draw.points.shape[0] ax_data.pcolorfast(x, y, dxy, cmap=plt.cm.gist_earth_r) # @UndefinedVariable ax_data.plot(values[0], values[1], "k.", markersize=1) ax_data.set_xlabel(labels[0]) ax_data.set_ylabel(labels[1]) ax_hist_x = plt.axes([0.1, 0.75, 0.63, 0.2], sharex=ax_data) ax_hist_x.hist(values[0], nbins, orientation="vertical", density=1) ax_hist_x.plot(x, px, "k-") ax_hist_x.yaxis.set_major_locator(plt.MaxNLocator(4, prune="both")) plt.setp( ax_hist_x.get_xticklabels(), visible=False, ) ax_hist_y = plt.axes([0.75, 0.1, 0.2, 0.63], sharey=ax_data) ax_hist_y.hist(values[1], nbins, orientation="horizontal", density=1) ax_hist_y.plot(py, y, "k-") ax_hist_y.xaxis.set_major_locator(plt.MaxNLocator(4, prune="both")) plt.setp(ax_hist_y.get_yticklabels(), visible=False) def plot_traces( state: MCMCDraw, vars: Optional[List[int]] = None, portion: Optional[float] = None, fig=None, max_traces=6 ): import matplotlib.pyplot as plt # TODO: Consider using masked chains in draw for traces rather than original # Can only plot traces for on the original state. Ignore any variables that are out of range. if vars: vars = [v for v in vars if v < state.Nvar] if fig is None: fig = plt.gcf() if vars is None: vars = list(range(min(state.Nvar, max_traces))) plt.clf() nw, nh = tile_axes(len(vars), fig=fig) fig.subplots_adjust(hspace=TRACE_HSPACE, wspace=TRACE_WSPACE) for k, var in enumerate(vars): axes = fig.add_subplot(nw, nh, k + 1) plot_trace(state, var=var, portion=portion, axes=axes) axes.set_yticklabels([]) # TODO: label traces on the plots to save space between plots # label = state.labels[var] # axes.text(0.93, 0.93, label, ha="right", va="top", transform=axes.transAxes, backgroundcolor=(1.0, 1.0, 1.0, 0.75)) # axes.set_ylabel("") row = k // nh if row != nw - 1: # print(f"row {row} of {nw} x {nh}") axes.set_xticklabels([]) axes.set_xlabel("") def plot_trace(state: MCMCDraw, var: int = 0, portion: Optional[float] = None, axes=None, fig=None): import matplotlib.pyplot as plt if axes is None: if fig is None: fig = plt.gcf() axes = fig.add_subplot(111) # TODO: We could reuse the same state.traces outputs for all traces. generations, chains = state.traces(portion=portion, thin=1, outliers=False) label = state.labels[var] axes.clear() axes.plot(generations, chains[:, :, var]) axes.set_xlabel("Generation number") axes.set_ylabel(label) def plot_logp(state: MCMCDraw, portion: Optional[float] = None, outliers: bool = False): from matplotlib.ticker import NullFormatter import matplotlib.pyplot as plt from scipy.stats import chi2 # Plot log likelihoods, stripping outliers genid, logp = state.gen_logp(portion=portion, outliers=outliers) width, height, margin, delta = 0.7, 0.75, 0.1, 0.01 trace = plt.axes([margin, 0.1, width, height]) trace.plot(genid, logp, ",", markersize=1) trace.set_xlabel("Generation number") trace.set_ylabel("Log likelihood at x[k]") plt.title("Log Likelihood History") # Plot log likelihood trend line from bumps.wsolve import wpolyfit x = genid y = np.mean(logp, axis=1) dy = np.std(logp, axis=1, ddof=1) p = wpolyfit(x, y, dy=dy, degree=1) px, dpx = p.ci(x, 1.0) label = f"slope={format_uncertainty(p.coeff[0], p.std[0])}" trace.plot(x, px, "k-", x, px + dpx, "k-.", x, px - dpx, "k-.") trace.text(0.02, 0.02, label, transform=trace.transAxes, va="bottom", ha="left") # Plot long likelihood histogram data = logp.flatten() data = data[np.isfinite(data)] hist = plt.axes([margin + width + delta, 0.1, 1 - 2 * margin - width - delta, height]) hist.hist(data, bins=40, orientation="horizontal", density=True) hist.set_ylim(trace.get_ylim()) null_formatter = NullFormatter() hist.xaxis.set_major_formatter(null_formatter) hist.yaxis.set_major_formatter(null_formatter) # Plot chisq fit to log likelihood histogram float_df, loc, scale = chi2.fit(-data, f0=state.Nvar) df = int(float_df + 0.5) # from scipy.stats import kstest # pval = kstest(-data, lambda x: chi2.cdf(x, df, loc, scale)) # with open("/tmp/chi", "a") as fd: # print("chi2 pars for llf", float_df, loc, scale, pval, file=fd) xmin, xmax = trace.get_ylim() x = np.linspace(xmin, xmax, 200) hist.plot(chi2.pdf(-x, df, loc, scale), x, "r") def tile_axes(n, size=None, fig=None): """ Creates a tile for the axes which covers as much area of the graph as possible while keeping the plot shape near the golden ratio. """ import matplotlib.pyplot as plt if size is None: if fig is None: fig = plt.gcf() size = fig.get_size_inches() figwidth, figheight = size # Golden ratio phi is the preferred dimension # phi = sqrt(5)/2 # # nw, nh is the number of tiles across and down respectively # w, h are the sizes of the tiles # # w,h = figwidth/nw, figheight/nh # # To achieve the golden ratio, set w/h to phi: # w/h = phi => figwidth/figheight*nh/nw = phi # => nh/nw = phi * figheight/figwidth # Must have enough tiles: # nh*nw > n => nw > n/nh # => nh**2 > n * phi * figheight/figwidth # => nh = floor(sqrt(n*phi*figheight/figwidth)) # => nw = ceil(n/nh) phi = math.sqrt(5) / 2 nh = int(math.floor(math.sqrt(n * phi * figheight / figwidth))) if nh < 1: nh = 1 nw = int(math.ceil(n / nh)) return nw, nh def plot_acceptance_rate(state: MCMCDraw, portion: Optional[float] = None): from matplotlib import pyplot as plt genid, AR = state.acceptance_rate(portion=portion) plt.plot(genid, AR) plt.xlabel("Generation #") plt.ylabel("Acceptance rate (%)") plt.title("DREAM acceptance rate by generation") bumps-bumps-2b313fd/bumps/dream/walk.py000066400000000000000000000073241521754644000201360ustar00rootroot00000000000000# This program is in the public domain # Author: Paul Kienzle """ Random walk functions. :function:`walk` simulates a mean-reverting random walk. """ # This code was developed to test outlier detection __all__ = ["walk"] from numpy import asarray, ones_like, nan, isnan from . import util def walk(n=1000, mu=0, sigma=1, alpha=0.01, s0=nan): """ Mean reverting random walk. Returns an array of n-1 steps in the following process:: s[i] = s[i-1] + alpha*(mu-s[i-1]) + e[i] with e ~ N(0,sigma). The parameters are:: *n* walk length *s0* starting value, defaults to N(mu,sigma) *mu* target mean, defaults to 0 *sigma* volatility *alpha* in [0,1] reversion rate Use alpha=0 for a pure Gaussian random walk or alpha=1 independent samples about the mean. If *mu* is a vector, multiple streams are run in parallel. In this case *s0*, *sigma* and *alpha* can either be scalars or vectors. If *mu* is an array, the target value is non-stationary, and the parameter *n* is ignored. Note: the default starting value should be selected from a distribution whose width depends on alpha. N(mu,sigma) is too narrow. This effect is illustrated in :function:`demo`, where the following choices of sigma and alpha give approximately the same histogram:: sigma = [0.138, 0.31, 0.45, 0.85, 1] alpha = [0.01, 0.05, 0.1, 0.5, 1] """ s0, mu, sigma, alpha = [asarray(v) for v in (s0, mu, sigma, alpha)] nchains = mu.shape[0] if mu.ndim > 0 else 1 if mu.ndim < 2: if isnan(s0): s0 = mu + util.rng.randn(nchains) * sigma s = [s0 * ones_like(mu)] for i in range(n - 1): s.append(s[-1] + alpha * (mu - s[-1]) + sigma * util.rng.randn(nchains)) elif mu.ndim == 2: if isnan(s0): s0 = mu[0] + util.rng.randn(nchains) * sigma s = [s0 * ones_like(mu[0])] for i in range(mu.shape[1]): s.append(s[-1] + alpha * (mu[i] - s[-1]) + sigma * util.rng.randn(nchains)) else: raise ValueError("mu must be scalar, vector or 2D array") return asarray(s) def demo(): """ Example showing the relationship between alpha and sigma in the random walk posterior distribution. The lag 1 autocorrelation coefficient R^2 is approximately 1-alpha. """ import numpy as np from numpy import mean, std, sum import matplotlib.pyplot as plt from matplotlib.ticker import MaxNLocator np.random.seed(10) # Pick a pretty starting point # Generate chains n = 5000 mu = [0, 5, 10, 15, 20] sigma = [0.138, 0.31, 0.45, 0.85, 1] alpha = [0.01, 0.05, 0.1, 0.5, 1] chains = walk(n, mu=mu, sigma=sigma, alpha=alpha) # Compute lag 1 correlation coefficient m, s = mean(chains, axis=0), std(chains, ddof=1, axis=0) r2 = sum((chains[1:] - m) * (chains[:-1] - m), axis=0) / ((n - 2) * s**2) r2[abs(r2) < 0.01] = 0 # Plot chains ax_data = plt.axes([0.05, 0.05, 0.65, 0.9]) # x,y,w,h ax_data.plot(chains) textkw = dict( xytext=(30, 0), textcoords="offset points", verticalalignment="center", backgroundcolor=(0.8, 0.8, 0.8, 0.8) ) label = r"$\ \alpha\,%.2f\ \ \sigma\,%.3f\ \ " r"R^2\,%.2f\ \ avg\,%.2f\ \ std\,%.2f\ $" for m, s, a, r2, em, es in zip(mu, sigma, alpha, r2, m, s): plt.annotate(label % (a, s, r2, em - m, es), xy=(0, m), **textkw) # Plot histogram ax_hist = plt.axes([0.75, 0.05, 0.2, 0.9], sharey=ax_data) ax_hist.hist(chains.flatten(), 100, orientation="horizontal") plt.setp(ax_hist.get_yticklabels(), visible=False) ax_hist.xaxis.set_major_locator(MaxNLocator(3)) plt.show() if __name__ == "__main__": demo() bumps-bumps-2b313fd/bumps/dump.py000066400000000000000000000206671521754644000170620ustar00rootroot00000000000000""" Print a summary of an hdf file to the console. """ import sys import numpy as np import h5py as h5 # Typing support from typing import Iterator, Union from pathlib import Path def _str(s): if hasattr(s, "dtype") and s.dtype.kind == "V": return s.tobytes().rstrip(b"\x00").decode("ascii") return s.decode("utf-8") if isinstance(s, bytes) else s def summarystr(group, indent=0, show_attrs=True, recursive=True): # type: (h5.Group, int, bool, bool) -> str """ Return the structure of the HDF 5 tree as a string. *group* is the starting group. *indent* is the indent for each line. *show_attrs* is False if attributes should be hidden *recursive* is False to show only the current level of the tree. """ return "\n".join(_tree_format(group, indent, show_attrs, recursive)) def summary(group, indent=0, show_attrs=True, recursive=True): # type: (h5.Node, int, bool, bool) -> None """ Print the structure of an HDF5 tree. *group* is the starting group. *indent* is the indent for each line. *show_attrs* is False if attributes should be hidden *recursive* is False to show only the current level of the tree. """ for s in _tree_format(group, indent, show_attrs, recursive): print(s) def _tree_format(node, indent, show_attrs, recursive): # type: (h5.Node, int, bool, bool) -> Iterator[str] """ Return an iterator for the lines in a formatted HDF5 tree. Individual lines are not terminated by newline. """ # Find fields and subgroups within the group; do this ahead of time # so that we can show all fields before any subgroups. items = [(name, hasattr(child, "dtype")) for (name, child) in node.items()] groupnames = [name for (name, isdata) in items if not isdata] datasets = [name for (name, isdata) in items if isdata] # Yield group as "nodename(nxclass)" yield "".join((" " * indent, _group_str(node))) # Yield group attributes as " @attr: value" indent += 2 if show_attrs: for s in _yield_attrs(node, indent): yield s # Yield fields as " field[NxM]: value" for fieldname in sorted(datasets): field = node[fieldname] # Short circuit links path = "/".join((node.name, fieldname)) if "target" in field.attrs and _str(field.attrs["target"]) != path: yield "".join((" " * indent, fieldname, " -> ", _str(field.attrs["target"]))) continue # Format field dimensions # print fieldname,field,field.shape,field.dtype,field.attrs['format'] ndim = len(field.shape) if ndim > 1 or (ndim == 1 and field.shape[0] > 1): shape = "[" + "x".join(str(dim) for dim in field.shape) + "]" else: shape = "" # shape = '['+'x'.join( str(dim) for dim in field.shape)+']'+str(field.dtype) # Format string or numeric value # if 'S' in field.attrs['format']: if field.dtype.kind in ("S", "O", "V"): if ndim == 0: # raise ValueError("zero dimensions on string?") value = _limited_str(_str(field[()])) elif ndim == 1: if field.shape[0] == 1: value = _limited_str(_str(field[0])) else: values = field[:] if field.shape[0] <= 5 else field[:5] values = [_limited_str(_str(v), width=10) for v in values] if field.shape[0] > 5: values.append("...") value = ", ".join(values) value = "[" + value + "]" else: value = "[[...]]" elif field.dtype.kind in ("V"): value = f"unknown type {field.dtype.kind}" else: size = np.prod(field.shape) if ndim == 0: value = "%g" % field[()] elif ndim == 1: if size == 0: value = "" elif size == 1: value = "%g" % field[0] elif size <= 6: value = " ".join("%g" % v for v in field[:]) else: value = " ".join("%g" % v for v in field[:6]) + " ..." value = "[" + value + "]" else: if size == 0: value = "" elif field.shape[-1] <= 6: x = field[:].flatten()[: field.shape[-1]] value = " " + " ".join("%g" % v for v in x) else: # print('testing', fieldname,field[:]) y = field[:].flatten()[:6] value = " " + " ".join("%g" % v for v in y) + " ..." value = "[[" + value + "], ...]" dtype = " " + str(field.dtype) units = " " + field.attrs.get("units", "") if not show_attrs else "" # Maybe using Angstroms in units try: units = units.decode("UTF-8") except AttributeError: pass except UnicodeDecodeError: units = units.decode("ISO-8859-1") try: value = value.decode("UTF-8") except AttributeError: pass except UnicodeDecodeError: value = value.decode("ISO-8859-1") # Yield field: value yield "".join((" " * indent, fieldname, shape, dtype, units, ": ", value)) # Yield attributes if show_attrs: for s in _yield_attrs(field, indent + 2): yield s # Yield groups. # If recursive, show group details, otherwise just show name. if recursive: for groupname in sorted(groupnames): group = node[groupname] # Short circuit links path = "/".join((node.name, groupname)) if "target" in group.attrs and _str(group.attrs["target"]) != path: yield "".join((" " * indent, groupname, " -> ", _str(group.attrs["target"]))) continue for s in _tree_format(group, indent, show_attrs, recursive): yield s else: for groupname in sorted(groupnames): group = node[groupname] # Short circuit links path = "/".join((node.name, groupname)) if "target" in group.attrs and _str(group.attrs["target"]) != path: yield "".join((" " * indent, groupname, " -> ", _str(group.attrs["target"]))) continue yield "".join((" " * indent, _group_str(node[g]))) def _yield_attrs(node, indent): # type: (Union[h5.Group, h5.Dataset], int) -> Iterator[str] """ Iterate over the attribute values of the node, excluding NX_class. """ # print "dumping",node.name,"attrs",node.attrs.keys() for k in sorted(node.attrs.keys()): if k not in ("NX_class", "target", "binary", "byteorder", "dtype", "format", "shape"): v = _str(node.attrs[k]) vstr = _limited_str(v) if isinstance(v, str) else str(v) yield "".join((" " * indent, "@", k, ": ", vstr)) def _group_str(node): # type: (h5.Group) -> str """ Return the name and nexus class of a node. """ if node.name == "/": return "root" nxclass = "(" + _str(node.attrs["NX_class"]) + ")" if "NX_class" in node.attrs else "" return node.name.split("/")[-1] + nxclass def _limited_str(s, width=60): # type: (str, int) -> str """ Returns the string trimmed to a maximum of one line of width+3 characters, with ... substituted for any trimmed characters. Leading and trailing blanks are removed. """ if "\n" in s: ret = s.split("\n")[0] if len(ret) > width: ret = ret[:width] + "..." ret += "\\n..." else: ret = s if len(s) < width else s[:width] + "..." # If it is a string that looks like a float then wrap it in quotes. try: float(ret) ret = f'"{ret}"' except ValueError: ... return ret def summarize_file(filename: Union[Path, str], show_attrs: bool = True, indent: int = 0): print(f"=== {filename} ===") with h5.File(filename, "r") as fd: summary(fd, indent=indent, show_attrs=show_attrs, recursive=True) def main(): """ Command line interface to file dump. """ # TODO: use option parser if len(sys.argv) <= 1: print("Usage: {sys.argv[0]} file...") sys.exit(1) for filename in sys.argv[1:]: summarize_file(filename) if __name__ == "__main__": main() bumps-bumps-2b313fd/bumps/errplot.py000066400000000000000000000127641521754644000176030ustar00rootroot00000000000000""" Estimate model uncertainty from random sample. MCMC uncertainty analysis gives the uncertainty on the model parameters rather than the model itself. For example, when fitting a line to a set of data, the uncertainty on the slope and the intercept does not directly give you the uncertainty in the expected value of *y* for a given value of *x*. The routines in bumps.errplot allow you to generate confidence intervals on the model using a random sample of MCMC parameters. After calculating the model *y* values for each sample, one can generate 68% and 95% contours for a set of sampling points *x*. This can apply even to models which are not directly measured. For example, in scattering inverse problems the scattered intensity is the value measured, but the fitting parameters describe the real space model that is being probed. It is the uncertainty in the real space model that is of primary interest. Since bumps knows only the probability of seeing the measured value given the input parameters, it is up to the model itself to calculate and display the confidence intervals on the model and the expected values for the data points. This is done using the :mod:`bumps.plugin` architecture, so application writers can provide the appropriate functions for their data types. Eventually this capability will move to the model definition so that different types of models can be processed in the same fit. For a completed MCMC run, four steps are required: #. reload the fitting problem and the MCMC state #. select a set of sample points #. evaluate model confidence intervals from sample points #. show model confidence intervals :func:`reload_errors` performs steps 1, 2 and 3, returning *errs*. If the fitting problem and the MCMC state are already loaded, then use :func:`calc_errors_from_state` to perform steps 2 and 3, returning *errs*. If alternative sampling is desired, then use :func:`calc_errors` on a given set of points to perform step 3, returning *errs*. Once *errs* has been calculated and returned by one of these methods, call :func:`show_errors` to perform step 4. """ __all__ = ["reload_errors", "calc_errors_from_state", "calc_errors", "show_errors"] import os import traceback import logging from typing import Optional import numpy as np from .dream.state import load_state from . import plugin from .fitproblem import load_problem, load_pars def reload_errors(model, store, nshown=50, random=True): """ Reload the MCMC state and compute the model confidence intervals. The loaded error data is a sample from the fit space according to the fit parameter uncertainty. This is a subset of the samples returned by the DREAM MCMC sampling process. *model* is the name of the model python file *store* is the name of the store directory containing the dream results *nshown* and *random* are as for :func:`calc_errors_from_state`. Returns *errs* for :func:`show_errors`. """ problem = load_problem(model) load_pars(problem, os.path.join(store, problem.name + ".par")) state = load_state(os.path.join(store, problem.name)) state.mark_outliers() points = error_points_from_state(state, nshown=nshown, random=random) return calc_errors(problem, points) def error_points_from_state(state, nshown=50, random=True, portion: Optional[float] = None): """ Return a set of points from the state for calculating errors. *nshown* is the number of samples to include from the state. *random* is True if the samples are randomly selected, or False if the most recent samples should be used. Use random if you have poor mixing (i.e., the parameters tend to stay fixed from generation to generation), but not random if your burn-in was too short, and you want to select from the end. Returns *points* for :func:`calc_errors`. """ points = state.draw(portion=portion).points if points.shape[0] < nshown: nshown = points.shape[0] # randomize the draw; skip the last point since state.keep_best() put # the best point at the end. if random: points = points[np.random.permutation(len(points) - 1)] return points[-nshown:-1] def calc_errors_from_state(problem, state, nshown=50, random=True, portion: Optional[float] = None): """ Compute confidence regions for a problem from the Align the sample profiles and compute the residual difference from the measured data for a set of points returned from DREAM. Returns *errs* for :func:`show_errors`. """ points = error_points_from_state(state, nshown=nshown, random=random, portion=portion) return calc_errors(problem, points) def calc_errors(problem, points): """ Align the sample profiles and compute the residual difference from the measured data for a set of points. The return value is arbitrary. It is passed to the :func:`show_errors` plugin for the application. Returns *errs* for :func:`show_errors`. """ original = problem.getp() try: ret = plugin.calc_errors(problem, points) except Exception: info = ["error calculating distribution on model", traceback.format_exc()] logging.error("\n".join(info)) ret = None finally: problem.setp(original) return ret # Note: should be figfile=None rather than save=None def show_errors(errs, fig=None, save=None): """ Display the confidence regions returned by :func:`calc_errors`. The content of *errs* depends on the active plugin. """ return plugin.show_errors(errs, fig=fig, save=save) bumps-bumps-2b313fd/bumps/fitproblem.py000066400000000000000000001467121521754644000202600ustar00rootroot00000000000000""" Interface between the models and the fitters. :class:`Fitness` defines the interface that model evaluators can follow. These models can be bundled together into a :func:`FitProblem` and sent to :class:`bumps.fitters.FitDriver` for optimization and uncertainty analysis. Summary of problem attributes:: # Used by fitters nllf(p: Optional[Vector]) -> float # main calculation bounds() -> Tuple(Vector, Vector) # or equivalent sequence setp(p: Vector) -> None getp() -> Vector residuals() -> Vector # for LM, MPFit parameter_residuals() -> Vector # for LM, MPFit constraints_nllf() -> float # for LM, MPFit; constraint cost is spread across the individual residuals randomize() -> None # for multistart resynth_data() -> None # for Monte Carlo resampling of maximum likelihood restore_data() -> None # for Monte Carlo resampling of maximum likelihood name: str # DREAM uses this chisq() -> float chisq_str() -> str labels() -> List[str] summarize() -> str show() -> None load(input_path: str) -> None save(output_path: str) -> None plot(figfile: str, view: str) -> None # Set/used by bumps.cli model_reset() -> None # called by load_problem path: str # set by load_problem name: str # set by load_problem title: str = filename # set by load_problem options: List[str] # from sys.argv[1:] undefined:List[int] # when loading a save .par file, these parameters weren't defined store: str # set by make_store output_path: str # set by make_store simulate_data(noise: float) -> None # for --simulate in opts cov() -> Matrix # for --cov in opts """ __all__ = ["Fitness", "FitProblem", "CovarianceMixin", "load_problem"] from contextlib import contextmanager from dataclasses import dataclass import logging import re import sys import traceback from typing import Generic, TypeVar, Union, Optional import warnings from pathlib import Path import numpy as np from numpy import inf, isnan, nan from scipy.stats import chi2 from . import parameter, util from .parameter import to_dict, Parameter, Variable, tag_all, priors from .util import format_uncertainty # Abstract base class: # can use "isinstance" to check if a class implements the protocol @util.runtime_checkable @dataclass(init=False) class Fitness(util.Protocol): """ Manage parameters, data, and theory function evaluation. See :ref:`fitness` for a detailed explanation. """ def parameters(self) -> util.List[Parameter]: """ return the parameters in the model. model parameters are a hierarchical structure of lists and dictionaries. """ raise NotImplementedError() def update(self): """ Called when parameters have been updated. Any cached values will need to be cleared and the model reevaluated. """ pass def numpoints(self): """ Return the number of data points. """ raise NotImplementedError() def nllf(self): """ Return the negative log likelihood value of the current parameter set. """ raise NotImplementedError() def resynth_data(self): """ Generate fake data based on uncertainties in the real data. For Monte Carlo resynth-refit uncertainty analysis. Bootstrapping? """ raise NotImplementedError() def restore_data(self): """ Restore the original data in the model (after resynth). """ raise NotImplementedError() def residuals(self): """ Return residuals for current theory minus data. Used for Levenburg-Marquardt, and for plotting. """ raise NotImplementedError() def save(self, basename): """ Save the model to a file based on basename+extension. This will point to a path to a directory on a remote machine; don't make any assumptions about information stored on the server. Return the set of files saved so that the monitor software can make a pretty web page. """ pass def plot(self, view="linear"): """ Plot the model to the current figure. You only get one figure, but you can make it as complex as you want. This will be saved as a png on the server, and composed onto a results web page. """ pass def no_constraints() -> float: """default constraints function for FitProblem""" return 0 # The functions below would be methods of Fitness if it were a # base class rather than a protocol. def fitness_parameters(fitness: Fitness) -> util.List[Parameter]: """ Return a list of fittable (non-fixed) parameters in the model """ parameters = parameter.unique(fitness.parameters()) return [p for p in parameters if isinstance(getattr(p, "slot", None), parameter.Variable) and not p.fixed] def fitness_chisq(fitness: Fitness) -> str: """ Return a string representing the chisq equivalent of the nllf for a single dataset. Unlike FitProblem.chisq_str, this does not include parameter uncertainty or constraint penalty. """ npars = len(fitness_parameters(fitness)) dof = fitness.numpoints() - npars chisq_norm, _ = nllf_scale(dof=dof, npars=npars, norm=True) # TODO: Check if parameters to fitness are in feasible region before computing nllf? chisq = fitness.nllf() * chisq_norm return chisq def fitness_chisq_str(fitness: Fitness) -> str: """ Return a string representing the chisq equivalent of the nllf for a single dataset. Unlike FitProblem.chisq_str, this does not include parameter uncertainty or constraint penalty. """ npars = len(fitness_parameters(fitness)) dof = fitness.numpoints() - npars chisq_norm, chisq_err = nllf_scale(dof=dof, npars=npars, norm=True) # TODO: Check if parameters are in feasible region before computing nllf? chisq = fitness.nllf() * chisq_norm text = format_uncertainty(chisq, chisq_err) # return f"{text} {fitness.nllf()=:.15e} {chisq_norm=:.15e} {chisq_err=:.15e} {dof=}" return text def fitness_show_parameters(fitness: Fitness, subs: util.Optional[util.Dict[util.Any, Parameter]] = None): """Print the available parameters to the console as a tree.""" print(parameter.format(fitness.parameters(), freevars=subs)) print("[chisq=%s, nllf=%g]" % (fitness_chisq_str(fitness), fitness.nllf())) FitnessType = TypeVar("FitnessType", bound=Fitness) class CovarianceMixin: """ Add methods for *cov*, *show_cov* and *show_err* to a bumps problem definition. This is done as a mixin because not all problems are FitProblem. See for example :class:`bumps.pdfwrapper.PDF`. """ # Note: if caching is implemented for covariance, make sure it is cleared on setp # This will be difficult to do as a mixin. def cov(self, x): r""" Return an estimate of the covariance of the fit. Depending on the fitter and the problem, this may be computed from existing evaluations within the fitter, or from numerical differentiation around the minimum. If the problem has residuals available, then the covariance is derived from the Jacobian:: x = fit.problem.getp() J = bumps.lsqerror.jacobian(fit.problem, x) cov = bumps.lsqerror.jacobian_cov(J) Otherwise, the numerical differentiation will use the Hessian estimated from nllf:: x = fit.problem.getp() H = bumps.lsqerror.hessian(fit.problem, x) cov = bumps.lsqerror.hessian_cov(H) """ # Use Jacobian if residuals are available because it is faster # to compute. Otherwise punt and use Hessian. The has_residuals # attribute should be True if present. It may be false if # the problem defines a residuals method but doesn't really # have residuals (e.g. to allow levenberg-marquardt to run even # though it is not fitting a sum-square problem). from bumps import lsqerror if hasattr(self, "has_residuals"): has_residuals = self.has_residuals else: has_residuals = hasattr(self, "residuals") if has_residuals: J = lsqerror.jacobian(self, x) # print("Jacobian", J) return lsqerror.jacobian_cov(J) else: H = lsqerror.hessian(self, x) # print("Hessian", H) return lsqerror.hessian_cov(H) def show_cov(self, x, cov): maxn = 1000 # max array dims to print cov_str = np.array2string( cov, max_line_width=20 * maxn, threshold=maxn * maxn, precision=6, # suppress_small=True, separator=", ", ) print("=== Covariance matrix ===") print(cov_str) print("=========================") def show_err(self, x, dx): """ Display the error approximation from the covariance matrix. *err* is the standard deviation computed from the covariance matrix. It is available as *result.dx* from the simple fitter, or using:: from bumps import lsqerror dx = lsqerror.stderr(problem.cov(x)) Warning: cost to compute cov grows as the cube of the number of parameters. """ # TODO: need cheaper uncertainty estimate # Note: error estimated from hessian diagonal is insufficient. print("=== Uncertainty from curvature: name value(unc.) ===") for k, v, dv in zip(self.labels(), x, dx): print(f"{k:>40s} {format_uncertainty(v, dv):<15s}") print("=" * 58) # The default penalty nllf has to be big enough that it won't be swamped by # the nllf of the residuals even for a 100 Mpixel image, but not so # big that the distance to the boundary is not lost in the floating point precision. PENALTY_NLLF = 1e12 # TODO: add filename to fitproblem so we don't have to coordinate it elsewhere? @dataclass(init=False, eq=False) class FitProblem(Generic[FitnessType], CovarianceMixin): r""" *models* is a sequence of :class:`Fitness` instances. Note that they do not need to all be of the same class. *weights* is an optional scale factor for each model. A weighted fit returns nllf $L = \sum w_k^2 L_k$. If an individual nllf is the sum squared residuals then this is equivalent to scaling the measurement uncertainty by $1/w$. Unless the measurement uncertainty is unknown, weights should be in [0, 1], representing an unknown systematic uncertainty spread across the individual measurements. *freevars* is :class:`.parameter.FreeVariables` instance defining the per-model parameter assignments. See :ref:`freevariables` for details. Additional parameters: *name* name of the problem *constraints* is a list of Constraint objects, which have a method to calculate the nllf for that constraint. Also supports an alternate form which cannot be serialized: A function which returns the negative log likelihood of seeing the parameters independent from the fitness function. Use this for example to check for feasible regions of the search space, or to add constraints that cannot be easily calculated per parameter. Ideally, the constraints nllf will increase as you go farther from the feasible region so that the fit will be directed toward feasible values. *penalty_nllf* is the nllf to use for *fitness* when *constraints* or model parameter bounds are not satisfied. The total nllf is the squared distance from the boundary plus the penalty so that the derivative points the search back to the feasible region. The penalty should be larger than any nllf you might see near the boundary so that the fit doesn't get stuck outside, but small enough that penalty plus distance is different from penalty. The default is 1e12. Total nllf is the sum of the parameter nllf, the constraints nllf and the depending on whether constraints is greater than soft_limit, either the fitness nllf or the penalty nllf. New in 0.9.0: weights are now squared when computing the sum rather than linear. """ # TODO: problem.path is set by fitproblem.load_problem(); should we add it as standard? name: util.Optional[str] models: util.List[FitnessType] freevars: util.Optional[parameter.FreeVariables] weights: util.Union[util.List[float], util.Literal[None]] constraints: util.Optional[util.Sequence[parameter.Constraint]] penalty_nllf: util.Union[float, util.Literal["inf"]] # The type is not quite correct. The models do not need to be of the same class _models: util.List[FitnessType] _priors: util.List[Parameter] _parameters: util.List[Parameter] _parameters_by_id: util.Dict[str, Parameter] _dof: float = np.nan # not a schema field, and is not used in __init__ _active_model_index: int = 0 # _all_constraints: util.List[util.Union[Parameter, Expression]] def __init__( self, models: util.Union[FitnessType, util.List[FitnessType]], weights=None, name=None, constraints=None, penalty_nllf=None, freevars=None, auto_tag=False, ): if not isinstance(models, (list, tuple)): models = [models] if callable(constraints): raise TypeError("Callable constraints function is no longer supported. Instead use a list of comparisons.") self.constraints = constraints if constraints is not None else [] if freevars is None: names = ["M%d" % i for i, _ in enumerate(models)] freevars = parameter.FreeVariables(names=names) self.freevars = freevars if auto_tag: for index, model in enumerate(models): model_name = model.name if model.name is not None else f"Model{index}" tag_all(model.parameters(), model_name) self._models = models if weights is None: weights = [1.0 for _ in models] self.weights = weights self.penalty_nllf = float(penalty_nllf) if penalty_nllf is not None else PENALTY_NLLF self.name = name # Do these steps last so that it has all of the attributes initialized. self.model_reset() # sets self._all_constraints self.set_active_model(0) # Set the active model to model 0 @property def fitness(self): warnings.warn("Deprecated: use of problem.fitness will be removed at some point") if len(self._models) == 1: return self._models[0] raise ValueError("problem.fitness is not defined") @property def dof(self): return self._dof @property def num_models(self): return len(self._models) @property def models(self): """Iterate over models, with free parameters set from model values""" try: for index in range(len(self._models)): index, model = self._switch_model(index) yield model finally: # Restore the active model after cycling, even if interrupted self._switch_model(self._active_model_index) # TODO: deprecate set_active_model and push_model # set_active_model is only used once in the wx gui. push_model is only needed # if we have a notion of active model. # noinspection PyAttributeOutsideInit def set_active_model(self, index): """ Fetch model *index* with the appropriate free variables substituted. This will remain the active model until a new active model is selected. Operations like chisq_str() or plot() which cycle through the models will restore the parameters upon completion. """ index, model = self._switch_model(index) self._active_model_index, self.active_model = index, model return model @contextmanager def push_model(self, index): """ Fetch model *index* with the appropriate free variables substituted. On completion of the context, restore the parameters for the active model. """ try: index, model = self._switch_model(index) yield model finally: # Restore the active model after cycling, even if interrupted self._switch_model(self._active_model_index) def _switch_model(self, index): # print(f"switching to {index} with freevars={bool(self.freevars)}") if not (-len(self._models) <= index < len(self._models)): raise IndexError(f"Index {index} invalid when only {len(self._models)} models") if index < 0: index = len(self._models) + index # TODO: FreeVariables destroys caching within the model. Replace it. # TODO: Only update parameters if index has changed. We can track this in freevars. self.freevars.set_model(index) if self.freevars: # Clear model cache when updating the parameters getattr(self._models[index], "update", lambda: None)() return index, self._models[index] def model_parameters(self): """Return parameters from all models""" pars = {} # Note: the self.models iterator plugs the free variables into # the model in turn, so no need to walk self.freevars directly. # The model.update() function is called each time, so whatever # caching is happening in the model is cleared, and it knows that # new parameter values have been inserted. pars["models"] = [f.parameters() for f in self.models] free = self.freevars.parameters() if free: pars["freevars"] = free return pars # TODO: no longer used? def to_dict(self): return { "type": type(self).__name__, "name": self.name, "models": to_dict(self._models), "weights": self.weights, "penalty_nllf": self.penalty_nllf, # TODO: constraints may be a function. "constraints": to_dict(self.constraints), "freevars": to_dict(self.freevars), } def __repr__(self): return "FitProblem(name=%s)" % self.name def valid(self, pvec): """Return true if the point is in the feasible region""" return all(v in p.prior for p, v in zip(self._parameters, pvec)) def setp(self, pvec): """ Set a new value for the parameters into the model. If the model is valid, calls model_update to signal that the model should be recalculated. Returns True if the value is valid and the parameters were set, otherwise returns False. """ # print("Calling setp with", pvec, self._parameters) for v, p in zip(pvec, self._parameters): p.value = v # TODO: setp_hook is a hack to support parameter expressions in sasview # Don't depend on this existing long term. setp_hook = getattr(self, "setp_hook", no_constraints) setp_hook() self.model_update() def getp(self): """ Returns the current value of the parameter vector. """ return np.array([p.value for p in self._parameters], "d") def bounds(self): """Return the bounds for each parameter as a 2 x N array""" limits = [p.prior.limits for p in self._parameters] return np.array(limits, "d").T if limits else np.empty((2, 0)) def randomize(self, n=None): """ Generates a random model. *randomize()* sets the model to a random value. *randomize(n)* returns a population of *n* random models. For indefinite bounds, the random population distribution is centered on initial value of the parameter, or 1. if the initial parameter is not finite. """ # TODO: split into two: randomize and random_pop if n is None: self.setp(self.randomize(n=1)[0]) return # Not returning anything since no n is requested # TODO: apply hard limits on parameters target = self.getp() target[~np.isfinite(target)] = 1.0 pop = [p.prior.random(n, target=v) for p, v in zip(self._parameters, target)] return np.array(pop).T def nllf(self, pvec=None) -> float: """ Compute the cost function for a new parameter set p. This is not simply the sum-squared residuals, but instead is the negative log likelihood of seeing the data given the model parameters plus the negative log likelihood of seeing the model parameters. The value is used for a likelihood ratio test so normalization constants can be ignored. There is an additional penalty value provided by the model which can be used to implement inequality constraints. Any penalty should be large enough that it is effectively excluded from the parameter space returned from uncertainty analysis. The model is not actually calculated if any of the parameters are out of bounds, or any of the constraints are not satisfied, but instead are assigned a value of *penalty_nllf*. This will prevent expensive models from spending time computing values in the unfeasible region. """ if pvec is not None: # Note that valid() only checks that the fit parameters are in the bounding box. # It doesn't check that all the constraints are satisfied. To do that we would # have to use setp on the vector then loop over p._priors, resetting to the # original pvec if the new pvec is invalid. if self.valid(pvec): self.setp(pvec) else: return inf pparameter, pconstraints, pmodel, failing = self._nllf_components() # Note: pmodel is zero if any constraints are failing. In that case the cost # will be the squared distance from the boundary of the feasible region for # the breaking parameters so that gradient descent can guide us back to the # feasible region. The penalty would ideally be greater than any value of # pmodel inside the boundary (otherwise the fitter may prefer the point in the # infeasible region) but it has to be small enough that adding a small distance # changes the penalty value (otherwise the slope outside the boundary is zero). # Currently using 1e12 as the default penalty. This will be too small for many problems # but any larger and the derivatives will break. # TODO: Drop the penalty term, either by rewriting the fitters so they know that # the constraints are broken, or rewriting the constraints to simple box constraints. cost = pparameter + pconstraints + (self.penalty_nllf if failing else pmodel) # print(f"prior:{float(pparameter)} + constraint:{float(pconstraints)} + nllf:{float(pmodel)} => {float(cost)} at [{pvec=}]") if isnan(cost): # TODO: make sure errors get back to the user # print "point evaluates to nan" # print parameter.summarize(self._parameters) return inf return cost def parameter_nllf(self) -> util.Tuple[float, util.List[str]]: """ Returns negative log likelihood of seeing parameters p. """ failing = [] nllf = 0.0 for p in self._priors: p_nllf = p.nllf() nllf += p_nllf if p_nllf == np.inf: failing.append(str(p)) # print("; ".join(f"{p}({p.value})={p.nllf()}" for p in self._priors)) return nllf, failing def parameter_residuals(self): """ Returns negative log likelihood of seeing parameters p. """ return [p.residual() for p in self._priors] def chisq(self, nllf: Union[float, util.NDArray, None] = None, norm: bool = True, compact: bool = True): """ Returns chisq as a floating point value. See documentation for :meth:`chisq_str`. """ chisq_norm, chisq_err = nllf_scale(dof=self.dof, npars=len(self._parameters), norm=norm) if nllf is None: pparameter, pconstraints, pmodel, failing = self._nllf_components() nllf = pmodel + pparameter + pconstraints + (self.penalty_nllf if failing else 0) return nllf * chisq_norm # TODO: Too many versions of chisq about. # Note: norm and compact are no longer used in bumps, so they are not documented def chisq_str(self, nllf: Optional[float] = None, norm: bool = True, compact: bool = True): """ Return a string representing the chisq equivalent of the nllf. If *nllf* is provided then use that instead of calling the model evaluator. Fail if *compact* is False. If the model has strictly gaussian independent uncertainties then the negative log likelihood function will return 0.5*sum(residuals**2), which is 1/2*chisq. Since we are printing normalized chisq, we multiply the model nllf by 2/DOF before displaying the value. This is different from the problem nllf function, which includes the cost of the cost of the penalty constraints in the total nllf. Parameter priors, if any, are treated as independent models in the total nllf. The constraint value is displayed separately. **Deprecated**: *norm:bool* and *compact:bool* are ignored. """ chisq_norm, chisq_err = nllf_scale(dof=self.dof, npars=len(self._parameters), norm=norm) failing = [] if nllf is None: pparameter, pconstraints, pmodel, failing = self._nllf_components() nllf = pmodel + pparameter + pconstraints + (self.penalty_nllf if failing else 0) # print(f"{pmodel=} {pparameter=} {pconstraints=} {nllf=} {chisq_norm=} {chisq_err=}") if failing: # Text is used in a context like f"χ² = {text}". We are not printing the list # of failing constraints since parameter names are arbitrarily long. text = "NaN [out of bounds]" else: text = format_uncertainty(nllf * chisq_norm, chisq_err) # return f"{text} p={self.getp()} => {pmodel:.15e}" return text def _nllf_components(self) -> util.Tuple[float, float, float, util.List[str]]: try: pparameter, bad_priors = self.parameter_nllf() pconstraints, bad_constraints = self.constraints_nllf() failing = bad_priors + bad_constraints # If constraints are failing don't bother to compute nllf. # Using pvalue = zero rather than NaN so that handling of penalties is easier. pmodel = self.model_nllf() if len(failing) == 0 else 0.0 if isnan(pparameter): # TODO: make sure errors get back to the user info = ["Parameter nllf is wrong"] info += ["%s %g" % (p, p.nllf()) for p in self._priors] logging.error("\n ".join(info)) return pparameter, pconstraints, pmodel, failing except Exception: # TODO: make sure errors get back to the user info = (traceback.format_exc(), parameter.summarize(self._parameters)) logging.error("\n".join(info)) return nan, nan, nan, [] def __call__(self, pvec=None): """ Problem cost function. Returns the negative log likelihood scaled by DOF so that the result looks like the familiar normalized chi-squared. These scale factors will not affect the value of the minimum, though some care will be required when interpreting the uncertainty. """ return 2 * self.nllf(pvec) / self.dof def summarize(self): """Return a table of current parameter values with range bars.""" return parameter.summarize(self._parameters) @property def parameters(self): """Return the list of fitted parameters.""" return self._parameters def labels(self) -> util.List[str]: """Return the list of labels, one per fitted parameter.""" return [p.name for p in self._parameters] # noinspection PyAttributeOutsideInit def model_reset(self): """ Prepare for the fit. This sets the parameters and the bounds properties that the solver is expecting from the fittable object. We also compute the degrees of freedom so that we can return a normalized fit likelihood. If the set of fit parameters changes, then model_reset must be called. """ # print("In model reset with", self.model_parameters()) all_parameters = parameter.unique(self.model_parameters()) # Check priors broken = [] for p in all_parameters: if not hasattr(p, "reset_prior"): continue p.reset_prior() limits = p.prior.limits value = p.value if (limits[0] > value) or (value > limits[1]): broken.append(f"{p}={value} is outside {limits}") elif not np.isfinite(p.prior.nllf(value)): broken.append(f"{p}={value} is outside {p.prior}") # Check other constraints. broken.extend([f"{c} fails" for c in self.constraints if float(c) > 0]) # Show broken constraints. Note that warnings.warn() will only show up once, # so this acts as a pre-check of the model when running in batch mode on the # command line. if len(broken) > 0: warnings.warn("Unsatisfied constraints: [%s]" % (",\n".join(broken))) # TODO: return broken_constraints from reset() rather than setting state self.broken_constraints = broken # Collect all fitting parameters pars = [] pars_by_id = {} for p in all_parameters: if hasattr(p, "id"): pars_by_id[p.id] = p slot = getattr(p, "slot", None) if isinstance(slot, Variable) and not p.fixed: pars.append(p) self._parameters = pars self._parameters_by_id = pars_by_id # Collect all priors that need to be evaluated self._priors = priors(all_parameters) # The degrees of freedom is the number of data points minus the number of fit # parameters. Gaussian priors on the parameters are treated as a simultaneous # fit to the data plus a fit of the parameter to its previously measured value. # That is, each prior adds a single data point to the fit without changing the # number of free parameters, thus increasing DOF by 1. Again, the target value # for a fit with no systematic error should be the number of degrees of freedom. # Uniform priors do not modify the degrees of freedom, not even soft-bounded uniform. self._dof = self.model_points() + sum(p.prior.dof for p in self._priors) - len(self._parameters) if self.dof <= 0: warnings.warn( f"Need more data points (currently: {self.model_points()}) than fitting parameters ({len(self._parameters)})" ) def model_points(self): """Return number of points in all models""" return sum(f.numpoints() for f in self.models) def model_update(self): """Let all models know they need to be recalculated""" # The self.models iterator calls update for each model if there are # free variable substitutions, so no need to update here. if self.freevars: return for f in self.models: getattr(f, "update", lambda: None)() def model_nllf(self): """Return cost function for all data sets""" # print("In model nllf with", self.getp()) return sum(w**2 * f.nllf() for w, f in zip(self.weights, self.models)) def constraints_nllf(self) -> util.Tuple[float, util.List[str]]: """Return the cost function for all constraints""" failing = [] nllf = 0.0 # Process the list of inequality constraints for c in self.constraints: # TODO: convert to list of residuals for Levenberg-Marquardt c_nllf = float(c) ** 2 nllf += c_nllf if c_nllf > 0: failing.append(str(c)) return nllf, failing def simulate_data(self, noise=None): """Simulate data with added noise""" for f in self.models: f.simulate_data(noise=noise) def resynth_data(self): """Resynthesize data with noise from the uncertainty estimates.""" for f in self.models: f.resynth_data() def restore_data(self): """Restore original data after resynthesis.""" for f in self.models: f.restore_data() @property def has_residuals(self): """ True if all underlying fitness functions define residuals. """ return all(hasattr(f, "residuals") for f in self.models) def residuals(self): resid = np.hstack([w * f.residuals() for w, f in zip(self.weights, self.models)]) return resid def save(self, basename): for i, f in enumerate(self.models): if hasattr(f, "save"): f.save(basename + "-%d" % (i + 1)) def show(self): for i, f in enumerate(self.models): print("-- Model %d %s" % (i, getattr(f, "name", ""))) fitness_show_parameters(f, subs=self.freevars.get_model(i)) print("[overall chisq=%s, nllf=%g]" % (self.chisq_str(), self.nllf())) def plot(self, p=None, fignum=1, figfile=None, view=None, model_indices=None): # TODO: remove duplicate logic from FitProblem.plot() and api.get_data_plot import matplotlib.pyplot as plt if p is not None: self.setp(p) # Don't show the figure calling problem.plot(figfile=base_file), as is # done during --export=path. If called as problem.plot(), as from a jupyter # notebook, then swe should show the plot. show_fig = not figfile # Overall chisq overall_chisq_str = self.chisq_str() for i, f in enumerate(self.models): outfile = f"{figfile}-model{i}.png" if figfile else None if model_indices is not None and i not in model_indices: continue backend = "matplotlib" if hasattr(f, "plot") else "plotly" if hasattr(f, "plotly") else None # backend = "plotly" if hasattr(f, "plotly") else "matplotlib" if hasattr(f, "plot") else None if backend is None: continue # TODO: duplicated in bumps.webserver.server.api._get_data_plot_mpl() if self.num_models > 1: chisq_str = fitness_chisq_str(f) chisq = f"χ² = {chisq_str}; overall {overall_chisq_str}" title = f"Model {i+1}: {f.name}" else: chisq = f"χ² = {overall_chisq_str}" title = f"{f.name}" fontsize = 16 if backend == "plotly": fig = f.plotly() # TODO: text offset of (x=0.5em, y=0.5ex) text_offset = 0.01 # portion of graph axis length font = dict(size=16) fig.add_annotation( x=text_offset, y=1 + text_offset, xanchor="left", yanchor="bottom", xref="paper", yref="paper", text=title, showarrow=False, font=font, ) fig.add_annotation( x=1 - text_offset, y=1 + text_offset, xanchor="right", yanchor="bottom", xref="paper", yref="paper", text=chisq, showarrow=False, font=font, ) if outfile: # Note: requires "pip install kaleido" # Note: much slower than matplotlib fig.write_image(outfile) if show_fig: # Try to guess whether we are in a jupyter notebook before deciding how # to render the plot. # TODO: gather all figures into one tab when rendering to the browser import sys jupyter = "ipykernel" in sys.modules renderer = None if jupyter else "browser" fig.show(renderer) continue # If not plotly then we must be using matplotlib. # Note that during api.export our matplotlib backend is 'agg' so no plot will show. fig = plt.figure(i + fignum) f.plot(view=view) # Make room for model name and chisq on the top of the plot # TODO: attach margins to canvas resize_event so that margins are fixed h, w = fig.get_size_inches() h_ex = h * 72 / fontsize # (h in * 72 pt/in) / (fontsize pt/ex) = height in ex text_offset = 0.5 / h_ex # 1/2 ex above and below the text top = 1 - 2 / h_ex # leave 2 ex at the top of the figure plt.subplots_adjust(top=top) # Add model name and chisq transform = fig.transFigure x, y = text_offset, 1 - text_offset ha, va = "left", "top" fig.text(x, y, title, transform=transform, va=va, ha=ha, fontsize=fontsize) x, y = 1 - text_offset, 1 - text_offset ha, va = "right", "top" fig.text(x, y, chisq, transform=transform, va=va, ha=ha, fontsize=fontsize) # plt.suptitle("Model %d - %s" % (i, f.name)) # plt.text(0.01, 0.01, "chisq=%s" % fitness_chisq_str(f), transform=plt.gca().transAxes) if outfile: plt.savefig(outfile, format="png") # Note: restore default behaviour of getstate/setstate rather than # inheriting from BaseFitProblem def __getstate__(self): return self.__dict__ def __setstate__(self, state): self.__dict__ = state PARS_PATTERN = re.compile(r"^(?P