pax_global_header00006660000000000000000000000064150123513350014510gustar00rootroot0000000000000052 comment=e9bba2a6566abf0e0a788dc06c56a468f8400d1f datalad-container-1.2.6/000077500000000000000000000000001501235133500150705ustar00rootroot00000000000000datalad-container-1.2.6/.appveyor.yml000066400000000000000000000231521501235133500175410ustar00rootroot00000000000000# This CI setup provides a largely homogeneous configuration across all # major platforms (Windows, MacOS, and Linux). The aim of this test setup is # to create a "native" platform experience, using as few cross-platform # helper tools as possible. # # On Linux/Mac a venv is used for testing. The effective virtual env # is available under ~/VENV. # # All workers support remote login. Login details are shown at the top of each # CI run log. # # - Linux/Mac workers (via SSH): # # - A permitted SSH key must be defined in an APPVEYOR_SSH_KEY environment # variable (via the appveyor project settings) # # - SSH login info is given in the form of: 'appveyor@67.225.164.xx -p 22xxx' # # - Login with: # # ssh -o StrictHostKeyChecking=no # # - to prevent the CI run from exiting, `touch` a file named `BLOCK` in the # user HOME directory (current directory directly after login). The session # will run until the file is removed (or 60 min have passed) # # - Windows workers (via RDP): # # - An RDP password should be defined in an APPVEYOR_RDP_PASSWORD environment # variable (via the appveyor project settings), or a random password is used # every time # # - RDP login info is given in the form of IP:PORT # # - Login with: # # xfreerdp /cert:ignore /dynamic-resolution /u:appveyor /p: /v: # # - to prevent the CI run from exiting, create a textfile named `BLOCK` on the # Desktop (a required .txt extension will be added automatically). The session # will run until the file is removed (or 60 min have passed) # # - in a terminal execute, for example, `C:\datalad_debug.bat 39` to set up the # environment to debug in a Python 3.8 session (should generally match the # respective CI run configuration). # do not make repository clone cheap: interfers with versioneer shallow_clone: false environment: DATALAD_TESTS_SSH: 1 # Do not use `image` as a matrix dimension, to have fine-grained control over # what tests run on which platform # The ID variable had no impact, but sorts first in the CI run overview # an intelligible name can help to locate a specific test run # All of these are common to all matrix runs ATM, so pre-defined here and to be overloaded if needed DTS: datalad_container APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004 INSTALL_SYSPKGS: python3-venv xz-utils jq # system git-annex is way too old, use better one INSTALL_GITANNEX: git-annex -m deb-url --url http://snapshot.debian.org/archive/debian/20210906T204127Z/pool/main/g/git-annex/git-annex_8.20210903-1_amd64.deb CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov matrix: # List a CI run for each platform first, to have immediate access when there # is a need for debugging # Ubuntu core tests - ID: Ubu # The same but with the oldest supported Python. - ID: Ubu-3.8 PY: '3.8' # The same but removing busybox first - triggers different code paths in the tests - ID: Ubu-nobusybox BEFORE_CMD: docker rmi busybox:latest # Windows core tests #- ID: WinP39core # # ~35 min # DTS: datalad_container # APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 # # Python version specification is non-standard on windows # PY: 39-x64 # INSTALL_GITANNEX: git-annex -m datalad/packages ## MacOS core tests #- ID: MacP38core # DTS: datalad_container # APPVEYOR_BUILD_WORKER_IMAGE: macOS # PY: 3.8 # INSTALL_GITANNEX: git-annex # DATALAD_LOCATIONS_SOCKETS: /Users/appveyor/DLTMP/sockets # CODECOV_BINARY: https://uploader.codecov.io/latest/macos/codecov matrix: allow_failures: - KNOWN2FAIL: 1 # it is OK to specify paths that may not exist for a particular test run cache: # pip cache - C:\Users\appveyor\AppData\Local\pip\Cache -> .appveyor.yml - /home/appveyor/.cache/pip -> .appveyor.yml # TODO: where is the cache on macOS? #- /Users/appveyor/.cache/pip -> .appveyor.yml # TODO: Can we cache `brew`? #- /usr/local/Cellar #- /usr/local/bin # turn of support for MS project build support (not needed) build: off # init cannot use any components from the repo, because it runs prior to # cloning it init: # remove windows 260-char limit on path names - cmd: powershell Set-Itemproperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name LongPathsEnabled -value 1 # enable developer mode on windows # this should enable mklink without admin privileges, but it doesn't seem to work #- cmd: powershell tools\ci\appveyor_enable_windevmode.ps1 # enable RDP access on windows (RDP password is in appveyor project config) # this is relatively expensive (1-2min), but very convenient to jump into any build at any time - cmd: powershell.exe iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) # enable external SSH access to CI worker on all other systems # needs APPVEYOR_SSH_KEY defined in project settings (or environment) - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - # Identity setup - git config --global user.email "test@appveyor.land" - git config --global user.name "Appveyor Almighty" # Scratch space - cmd: md C:\DLTMP # we place the "unix" one into the user's HOME to avoid git-annex issues on MacOSX # gh-5291 - sh: mkdir ~/DLTMP # and use that scratch space to get short paths in test repos # (avoiding length-limits as much as possible) - cmd: "set TMP=C:\\DLTMP" - cmd: "set TEMP=C:\\DLTMP" - sh: export TMPDIR=~/DLTMP # docker login to get "personalized" rate limit (rather than IP-based) - sh: docker login -p "$DOCKERHUB_TOKEN" -u "$DOCKERHUB_USERNAME" install: # place a debug setup helper at a convenient location - cmd: copy tools\ci\appveyor_env_setup.bat C:\\datalad_debug.bat # Missing system software - sh: "[ -n \"$INSTALL_SYSPKGS\" ] && ( [ \"x${APPVEYOR_BUILD_WORKER_IMAGE}\" = \"xmacOS\" ] && brew install -q ${INSTALL_SYSPKGS} || { sudo apt-get update -y && sudo apt-get install --no-install-recommends -y ${INSTALL_SYSPKGS}; } ) || true" # If a particular Python version is requested, use env setup (using the # appveyor provided environments/installation). # Otherwise create a venv using the default Python 3, to enable uniform # use of python/pip executables below - sh: "[ \"x$PY\" != x ] && . ${HOME}/venv${PY}/bin/activate || python3 -m venv ${HOME}/dlvenv && . ${HOME}/dlvenv/bin/activate; ln -s \"$VIRTUAL_ENV\" \"${HOME}/VENV\"" - cmd: "set PATH=C:\\Python%PY%;C:\\Python%PY%\\Scripts;%PATH%" # deploy the datalad installer, override version via DATALAD_INSTALLER_VERSION - cmd: IF DEFINED DATALAD_INSTALLER_VERSION ( python -m pip install -q "datalad-installer%DATALAD_INSTALLER_VERSION%" ) ELSE ( python -m pip install -q datalad-installer ) - sh: python -m pip install datalad-installer${DATALAD_INSTALLER_VERSION:-} - pip install wheel # setup neurodebian, needs update of sources.list when base release changes - sh: "echo $ID | grep -q '^Ubu' && wget -O- http://neuro.debian.net/lists/focal.us-nh.full | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list && ( sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 || { wget -q -O- http://neuro.debian.net/_static/neuro.debian.net.asc | sudo apt-key add -; } )" # Missing system software - sh: "[ -z \"$INSTALL_SYSPKGS\" ] || { if [ \"x${APPVEYOR_BUILD_WORKER_IMAGE}\" = \"xmacOS\" ]; then brew install -q ${INSTALL_SYSPKGS}; else sudo apt-get update -y -qq --allow-releaseinfo-change && sudo apt-get install -qq --no-install-recommends -y ${INSTALL_SYSPKGS}; fi }" # Install singularity - sh: tools/ci/install-singularity.sh # Install git-annex on windows, otherwise INSTALL_SYSPKGS can be used # deploy git-annex, if desired - cmd: IF DEFINED INSTALL_GITANNEX datalad-installer --sudo ok %INSTALL_GITANNEX% - sh: "[ -n \"${INSTALL_GITANNEX}\" ] && datalad-installer --sudo ok ${INSTALL_GITANNEX}" # in case of a snapshot installation, use the following approach to adjust # the PATH as necessary #- sh: "[ -n \"${INSTALL_GITANNEX}\" ] && datalad-installer -E ${HOME}/dlinstaller_env.sh --sudo ok ${INSTALL_GITANNEX}" # add location of datalad installer results to PATH #- sh: "[ -f ${HOME}/dlinstaller_env.sh ] && . ${HOME}/dlinstaller_env.sh || true" #before_build: # build_script: - python -m pip install -q -r requirements-devel.txt - python -m pip install . #after_build: # before_test: # simple call to see if datalad and git-annex are installed properly - datalad wtf # remove busybox:latest so tests could fetch/drop it as needed - sh: "[ -n \"${BEFORE_CMD}\" ] && ${BEFORE_CMD} || :" test_script: # run tests on installed module, not source tree files - cmd: md __testhome__ - sh: mkdir __testhome__ - cd __testhome__ # run test selecion (--traverse-namespace needed from Python 3.8 onwards) - cmd: python -m pytest -s -v -m "not (turtle)" --doctest-modules --cov=datalad_container --pyargs %DTS% - sh: python -m pytest -s -v -m "not (turtle)" --doctest-modules --cov=datalad_container --pyargs ${DTS} after_test: - python -m coverage xml - cmd: curl -fsSL -o codecov.exe "https://uploader.codecov.io/latest/windows/codecov.exe" - cmd: .\codecov.exe -f "coverage.xml" - sh: "curl -Os $CODECOV_BINARY" - sh: chmod +x codecov - sh: ./codecov #on_success: # #on_failure: # on_finish: # conditionally block the exit of a CI run for direct debugging - sh: while [ -f ~/BLOCK ]; do sleep 5; done - cmd: powershell.exe while ((Test-Path "C:\Users\\appveyor\\Desktop\\BLOCK.txt")) { Start-Sleep 5 } datalad-container-1.2.6/.codeclimate.yml000066400000000000000000000004111501235133500201360ustar00rootroot00000000000000version: "2" checks: file-lines: config: threshold: 500 plugins: bandit: enabled: true checks: assert_used: enabled: false exclude_patterns: - "_datalad_buildsupport/" - "versioneer.py" - "*/_version.py" - "tools/" - "**/tests/" datalad-container-1.2.6/.codespellrc000066400000000000000000000002261501235133500173700ustar00rootroot00000000000000[codespell] skip = .venv,venvs,.git,build,*.egg-info,*.lock,.asv,.mypy_cache,.tox,fixtures,_version.py,*.pem # ignore-words-list = # exclude-file = datalad-container-1.2.6/.datalad-release-action.yaml000066400000000000000000000012461501235133500223200ustar00rootroot00000000000000fragment_directory: changelog.d # Categories must be listed in descending order of precedence for determining # what category to apply to a PR with multiple labels. # The category names must align with the categories in changelog.d/scriv.ini categories: - name: πŸ’₯ Breaking Changes bump: major label: major - name: πŸš€ Enhancements and New Features bump: minor label: minor - name: πŸ› Bug Fixes label: patch - name: πŸ”© Dependencies label: dependencies - name: πŸ“ Documentation label: documentation - name: 🏠 Internal label: internal - name: 🏎 Performance label: performance - name: πŸ§ͺ Tests label: tests datalad-container-1.2.6/.gitattributes000066400000000000000000000000531501235133500177610ustar00rootroot00000000000000datalad_container/_version.py export-subst datalad-container-1.2.6/.github/000077500000000000000000000000001501235133500164305ustar00rootroot00000000000000datalad-container-1.2.6/.github/dependabot.yml000066400000000000000000000004251501235133500212610ustar00rootroot00000000000000# This action keeps the versions of all github actions up-to-date version: 2 updates: - package-ecosystem: github-actions directory: / schedule: interval: weekly commit-message: prefix: "[gh-actions]" include: scope labels: - internal datalad-container-1.2.6/.github/workflows/000077500000000000000000000000001501235133500204655ustar00rootroot00000000000000datalad-container-1.2.6/.github/workflows/add-changelog-snippet.yml000066400000000000000000000017541501235133500253540ustar00rootroot00000000000000name: Add changelog.d snippet on: pull_request_target: # Run whenever the PR is pushed to, receives a label, or is created with # one or more labels: types: [synchronize, labeled] # Prevent the workflow from running multiple jobs at once when a PR is created # with multiple labels: concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} cancel-in-progress: true jobs: add: runs-on: ubuntu-latest # Only run on PRs that have the "CHANGELOG-missing" label: if: contains(github.event.pull_request.labels.*.name, 'CHANGELOG-missing') steps: - name: Check out repository uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} - name: Add changelog snippet uses: datalad/release-action/add-changelog-snippet@master with: token: ${{ secrets.GITHUB_TOKEN }} rm-labels: CHANGELOG-missing datalad-container-1.2.6/.github/workflows/codespell.yml000066400000000000000000000005431501235133500231640ustar00rootroot00000000000000--- name: Codespell on: push: branches: [master] pull_request: branches: [master] permissions: contents: read jobs: codespell: name: Check for spelling errors runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - name: Codespell uses: codespell-project/actions-codespell@v2 datalad-container-1.2.6/.github/workflows/docbuild.yml000066400000000000000000000011541501235133500227760ustar00rootroot00000000000000name: docs on: [push, pull_request] jobs: build: runs-on: ubuntu-latest steps: - name: Set up environment run: | git config --global user.email "test@github.land" git config --global user.name "GitHub Almighty" - uses: actions/checkout@v4 - name: Set up Python 3.9 uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install dependencies run: | python -m pip install --upgrade pip setuptools pip install -r requirements-devel.txt pip install . - name: Build docs run: | make -C docs html datalad-container-1.2.6/.github/workflows/release.yml000066400000000000000000000020341501235133500226270ustar00rootroot00000000000000name: Auto-release on PR merge on: pull_request_target: branches: # Create a release whenever a PR is merged into one of these branches: - master types: - closed jobs: release: runs-on: ubuntu-latest # Only run for merged PRs with the "release" label: if: github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release') steps: - name: Checkout source uses: actions/checkout@v4 with: # Check out all history so that the previous release tag can be # found: fetch-depth: 0 - name: Prepare release uses: datalad/release-action/release@master with: token: ${{ secrets.GITHUB_TOKEN }} pypi-token: ${{ secrets.PYPI_TOKEN }} pre-tag: | version_file=datalad_container/version.py printf '__version__ = "%s"\n' "$new_version" > "$version_file" git commit -m "Update __version__ to $new_version" "$version_file" # vim:set et sts=2: datalad-container-1.2.6/.gitignore000066400000000000000000000001721501235133500170600ustar00rootroot00000000000000.pybuild/ .coverage /.tox *.egg-info *.py[coe] .#* .*.swp docs/build docs/source/generated build # manpage .idea/ venvs/ datalad-container-1.2.6/.noannex000066400000000000000000000000001501235133500165250ustar00rootroot00000000000000datalad-container-1.2.6/.readthedocs.yaml000066400000000000000000000010531501235133500203160ustar00rootroot00000000000000# .readthedocs.yaml # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details # Required version: 2 # Set the version of Python and other tools you might need build: os: ubuntu-22.04 tools: python: "3.10" # Build documentation in the docs/ directory with Sphinx sphinx: configuration: docs/source/conf.py formats: all # Optionally declare the Python requirements required to build your docs python: install: - path: . method: pip - requirements: requirements-devel.txt datalad-container-1.2.6/CHANGELOG.md000066400000000000000000000345221501235133500167070ustar00rootroot00000000000000 # 1.2.6 (2025-05-18) ## πŸ› Bug Fixes - MNT: Account for a number of deprecations in core. [PR #268](https://github.com/datalad/datalad-container/pull/268) (by [@adswa](https://github.com/adswa)) # 1.2.5 (2024-01-17) ## 🏠 Internal - Run isort across entire codebase to harmonize imports order/appearance. https://github.com/datalad/datalad-container/260 (by @yarikoptic) # 1.2.4 (2024-01-17) ## πŸš€ Enhancements and New Features - A new placeholder `{python}` is supported by container execution. It resolves to the Python interpreter executable running DataLad on container execution. This solves portability issues with the previous approach of hard-coding a command name on container configuration. Fixes https://github.com/datalad/datalad-container/issues/226 via https://github.com/datalad/datalad-container/pull/227 (by @mih) # 1.2.3 (2023-10-02) ## 🏠 Internal - Add [extras] extras_require with datalad-metalad and add all those extras to [devel]. [PR #215](https://github.com/datalad/datalad-container/pull/215) (by [@yarikoptic](https://github.com/yarikoptic)) - Robustify installation of singularity (install libfuse2). [PR #221](https://github.com/datalad/datalad-container/pull/221) (by [@yarikoptic](https://github.com/yarikoptic)) # 1.2.2 (2023-08-09) ## πŸ› Bug Fixes - BF: make it [] in case of None being returned. [PR #217](https://github.com/datalad/datalad-container/pull/217) (by [@yarikoptic](https://github.com/yarikoptic)) # 1.2.1 (2023-06-09) ## πŸ› Bug Fixes - Capture stderr as well while trying for singularity or apptainer to avoid spurious stderr display. [PR #208](https://github.com/datalad/datalad-container/pull/208) (by [@yarikoptic](https://github.com/yarikoptic)) - BF: by default stop containers-run on error, to not proceed to save. [PR #209](https://github.com/datalad/datalad-container/pull/209) (by [@yarikoptic](https://github.com/yarikoptic)) # 1.2.0 (2023-05-25) ## πŸš€ Enhancements and New Features - Add metalad extractor using `singularity inspect`. Fixes https://github.com/datalad/datalad-container/issues/198 via https://github.com/datalad/datalad-container/pull/200 (by @asmacdo ) - Add `--extra-inputs` to `containers-add`. Fixes [#189](https://github.com/datalad/datalad-container/issues/189) via [PR #190](https://github.com/datalad/datalad-container/pull/190) (by [@nobodyinperson](https://github.com/nobodyinperson)) ## πŸ› Bug Fixes - Make `datalad_container.adapters.docker save` assume `latest` if no image version given. Fixes [#105](https://github.com/datalad/datalad-container/issues/105) via [PR #206](https://github.com/datalad/datalad-container/pull/206) (by [@jwodder](https://github.com/jwodder)) ## 🏠 Internal - Eliminate use of distutils. [PR #203](https://github.com/datalad/datalad-container/pull/203) (by [@jwodder](https://github.com/jwodder)) - Add codespell action,config and fix 1 typo. [PR #207](https://github.com/datalad/datalad-container/pull/207) (by [@yarikoptic](https://github.com/yarikoptic)) # 1.1.9 (2023-02-06) ## 🏠 Internal - Fix the "bump" level for breaking changes in .datalad-release-action.yaml. [PR #186](https://github.com/datalad/datalad-container/pull/186) (by [@jwodder](https://github.com/jwodder)) - Account for move of @eval_results in datalad core. [PR #192](https://github.com/datalad/datalad-container/pull/192) (by [@yarikoptic](https://github.com/yarikoptic)) - scriv.ini: Provide full relative path to the templates. [PR #193](https://github.com/datalad/datalad-container/pull/193) (by [@yarikoptic](https://github.com/yarikoptic)) ## πŸ§ͺ Tests - Install Singularity 3 from an official .deb, use newer ubuntu (jammy) on travis. [PR #188](https://github.com/datalad/datalad-container/pull/188) (by [@bpoldrack](https://github.com/bpoldrack)) # 1.1.8 (Mon Oct 10 2022) #### πŸ› Bug Fix - Replace `simplejson` with `json` [#182](https://github.com/datalad/datalad-container/pull/182) ([@christian-monch](https://github.com/christian-monch)) #### πŸ“ Documentation - codespell fix some typos [#184](https://github.com/datalad/datalad-container/pull/184) ([@yarikoptic](https://github.com/yarikoptic)) #### πŸ§ͺ Tests - Reenabling tests using SingularityHub [#180](https://github.com/datalad/datalad-container/pull/180) ([@yarikoptic](https://github.com/yarikoptic)) #### Authors: 2 - Christian MΓΆnch ([@christian-monch](https://github.com/christian-monch)) - Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) --- # 1.1.7 (Tue Aug 30 2022) #### πŸ› Bug Fix - DOC: Set language in Sphinx config to en [#178](https://github.com/datalad/datalad-container/pull/178) ([@adswa](https://github.com/adswa)) #### πŸ§ͺ Tests - nose -> pytest, isort imports in tests, unify requirements-devel to correspond to the form as in core [#179](https://github.com/datalad/datalad-container/pull/179) ([@yarikoptic](https://github.com/yarikoptic)) #### Authors: 2 - Adina Wagner ([@adswa](https://github.com/adswa)) - Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) --- # 1.1.6 (Mon Apr 11 2022) #### πŸ› Bug Fix - BF: Disable subdataset result rendering [#175](https://github.com/datalad/datalad-container/pull/175) ([@adswa](https://github.com/adswa)) - DOC: A few typos in comments/docstrings [#173](https://github.com/datalad/datalad-container/pull/173) ([@yarikoptic](https://github.com/yarikoptic)) - Update badges [#172](https://github.com/datalad/datalad-container/pull/172) ([@mih](https://github.com/mih)) - Build docs in standard workflow, not with travis [#171](https://github.com/datalad/datalad-container/pull/171) ([@mih](https://github.com/mih)) - Make six obsolete [#170](https://github.com/datalad/datalad-container/pull/170) ([@mih](https://github.com/mih)) - Adopt standard extension setup [#169](https://github.com/datalad/datalad-container/pull/169) ([@mih](https://github.com/mih) [@jwodder](https://github.com/jwodder) [@yarikoptic](https://github.com/yarikoptic)) - Adopt standard appveyor config [#167](https://github.com/datalad/datalad-container/pull/167) ([@mih](https://github.com/mih)) - Clarify documentation for docker usage [#164](https://github.com/datalad/datalad-container/pull/164) ([@mih](https://github.com/mih)) - Strip unsupported scenarios from travis [#166](https://github.com/datalad/datalad-container/pull/166) ([@mih](https://github.com/mih)) - WIP: Implement the actual command "containers" [#2](https://github.com/datalad/datalad-container/pull/2) ([@mih](https://github.com/mih) [@bpoldrack](https://github.com/bpoldrack)) - Stop using deprecated Repo.add_submodule() [#161](https://github.com/datalad/datalad-container/pull/161) ([@mih](https://github.com/mih)) - BF:Docs: replace incorrect dashes with spaces in command names [#154](https://github.com/datalad/datalad-container/pull/154) ([@loj](https://github.com/loj)) #### ⚠️ Pushed to `master` - Adjust test to acknowledge reckless behavior ([@mih](https://github.com/mih)) - Slightly relax tests to account for upcoming remove() change ([@mih](https://github.com/mih)) #### πŸ“ Documentation - Mention that could be installed from conda-forge [#177](https://github.com/datalad/datalad-container/pull/177) ([@yarikoptic](https://github.com/yarikoptic)) #### Authors: 6 - Adina Wagner ([@adswa](https://github.com/adswa)) - Benjamin Poldrack ([@bpoldrack](https://github.com/bpoldrack)) - John T. Wodder II ([@jwodder](https://github.com/jwodder)) - Laura Waite ([@loj](https://github.com/loj)) - Michael Hanke ([@mih](https://github.com/mih)) - Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) --- # 1.1.5 (Mon Jun 07 2021) #### πŸ› Bug Fix - BF: fix special remotes without "externaltype" [#156](https://github.com/datalad/datalad-container/pull/156) ([@loj](https://github.com/loj)) #### Authors: 1 - Laura Waite ([@loj](https://github.com/loj)) --- # 1.1.4 (Mon Apr 19 2021) #### πŸ› Bug Fix - BF+RF: no need to pandoc long description for pypi + correctly boost MODULE/version.py for the release [#152](https://github.com/datalad/datalad-container/pull/152) ([@yarikoptic](https://github.com/yarikoptic)) #### Authors: 1 - Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) --- # 1.1.3 (Thu Apr 15 2021) #### πŸ› Bug Fix - Set up workflow with auto for releasing & PyPI uploads [#151](https://github.com/datalad/datalad-container/pull/151) ([@yarikoptic](https://github.com/yarikoptic)) - TST: docker_adapter: Skip tests if 'docker pull' in setup fails [#148](https://github.com/datalad/datalad-container/pull/148) ([@kyleam](https://github.com/kyleam)) #### 🏠 Internal - ENH: containers-add-dhub - add multiple images/tags/repositories from docker hub [#135](https://github.com/datalad/datalad-container/pull/135) ([@kyleam](https://github.com/kyleam) [@yarikoptic](https://github.com/yarikoptic)) #### Authors: 2 - Kyle Meyer ([@kyleam](https://github.com/kyleam)) - Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) --- # 1.1.2 (January 16, 2021) -- - Replace use of `mock` with `unittest.mock` as we do no longer support Python 2 # 1.1.1 (January 03, 2021) -- - Drop use of `Runner` (to be removed in datalad 0.14.0) in favor of `WitlessRunner` # 1.1.0 (October 30, 2020) -- - Datalad version 0.13.0 or later is now required. - In the upcoming 0.14.0 release of DataLad, the datalad special remote will have built-in support for "shub://" URLs. If `containers-add` detects support for this feature, it will now add the "shub://" URL as is rather than resolving the URL itself. This avoids registering short-lived URLs, allowing the image to be retrieved later with `datalad get`. - `containers-run` learned to install necessary subdatasets when asked to execute a container from underneath an uninstalled subdataset. # 1.0.1 (June 23, 2020) -- - Prefer `datalad.core.local.run` to `datalad.interface.run`. The latter has been marked as obsolete since DataLad v0.12 (our minimum requirement) and will be removed in DataLad's next feature release. # 1.0.0 (Feb 23, 2020) -- not-as-a-shy-one Extension is pretty stable so releasing as 1. MAJOR release, so we could start tracking API breakages and enhancements properly. - Drops support for Python 2 and DataLad prior 0.12 # 0.5.2 (Nov 12, 2019) -- ### Fixes - The Docker adapter unconditionally called `docker run` with `--interactive` and `--tty` even when stdin was not attached to a TTY, leading to an error. # 0.5.1 (Nov 08, 2019) -- ### Fixes - The Docker adapter, which is used for the "dhub://" URL scheme, assumed the Python executable was spelled "python". - A call to DataLad's `resolve_path` helper assumed a string return value, which isn't true as of the latest DataLad release candidate, 0.12.0rc6. # 0.5.0 (Jul 12, 2019) -- damn-you-malicious-users ### New features - The default result renderer for `containers-list` is now a custom renderer that includes the container name in the output. ### Fixes - Temporarily skip two tests relying on SingularityHub -- it is down. # 0.4.0 (May 29, 2019) -- run-baby-run The minimum required DataLad version is now 0.11.5. ### New features - The call format gained the "{img_dspath}" placeholder, which expands to the relative path of the dataset that contains the image. This is useful for pointing to a wrapper script that is bundled in the same subdataset as a container. - `containers-run` now passes the container image to `run` via its `extra_inputs` argument so that a run command's "{inputs}" field is restricted to inputs that the caller explicitly specified. - During execution, `containers-run` now sets the environment variable `DATALAD_CONTAINER_NAME` to the name of the container. ### Fixes - `containers-run` mishandled paths when called from a subdirectory. - `containers-run` didn't provide an informative error message when `cmdexec` contained an unknown placeholder. - `containers-add` ignores the `--update` flag when the container doesn't yet exist, but it confusingly still used the word "update" in the commit message. # 0.3.1 (Mar 05, 2019) -- Upgrayeddd ### Fixes - `containers-list` recursion actually does recursion. # 0.3.0 (Mar 05, 2019) -- Upgrayedd ### API changes - `containers-list` no longer lists containers from subdatasets by default. Specify `--recursive` to do so. - `containers-run` no longer considers subdataset containers in its automatic selection of a container name when no name is specified. If the current dataset has one container, that container is selected. Subdataset containers must always be explicitly specified. ### New features - `containers-add` learned to update a previous container when passed `--update`. - `containers-add` now supports Singularity's "docker://" scheme in the URL. - To avoid unnecessary recursion into subdatasets, `containers-run` now decides to look for containers in subdatasets based on whether the name has a slash (which is true of all subdataset containers). # 0.2.2 (Dec 19, 2018) -- The more the merrier - list/use containers recursively from installed subdatasets - Allow to specify container by path rather than just by name - Adding a container from local filesystem will copy it now # 0.2.1 (Jul 14, 2018) -- Explicit lyrics - Add support `datalad run --explicit`. # 0.2 (Jun 08, 2018) -- Docker - Initial support for adding and running Docker containers. - Add support `datalad run --sidecar`. - Simplify storage of `call_fmt` arguments in the Git config, by benefiting from `datalad run` being able to work with single-string compound commands. # 0.1.2 (May 28, 2018) -- The docs - Basic beginner documentation # 0.1.1 (May 22, 2018) -- The fixes ### New features - Add container images straight from singularity-hub, no need to manually specify `--call-fmt` arguments. ### API changes - Use "name" instead of "label" for referring to a container (e.g. `containers-run -n ...` instead of `containers-run -l`. ### Fixes - Pass relative container path to `datalad run`. - `containers-run` no longer hides `datalad run` failures. # 0.1 (May 19, 2018) -- The Release - Initial release with basic functionality to add, remove, and list containers in a dataset, plus a `run` command wrapper that injects the container image as an input dependency of a command call. datalad-container-1.2.6/CONTRIBUTORS000066400000000000000000000001661501235133500167530ustar00rootroot00000000000000The following people have contributed to this project: Benjamin Poldrack Kyle Meyer Michael Hanke Yaroslav Halchenko datalad-container-1.2.6/COPYING000066400000000000000000000024241501235133500161250ustar00rootroot00000000000000# Main Copyright/License DataLad, including all examples, code snippets and attached documentation is covered by the MIT license. The MIT License Copyright (c) 2018- DataLad Team Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. See CONTRIBUTORS file for a full list of contributors. datalad-container-1.2.6/MANIFEST.in000066400000000000000000000001741501235133500166300ustar00rootroot00000000000000include CONTRIBUTORS LICENSE versioneer.py graft _datalad_buildsupport graft docs prune docs/build global-exclude *.py[cod] datalad-container-1.2.6/Makefile000066400000000000000000000007741501235133500165400ustar00rootroot00000000000000PYTHON ?= python clean: $(PYTHON) setup.py clean rm -rf dist build bin docs/build docs/source/generated *.egg-info -find . -name '*.pyc' -delete -find . -name '__pycache__' -type d -delete release-pypi: # avoid upload of stale builds test ! -e dist $(PYTHON) setup.py sdist bdist_wheel twine upload dist/* update-buildsupport: git subtree pull \ -m "Update DataLad build helper" \ --squash \ --prefix _datalad_buildsupport \ https://github.com/datalad/datalad-buildsupport.git \ master datalad-container-1.2.6/README.md000066400000000000000000000074571501235133500163640ustar00rootroot00000000000000 ____ _ _ _ | _ \ __ _ | |_ __ _ | | __ _ __| | | | | | / _` || __| / _` || | / _` | / _` | | |_| || (_| || |_ | (_| || |___ | (_| || (_| | |____/ \__,_| \__| \__,_||_____| \__,_| \__,_| Container [![Build status](https://ci.appveyor.com/api/projects/status/k4eyq1yygcvwf7wk/branch/master?svg=true)](https://ci.appveyor.com/project/mih/datalad-container/branch/master) [![Travis tests status](https://app.travis-ci.com/datalad/datalad-container.svg?branch=master)](https://app.travis-ci.com/datalad/datalad-container) [![codecov.io](https://codecov.io/github/datalad/datalad-container/coverage.svg?branch=master)](https://codecov.io/github/datalad/datalad-container?branch=master) [![Documentation](https://readthedocs.org/projects/datalad-container/badge/?version=latest)](http://datalad-container.rtfd.org) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![GitHub release](https://img.shields.io/github/release/datalad/datalad-container.svg)](https://GitHub.com/datalad/datalad-container/releases/) [![PyPI version fury.io](https://badge.fury.io/py/datalad-container.svg)](https://pypi.python.org/pypi/datalad-container/) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3368666.svg)](https://doi.org/10.5281/zenodo.3368666) ![Conda](https://anaconda.org/conda-forge/datalad-container/badges/version.svg) This extension enhances DataLad (http://datalad.org) for working with computational containers. Please see the [extension documentation](http://datalad-container.rtfd.org) for a description on additional commands and functionality. For general information on how to use or contribute to DataLad (and this extension), please see the [DataLad website](http://datalad.org) or the [main GitHub project page](http://datalad.org). ## Installation Before you install this package, please make sure that you [install a recent version of git-annex](https://git-annex.branchable.com/install). Afterwards, install the latest version of `datalad-container` from [PyPi](https://pypi.org/project/datalad-container). It is recommended to use a dedicated [virtualenv](https://virtualenv.pypa.io): # create and enter a new virtual environment (optional) virtualenv --system-site-packages --python=python3 ~/env/datalad . ~/env/datalad/bin/activate # install from PyPi pip install datalad_container It is also available for conda package manager from conda-forge: conda install -c conda-forge datalad-container ## Support The documentation of this project is found here: http://docs.datalad.org/projects/container All bugs, concerns and enhancement requests for this software can be submitted here: https://github.com/datalad/datalad-container/issues If you have a problem or would like to ask a question about how to use DataLad, please [submit a question to NeuroStars.org](https://neurostars.org/tags/datalad) with a ``datalad`` tag. NeuroStars.org is a platform similar to StackOverflow but dedicated to neuroinformatics. All previous DataLad questions are available here: http://neurostars.org/tags/datalad/ ## Acknowledgements DataLad development is supported by a US-German collaboration in computational neuroscience (CRCNS) project "DataGit: converging catalogues, warehouses, and deployment logistics into a federated 'data distribution'" (Halchenko/Hanke), co-funded by the US National Science Foundation (NSF 1429999) and the German Federal Ministry of Education and Research (BMBF 01GQ1411). Additional support is provided by the German federal state of Saxony-Anhalt and the European Regional Development Fund (ERDF), Project: Center for Behavioral Brain Sciences, Imaging Platform. This work is further facilitated by the ReproNim project (NIH 1P41EB019936-01A1). datalad-container-1.2.6/_datalad_buildsupport/000077500000000000000000000000001501235133500214355ustar00rootroot00000000000000datalad-container-1.2.6/_datalad_buildsupport/__init__.py000066400000000000000000000010211501235133500235400ustar00rootroot00000000000000# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the DataLad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Python package for functionality needed at package 'build' time by DataLad and its extensions __init__ here should be really minimalistic, not import submodules by default and submodules should also not require heavy dependencies. """ __version__ = '0.1' datalad-container-1.2.6/_datalad_buildsupport/formatters.py000066400000000000000000000247401501235133500242040ustar00rootroot00000000000000# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the DataLad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## import argparse import datetime import re class ManPageFormatter(argparse.HelpFormatter): # This code was originally distributed # under the same License of Python # Copyright (c) 2014 Oz Nahum Tiram def __init__(self, prog, indent_increment=2, max_help_position=4, width=1000000, section=1, ext_sections=None, authors=None, version=None ): super(ManPageFormatter, self).__init__( prog, indent_increment=indent_increment, max_help_position=max_help_position, width=width) self._prog = prog self._section = 1 self._today = datetime.date.today().strftime('%Y\\-%m\\-%d') self._ext_sections = ext_sections self._version = version def _get_formatter(self, **kwargs): return self.formatter_class(prog=self.prog, **kwargs) def _markup(self, txt): return txt.replace('-', '\\-') def _underline(self, string): return "\\fI\\s-1" + string + "\\s0\\fR" def _bold(self, string): if not string.strip().startswith('\\fB'): string = '\\fB' + string if not string.strip().endswith('\\fR'): string = string + '\\fR' return string def _mk_synopsis(self, parser): self.add_usage(parser.usage, parser._actions, parser._mutually_exclusive_groups, prefix='') usage = self._format_usage(None, parser._actions, parser._mutually_exclusive_groups, '') # replace too long list of commands with a single placeholder usage = re.sub(r'{[^]]*?create,.*?}', ' COMMAND ', usage, flags=re.MULTILINE) # take care of proper wrapping usage = re.sub(r'\[([-a-zA-Z0-9]*)\s([a-zA-Z0-9{}|_]*)\]', r'[\1\~\2]', usage) usage = usage.replace('%s ' % self._prog, '') usage = '.SH SYNOPSIS\n.nh\n.HP\n\\fB%s\\fR %s\n.hy\n' % (self._markup(self._prog), usage) return usage def _mk_title(self, prog): name_version = "{0} {1}".format(prog, self._version) return '.TH "{0}" "{1}" "{2}" "{3}"\n'.format( prog, self._section, self._today, name_version) def _mk_name(self, prog, desc): """ this method is in consistent with others ... it relies on distribution """ desc = desc.splitlines()[0] if desc else 'it is in the name' # ensure starting lower case desc = desc[0].lower() + desc[1:] return '.SH NAME\n%s \\- %s\n' % (self._bold(prog), desc) def _mk_description(self, parser): desc = parser.description desc = '\n'.join(desc.splitlines()[1:]) if not desc: return '' desc = desc.replace('\n\n', '\n.PP\n') # sub-section headings desc = re.sub(r'^\*(.*)\*$', r'.SS \1', desc, flags=re.MULTILINE) # italic commands desc = re.sub(r'^ ([-a-z]*)$', r'.TP\n\\fI\1\\fR', desc, flags=re.MULTILINE) # deindent body text, leave to troff viewer desc = re.sub(r'^ (\S.*)\n', '\\1\n', desc, flags=re.MULTILINE) # format NOTEs as indented paragraphs desc = re.sub(r'^NOTE\n', '.TP\nNOTE\n', desc, flags=re.MULTILINE) # deindent indented paragraphs after heading setup desc = re.sub(r'^ (.*)$', '\\1', desc, flags=re.MULTILINE) return '.SH DESCRIPTION\n%s\n' % self._markup(desc) def _mk_footer(self, sections): if not hasattr(sections, '__iter__'): return '' footer = [] for section, value in sections.items(): part = ".SH {}\n {}".format(section.upper(), value) footer.append(part) return '\n'.join(footer) def format_man_page(self, parser): page = [] page.append(self._mk_title(self._prog)) page.append(self._mk_name(self._prog, parser.description)) page.append(self._mk_synopsis(parser)) page.append(self._mk_description(parser)) page.append(self._mk_options(parser)) page.append(self._mk_footer(self._ext_sections)) return ''.join(page) def _mk_options(self, parser): formatter = parser._get_formatter() # positionals, optionals and user-defined groups for action_group in parser._action_groups: formatter.start_section(None) formatter.add_text(None) formatter.add_arguments(action_group._group_actions) formatter.end_section() # epilog formatter.add_text(parser.epilog) # determine help from format above help = formatter.format_help() # add spaces after comma delimiters for easier reformatting help = re.sub(r'([a-z]),([a-z])', '\\1, \\2', help) # get proper indentation for argument items help = re.sub(r'^ (\S.*)\n', '.TP\n\\1\n', help, flags=re.MULTILINE) # deindent body text, leave to troff viewer help = re.sub(r'^ (\S.*)\n', '\\1\n', help, flags=re.MULTILINE) return '.SH OPTIONS\n' + help def _format_action_invocation(self, action, doubledash='--'): if not action.option_strings: metavar, = self._metavar_formatter(action, action.dest)(1) return metavar else: parts = [] # if the Optional doesn't take a value, format is: # -s, --long if action.nargs == 0: parts.extend([self._bold(action_str) for action_str in action.option_strings]) # if the Optional takes a value, format is: # -s ARGS, --long ARGS else: default = self._underline(action.dest.upper()) args_string = self._format_args(action, default) for option_string in action.option_strings: parts.append('%s %s' % (self._bold(option_string), args_string)) return ', '.join(p.replace('--', doubledash) for p in parts) class RSTManPageFormatter(ManPageFormatter): def _get_formatter(self, **kwargs): return self.formatter_class(prog=self.prog, **kwargs) def _markup(self, txt): # put general tune-ups here return txt def _underline(self, string): return "*{0}*".format(string) def _bold(self, string): return "**{0}**".format(string) def _mk_synopsis(self, parser): self.add_usage(parser.usage, parser._actions, parser._mutually_exclusive_groups, prefix='') usage = self._format_usage(None, parser._actions, parser._mutually_exclusive_groups, '') usage = usage.replace('%s ' % self._prog, '') usage = 'Synopsis\n--------\n::\n\n %s %s\n' \ % (self._markup(self._prog), usage) return usage def _mk_title(self, prog): # and an easy to use reference point title = ".. _man_%s:\n\n" % prog.replace(' ', '-') title += "{0}".format(prog) title += '\n{0}\n\n'.format('=' * len(prog)) return title def _mk_name(self, prog, desc): return '' def _mk_description(self, parser): desc = parser.description if not desc: return '' return 'Description\n-----------\n%s\n' % self._markup(desc) def _mk_footer(self, sections): if not hasattr(sections, '__iter__'): return '' footer = [] for section, value in sections.items(): part = "\n{0}\n{1}\n{2}\n".format( section, '-' * len(section), value) footer.append(part) return '\n'.join(footer) def _mk_options(self, parser): # this non-obvious maneuver is really necessary! formatter = self.__class__(self._prog) # positionals, optionals and user-defined groups for action_group in parser._action_groups: formatter.start_section(None) formatter.add_text(None) formatter.add_arguments(action_group._group_actions) formatter.end_section() # epilog formatter.add_text(parser.epilog) # determine help from format above option_sec = formatter.format_help() return '\n\nOptions\n-------\n{0}'.format(option_sec) def _format_action(self, action): # determine the required width and the entry label action_header = self._format_action_invocation(action, doubledash='-\\\\-') if action.help: help_text = self._expand_help(action) help_lines = self._split_lines(help_text, 80) help = ' '.join(help_lines) else: help = '' # return a single string return '{0}\n{1}\n{2}\n\n'.format( action_header, '~' * len(action_header), help) def cmdline_example_to_rst(src, out=None, ref=None): if out is None: from io import StringIO out = StringIO() # place header out.write('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n') if ref: # place cross-ref target out.write('.. {0}:\n\n'.format(ref)) # parser status vars inexample = False incodeblock = False for line in src: if line.startswith('#% EXAMPLE START'): inexample = True incodeblock = False continue if not inexample: continue if line.startswith('#% EXAMPLE END'): break if not inexample: continue if line.startswith('#%'): incodeblock = not incodeblock if incodeblock: out.write('\n.. code-block:: sh\n\n') continue if not incodeblock and line.startswith('#'): out.write(line[(min(2, len(line) - 1)):]) continue if incodeblock: if not line.rstrip().endswith('#% SKIP'): out.write(' %s' % line) continue if not len(line.strip()): continue else: raise RuntimeError("this should not happen") return out datalad-container-1.2.6/_datalad_buildsupport/setup.py000066400000000000000000000227251501235133500231570ustar00rootroot00000000000000# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the DataLad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## import datetime import os from os.path import ( dirname, join as opj, ) from setuptools import Command from setuptools.config import read_configuration from setuptools.errors import OptionError import versioneer from . import formatters as fmt class BuildManPage(Command): # The BuildManPage code was originally distributed # under the same License of Python # Copyright (c) 2014 Oz Nahum Tiram description = 'Generate man page from an ArgumentParser instance.' user_options = [ ('manpath=', None, 'output path for manpages (relative paths are relative to the ' 'datalad package)'), ('rstpath=', None, 'output path for RST files (relative paths are relative to the ' 'datalad package)'), ('parser=', None, 'module path to an ArgumentParser instance' '(e.g. mymod:func, where func is a method or function which return' 'a dict with one or more arparse.ArgumentParser instances.'), ('cmdsuite=', None, 'module path to an extension command suite ' '(e.g. mymod:command_suite) to limit the build to the contained ' 'commands.'), ] def initialize_options(self): self.manpath = opj('build', 'man') self.rstpath = opj('docs', 'source', 'generated', 'man') self.parser = 'datalad.cmdline.main:setup_parser' self.cmdsuite = None def finalize_options(self): if self.manpath is None: raise OptionError('\'manpath\' option is required') if self.rstpath is None: raise OptionError('\'rstpath\' option is required') if self.parser is None: raise OptionError('\'parser\' option is required') mod_name, func_name = self.parser.split(':') fromlist = mod_name.split('.') try: mod = __import__(mod_name, fromlist=fromlist) self._parser = getattr(mod, func_name)( ['datalad'], formatter_class=fmt.ManPageFormatter, return_subparsers=True, # ignore extensions only for the main package to avoid pollution # with all extension commands that happen to be installed help_ignore_extensions=self.distribution.get_name() == 'datalad') except ImportError as err: raise err if self.cmdsuite: mod_name, suite_name = self.cmdsuite.split(':') mod = __import__(mod_name, fromlist=mod_name.split('.')) suite = getattr(mod, suite_name) self.cmdlist = [c[2] if len(c) > 2 else c[1].replace('_', '-').lower() for c in suite[1]] self.announce('Writing man page(s) to %s' % self.manpath) self._today = datetime.date.today() @classmethod def handle_module(cls, mod_name, **kwargs): """Module specific handling. This particular one does 1. Memorize (at class level) the module name of interest here 2. Check if 'datalad.extensions' are specified for the module, and then analyzes them to obtain command names it provides If cmdline commands are found, its entries are to be used instead of the ones in datalad's _parser. Parameters ---------- **kwargs: all the kwargs which might be provided to setuptools.setup """ cls.mod_name = mod_name exts = kwargs.get('entry_points', {}).get('datalad.extensions', []) for ext in exts: assert '=' in ext # should be label=module:obj ext_label, mod_obj = ext.split('=', 1) assert ':' in mod_obj # should be module:obj mod, obj = mod_obj.split(':', 1) assert mod_name == mod # AFAIK should be identical mod = __import__(mod_name) if hasattr(mod, obj): command_suite = getattr(mod, obj) assert len(command_suite) == 2 # as far as I see it if not hasattr(cls, 'cmdline_names'): cls.cmdline_names = [] cls.cmdline_names += [ cmd for _, _, cmd, _ in command_suite[1] ] def run(self): dist = self.distribution #homepage = dist.get_url() #appname = self._parser.prog appname = 'datalad' cfg = read_configuration( opj(dirname(dirname(__file__)), 'setup.cfg'))['metadata'] sections = { 'Authors': """{0} is developed by {1} <{2}>.""".format( appname, cfg['author'], cfg['author_email']), } for cls, opath, ext in ((fmt.ManPageFormatter, self.manpath, '1'), (fmt.RSTManPageFormatter, self.rstpath, 'rst')): if not os.path.exists(opath): os.makedirs(opath) for cmdname in getattr(self, 'cmdline_names', list(self._parser)): if hasattr(self, 'cmdlist') and cmdname not in self.cmdlist: continue p = self._parser[cmdname] cmdname = "{0}{1}".format( 'datalad ' if cmdname != 'datalad' else '', cmdname) format = cls( cmdname, ext_sections=sections, version=versioneer.get_version()) formatted = format.format_man_page(p) with open(opj(opath, '{0}.{1}'.format( cmdname.replace(' ', '-'), ext)), 'w') as f: f.write(formatted) class BuildRSTExamplesFromScripts(Command): description = 'Generate RST variants of example shell scripts.' user_options = [ ('expath=', None, 'path to look for example scripts'), ('rstpath=', None, 'output path for RST files'), ] def initialize_options(self): self.expath = opj('docs', 'examples') self.rstpath = opj('docs', 'source', 'generated', 'examples') def finalize_options(self): if self.expath is None: raise OptionError('\'expath\' option is required') if self.rstpath is None: raise OptionError('\'rstpath\' option is required') self.announce('Converting example scripts') def run(self): opath = self.rstpath if not os.path.exists(opath): os.makedirs(opath) from glob import glob for example in glob(opj(self.expath, '*.sh')): exname = os.path.basename(example)[:-3] with open(opj(opath, '{0}.rst'.format(exname)), 'w') as out: fmt.cmdline_example_to_rst( open(example), out=out, ref='_example_{0}'.format(exname)) class BuildConfigInfo(Command): description = 'Generate RST documentation for all config items.' user_options = [ ('rstpath=', None, 'output path for RST file'), ] def initialize_options(self): self.rstpath = opj('docs', 'source', 'generated', 'cfginfo') def finalize_options(self): if self.rstpath is None: raise OptionError('\'rstpath\' option is required') self.announce('Generating configuration documentation') def run(self): opath = self.rstpath if not os.path.exists(opath): os.makedirs(opath) from datalad.dochelpers import _indent from datalad.interface.common_cfg import definitions as cfgdefs categories = { 'global': {}, 'local': {}, 'dataset': {}, 'misc': {} } for term, v in cfgdefs.items(): categories[v.get('destination', 'misc')][term] = v for cat in categories: with open(opj(opath, '{}.rst.in'.format(cat)), 'w') as rst: rst.write('.. glossary::\n') for term, v in sorted(categories[cat].items(), key=lambda x: x[0]): rst.write(_indent(term, '\n ')) qtype, docs = v.get('ui', (None, {})) desc_tmpl = '\n' if 'title' in docs: desc_tmpl += '{title}:\n' if 'text' in docs: desc_tmpl += '{text}\n' if 'default' in v: default = v['default'] if hasattr(default, 'replace'): # protect against leaking specific home dirs v['default'] = default.replace(os.path.expanduser('~'), '~') desc_tmpl += 'Default: {default}\n' if 'type' in v: type_ = v['type'] if hasattr(type_, 'long_description'): type_ = type_.long_description() else: type_ = type_.__name__ desc_tmpl += '\n[{type}]\n' v['type'] = type_ if desc_tmpl == '\n': # we need something to avoid joining terms desc_tmpl += 'undocumented\n' v.update(docs) rst.write(_indent(desc_tmpl.format(**v), ' ')) datalad-container-1.2.6/changelog.d/000077500000000000000000000000001501235133500172415ustar00rootroot00000000000000datalad-container-1.2.6/changelog.d/scriv.ini000066400000000000000000000005541501235133500210740ustar00rootroot00000000000000[scriv] fragment_directory = changelog.d entry_title_template = file: changelog.d/templates/entry_title.md.j2 new_fragment_template = file: changelog.d/templates/new_fragment.md.j2 format = md categories = πŸ’₯ Breaking Changes, πŸš€ Enhancements and New Features, πŸ› Bug Fixes, πŸ”© Dependencies, πŸ“ Documentation, 🏠 Internal, 🏎 Performance, πŸ§ͺ Tests datalad-container-1.2.6/changelog.d/templates/000077500000000000000000000000001501235133500212375ustar00rootroot00000000000000datalad-container-1.2.6/changelog.d/templates/entry_title.md.j2000066400000000000000000000001121501235133500244270ustar00rootroot00000000000000{{ version if version else "VERSION" }} ({{ date.strftime('%Y-%m-%d') }}) datalad-container-1.2.6/changelog.d/templates/new_fragment.md.j2000066400000000000000000000005471501235133500245550ustar00rootroot00000000000000 {% for cat in config.categories -%} {% endfor -%} datalad-container-1.2.6/datalad_container/000077500000000000000000000000001501235133500205245ustar00rootroot00000000000000datalad-container-1.2.6/datalad_container/__init__.py000066400000000000000000000036321501235133500226410ustar00rootroot00000000000000"""DataLad container extension""" __docformat__ = 'restructuredtext' # Imported to set singularity/apptainer version commands at init import datalad_container.extractors._load_singularity_versions # noqa # defines a datalad command suite # this symbold must be identified as a setuptools entrypoint # to be found by datalad command_suite = ( # description of the command suite, displayed in cmdline help "Containerized environments", [ # specification of a command, any number of commands can be defined ( # importable module that contains the command implementation 'datalad_container.containers_list', # name of the command class implementation in above module 'ContainersList', 'containers-list', 'containers_list', ), ( 'datalad_container.containers_remove', # name of the command class implementation in above module 'ContainersRemove', 'containers-remove', 'containers_remove', ), ( 'datalad_container.containers_add', # name of the command class implementation in above module 'ContainersAdd', 'containers-add', 'containers_add', ), ( 'datalad_container.containers_run', 'ContainersRun', 'containers-run', 'containers_run', ) ] ) from os.path import join as opj from datalad.support.constraints import EnsureStr from datalad.support.extensions import register_config register_config( 'datalad.containers.location', 'Container location', description='path within the dataset where to store containers', type=EnsureStr(), default=opj(".datalad", "environments"), dialog='question', scope='dataset', ) from . import _version __version__ = _version.get_versions()['version'] datalad-container-1.2.6/datalad_container/_version.py000066400000000000000000000600441501235133500227260ustar00rootroot00000000000000 # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. # Generated by versioneer-0.29 # https://github.com/python-versioneer/python-versioneer """Git implementation of _version.py.""" import errno import functools import os import re import subprocess import sys from typing import ( Any, Callable, Dict, List, Optional, Tuple, ) def get_keywords() -> Dict[str, str]: """Get the keywords needed to look up the version information.""" # these strings will be replaced by git during git-archive. # setup.py/versioneer.py will grep for the variable names, so they must # each be defined on a line of their own. _version.py will just call # get_keywords(). git_refnames = " (tag: 1.2.6, origin/master, origin/HEAD, master)" git_full = "e9bba2a6566abf0e0a788dc06c56a468f8400d1f" git_date = "2025-05-18 12:30:21 +0000" keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} return keywords class VersioneerConfig: """Container for Versioneer configuration parameters.""" VCS: str style: str tag_prefix: str parentdir_prefix: str versionfile_source: str verbose: bool def get_config() -> VersioneerConfig: """Create, populate and return the VersioneerConfig() object.""" # these strings are filled in when 'setup.py versioneer' creates # _version.py cfg = VersioneerConfig() cfg.VCS = "git" cfg.style = "pep440" cfg.tag_prefix = "" cfg.parentdir_prefix = "" cfg.versionfile_source = "datalad_container/_version.py" cfg.verbose = False return cfg class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" LONG_VERSION_PY: Dict[str, str] = {} HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs: str, method: str) -> Callable: # decorator """Create decorator to mark a method as the handler of a VCS.""" def decorate(f: Callable) -> Callable: """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f return decorate def run_command( commands: List[str], args: List[str], cwd: Optional[str] = None, verbose: bool = False, hide_stderr: bool = False, env: Optional[Dict[str, str]] = None, ) -> Tuple[Optional[str], Optional[int]]: """Call the given command(s).""" assert isinstance(commands, list) process = None popen_kwargs: Dict[str, Any] = {} if sys.platform == "win32": # This hides the console window if pythonw.exe is used startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW popen_kwargs["startupinfo"] = startupinfo for command in commands: try: dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git process = subprocess.Popen([command] + args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None), **popen_kwargs) break except OSError as e: if e.errno == errno.ENOENT: continue if verbose: print("unable to run %s" % dispcmd) print(e) return None, None else: if verbose: print("unable to find command, tried %s" % (commands,)) return None, None stdout = process.communicate()[0].strip().decode() if process.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) print("stdout was %s" % stdout) return None, process.returncode return stdout, process.returncode def versions_from_parentdir( parentdir_prefix: str, root: str, verbose: bool, ) -> Dict[str, Any]: """Try to determine the version from the parent directory name. Source tarballs conventionally unpack into a directory that includes both the project name and a version string. We will also support searching up two directory levels for an appropriately named parent directory """ rootdirs = [] for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: print("Tried directories %s but none started with prefix %s" % (str(rootdirs), parentdir_prefix)) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @register_vcs_handler("git", "get_keywords") def git_get_keywords(versionfile_abs: str) -> Dict[str, str]: """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from # _version.py. keywords: Dict[str, str] = {} try: with open(versionfile_abs, "r") as fobj: for line in fobj: if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["full"] = mo.group(1) if line.strip().startswith("git_date ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["date"] = mo.group(1) except OSError: pass return keywords @register_vcs_handler("git", "keywords") def git_versions_from_keywords( keywords: Dict[str, str], tag_prefix: str, verbose: bool, ) -> Dict[str, Any]: """Get version information from git keywords.""" if "refnames" not in keywords: raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because # it's been around since git-1.5.3, and it's too difficult to # discover which version we're using, or to work around using an # older one. date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d # expansion behaves like git log --decorate=short and strips out the # refs/heads/ and refs/tags/ prefixes that would let us distinguish # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] # Filter out refs that exactly match prefix or that don't start # with a number once the prefix is stripped (mostly a concern # when prefix is '') if not re.match(r'\d', r): continue if verbose: print("picking %s" % r) return {"version": r, "full-revisionid": keywords["full"].strip(), "dirty": False, "error": None, "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") return {"version": "0+unknown", "full-revisionid": keywords["full"].strip(), "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs( tag_prefix: str, root: str, verbose: bool, runner: Callable = run_command ) -> Dict[str, Any]: """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* expanded, and _version.py hasn't already been rewritten with a short version string, meaning we're inside a checked out source tree. """ GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] # GIT_DIR can interfere with correct operation of Versioneer. # It may be intended to be passed to the Versioneer-versioned project, # but that should not change where we get our version from. env = os.environ.copy() env.pop("GIT_DIR", None) runner = functools.partial(runner, env=env) _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) raise NotThisMethod("'git rev-parse --git-dir' returned error") # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) describe_out, rc = runner(GITS, [ "describe", "--tags", "--dirty", "--always", "--long", "--match", f"{tag_prefix}[[:digit:]]*" ], cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() pieces: Dict[str, Any] = {} pieces["long"] = full_out pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) # --abbrev-ref was added in git-1.6.3 if rc != 0 or branch_name is None: raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") branch_name = branch_name.strip() if branch_name == "HEAD": # If we aren't exactly on a branch, pick a branch which represents # the current commit. If all else fails, we are on a branchless # commit. branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) # --contains was added in git-1.5.4 if rc != 0 or branches is None: raise NotThisMethod("'git branch --contains' returned error") branches = branches.split("\n") # Remove the first line if we're running detached if "(" in branches[0]: branches.pop(0) # Strip off the leading "* " from the list of branches. branches = [branch[2:] for branch in branches] if "master" in branches: branch_name = "master" elif not branches: branch_name = None else: # Pick the first branch that is returned. Good or bad. branch_name = branches[0] pieces["branch"] = branch_name # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out # look for -dirty suffix dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: git_describe = git_describe[:git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%s'" % describe_out) return pieces # tag full_tag = mo.group(1) if not full_tag.startswith(tag_prefix): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" % (full_tag, tag_prefix)) return pieces pieces["closest-tag"] = full_tag[len(tag_prefix):] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) # commit: short hex revision ID pieces["short"] = mo.group(3) else: # HEX: no tags pieces["closest-tag"] = None out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces def plus_or_dot(pieces: Dict[str, Any]) -> str: """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" def render_pep440(pieces: Dict[str, Any]) -> str: """Build up version string, with post-release "local version identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty Exceptions: 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += plus_or_dot(pieces) rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_branch(pieces: Dict[str, Any]) -> str: """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . The ".dev0" means not master branch. Note that .dev0 sorts backwards (a feature branch will appear "older" than the master branch). Exceptions: 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0" if pieces["branch"] != "master": rendered += ".dev0" rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def pep440_split_post(ver: str) -> Tuple[str, Optional[int]]: """Split pep440 version string at the post-release segment. Returns the release segments before the post-release and the post-release version number (or -1 if no post-release segment is present). """ vc = str.split(ver, ".post") return vc[0], int(vc[1] or 0) if len(vc) == 2 else None def render_pep440_pre(pieces: Dict[str, Any]) -> str: """TAG[.postN.devDISTANCE] -- No -dirty. Exceptions: 1: no tags. 0.post0.devDISTANCE """ if pieces["closest-tag"]: if pieces["distance"]: # update the post release segment tag_version, post_version = pep440_split_post(pieces["closest-tag"]) rendered = tag_version if post_version is not None: rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) else: rendered += ".post0.dev%d" % (pieces["distance"]) else: # no commits, use the tag as the version rendered = pieces["closest-tag"] else: # exception #1 rendered = "0.post0.dev%d" % pieces["distance"] return rendered def render_pep440_post(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that .dev0 sorts backwards (a dirty tree will appear "older" than the corresponding clean one), but you shouldn't be releasing software with -dirty anyways. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%s" % pieces["short"] else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += "+g%s" % pieces["short"] return rendered def render_pep440_post_branch(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . The ".dev0" means not master branch. Exceptions: 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%s" % pieces["short"] if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += "+g%s" % pieces["short"] if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_old(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" return rendered def render_git_describe(pieces: Dict[str, Any]) -> str: """TAG[-DISTANCE-gHEX][-dirty]. Like 'git describe --tags --dirty --always'. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"]: rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render_git_describe_long(pieces: Dict[str, Any]) -> str: """TAG-DISTANCE-gHEX[-dirty]. Like 'git describe --tags --dirty --always -long'. The distance/hash is unconditional. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: """Render the given version pieces into the requested style.""" if pieces["error"]: return {"version": "unknown", "full-revisionid": pieces.get("long"), "dirty": None, "error": pieces["error"], "date": None} if not style or style == "default": style = "pep440" # the default if style == "pep440": rendered = render_pep440(pieces) elif style == "pep440-branch": rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) elif style == "pep440-post-branch": rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": rendered = render_git_describe(pieces) elif style == "git-describe-long": rendered = render_git_describe_long(pieces) else: raise ValueError("unknown style '%s'" % style) return {"version": rendered, "full-revisionid": pieces["long"], "dirty": pieces["dirty"], "error": None, "date": pieces.get("date")} def get_versions() -> Dict[str, Any]: """Get version information or return default if unable to do so.""" # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have # __file__, we can work backwards from there to the root. Some # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which # case we can only use expanded keywords. cfg = get_config() verbose = cfg.verbose try: return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) except NotThisMethod: pass try: root = os.path.realpath(__file__) # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. for _ in cfg.versionfile_source.split('/'): root = os.path.dirname(root) except NameError: return {"version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to find root of source tree", "date": None} try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) return render(pieces, cfg.style) except NotThisMethod: pass try: if cfg.parentdir_prefix: return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) except NotThisMethod: pass return {"version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to compute version", "date": None} datalad-container-1.2.6/datalad_container/adapters/000077500000000000000000000000001501235133500223275ustar00rootroot00000000000000datalad-container-1.2.6/datalad_container/adapters/__init__.py000066400000000000000000000000001501235133500244260ustar00rootroot00000000000000datalad-container-1.2.6/datalad_container/adapters/docker.py000066400000000000000000000202361501235133500241530ustar00rootroot00000000000000"""Work with Docker images as local paths. This module provides support for saving a Docker image in a local directory and then loading it on-the-fly before calling `docker run ...`. The motivation for this is that it allows the components of an image to be tracked as objects in a DataLad dataset. Run `python -m datalad_container.adapters.docker --help` for details about the command-line interface. """ import hashlib import json import logging import os import os.path as op import subprocess as sp import sys import tarfile import tempfile from datalad.utils import on_windows lgr = logging.getLogger("datalad.containers.adapters.docker") # Note: A dockerpy dependency probably isn't worth it in the current # state but is worth thinking about if this module gets more # complicated. # FIXME: These functions assume that there is a "docker" on the path # that can be managed by a non-root user. At the least, this should # be documented somewhere. def save(image, path): """Save and extract a docker image to a directory. Parameters ---------- image : str A unique identifier for a docker image. path : str A directory to extract the image to. """ # Use a temporary file because docker save (or actually tar underneath) # complains that stdout needs to be redirected if we use Popen and PIPE. if ":" not in image: image = f"{image}:latest" with tempfile.NamedTemporaryFile() as stream: # Windows can't write to an already opened file stream.close() sp.check_call(["docker", "save", "-o", stream.name, image]) with tarfile.open(stream.name, mode="r:") as tar: if not op.exists(path): lgr.debug("Creating new directory at %s", path) os.makedirs(path) elif os.listdir(path): raise OSError("Directory {} is not empty".format(path)) def is_within_directory(directory, target): abs_directory = os.path.abspath(directory) abs_target = os.path.abspath(target) prefix = os.path.commonprefix([abs_directory, abs_target]) return prefix == abs_directory def safe_extract(tar, path=".", members=None, *, numeric_owner=False): for member in tar.getmembers(): member_path = os.path.join(path, member.name) if not is_within_directory(path, member_path): raise Exception("Attempted Path Traversal in Tar File") tar.extractall(path, members, numeric_owner=numeric_owner) safe_extract(tar, path=path) lgr.info("Saved %s to %s", image, path) def _list_images(): out = sp.check_output( ["docker", "images", "--all", "--quiet", "--no-trunc"]) return out.decode().splitlines() def get_image(path, repo_tag=None, config=None): """Return the image ID of the image extracted at `path`. """ manifest_path = op.join(path, "manifest.json") with open(manifest_path) as fp: manifest = json.load(fp) if repo_tag is not None: manifest = [img for img in manifest if repo_tag in (img.get("RepoTags") or [])] if config is not None: manifest = [img for img in manifest if img["Config"].startswith(config)] if len(manifest) == 0: raise ValueError(f"No matching images found in {manifest_path}") elif len(manifest) > 1: raise ValueError( f"Multiple images found in {manifest_path}; disambiguate with" " --repo-tag or --config" ) with open(op.join(path, manifest[0]["Config"]), "rb") as stream: return hashlib.sha256(stream.read()).hexdigest() def load(path, repo_tag, config): """Load the Docker image from `path`. Parameters ---------- path : str A directory with an extracted tar archive. repo_tag : str or None `image:tag` of image to load config : str or None "Config" value or prefix of image to load Returns ------- The image ID (str) """ # FIXME: If we load a dataset, it may overwrite the current tag. Say that # (1) a dataset has a saved neurodebian:latest from a month ago, (2) a # newer neurodebian:latest has been pulled, and (3) the old image have been # deleted (e.g., with 'docker image prune --all'). Given all three of these # things, loading the image from the dataset will tag the old neurodebian # image as the latest. image_id = "sha256:" + get_image(path, repo_tag, config) if image_id not in _list_images(): lgr.debug("Loading %s", image_id) cmd = ["docker", "load"] p = sp.Popen(cmd, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE) with tarfile.open(fileobj=p.stdin, mode="w|", dereference=True) as tar: tar.add(path, arcname="") out, err = p.communicate() return_code = p.poll() if return_code: lgr.warning("Running %r failed: %s", cmd, err.decode()) raise sp.CalledProcessError(return_code, cmd, output=out) else: lgr.debug("Image %s is already present", image_id) if image_id not in _list_images(): raise RuntimeError( "docker image {} was not successfully loaded".format(image_id)) return image_id # Command-line def cli_save(namespace): save(namespace.image, namespace.path) def cli_run(namespace): image_id = load(namespace.path, namespace.repo_tag, namespace.config) prefix = ["docker", "run", # FIXME: The -v/-w settings are convenient for testing, but they # should be configurable. "-v", "{}:/tmp".format(os.getcwd()), "-w", "/tmp", "--rm", "--interactive"] if not on_windows: # Make it possible for the output files to be added to the # dataset without the user needing to manually adjust the # permissions. prefix.extend(["-u", "{}:{}".format(os.getuid(), os.getgid())]) if sys.stdin.isatty(): prefix.append("--tty") prefix.append(image_id) cmd = prefix + namespace.cmd lgr.debug("Running %r", cmd) sp.check_call(cmd) def main(args): import argparse parser = argparse.ArgumentParser( prog="python -m datalad_container.adapters.docker", description="Work with Docker images as local paths") parser.add_argument( "-v", "--verbose", action="store_true") subparsers = parser.add_subparsers(title="subcommands") # Don't continue without a subcommand. subparsers.required = True subparsers.dest = "command" parser_save = subparsers.add_parser( "save", help="save and extract a Docker image to a directory") parser_save.add_argument( "image", metavar="NAME", help="image to save") parser_save.add_argument( "path", metavar="PATH", help="directory to save image in") parser_save.set_defaults(func=cli_save) # TODO: Add command for updating an archive directory. parser_run = subparsers.add_parser( "run", help="run a command with a directory's image") parser_run.add_argument( "--repo-tag", metavar="IMAGE:TAG", help="Tag of image to load" ) parser_run.add_argument( "--config", metavar="IDPREFIX", help="Config value or prefix of image to load" ) parser_run.add_argument( "path", metavar="PATH", help="run the image in this directory") parser_run.add_argument( "cmd", metavar="CMD", nargs=argparse.REMAINDER, help="command to execute") parser_run.set_defaults(func=cli_run) namespace = parser.parse_args(args[1:]) logging.basicConfig( level=logging.DEBUG if namespace.verbose else logging.INFO, format="%(message)s") namespace.func(namespace) if __name__ == "__main__": try: main(sys.argv) except Exception as exc: lgr.exception("Failed to execute %s", sys.argv) if isinstance(exc, sp.CalledProcessError): excode = exc.returncode else: excode = 1 sys.exit(excode) datalad-container-1.2.6/datalad_container/adapters/tests/000077500000000000000000000000001501235133500234715ustar00rootroot00000000000000datalad-container-1.2.6/datalad_container/adapters/tests/__init__.py000066400000000000000000000000001501235133500255700ustar00rootroot00000000000000datalad-container-1.2.6/datalad_container/adapters/tests/test_docker.py000066400000000000000000000104141501235133500263510ustar00rootroot00000000000000import json import os.path as op import sys from shutil import ( unpack_archive, which, ) import pytest from datalad.cmd import ( StdOutCapture, WitlessRunner, ) from datalad.support.exceptions import CommandError from datalad.tests.utils_pytest import ( SkipTest, assert_in, assert_raises, eq_, ok_exists, with_tempfile, with_tree, ) import datalad_container.adapters.docker as da if not which("docker"): raise SkipTest("'docker' not found on path") def call(args, **kwds): return WitlessRunner().run( [sys.executable, "-m", "datalad_container.adapters.docker"] + args, **kwds) def list_images(args): cmd = ["docker", "images", "--quiet", "--no-trunc"] + args res = WitlessRunner().run(cmd, protocol=StdOutCapture) return res["stdout"].strip().split() def images_exist(args): return bool(list_images(args)) @with_tempfile def test_docker_save_doesnt_exist(path=None): image_name = "idonotexistsurely" if images_exist([image_name]): raise SkipTest("Image wasn't supposed to exist, but does: {}" .format(image_name)) with assert_raises(CommandError): call(["save", image_name, path]) class TestAdapterBusyBox(object): @classmethod def setup_class(cls): cls.image_name = "busybox:latest" if images_exist([cls.image_name]): cls.image_existed = True else: cls.image_existed = False try: WitlessRunner().run(["docker", "pull", cls.image_name]) except CommandError: # This is probably due to rate limiting. raise SkipTest("Plain `docker pull` failed; skipping") @classmethod def teardown_class(cls): if not cls.image_existed and images_exist([cls.image_name]): WitlessRunner().run(["docker", "rmi", cls.image_name]) @with_tempfile(mkdir=True) def test_save_and_run(self, path=None): image_dir = op.join(path, "image") call(["save", self.image_name, image_dir]) ok_exists(op.join(image_dir, "manifest.json")) img_ids = list_images([self.image_name]) assert len(img_ids) == 1 eq_("sha256:" + da.get_image(image_dir), img_ids[0]) if not self.image_existed: WitlessRunner().run(["docker", "rmi", self.image_name]) out = call(["run", image_dir, "ls"], cwd=path, protocol=StdOutCapture) assert images_exist([self.image_name]) assert_in("image", out["stdout"]) @with_tree({"foo": "content"}) def test_containers_run(self, path=None): if self.image_existed: raise SkipTest( "Not pulling with containers-run due to existing image: {}" .format(self.image_name)) from datalad.api import Dataset ds = Dataset(path).create(force=True) ds.save(path="foo") ds.containers_add("bb", url="dhub://" + self.image_name) out = WitlessRunner(cwd=ds.path).run( ["datalad", "containers-run", "-n", "bb", "cat foo"], protocol=StdOutCapture) assert_in("content", out["stdout"]) # Data can be received on stdin. with (ds.pathobj / "foo").open() as ifh: out = WitlessRunner(cwd=ds.path).run( ["datalad", "containers-run", "-n", "bb", "cat"], protocol=StdOutCapture, stdin=ifh) assert_in("content", out["stdout"]) def test_load_multi_image(tmp_path): for v in ["3.15", "3.16", "3.17"]: WitlessRunner().run(["docker", "pull", f"alpine:{v}"]) WitlessRunner().run(["docker", "save", "alpine", "-o", str(tmp_path / "alpine.tar")]) unpack_archive(tmp_path / "alpine.tar", tmp_path / "alpine") with pytest.raises(CommandError): call(["run", str(tmp_path / "alpine"), "ls"]) call(["run", "--repo-tag", "alpine:3.16", str(tmp_path / "alpine"), "ls"]) def test_save_multi_image(tmp_path): for v in ["3.15", "3.16", "latest"]: WitlessRunner().run(["docker", "pull", f"alpine:{v}"]) call(["save", "alpine", str(tmp_path)]) with (tmp_path / "manifest.json").open() as fp: manifest = json.load(fp) assert len(manifest) == 1 assert manifest[0]["RepoTags"] == ["alpine:latest"] datalad-container-1.2.6/datalad_container/conftest.py000066400000000000000000000001741501235133500227250ustar00rootroot00000000000000from datalad.conftest import setup_package from .tests.fixtures import * # noqa: F401, F403 # lgtm [py/polluting-import] datalad-container-1.2.6/datalad_container/containers_add.py000066400000000000000000000360101501235133500240530ustar00rootroot00000000000000"""Add a container environment to a dataset""" __docformat__ = 'restructuredtext' import json import logging import os import os.path as op import re from pathlib import ( Path, PurePosixPath, ) from shutil import copyfile from datalad.cmd import WitlessRunner from datalad.distribution.dataset import ( EnsureDataset, datasetmethod, require_dataset, ) from datalad.interface.base import ( Interface, build_doc, eval_results, ) from datalad.interface.results import get_status_dict from datalad.support.constraints import ( EnsureNone, EnsureStr, ) from datalad.support.exceptions import InsufficientArgumentsError from datalad.support.param import Parameter from .utils import get_container_configuration lgr = logging.getLogger("datalad.containers.containers_add") # The DataLad special remote has built-in support for Singularity Hub URLs. Let # it handle shub:// URLs if it's available. _HAS_SHUB_DOWNLOADER = True try: import datalad.downloaders.shub except ImportError: lgr.debug("DataLad's shub downloader not found. " "Custom handling for shub:// will be used") _HAS_SHUB_DOWNLOADER = False def _resolve_img_url(url): """Takes a URL and tries to resolve it to an actual download URL that `annex addurl` can handle""" if not _HAS_SHUB_DOWNLOADER and url.startswith('shub://'): # TODO: Remove this handling once the minimum DataLad version is at # least 0.14. lgr.debug('Query singularity-hub for image download URL') import requests req = requests.get( 'https://www.singularity-hub.org/api/container/{}'.format( url[7:])) shub_info = json.loads(req.text) url = shub_info['image'] return url def _guess_call_fmt(ds, name, url): """Helper to guess a container exec setup based on - a name (to be able to look up more config - a plain url to make inference based on the source location Should return `None` is no guess can be made. """ if url is None: return None elif url.startswith('shub://') or url.startswith('docker://'): return 'singularity exec {img} {cmd}' elif url.startswith('dhub://'): # {python} is replaced with sys.executable on *execute* return '{python} -m datalad_container.adapters.docker run {img} {cmd}' def _ensure_datalad_remote(repo): """Initialize and enable datalad special remote if it isn't already.""" dl_remote = None for info in repo.get_special_remotes().values(): if info.get("externaltype") == "datalad": dl_remote = info["name"] break if not dl_remote: from datalad.consts import DATALAD_SPECIAL_REMOTE from datalad.customremotes.base import init_datalad_remote init_datalad_remote(repo, DATALAD_SPECIAL_REMOTE, autoenable=True) elif repo.is_special_annex_remote(dl_remote, check_if_known=False): lgr.debug("datalad special remote '%s' is already enabled", dl_remote) else: lgr.debug("datalad special remote '%s' found. Enabling", dl_remote) repo.enable_remote(dl_remote) @build_doc # all commands must be derived from Interface class ContainersAdd(Interface): # first docstring line is used a short description in the cmdline help # the rest is put in the verbose help and manpage """Add a container to a dataset """ # parameters of the command, must be exhaustive _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), doc="""specify the dataset to add the container to. If no dataset is given, an attempt is made to identify the dataset based on the current working directory""", constraints=EnsureDataset() | EnsureNone() ), name=Parameter( args=("name",), doc="""The name to register the container under. This also determines the default location of the container image within the dataset.""", metavar="NAME", constraints=EnsureStr(), ), url=Parameter( args=("-u", "--url"), doc="""A URL (or local path) to get the container image from. If the URL scheme is one recognized by Singularity (e.g., 'shub://neurodebian/dcm2niix:latest' or 'docker://debian:stable-slim'), a command format string for Singularity-based execution will be auto-configured when [CMD: --call-fmt CMD][PY: call_fmt PY] is not specified. For Docker-based container execution with the URL scheme 'dhub://', the rest of the URL will be interpreted as the argument to 'docker pull', the image will be saved to a location specified by `name`, and the call format will be auto-configured to run docker, unless overwritten. The auto-configured call to docker run mounts the CWD to '/tmp' and sets the working directory to '/tmp'.""", metavar="URL", constraints=EnsureStr() | EnsureNone(), ), # TODO: The "prepared command stuff should ultimately go somewhere else # (probably datalad-run). But first figure out, how exactly to address # container datasets call_fmt=Parameter( args=("--call-fmt",), doc="""Command format string indicating how to execute a command in this container, e.g. "singularity exec {img} {cmd}". Where '{img}' is a placeholder for the path to the container image and '{cmd}' is replaced with the desired command. Additional placeholders: '{img_dspath}' is relative path to the dataset containing the image, '{img_dirpath}' is the directory containing the '{img}'. '{python}' expands to the path of the Python executable that is running the respective DataLad session, for example a 'datalad containers-run' command. """, metavar="FORMAT", constraints=EnsureStr() | EnsureNone(), ), extra_input=Parameter( args=("--extra-input",), doc="""Additional file the container invocation depends on (e.g. overlays used in --call-fmt). Can be specified multiple times. Similar to --call-fmt, the placeholders {img_dspath} and {img_dirpath} are available. Will be stored in the dataset config and later added alongside the container image to the `extra_inputs` field in the run-record and thus automatically be fetched when needed. """, action="append", default=[], metavar="FILE", # Can't use EnsureListOf(str) yet as it handles strings as iterables... # See this PR: https://github.com/datalad/datalad/pull/7267 # constraints=EnsureListOf(str) | EnsureNone(), ), image=Parameter( args=("-i", "--image"), doc="""Relative path of the container image within the dataset. If not given, a default location will be determined using the `name` argument.""", metavar="IMAGE", constraints=EnsureStr() | EnsureNone(), ), update=Parameter( args=("--update",), action="store_true", doc="""Update the existing container for `name`. If no other options are specified, URL will be set to 'updateurl', if configured. If a container with `name` does not already exist, this option is ignored.""" ) ) @staticmethod @datasetmethod(name='containers_add') @eval_results def __call__(name, url=None, dataset=None, call_fmt=None, image=None, update=False, extra_input=None): if not name: raise InsufficientArgumentsError("`name` argument is required") ds = require_dataset(dataset, check_installed=True, purpose='add container') runner = WitlessRunner() # prevent madness in the config file if not re.match(r'^[0-9a-zA-Z-]+$', name): raise ValueError( "Container names can only contain alphanumeric characters " "and '-', got: '{}'".format(name)) container_cfg = get_container_configuration(ds, name) if 'image' in container_cfg: if not update: yield get_status_dict( action="containers_add", ds=ds, logger=lgr, status="impossible", message=("Container named %r already exists. " "Use --update to reconfigure.", name)) return if not (url or image or call_fmt): # No updated values were provided. See if an update url is # configured (currently relevant only for Singularity Hub). url = container_cfg.get("updateurl") if not url: yield get_status_dict( action="containers_add", ds=ds, logger=lgr, status="impossible", message="No values to update specified") return call_fmt = call_fmt or container_cfg.get("cmdexec") image = image or container_cfg.get("image") if not image: loc_cfg_var = "datalad.containers.location" container_loc = \ ds.config.obtain( loc_cfg_var, # if not False it would actually modify the # dataset config file -- undesirable store=False, ) image = op.join(ds.path, container_loc, name, 'image') else: image = op.join(ds.path, image) result = get_status_dict( action="containers_add", path=image, type="file", logger=lgr, ) if call_fmt is None: # maybe built in knowledge can help call_fmt = _guess_call_fmt(ds, name, url) # collect bits for a final and single save() call to_save = [] imgurl = url was_updated = False if url: if update and op.lexists(image): was_updated = True # XXX: check=False is used to avoid dropping the image. It # should use drop=False if remove() gets such an option (see # DataLad's gh-2673). for r in ds.remove(image, reckless='availability', return_type="generator"): yield r imgurl = _resolve_img_url(url) lgr.debug('Attempt to obtain container image from: %s', imgurl) if url.startswith("dhub://"): from .adapters import docker docker_image = url[len("dhub://"):] lgr.debug( "Running 'docker pull %s and saving image to %s", docker_image, image) runner.run(["docker", "pull", docker_image]) docker.save(docker_image, image) elif url.startswith("docker://"): image_dir, image_basename = op.split(image) if not image_basename: raise ValueError("No basename in path {}".format(image)) if image_dir and not op.exists(image_dir): os.makedirs(image_dir) lgr.info("Building Singularity image for %s " "(this may take some time)", url) runner.run(["singularity", "build", image_basename, url], cwd=image_dir or None) elif op.exists(url): lgr.info("Copying local file %s to %s", url, image) image_dir = op.dirname(image) if image_dir and not op.exists(image_dir): os.makedirs(image_dir) copyfile(url, image) else: if _HAS_SHUB_DOWNLOADER and url.startswith('shub://'): _ensure_datalad_remote(ds.repo) try: ds.repo.add_url_to_file(image, imgurl) except Exception as e: result["status"] = "error" result["message"] = str(e) yield result # TODO do we have to take care of making the image executable # if --call_fmt is not provided? to_save.append(image) # continue despite a remote access failure, the following config # setting will enable running the command again with just the name # given to ease a re-run if not op.lexists(image): result["status"] = "error" result["message"] = ('no image at %s', image) yield result return # store configs cfgbasevar = "datalad.containers.{}".format(name) if imgurl != url: # store originally given URL, as it resolves to something # different and maybe can be used to update the container # at a later point in time ds.config.set("{}.updateurl".format(cfgbasevar), url) # force store the image, and prevent multiple entries ds.config.set( "{}.image".format(cfgbasevar), # always store a POSIX path, relative to dataset root str(PurePosixPath(Path(image).relative_to(ds.pathobj))), force=True) if call_fmt: ds.config.set( "{}.cmdexec".format(cfgbasevar), call_fmt, force=True) # --extra-input sanity check # TODO: might also want to do that for --call-fmt above? extra_input_placeholders = dict(img_dirpath="", img_dspath="") for xi in (extra_input or []): try: xi.format(**extra_input_placeholders) except KeyError as exc: yield get_status_dict( action="containers_add", ds=ds, logger=lgr, status="error", message=("--extra-input %r contains unknown placeholder %s. " "Available placeholders: %s", repr(xi), exc, ', '.join(extra_input_placeholders))) return # actually setting --extra-input config cfgextravar = "{}.extra-input".format(cfgbasevar) if ds.config.get(cfgextravar) is not None: ds.config.unset(cfgextravar) for xi in (extra_input or []): ds.config.add(cfgextravar, xi) # store changes to_save.append(op.join(".datalad", "config")) for r in ds.save( path=to_save, message="[DATALAD] {do} containerized environment '{name}'".format( do="Update" if was_updated else "Configure", name=name)): yield r result["status"] = "ok" yield result datalad-container-1.2.6/datalad_container/containers_list.py000066400000000000000000000101271501235133500242770ustar00rootroot00000000000000"""List known container environments of a dataset""" __docformat__ = 'restructuredtext' import logging import os.path as op import datalad.support.ansi_colors as ac from datalad.coreapi import subdatasets from datalad.distribution.dataset import ( Dataset, EnsureDataset, datasetmethod, require_dataset, ) from datalad.interface.base import ( Interface, build_doc, eval_results, ) from datalad.interface.common_opts import recursion_flag from datalad.interface.results import get_status_dict from datalad.interface.utils import default_result_renderer from datalad.support.constraints import EnsureNone from datalad.support.param import Parameter from datalad.ui import ui from datalad_container.utils import get_container_configuration lgr = logging.getLogger("datalad.containers.containers_list") @build_doc # all commands must be derived from Interface class ContainersList(Interface): # first docstring line is used a short description in the cmdline help # the rest is put in the verbose help and manpage """List containers known to a dataset """ result_renderer = 'tailored' # parameters of the command, must be exhaustive _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), doc="""specify the dataset to query. If no dataset is given, an attempt is made to identify the dataset based on the current working directory""", constraints=EnsureDataset() | EnsureNone()), contains=Parameter( args=('--contains',), metavar='PATH', action='append', doc="""when operating recursively, restrict the reported containers to those from subdatasets that contain the given path (i.e. the subdatasets that are reported by :command:`datalad subdatasets --contains=PATH`). Top-level containers are always reported."""), recursive=recursion_flag, ) @staticmethod @datasetmethod(name='containers_list') @eval_results def __call__(dataset=None, recursive=False, contains=None): ds = require_dataset(dataset, check_installed=True, purpose='list containers') refds = ds.path if recursive: for sub in ds.subdatasets( contains=contains, on_failure='ignore', return_type='generator', result_renderer='disabled'): subds = Dataset(sub['path']) if subds.is_installed(): for c in subds.containers_list(recursive=recursive, return_type='generator', on_failure='ignore', result_filter=None, result_renderer=None, result_xfm=None): c['name'] = sub['gitmodule_name'] + '/' + c['name'] c['refds'] = refds yield c # all info is in the dataset config! containers = get_container_configuration(ds) for k, v in containers.items(): if 'image' not in v: # there is no container location configured continue res = get_status_dict( status='ok', action='containers', name=k, type='file', path=op.join(ds.path, v.pop('image')), refds=refds, parentds=ds.path, # TODO #state='absent' if ... else 'present' **v) yield res @staticmethod def custom_result_renderer(res, **kwargs): if res["action"] != "containers": default_result_renderer(res) else: ui.message( "{name} -> {path}" .format(name=ac.color_word(res["name"], ac.MAGENTA), path=op.relpath(res["path"], res["refds"]))) datalad-container-1.2.6/datalad_container/containers_remove.py000066400000000000000000000072771501235133500246350ustar00rootroot00000000000000"""Remove a container environment from a dataset""" __docformat__ = 'restructuredtext' import logging import os.path as op from datalad.distribution.dataset import ( EnsureDataset, datasetmethod, require_dataset, ) from datalad.interface.base import ( Interface, build_doc, eval_results, ) from datalad.interface.results import get_status_dict from datalad.support.constraints import ( EnsureNone, EnsureStr, ) from datalad.support.param import Parameter from datalad.utils import rmtree from datalad_container.utils import get_container_configuration lgr = logging.getLogger("datalad.containers.containers_remove") @build_doc # all commands must be derived from Interface class ContainersRemove(Interface): # first docstring line is used a short description in the cmdline help # the rest is put in the verbose help and manpage """Remove a known container from a dataset This command is only removing a container from the committed Dataset configuration (configuration scope ``branch``). It will not modify any other configuration scopes. This command is *not* dropping the container image associated with the removed record, because it may still be needed for other dataset versions. In order to drop the container image, use the 'drop' command prior to removing the container configuration. """ # parameters of the command, must be exhaustive _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), doc="""specify the dataset from removing a container. If no dataset is given, an attempt is made to identify the dataset based on the current working directory""", constraints=EnsureDataset() | EnsureNone()), name=Parameter( args=("name",), doc="""name of the container to remove""", metavar="NAME", constraints=EnsureStr(), ), remove_image=Parameter( args=("-i", "--remove-image",), doc="""if set, remove container image as well. Even with this flag, the container image content will not be dropped. Use the 'drop' command explicitly before removing the container configuration.""", action="store_true", ), ) @staticmethod @datasetmethod(name='containers_remove') @eval_results def __call__(name, dataset=None, remove_image=False): ds = require_dataset(dataset, check_installed=True, purpose='remove a container') res = get_status_dict( ds=ds, action='containers_remove', logger=lgr) container_cfg = get_container_configuration(ds, name) to_save = [] if remove_image and 'image' in container_cfg: imagepath = ds.pathobj / container_cfg['image'] # we use rmtree() and not .unlink(), because # the image could be more than a single file underneath # this location (e.g., docker image dumps) rmtree(imagepath) # at the very end, save() will take care of committing # any removal that just occurred to_save.append(imagepath) if container_cfg: ds.config.remove_section( f'datalad.containers.{name}', scope='branch', reload=True) res['status'] = 'ok' to_save.append(op.join('.datalad', 'config')) else: res['status'] = 'notneeded' if to_save: for r in ds.save( path=to_save, message='[DATALAD] Remove container {}'.format(name)): yield r yield res datalad-container-1.2.6/datalad_container/containers_run.py000066400000000000000000000163711501235133500241370ustar00rootroot00000000000000"""Drop-in replacement for `datalad run` for command execution in a container""" __docformat__ = 'restructuredtext' import logging import os.path as op import sys from datalad.core.local.run import ( Run, get_command_pwds, normalize_command, run_command, ) from datalad.distribution.dataset import ( datasetmethod, require_dataset, ) from datalad.interface.base import ( Interface, build_doc, eval_results, ) from datalad.interface.results import get_status_dict from datalad.support.param import Parameter from datalad.utils import ensure_iter from datalad_container.find_container import find_container_ lgr = logging.getLogger("datalad.containers.containers_run") # Environment variable to be set during execution to possibly # inform underlying shim scripts about the original name of # the container CONTAINER_NAME_ENVVAR = 'DATALAD_CONTAINER_NAME' _run_params = dict( Run._params_, container_name=Parameter( args=('-n', '--container-name',), metavar="NAME", doc="""Specify the name of or a path to a known container to use for execution, in case multiple containers are configured."""), ) @build_doc # all commands must be derived from Interface class ContainersRun(Interface): # first docstring line is used a short description in the cmdline help # the rest is put in the verbose help and manpage """Drop-in replacement of 'run' to perform containerized command execution Container(s) need to be configured beforehand (see containers-add). If no container is specified and only one container is configured in the current dataset, it will be selected automatically. If more than one container is registered in the current dataset or to access containers from subdatasets, the container has to be specified. A command is generated based on the input arguments such that the container image itself will be recorded as an input dependency of the command execution in the `run` record in the git history. During execution the environment variable {name_envvar} is set to the name of the used container. """ _docs_ = dict( name_envvar=CONTAINER_NAME_ENVVAR ) _params_ = _run_params # Analogous to 'run' command - stop on first error on_failure = 'stop' @staticmethod @datasetmethod(name='containers_run') @eval_results def __call__(cmd, container_name=None, dataset=None, inputs=None, outputs=None, message=None, expand=None, explicit=False, sidecar=None): from unittest.mock import \ patch # delayed, since takes long (~600ms for yoh) pwd, _ = get_command_pwds(dataset) ds = require_dataset(dataset, check_installed=True, purpose='run a containerized command execution') # this following block locates the target container. this involves a # configuration look-up. This is not using # get_container_configuration(), because it needs to account for a # wide range of scenarios, including the installation of the dataset(s) # that will eventually provide (the configuration) for the container. # However, internally this is calling `containers_list()`, which is # using get_container_configuration(), so any normalization of # configuration on-read, get still be implemented in this helper. container = None for res in find_container_(ds, container_name): if res.get("action") == "containers": container = res else: yield res assert container, "bug: container should always be defined here" image_path = op.relpath(container["path"], pwd) # container record would contain path to the (sub)dataset containing # it. If not - take current dataset, as it must be coming from it image_dspath = op.relpath(container.get('parentds', ds.path), pwd) # sure we could check whether the container image is present, # but it might live in a subdataset that isn't even installed yet # let's leave all this business to `get` that is called by `run` cmd = normalize_command(cmd) # expand the command with container execution if 'cmdexec' in container: callspec = container['cmdexec'] # Temporary kludge to give a more helpful message if callspec.startswith("["): import json try: json.loads(callspec) except json.JSONDecodeError: pass # Never mind, false positive. else: raise ValueError( 'cmdexe {!r} is in an old, unsupported format. ' 'Convert it to a plain string.'.format(callspec)) try: cmd_kwargs = dict( # point to the python installation that runs *this* code # we know that it would have things like the docker # adaptor installed with this extension package python=sys.executable, img=image_path, cmd=cmd, img_dspath=image_dspath, img_dirpath=op.dirname(image_path) or ".", ) cmd = callspec.format(**cmd_kwargs) except KeyError as exc: yield get_status_dict( 'run', ds=ds, status='error', message=( 'Unrecognized cmdexec placeholder: %s. ' 'See containers-add for information on known ones: %s', exc, ", ".join(cmd_kwargs))) return else: # just prepend and pray cmd = container['path'] + ' ' + cmd extra_inputs = [] for extra_input in ensure_iter(container.get("extra-input",[]), set): try: xi_kwargs = dict( img_dspath=image_dspath, img_dirpath=op.dirname(image_path) or ".", ) extra_inputs.append(extra_input.format(**xi_kwargs)) except KeyError as exc: yield get_status_dict( 'run', ds=ds, status='error', message=( 'Unrecognized extra_input placeholder: %s. ' 'See containers-add for information on known ones: %s', exc, ", ".join(xi_kwargs))) return lgr.debug("extra_inputs = %r", extra_inputs) with patch.dict('os.environ', {CONTAINER_NAME_ENVVAR: container['name']}): # fire! for r in run_command( cmd=cmd, dataset=dataset or (ds if ds.path == pwd else None), inputs=inputs, extra_inputs=[image_path] + extra_inputs, outputs=outputs, message=message, expand=expand, explicit=explicit, sidecar=sidecar): yield r datalad-container-1.2.6/datalad_container/extractors/000077500000000000000000000000001501235133500227225ustar00rootroot00000000000000datalad-container-1.2.6/datalad_container/extractors/__init__.py000066400000000000000000000000011501235133500250220ustar00rootroot00000000000000 datalad-container-1.2.6/datalad_container/extractors/_load_singularity_versions.py000066400000000000000000000015041501235133500307340ustar00rootroot00000000000000""" Importing this file extends datalad.support.external_version: Adds: - external_versions["cmd:apptainer"] - external_versions["cmd:singularity"] """ from datalad.cmd import ( StdOutErrCapture, WitlessRunner, ) from datalad.support.external_versions import external_versions def __get_apptainer_version(): version = WitlessRunner().run("apptainer --version", protocol=StdOutErrCapture)['stdout'].strip() return version.split("apptainer version ")[1] def __get_singularity_version(): return WitlessRunner().run("singularity version", protocol=StdOutErrCapture)['stdout'].strip() # Load external_versions and patch with "cmd:singularity" and "cmd:apptainer" external_versions.add("cmd:apptainer", func=__get_apptainer_version) external_versions.add("cmd:singularity", func=__get_singularity_version) datalad-container-1.2.6/datalad_container/extractors/metalad_container.py000066400000000000000000000054141501235133500267510ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Metadata extractors for Container Images stored in Datalad's own core storage""" import json import logging import subprocess import time from uuid import UUID from datalad.support.external_versions import ( UnknownVersion, external_versions, ) from datalad_metalad import get_file_id from datalad_metalad.extractors.base import ( DataOutputCategory, ExtractorResult, FileMetadataExtractor, ) from datalad_container.utils import get_container_command CURRENT_VERSION = "0.0.1" lgr = logging.getLogger('datalad.metadata.extractors.metalad_container') class MetaladContainerInspect(FileMetadataExtractor): """ Populates metadata singularity/apptainer version and `inspect` output. """ def get_data_output_category(self) -> DataOutputCategory: return DataOutputCategory.IMMEDIATE def is_content_required(self) -> bool: return True def get_id(self) -> UUID: # Nothing special, made this up - asmacdo return UUID('3a28cca6-b7a1-11ed-b106-fc3497650c92') @staticmethod def get_version() -> str: return CURRENT_VERSION def extract(self, _=None) -> ExtractorResult: container_command = get_container_command() return ExtractorResult( extractor_version=self.get_version(), extraction_parameter=self.parameter or {}, extraction_success=True, datalad_result_dict={ "type": "container", "status": "ok" }, immediate_data={ "@id": get_file_id(dict( path=self.file_info.path, type=self.file_info.type)), "type": self.file_info.type, "path": self.file_info.intra_dataset_path, "content_byte_size": self.file_info.byte_size, "comment": f"SingularityInspect extractor executed at {time.time()}", "container_system": container_command, "container_system_version": str(external_versions[container_command]), "container_inspect": self._container_inspect(container_command, self.file_info.path), }) @staticmethod def _container_inspect(command, path) -> str: data = subprocess.run( [command, "inspect", "--json", path], check=True, stdout=subprocess.PIPE).stdout.decode() return json.loads(data) datalad-container-1.2.6/datalad_container/extractors/tests/000077500000000000000000000000001501235133500240645ustar00rootroot00000000000000datalad-container-1.2.6/datalad_container/extractors/tests/__init__.py000066400000000000000000000000001501235133500261630ustar00rootroot00000000000000datalad-container-1.2.6/datalad_container/extractors/tests/test_metalad_container.py000066400000000000000000000047711501235133500311570ustar00rootroot00000000000000import subprocess import pytest from datalad.support.external_versions import external_versions # Early detection before we try to import meta_extract from datalad.tests.utils_pytest import SkipTest if not external_versions["datalad_metalad"]: raise SkipTest("skipping metalad tests") from datalad.api import meta_extract from datalad.tests.utils_pytest import ( SkipTest, with_tempfile, ) from datalad_container.utils import get_container_command try: container_command = get_container_command() except RuntimeError: raise SkipTest("skipping singularity/apptainer tests") # Must come after skiptest or imports will not work from datalad_container.extractors.metalad_container import ( MetaladContainerInspect, ) @with_tempfile def test__container_inspect_nofile(path=None): """Singularity causes CalledProcessError if path DNE.""" with pytest.raises(subprocess.CalledProcessError): MetaladContainerInspect._container_inspect(container_command, path) def test__container_inspect_valid(singularity_test_image): """Call inspect on a valid singularity container image.""" result = MetaladContainerInspect._container_inspect( container_command, singularity_test_image["img_path"], ) expected_result = { 'data': { 'attributes': { 'labels': { 'org.label-schema.build-date': 'Sat,_19_May_2018_07:06:48_+0000', 'org.label-schema.build-size': '62MB', 'org.label-schema.schema-version': '1.0', 'org.label-schema.usage.singularity.deffile': 'Singularity.testhelper', 'org.label-schema.usage.singularity.deffile.bootstrap': 'docker', 'org.label-schema.usage.singularity.deffile.from': 'debian:stable-slim', 'org.label-schema.usage.singularity.version': '2.5.0-feature-squashbuild-secbuild-2.5.0.gddf62fb5' } } }, 'type': 'container' } assert result == expected_result def test_extract(singularity_test_image): ds = singularity_test_image["ds"] path = singularity_test_image["img_path"] result = meta_extract(dataset=ds, extractorname="container_inspect", path=path) assert len(result) == 1 assert result[0]["metadata_record"]["extracted_metadata"] assert result[0]["metadata_record"]["extractor_name"] == 'container_inspect' assert result[0]["metadata_record"]["extractor_version"] == MetaladContainerInspect.get_version() datalad-container-1.2.6/datalad_container/find_container.py000066400000000000000000000151271501235133500240660ustar00rootroot00000000000000"""Support module for selecting a container from a dataset and its subdatasets. """ import logging from datalad.distribution.dataset import Dataset from datalad.utils import Path from datalad_container.containers_list import ContainersList lgr = logging.getLogger("datalad_container.find_container") def _list_containers(dataset, recursive, contains=None): return {c['name']: c for c in ContainersList.__call__(dataset=dataset, recursive=recursive, contains=contains, return_type='generator', on_failure='ignore', result_filter=None, result_renderer=None, result_xfm=None)} def _get_subdataset_container(ds, container_name): """Try to get subdataset container matching `container_name`. This is the primary function tried by find_container_() when the container name looks like it is from a subdataset (i.e. has a slash). Parameters ---------- ds : Dataset container_name : str Yields ------- Result records for any installed subdatasets and a containers-list record for the container, if any, found for `container_name`. """ name_parts = container_name.split('/') subds_names = name_parts[:-1] if Dataset(ds.pathobj / Path(*subds_names)).is_installed(): # This avoids unnecessary work in the common case, but it can result in # not installing the necessary subdatasets in the rare case that chain # of submodule names point to a subdataset path that is installed while # the actual submodule paths contains uninstalled parts. lgr.debug( "Subdataset for %s is probably installed. Skipping install logic", container_name) return curds = ds for name in subds_names: for sub in curds.subdatasets(return_type='generator'): if sub['gitmodule_name'] == name: path = sub['path'] yield from curds.get( path, get_data=False, on_failure='ignore', return_type='generator') curds = Dataset(path) break else: # There wasn't a submodule name chain that matched container_name. # Aside from an invalid name, the main case where this can happen # is when an image path is given for the container name. lgr.debug("Did not find submodule name %s in %s", name, curds) return containers = _list_containers(dataset=ds, recursive=True, contains=curds.path) res = containers.get(container_name) if res: yield res # Fallback functions tried by find_container_. These are called with the # current dataset, the container name, and a dictionary mapping the container # name to a record (as returned by containers-list). def _get_the_one_and_only(_, name, containers): if name is None: if len(containers) == 1: # no questions asked, take container and run return list(containers.values())[0] else: raise ValueError("Must explicitly specify container" " (known containers are: {})" .format(', '.join(containers))) def _get_container_by_name(_, name, containers): return containers.get(name) def _get_container_by_path(ds, name, containers): from datalad.distribution.dataset import resolve_path # Note: since datalad0.12.0rc6 resolve_path returns a Path object here, # which then fails to equal c['path'] below as this is taken from # config as a string container_path = str(resolve_path(name, ds)) container = [c for c in containers.values() if c['path'] == container_path] if len(container) == 1: return container[0] # Entry points def find_container_(ds, container_name=None): """Find the container in dataset `ds` specified by `container_name`. Parameters ---------- ds : Dataset Dataset to query. container_name : str or None Name in the form of how `containers-list -d ds -r` would report it (e.g., "s0/s1/cname"). Yields ------ The container record, as returned by containers-list. Before that record, it may yield records of other action types, in particular "install" records for subdatasets that were installed to try to get access to a subdataset container. Raises ------ ValueError if a uniquely matching container cannot be found. """ recurse = container_name and "/" in container_name if recurse: for res in _get_subdataset_container(ds, container_name): # Before the container record, the results may include install # records. Don't relay "notneeded" results to avoid noise. Also, # don't propagate install failures, which may be due to an image # path being given or a non-existent container, both cases that are # handled downstream. if res.get("status") == "ok": yield res if res.get("action") == "containers": return containers = _list_containers(dataset=ds, recursive=recurse) if not containers: raise ValueError("No known containers. Use containers-add") fns = [ _get_the_one_and_only, _get_container_by_name, _get_container_by_path, ] for fn in fns: lgr.debug("Trying to find container with %s", fn) container = fn(ds, container_name, containers) if container: yield container return raise ValueError( 'Container selection impossible: not specified, ambiguous ' 'or unknown (known containers are: {})' .format(', '.join(containers)) ) def find_container(ds, container_name=None): """Like `find_container_`, but just return the container record. """ # Note: This function was once used directly by containers_run(), but that # now uses the find_container_() generator function directly. Now # find_container() exists for compatibility with third-party tools # (reproman) and the test_find.py tests. for res in find_container_(ds, container_name): if res.get("action") == "containers": return res raise RuntimeError( "bug: find_container_() should return container or raise exception") datalad-container-1.2.6/datalad_container/tests/000077500000000000000000000000001501235133500216665ustar00rootroot00000000000000datalad-container-1.2.6/datalad_container/tests/__init__.py000066400000000000000000000000001501235133500237650ustar00rootroot00000000000000datalad-container-1.2.6/datalad_container/tests/fixtures/000077500000000000000000000000001501235133500235375ustar00rootroot00000000000000datalad-container-1.2.6/datalad_container/tests/fixtures/__init__.py000066400000000000000000000000661501235133500256520ustar00rootroot00000000000000from .singularity_image import singularity_test_image datalad-container-1.2.6/datalad_container/tests/fixtures/singularity_image.py000066400000000000000000000016441501235133500276320ustar00rootroot00000000000000from pathlib import Path import pytest from datalad.api import Dataset from datalad.tests.utils_pytest import with_tempfile from datalad_container.tests.utils import add_pyscript_image from datalad_container.utils import get_container_command TEST_IMG_URL = 'shub://datalad/datalad-container:testhelper' @pytest.fixture(scope="session") def singularity_test_image(tmp_path_factory: pytest.TempPathFactory) -> str: fixture_file_name = "fixture.sing" ds = Dataset(tmp_path_factory.mktemp("singularity_image")) ds.create(force=True) ds.containers_add( 'mycontainer', url=TEST_IMG_URL, image=fixture_file_name, ) img_path = ds.pathobj / fixture_file_name ds.get(img_path) return {"ds": ds, "img_path": img_path} @pytest.fixture(scope="session") def container_command(): """Not a very useful function other than to add session scope.""" return get_container_command() datalad-container-1.2.6/datalad_container/tests/test_add.py000066400000000000000000000024251501235133500240320ustar00rootroot00000000000000import pytest from datalad.api import ( Dataset, clone, ) from datalad.consts import DATALAD_SPECIAL_REMOTE from datalad.customremotes.base import init_datalad_remote from datalad.tests.utils_pytest import ( assert_false, assert_in, assert_not_in, with_tempfile, ) from datalad.utils import Path from datalad_container.containers_add import _ensure_datalad_remote # NOTE: At the moment, testing of the containers-add itself happens implicitly # via use in other tests. @with_tempfile def test_ensure_datalad_remote_init_and_enable_needed(path=None): ds = Dataset(path).create(force=True) repo = ds.repo assert_false(repo.get_remotes()) _ensure_datalad_remote(repo) assert_in("datalad", repo.get_remotes()) @pytest.mark.parametrize("autoenable", [False, True]) @with_tempfile def test_ensure_datalad_remote_maybe_enable(path=None, *, autoenable): path = Path(path) ds_a = Dataset(path / "a").create(force=True) init_datalad_remote(ds_a.repo, DATALAD_SPECIAL_REMOTE, autoenable=autoenable) ds_b = clone(source=ds_a.path, path=path / "b") repo = ds_b.repo if not autoenable: assert_not_in("datalad", repo.get_remotes()) _ensure_datalad_remote(repo) assert_in("datalad", repo.get_remotes())datalad-container-1.2.6/datalad_container/tests/test_containers.py000066400000000000000000000306321501235133500254500ustar00rootroot00000000000000import os.path as op from datalad.api import ( Dataset, containers_add, containers_list, containers_remove, install, ) from datalad.support.network import get_local_file_url from datalad.tests.utils_pytest import ( SkipTest, assert_equal, assert_in, assert_in_results, assert_not_in, assert_raises, assert_re_in, assert_result_count, assert_status, ok_, ok_clean_git, ok_file_has_content, serve_path_via_http, with_tempfile, with_tree, ) from datalad.utils import swallow_outputs from datalad_container.tests.utils import add_pyscript_image common_kwargs = {'result_renderer': 'disabled'} @with_tempfile def test_add_noop(path=None): ds = Dataset(path).create(**common_kwargs) ok_clean_git(ds.path) assert_raises(TypeError, ds.containers_add) # fails when there is no image assert_status( 'error', ds.containers_add('name', on_failure='ignore', **common_kwargs)) # no config change ok_clean_git(ds.path) # place a dummy "image" file with open(op.join(ds.path, 'dummy'), 'w') as f: f.write('some') ds.save('dummy', **common_kwargs) ok_clean_git(ds.path) # config will be added, as long as there is a file, even when URL access # fails res = ds.containers_add( 'broken', url='bogus-protocol://bogus-server', image='dummy', on_failure='ignore', **common_kwargs ) assert_status('ok', res) assert_result_count(res, 1, action='save', status='ok') @with_tempfile @with_tree(tree={"foo.img": "doesn't matter 0", "bar.img": "doesn't matter 1"}) def test_add_local_path(path=None, local_file=None): ds = Dataset(path).create(**common_kwargs) res = ds.containers_add(name="foobert", url=op.join(local_file, "foo.img")) foo_target = op.join(path, ".datalad", "environments", "foobert", "image") assert_result_count(res, 1, status="ok", type="file", path=foo_target, action="containers_add") # the image path configuration is always in POSIX format assert ds.config.get('datalad.containers.foobert.image') \ == '.datalad/environments/foobert/image' # We've just copied and added the file. assert_not_in(ds.repo.WEB_UUID, ds.repo.whereis(foo_target)) # We can force the URL to be added. (Note: This works because datalad # overrides 'annex.security.allowed-url-schemes' in its tests.) ds.containers_add(name="barry", url=get_local_file_url(op.join(local_file, "bar.img"))) bar_target = op.join(path, ".datalad", "environments", "barry", "image") assert_in(ds.repo.WEB_UUID, ds.repo.whereis(bar_target)) RAW_KWDS = dict(return_type='list', result_filter=None, result_renderer=None, result_xfm=None) @with_tempfile @with_tree(tree={'some_container.img': "doesn't matter"}) @serve_path_via_http def test_container_files(ds_path=None, local_file=None, url=None): # setup things to add # # Note: Since "adding" as a container doesn't actually call anything or use # the container in some way, but simply registers it, for testing any file # is sufficient. local_file = get_local_file_url(op.join(local_file, 'some_container.img')) # prepare dataset: ds = Dataset(ds_path).create(**common_kwargs) # non-default location: ds.config.add("datalad.containers.location", value=op.join(".datalad", "test-environments"), scope='branch') ds.save(message="Configure container mountpoint", **common_kwargs) # no containers yet: res = ds.containers_list(**RAW_KWDS) assert_result_count(res, 0) # add first "image": must end up at the configured default location target_path = op.join( ds.path, ".datalad", "test-environments", "first", "image") res = ds.containers_add(name="first", url=local_file, **common_kwargs) ok_clean_git(ds.repo) assert_result_count(res, 1, status="ok", type="file", path=target_path, action="containers_add") ok_(op.lexists(target_path)) res = ds.containers_list(**RAW_KWDS) assert_result_count(res, 1) assert_result_count( res, 1, name='first', type='file', action='containers', status='ok', path=target_path) # and kill it again # but needs name assert_raises(TypeError, ds.containers_remove) res = ds.containers_remove('first', remove_image=True, **common_kwargs) assert_status('ok', res) assert_result_count(ds.containers_list(**RAW_KWDS), 0) # image removed assert(not op.lexists(target_path)) @with_tree(tree={ "container.img": "container", "overlay1.img": "overlay 1", "overlay2.img": "overlay 2", }) def test_extra_inputs(ds_path=None): container_file = 'container.img' overlay1_file = 'overlay1.img' overlay2_file = 'overlay2.img' # prepare dataset: ds = Dataset(ds_path).create(force=True, **common_kwargs) ds.save(**common_kwargs) ds.containers_add( name="container", image=container_file, call_fmt="apptainer exec {img} {cmd}", **common_kwargs ) ds.containers_add( name="container-with-overlay", image=container_file, call_fmt="apptainer exec --overlay {img_dirpath}/overlay1.img {img} {cmd}", extra_input=[overlay1_file], **common_kwargs ) ds.containers_add( name="container-with-two-overlays", image=container_file, call_fmt="apptainer exec --overlay {img_dirpath}/overlay1.img --overlay {img_dirpath}/overlay2.img:ro {img} {cmd}", extra_input=[overlay1_file, overlay2_file], **common_kwargs ) res = ds.containers_list(**RAW_KWDS) assert_result_count(res, 3) assert_equal(ds.config.get("datalad.containers.container.extra-input"), None) assert_equal(ds.config.get("datalad.containers.container-with-overlay.extra-input",get_all=True), "overlay1.img") assert_equal(ds.config.get("datalad.containers.container-with-two-overlays.extra-input",get_all=True), ("overlay1.img", "overlay2.img")) @with_tempfile @with_tree(tree={'foo.img': "foo", 'bar.img': "bar"}) @serve_path_via_http def test_container_update(ds_path=None, local_file=None, url=None): url_foo = get_local_file_url(op.join(local_file, 'foo.img')) url_bar = get_local_file_url(op.join(local_file, 'bar.img')) img = op.join(".datalad", "environments", "foo", "image") ds = Dataset(ds_path).create(**common_kwargs) ds.containers_add(name="foo", call_fmt="call-fmt1", url=url_foo, **common_kwargs) # Abort without --update flag. res = ds.containers_add(name="foo", on_failure="ignore", **common_kwargs) assert_result_count(res, 1, action="containers_add", status="impossible") # Abort if nothing to update is specified. res = ds.containers_add(name="foo", update=True, on_failure="ignore", **common_kwargs) assert_result_count(res, 1, action="containers_add", status="impossible", message="No values to update specified") # Update call format. ds.containers_add(name="foo", update=True, call_fmt="call-fmt2", **common_kwargs) assert_equal(ds.config.get("datalad.containers.foo.cmdexec"), "call-fmt2") ok_file_has_content(op.join(ds.path, img), "foo") # Update URL/image. ds.drop(img, **common_kwargs) # Make sure it works even with absent content. res = ds.containers_add(name="foo", update=True, url=url_bar, **common_kwargs) assert_in_results(res, action="remove", status="ok") assert_in_results(res, action="save", status="ok") ok_file_has_content(op.join(ds.path, img), "bar") # the image path configuration is (still) always in POSIX format assert ds.config.get('datalad.containers.foo.image') \ == '.datalad/environments/foo/image' # Test commit message # In the above case it was updating existing image so should have "Update " get_commit_msg = lambda *args: ds.repo.format_commit('%B') assert_in("Update ", get_commit_msg()) # If we add a new image with update=True should say Configure res = ds.containers_add(name="foo2", update=True, url=url_bar, **common_kwargs) assert_in("Configure ", get_commit_msg()) @with_tempfile @with_tempfile @with_tree(tree={'some_container.img': "doesn't matter"}) def test_container_from_subdataset(ds_path=None, src_subds_path=None, local_file=None): # prepare a to-be subdataset with a registered container src_subds = Dataset(src_subds_path).create(**common_kwargs) src_subds.containers_add( name="first", url=get_local_file_url(op.join(local_file, 'some_container.img')), **common_kwargs ) # add it as subdataset to a super ds: ds = Dataset(ds_path).create(**common_kwargs) subds = ds.install("sub", source=src_subds_path, **common_kwargs) # add it again one level down to see actual recursion: subds.install("subsub", source=src_subds_path, **common_kwargs) # We come up empty without recursive: res = ds.containers_list(recursive=False, **RAW_KWDS) assert_result_count(res, 0) # query available containers from within super: res = ds.containers_list(recursive=True, **RAW_KWDS) assert_result_count(res, 2) assert_in_results(res, action="containers", refds=ds.path) # default location within the subdataset: target_path = op.join(subds.path, '.datalad', 'environments', 'first', 'image') assert_result_count( res, 1, name='sub/first', type='file', action='containers', status='ok', path=target_path, parentds=subds.path ) # not installed subdataset doesn't pose an issue: sub2 = ds.create("sub2", **common_kwargs) assert_result_count(ds.subdatasets(**common_kwargs), 2, type="dataset") ds.drop("sub2", reckless='availability', what='datasets', **common_kwargs) from datalad.tests.utils_pytest import assert_false assert_false(sub2.is_installed()) # same results as before, not crashing or somehow confused by a not present # subds: res = ds.containers_list(recursive=True, **RAW_KWDS) assert_result_count(res, 2) assert_result_count( res, 1, name='sub/first', type='file', action='containers', status='ok', path=target_path, parentds=subds.path ) # The default renderer includes the image names. with swallow_outputs() as out: ds.containers_list(recursive=True) lines = out.out.splitlines() assert_re_in("sub/first", lines) assert_re_in("sub/subsub/first", lines) # But we are careful not to render partial names from subdataset traversals # (i.e. we recurse with containers_list(..., result_renderer=None)). with assert_raises(AssertionError): assert_re_in("subsub/first", lines) @with_tempfile def test_list_contains(path=None): ds = Dataset(path).create(**common_kwargs) subds_a = ds.create("a", **common_kwargs) subds_b = ds.create("b", **common_kwargs) subds_a_c = subds_a.create("c", **common_kwargs) add_pyscript_image(subds_a_c, "in-c", "img") add_pyscript_image(subds_a, "in-a", "img") add_pyscript_image(subds_b, "in-b", "img") add_pyscript_image(ds, "in-top", "img") ds.save(recursive=True, **common_kwargs) assert_result_count(ds.containers_list(recursive=True, **RAW_KWDS), 4) assert_result_count( ds.containers_list(contains=["nowhere"], recursive=True, **RAW_KWDS), 1, name="in-top", action='containers') res = ds.containers_list(contains=[subds_a.path], recursive=True, **RAW_KWDS) assert_result_count(res, 3) assert_in_results(res, name="in-top") assert_in_results(res, name="a/in-a") assert_in_results(res, name="a/c/in-c") res = ds.containers_list(contains=[subds_a_c.path], recursive=True, **RAW_KWDS) assert_result_count(res, 3) assert_in_results(res, name="in-top") assert_in_results(res, name="a/in-a") assert_in_results(res, name="a/c/in-c") res = ds.containers_list(contains=[subds_b.path], recursive=True, **RAW_KWDS) assert_result_count(res, 2) assert_in_results(res, name="in-top") assert_in_results(res, name="b/in-b") datalad-container-1.2.6/datalad_container/tests/test_find.py000066400000000000000000000023031501235133500242150ustar00rootroot00000000000000import os.path as op from datalad.api import Dataset from datalad.tests.utils_pytest import ( assert_in, assert_in_results, assert_is_instance, assert_raises, assert_result_count, ok_clean_git, with_tree, ) from datalad_container.find_container import find_container @with_tree(tree={"sub": {"i.img": "doesn't matter"}}) def test_find_containers(path=None): ds = Dataset(path).create(force=True) ds.save(path=[op.join('sub', 'i.img')], message="dummy container") ds.containers_add("i", image=op.join('sub', 'i.img')) ok_clean_git(path) # find the only one res = find_container(ds) assert_is_instance(res, dict) assert_result_count([res], 1, status="ok", path=op.join(ds.path, "sub", "i.img")) # find by name res = find_container(ds, "i") assert_is_instance(res, dict) assert_result_count([res], 1, status="ok", path=op.join(ds.path, "sub", "i.img")) # find by path res = find_container(ds, op.join("sub", "i.img")) assert_is_instance(res, dict) assert_result_count([res], 1, status="ok", path=op.join(ds.path, "sub", "i.img")) # don't find another thing assert_raises(ValueError, find_container, ds, "nothere") datalad-container-1.2.6/datalad_container/tests/test_register.py000066400000000000000000000003021501235133500251160ustar00rootroot00000000000000from datalad.tests.utils_pytest import assert_result_count def test_register(): import datalad.api as da assert hasattr(da, 'containers_list') assert hasattr(da, 'containers_add') datalad-container-1.2.6/datalad_container/tests/test_run.py000066400000000000000000000360621501235133500241120ustar00rootroot00000000000000import os import os.path as op import pytest from datalad.api import ( Dataset, clone, containers_add, containers_list, containers_run, create, ) from datalad.cmd import ( StdOutCapture, WitlessRunner, ) from datalad.local.rerun import get_run_info from datalad.support.exceptions import IncompleteResultsError from datalad.support.network import get_local_file_url from datalad.tests.utils_pytest import ( assert_false, assert_in, assert_not_in_results, assert_raises, assert_repo_status, assert_result_count, ok_, ok_clean_git, ok_file_has_content, skip_if_no_network, with_tempfile, with_tree, ) from datalad.utils import ( Path, chpwd, on_windows, ) from datalad_container.tests.utils import add_pyscript_image testimg_url = 'shub://datalad/datalad-container:testhelper' common_kwargs = {'result_renderer': 'disabled'} @with_tree(tree={"dummy0.img": "doesn't matter 0", "dummy1.img": "doesn't matter 1"}) def test_run_mispecified(path=None): ds = Dataset(path).create(force=True, **common_kwargs) ds.save(path=["dummy0.img", "dummy1.img"], **common_kwargs) ok_clean_git(path) # Abort if no containers exist. with assert_raises(ValueError) as cm: ds.containers_run("doesn't matter", **common_kwargs) assert_in("No known containers", str(cm.value)) # Abort if more than one container exists but no container name is # specified. ds.containers_add("d0", image="dummy0.img", **common_kwargs) ds.containers_add("d1", image="dummy0.img", **common_kwargs) with assert_raises(ValueError) as cm: ds.containers_run("doesn't matter", **common_kwargs) assert_in("explicitly specify container", str(cm.value)) # Abort if unknown container is specified. with assert_raises(ValueError) as cm: ds.containers_run("doesn't matter", container_name="ghost", **common_kwargs) assert_in("Container selection impossible", str(cm.value)) @with_tree(tree={"i.img": "doesn't matter"}) def test_run_unknown_cmdexec_placeholder(path=None): ds = Dataset(path).create(force=True) ds.containers_add("i", image="i.img", call_fmt="{youdontknowme}", **common_kwargs) assert_result_count( ds.containers_run("doesn't matter", on_failure="ignore", **common_kwargs), 1, path=ds.path, action="run", status="error") @skip_if_no_network @with_tempfile @with_tempfile def test_container_files(path=None, super_path=None): ds = Dataset(path).create() cmd = ['dir'] if on_windows else ['ls'] # plug in a proper singularity image ds.containers_add( 'mycontainer', url=testimg_url, image='righthere', # the next one is auto-guessed #call_fmt='singularity exec {img} {cmd}' **common_kwargs ) assert_result_count( ds.containers_list(**common_kwargs), 1, path=op.join(ds.path, 'righthere'), name='mycontainer') ok_clean_git(path) def assert_no_change(res, path): # this command changed nothing # # Avoid specifying the action because it will change from "add" to # "save" in DataLad v0.12. assert_result_count( res, 1, status='notneeded', path=path, type='dataset') # now we can run stuff in the container # and because there is just one, we don't even have to name the container res = ds.containers_run(cmd, **common_kwargs) # container becomes an 'input' for `run` -> get request, but "notneeded" assert_result_count( res, 1, action='get', status='notneeded', path=op.join(ds.path, 'righthere'), type='file') assert_no_change(res, ds.path) # same thing as we specify the container by its name: res = ds.containers_run(cmd, container_name='mycontainer', **common_kwargs) # container becomes an 'input' for `run` -> get request, but "notneeded" assert_result_count( res, 1, action='get', status='notneeded', path=op.join(ds.path, 'righthere'), type='file') assert_no_change(res, ds.path) # we can also specify the container by its path: res = ds.containers_run(cmd, container_name=op.join(ds.path, 'righthere'), **common_kwargs) # container becomes an 'input' for `run` -> get request, but "notneeded" assert_result_count( res, 1, action='get', status='notneeded', path=op.join(ds.path, 'righthere'), type='file') assert_no_change(res, ds.path) # Now, test the same thing, but with this dataset being a subdataset of # another one: super_ds = Dataset(super_path).create(**common_kwargs) super_ds.install("sub", source=path, **common_kwargs) # When running, we don't discover containers in subdatasets with assert_raises(ValueError) as cm: super_ds.containers_run(cmd, **common_kwargs) assert_in("No known containers", str(cm.value)) # ... unless we need to specify the name res = super_ds.containers_run(cmd, container_name="sub/mycontainer", **common_kwargs) # container becomes an 'input' for `run` -> get request (needed this time) assert_result_count( res, 1, action='get', status='ok', path=op.join(super_ds.path, 'sub', 'righthere'), type='file') assert_no_change(res, super_ds.path) @with_tempfile @with_tree(tree={'some_container.img': "doesn't matter"}) def test_non0_exit(path=None, local_file=None): ds = Dataset(path).create(**common_kwargs) # plug in a proper singularity image ds.containers_add( 'mycontainer', url=get_local_file_url(op.join(local_file, 'some_container.img')), image='righthere', call_fmt="sh -c '{cmd}'", **common_kwargs ) ds.save(**common_kwargs) # record the effect in super-dataset ok_clean_git(path) # now we can run stuff in the container # and because there is just one, we don't even have to name the container ds.containers_run(['touch okfile'], **common_kwargs) assert_repo_status(path) # Test that regular 'run' behaves as expected -- it does not proceed to save upon error with pytest.raises(IncompleteResultsError): ds.run(['sh', '-c', 'touch nokfile && exit 1'], **common_kwargs) assert_repo_status(path, untracked=['nokfile']) (ds.pathobj / "nokfile").unlink() # remove for the next test assert_repo_status(path) # Now test with containers-run which should behave similarly -- not save in case of error with pytest.raises(IncompleteResultsError): ds.containers_run(['touch nokfile && exit 1'], **common_kwargs) # check - must have created the file but not saved anything since failed to run! assert_repo_status(path, untracked=['nokfile']) @with_tempfile @with_tree(tree={'some_container.img': "doesn't matter"}) def test_custom_call_fmt(path=None, local_file=None): ds = Dataset(path).create(**common_kwargs) subds = ds.create('sub', **common_kwargs) # plug in a proper singularity image subds.containers_add( 'mycontainer', url=get_local_file_url(op.join(local_file, 'some_container.img')), image='righthere', call_fmt='echo image={img} cmd={cmd} img_dspath={img_dspath} ' # and environment variable being set/propagated by default 'name=$DATALAD_CONTAINER_NAME', **common_kwargs, ) ds.save(**common_kwargs) # record the effect in super-dataset # Running should work fine either within sub or within super out = WitlessRunner(cwd=subds.path).run( ['datalad', 'containers-run', '-n', 'mycontainer', 'XXX'], protocol=StdOutCapture) assert_in('image=righthere cmd=XXX img_dspath=. name=mycontainer', out['stdout']) out = WitlessRunner(cwd=ds.path).run( ['datalad', 'containers-run', '-n', 'sub/mycontainer', 'XXX'], protocol=StdOutCapture) assert_in('image=sub/righthere cmd=XXX img_dspath=sub', out['stdout']) # Test within subdirectory of the super-dataset subdir = op.join(ds.path, 'subdir') os.mkdir(subdir) out = WitlessRunner(cwd=subdir).run( ['datalad', 'containers-run', '-n', 'sub/mycontainer', 'XXX'], protocol=StdOutCapture) assert_in('image=../sub/righthere cmd=XXX img_dspath=../sub', out['stdout']) @with_tree( tree={ "overlay1.img": "overlay1", "sub": { "containers": {"container.img": "image file"}, "overlays": {"overlay2.img": "overlay2", "overlay3.img": "overlay3"}, }, } ) def test_extra_inputs(path=None): ds = Dataset(path).create(force=True, **common_kwargs) subds = ds.create("sub", force=True, **common_kwargs) subds.containers_add( "mycontainer", image="containers/container.img", call_fmt="echo image={img} cmd={cmd} img_dspath={img_dspath} img_dirpath={img_dirpath} > out.log", extra_input=[ "overlay1.img", "{img_dirpath}/../overlays/overlay2.img", "{img_dspath}/overlays/overlay3.img", ], **common_kwargs ) ds.save(recursive=True, **common_kwargs) # record the entire tree of files etc ds.containers_run("XXX", container_name="sub/mycontainer", **common_kwargs) ok_file_has_content( os.path.join(ds.repo.path, "out.log"), "image=sub/containers/container.img", re_=True, ) commit_msg = ds.repo.call_git(["show", "--format=%B"]) cmd, runinfo = get_run_info(ds, commit_msg) assert set( [ "sub/containers/container.img", "overlay1.img", "sub/containers/../overlays/overlay2.img", "sub/overlays/overlay3.img", ] ) == set(runinfo.get("extra_inputs", set())) @skip_if_no_network @with_tree(tree={"subdir": {"in": "innards"}}) def test_run_no_explicit_dataset(path=None): ds = Dataset(path).create(force=True, **common_kwargs) ds.save(**common_kwargs) ds.containers_add("deb", url=testimg_url, call_fmt="singularity exec {img} {cmd}", **common_kwargs) # When no explicit dataset is given, paths are interpreted as relative to # the current working directory. # From top-level directory. with chpwd(path): containers_run("cat {inputs[0]} {inputs[0]} >doubled", inputs=[op.join("subdir", "in")], outputs=["doubled"], **common_kwargs) ok_file_has_content(op.join(path, "doubled"), "innardsinnards") # From under a subdirectory. subdir = op.join(ds.path, "subdir") with chpwd(subdir): containers_run("cat {inputs[0]} {inputs[0]} >doubled", inputs=["in"], outputs=["doubled"], **common_kwargs) ok_file_has_content(op.join(subdir, "doubled"), "innardsinnards") @with_tempfile def test_run_subdataset_install(path=None): path = Path(path) ds_src = Dataset(path / "src").create() # Repository setup # # . # |-- a/ # | |-- a2/ # | | `-- img # | `-- img # |-- b/ # | `-- b2/ # | `-- img # |-- c/ # | `-- c2/ # | `-- img # `-- d/ # `-- d2/ # `-- img ds_src_a = ds_src.create("a", **common_kwargs) ds_src_a2 = ds_src_a.create("a2", **common_kwargs) ds_src_b = Dataset(ds_src.pathobj / "b").create(**common_kwargs) ds_src_b2 = ds_src_b.create("b2", **common_kwargs) ds_src_c = ds_src.create("c", **common_kwargs) ds_src_c2 = ds_src_c.create("c2", **common_kwargs) ds_src_d = Dataset(ds_src.pathobj / "d").create(**common_kwargs) ds_src_d2 = ds_src_d.create("d2", **common_kwargs) ds_src.save(**common_kwargs) add_pyscript_image(ds_src_a, "in-a", "img") add_pyscript_image(ds_src_a2, "in-a2", "img") add_pyscript_image(ds_src_b2, "in-b2", "img") add_pyscript_image(ds_src_c2, "in-c2", "img") add_pyscript_image(ds_src_d2, "in-d2", "img") ds_src.save(recursive=True, **common_kwargs) ds_dest = clone(ds_src.path, str(path / "dest"), **common_kwargs) ds_dest_a2 = Dataset(ds_dest.pathobj / "a" / "a2") ds_dest_b2 = Dataset(ds_dest.pathobj / "b" / "b2") ds_dest_c2 = Dataset(ds_dest.pathobj / "c" / "c2") ds_dest_d2 = Dataset(ds_dest.pathobj / "d" / "d2") assert_false(ds_dest_a2.is_installed()) assert_false(ds_dest_b2.is_installed()) assert_false(ds_dest_c2.is_installed()) assert_false(ds_dest_d2.is_installed()) # Needed subdatasets are installed if container name is given... res = ds_dest.containers_run(["arg"], container_name="a/a2/in-a2", **common_kwargs) assert_result_count( res, 1, action="install", status="ok", path=ds_dest_a2.path) assert_result_count( res, 1, action="get", status="ok", path=str(ds_dest_a2.pathobj / "img")) ok_(ds_dest_a2.is_installed()) # ... even if the name and path do not match. res = ds_dest.containers_run(["arg"], container_name="b/b2/in-b2", **common_kwargs) assert_result_count( res, 1, action="install", status="ok", path=ds_dest_b2.path) assert_result_count( res, 1, action="get", status="ok", path=str(ds_dest_b2.pathobj / "img")) ok_(ds_dest_b2.is_installed()) # Subdatasets will also be installed if given an image path... res = ds_dest.containers_run(["arg"], container_name=str(Path("c/c2/img")), **common_kwargs) assert_result_count( res, 1, action="install", status="ok", path=ds_dest_c2.path) assert_result_count( res, 1, action="get", status="ok", path=str(ds_dest_c2.pathobj / "img")) ok_(ds_dest_c2.is_installed()) ds_dest.containers_run(["arg"], container_name=str(Path("d/d2/img")), **common_kwargs) # There's no install record if subdataset is already present. res = ds_dest.containers_run(["arg"], container_name="a/a2/in-a2", **common_kwargs) assert_not_in_results(res, action="install") @skip_if_no_network def test_nonPOSIX_imagepath(tmp_path): ds = Dataset(tmp_path).create(**common_kwargs) # plug in a proper singularity image ds.containers_add( 'mycontainer', url=testimg_url, **common_kwargs ) assert_result_count( ds.containers_list(**common_kwargs), 1, # we get a report in platform conventions path=str(ds.pathobj / '.datalad' / 'environments' / 'mycontainer' / 'image'), name='mycontainer') assert_repo_status(tmp_path) # now reconfigure the image path to look as if a version <= 1.2.4 # configured it on windows # this must still run across all platforms ds.config.set( 'datalad.containers.mycontainer.image', '.datalad\\environments\\mycontainer\\image', scope='branch', reload=True, ) ds.save(**common_kwargs) assert_repo_status(tmp_path) assert_result_count( ds.containers_list(**common_kwargs), 1, # we still get a report in platform conventions path=str(ds.pathobj / '.datalad' / 'environments' / 'mycontainer' / 'image'), name='mycontainer') res = ds.containers_run(['ls'], **common_kwargs) assert_result_count( res, 1, action='run', status='ok', ) datalad-container-1.2.6/datalad_container/tests/test_schemes.py000066400000000000000000000016571501235133500247370ustar00rootroot00000000000000import os.path as op from datalad.api import ( Dataset, containers_add, containers_list, containers_run, create, ) from datalad.cmd import ( StdOutCapture, WitlessRunner, ) from datalad.tests.utils_pytest import ( assert_result_count, ok_clean_git, ok_file_has_content, skip_if_no_network, with_tempfile, ) @skip_if_no_network @with_tempfile def test_docker(path=None): # Singularity's "docker://" scheme. ds = Dataset(path).create() ds.containers_add( "bb", url=("docker://busybox@sha256:" "7964ad52e396a6e045c39b5a44438424ac52e12e4d5a25d94895f2058cb863a0")) img = op.join(ds.path, ".datalad", "environments", "bb", "image") assert_result_count(ds.containers_list(), 1, path=img, name="bb") ok_clean_git(path) WitlessRunner(cwd=ds.path).run( ["datalad", "containers-run", "ls", "/singularity"], protocol=StdOutCapture) datalad-container-1.2.6/datalad_container/tests/utils.py000066400000000000000000000024141501235133500234010ustar00rootroot00000000000000import os import os.path as op import sys from datalad.api import containers_add from datalad.interface.common_cfg import dirs as appdirs from datalad.tests.utils_pytest import SkipTest from datalad.utils import chpwd def add_pyscript_image(ds, container_name, file_name): """Set up simple Python script as image. Parameters ---------- ds : Dataset container_name : str Add container with this name. file_name : str Write script to this file and use it as the image. """ ds_file = (ds.pathobj / file_name) ds_file.write_text("import sys\nprint(sys.argv)\n") ds.save(ds_file, message="Add dummy container") containers_add(container_name, image=str(ds_file), call_fmt=sys.executable + " {img} {cmd}", dataset=ds) def get_singularity_image(): imgname = 'datalad_container_singularity_testimg.simg' targetpath = op.join( appdirs.user_cache_dir, imgname) if op.exists(targetpath): return targetpath with chpwd(appdirs.user_cache_dir): os.system( 'singularity pull --name "{}" shub://datalad/datalad-container:testhelper'.format( imgname)) if op.exists(targetpath): return targetpath raise SkipTest datalad-container-1.2.6/datalad_container/utils.py000066400000000000000000000124731501235133500222450ustar00rootroot00000000000000"""Collection of common utilities""" from __future__ import annotations # the pathlib equivalent is only available in PY3.12 from os.path import lexists from pathlib import ( PurePath, PurePosixPath, PureWindowsPath, ) from datalad.distribution.dataset import Dataset from datalad.support.external_versions import external_versions def get_container_command(): for command in ["apptainer", "singularity"]: container_system_version = external_versions[f"cmd:{command}"] if container_system_version: return command else: raise RuntimeError("Did not find apptainer or singularity") def get_container_configuration( ds: Dataset, name: str | None = None, ) -> dict: """Report all container-related configuration in a dataset Such configuration is identified by the item name pattern:: datalad.containers.. Parameters ---------- ds: Dataset Dataset instance to report configuration on. name: str, optional If given, the reported configuration will be limited to the container with this exact name. In this case, only a single ``dict`` is returned, not nested dictionaries. Returns ------- dict Keys are the names of configured containers and values are dictionaries with their respective configuration items (with the ``datalad.containers..`` prefix removed from their keys). If `name` is given, only a single ``dict`` with the configuration items of the matching container is returned (i.e., there will be no outer ``dict`` with container names as keys). If not (matching) container configuration exists, and empty dictionary is returned. """ var_prefix = 'datalad.containers.' containers = {} # all info is in the dataset config! for var, value in ds.config.items(): if not var.startswith(var_prefix): # not an interesting variable continue var_comps = var.split('.') # container name is the 3rd after 'datalad'.'container'. cname = var_comps[2] if name and name != cname: # we are looking for a specific container's configuration # and this is not it continue # reconstruct config item name, anything after # datalad.containers.. ccfgname = '.'.join(var_comps[3:]) if not ccfgname: continue if ccfgname == 'image': # run image path normalization to get a relative path # in platform conventions, regardless of the input. # for now we report a str, because the rest of the code # is not using pathlib value = str(_normalize_image_path(value, ds)) cinfo = containers.get(cname, {}) cinfo[ccfgname] = value containers[cname] = cinfo return containers if name is None else containers.get(name, {}) def _normalize_image_path(path: str, ds: Dataset) -> PurePath: """Helper to standardize container image path handling Previously, container configuration would contain platform-paths for container image location (e.g., windows paths when added on windows, POSIX paths elsewhere). This made cross-platform reuse impossible out-of-the box, but it also means that such dataset are out there in unknown numbers. This helper inspects an image path READ FROM CONFIG(!) and ensures that it matches platform conventions (because all other arguments) also come in platform conventions. This enables standardizing the storage conventions to be POSIX-only (for the future). Parameters ---------- path: str A str-path, as read from the configuration, matching its conventions (relative path, pointing to a container image relative to the dataset's root). ds: Dataset This dataset's base path is used as a reference for resolving the relative image path to an absolute location on the file system. Returns ------- PurePath Relative path in platform conventions """ # we only need to act differently, when an incoming path is # windows. This is not possible to say with 100% confidence, # because a POSIX path can also contain a backslash. We support # a few standard cases where we CAN tell pathobj = None if '\\' not in path: # no windows pathsep, no problem pathobj = PurePosixPath(path) elif path.startswith(r'.datalad\\environments\\'): # this is the default location setup in windows conventions pathobj = PureWindowsPath(path) else: # let's assume it is windows for a moment if lexists(str(ds.pathobj / PureWindowsPath(path))): # if there is something on the filesystem for this path, # we can be reasonably sure that this is indeed a windows # path. This won't catch images in uninstalled subdataset, # but better than nothing pathobj = PureWindowsPath(path) else: # if we get here, we have no idea, and no means to verify # further hypotheses -- go with the POSIX assumption # and hope for the best pathobj = PurePosixPath(path) assert pathobj is not None # we report in platform-conventions return PurePath(pathobj) datalad-container-1.2.6/datalad_container/version.py000066400000000000000000000000261501235133500225610ustar00rootroot00000000000000__version__ = "1.2.6" datalad-container-1.2.6/docs/000077500000000000000000000000001501235133500160205ustar00rootroot00000000000000datalad-container-1.2.6/docs/Makefile000066400000000000000000000165251501235133500174710ustar00rootroot00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = -W SPHINXBUILD = sphinx-build PAPER = BUILDDIR = build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) endif # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " applehelp to make an Apple Help Book" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" @echo " coverage to run coverage check of the documentation (if enabled)" clean: rm -rf $(BUILDDIR)/* source/generated source/_extras/schema.json html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/datalad_container.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/datalad_container.qhc" applehelp: $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp @echo @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." @echo "N.B. You won't be able to view it unless you put it in" \ "~/Library/Documentation/Help or install it in your application" \ "bundle." devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/datalad_container" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/datalad_container" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." coverage: $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage @echo "Testing of coverage in the sources finished, look at the " \ "results in $(BUILDDIR)/coverage/python.txt." xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." datalad-container-1.2.6/docs/examples/000077500000000000000000000000001501235133500176365ustar00rootroot00000000000000datalad-container-1.2.6/docs/examples/basic_demo.sh000066400000000000000000000034761501235133500222710ustar00rootroot00000000000000#!/bin/sh # SKIP_IN_V6 set -e OLD_PWD=$PWD # BOILERPLATE #% EXAMPLE START # # Getting started # *************** # # The Datalad container extension provides a few commands to register # containers with a dataset and use them for execution of arbitrary # commands. In order to get going quickly, we only need a dataset # and a ready-made container. For this demo we will start with a # fresh dataset and a demo container from Singularity-Hub. #% # fresh dataset datalad create demo cd demo # register container straight from Singularity-Hub datalad containers-add my1st --url shub://datalad/datalad-container:testhelper #% # This will download the container image, add it to the dataset, and record # basic information on the container under its name "my1st" in the dataset's # configuration at ``.datalad/config``. # # Now we are all set to use this container for command execution. All it needs # is to swap the command `datalad run` with `datalad containers-run`. The # command is automatically executed in the registered container and the results # (if there are any) will be added to the dataset: #% datalad containers-run cp /etc/debian_version proof.txt #% # If there is more than one container registered, the desired container needs # to be specified via the ``--name`` option. Containers do not need to come from # Singularity-Hub, but can be local images too. Via the ``containers-add # --call-fmt`` option it is possible to configure how exactly a container # is being executed, or which local directories shall be made available to # a container. # # At the moment there is built-in support for Singularity and Docker, but other # container execution systems can be used together with custom helper scripts. #% EXAMPLE END testEquality() { assertEquals 1 1 } cd "$OLD_PWD" [ -n "$DATALAD_TESTS_RUNCMDLINE" ] && . shunit2 || true datalad-container-1.2.6/docs/source/000077500000000000000000000000001501235133500173205ustar00rootroot00000000000000datalad-container-1.2.6/docs/source/_static/000077500000000000000000000000001501235133500207465ustar00rootroot00000000000000datalad-container-1.2.6/docs/source/_static/datalad_logo.png000066400000000000000000000016761501235133500241000ustar00rootroot00000000000000‰PNG  IHDRddG»‹PΘA@βεncO €gw iYθ„AJ# a έ0H!Ω‚άS Ω‰X* HΥhοmη#A,Ωv6ή,Λ]i&RFkV2αLΔ³¬Φdν υƒγ‰†θΏπœ–ʰʏ6«ΡΔ ε>}†°]ΠSΫD| ~Šp~mX/άOTD~ U΅p4πa΄dQΚ/κΒ™Η£%»ΌxΎχ!ΝEΨx4δ₯ʡꈾdα[›…p8uΩ‘@ΘΖωhβ±[Ηi(mBB!‘γ ½χ‰PHόϊύζ2ͺ†δ?c―ΜΣ―!)^U–Ν™ˆμώβpωw~χ‰€ώ?gGΣx κ=όl9‹‘‘u9ά―‹aJ0$·ϋƒOŸτ8Δxh$pΤΑIENDB`‚datalad-container-1.2.6/docs/source/_templates/000077500000000000000000000000001501235133500214555ustar00rootroot00000000000000datalad-container-1.2.6/docs/source/_templates/autosummary/000077500000000000000000000000001501235133500240435ustar00rootroot00000000000000datalad-container-1.2.6/docs/source/_templates/autosummary/module.rst000066400000000000000000000006641501235133500260700ustar00rootroot00000000000000{% if fullname == 'datalad.api' -%} `{{ name }}` =={%- for c in name %}={%- endfor %} .. automodule:: datalad.api .. currentmodule:: datalad.api {% for item in members if not item.startswith('_') %} `{{ item }}` --{%- for c in item %}-{%- endfor %} .. autofunction:: {{ item }} {% endfor %} {% else -%} {{ fullname }} {{ underline }} .. automodule:: {{ fullname }} :members: :undoc-members: :show-inheritance: {% endif %} datalad-container-1.2.6/docs/source/acknowledgements.rst000066400000000000000000000021321501235133500234020ustar00rootroot00000000000000Acknowledgments *************** DataLad development is being performed as part of a US-German collaboration in computational neuroscience (CRCNS) project "DataGit: converging catalogues, warehouses, and deployment logistics into a federated 'data distribution'" (Halchenko_/Hanke_), co-funded by the US National Science Foundation (`NSF 1429999`_) and the German Federal Ministry of Education and Research (`BMBF 01GQ1411`_). Additional support is provided by the German federal state of Saxony-Anhalt and the European Regional Development Fund (ERDF), Project: `Center for Behavioral Brain Sciences`_, Imaging Platform DataLad is built atop the git-annex_ software that is being developed and maintained by `Joey Hess`_. .. _Halchenko: http://haxbylab.dartmouth.edu/ppl/yarik.html .. _Hanke: http://www.psychoinformatics.de .. _NSF 1429999: http://www.nsf.gov/awardsearch/showAward?AWD_ID=1429999 .. _BMBF 01GQ1411: http://www.gesundheitsforschung-bmbf.de/de/2550.php .. _Center for Behavioral Brain Sciences: http://cbbs.eu/en/ .. _git-annex: http://git-annex.branchable.com .. _Joey Hess: https://joeyh.name datalad-container-1.2.6/docs/source/changelog.rst000066400000000000000000000155011501235133500220030ustar00rootroot00000000000000.. This file is auto-converted from CHANGELOG.md (make update-changelog) -- do not edit Change log ********** :: ____ _ _ _ | _ \ __ _ | |_ __ _ | | __ _ __| | | | | | / _` || __| / _` || | / _` | / _` | | |_| || (_| || |_ | (_| || |___ | (_| || (_| | |____/ \__,_| \__| \__,_||_____| \__,_| \__,_| Container This is a high level and scarce summary of the changes between releases. We would recommend to consult log of the `DataLad git repository `__ for more details. 1.1.2 (January 16, 2021) – -------------------------- - Replace use of ``mock`` with ``unittest.mock`` as we do no longer support Python 2 1.1.1 (January 03, 2021) – -------------------------- - Drop use of ``Runner`` (to be removed in datalad 0.14.0) in favor of ``WitlessRunner`` 1.1.0 (October 30, 2020) – -------------------------- - Datalad version 0.13.0 or later is now required. - In the upcoming 0.14.0 release of DataLad, the datalad special remote will have built-in support for β€œshub://” URLs. If ``containers-add`` detects support for this feature, it will now add the β€œshub://” URL as is rather than resolving the URL itself. This avoids registering short-lived URLs, allowing the image to be retrieved later with ``datalad get``. - ``containers-run`` learned to install necessary subdatasets when asked to execute a container from underneath an uninstalled subdataset. 1.0.1 (June 23, 2020) – ----------------------- - Prefer ``datalad.core.local.run`` to ``datalad.interface.run``. The latter has been marked as obsolete since DataLad v0.12 (our minimum requirement) and will be removed in DataLad’s next feature release. 1.0.0 (Feb 23, 2020) – not-as-a-shy-one --------------------------------------- Extension is pretty stable so releasing as 1. MAJOR release, so we could start tracking API breakages and enhancements properly. - Drops support for Python 2 and DataLad prior 0.12 0.5.2 (Nov 12, 2019) – ---------------------- Fixes ~~~~~ - The Docker adapter unconditionally called ``docker run`` with ``--interactive`` and ``--tty`` even when stdin was not attached to a TTY, leading to an error. 0.5.1 (Nov 08, 2019) – ---------------------- .. _fixes-1: Fixes ~~~~~ - The Docker adapter, which is used for the β€œdhub://” URL scheme, assumed the Python executable was spelled β€œpython”. - A call to DataLad’s ``resolve_path`` helper assumed a string return value, which isn’t true as of the latest DataLad release candidate, 0.12.0rc6. 0.5.0 (Jul 12, 2019) – damn-you-malicious-users ----------------------------------------------- New features ~~~~~~~~~~~~ - The default result renderer for ``containers-list`` is now a custom renderer that includes the container name in the output. .. _fixes-2: Fixes ~~~~~ - Temporarily skip two tests relying on SingularityHub – it is down. 0.4.0 (May 29, 2019) – run-baby-run ----------------------------------- The minimum required DataLad version is now 0.11.5. .. _new-features-1: New features ~~~~~~~~~~~~ - The call format gained the β€œ{img_dspath}” placeholder, which expands to the relative path of the dataset that contains the image. This is useful for pointing to a wrapper script that is bundled in the same subdataset as a container. - ``containers-run`` now passes the container image to ``run`` via its ``extra_inputs`` argument so that a run command’s β€œ{inputs}” field is restricted to inputs that the caller explicitly specified. - During execution, ``containers-run`` now sets the environment variable ``DATALAD_CONTAINER_NAME`` to the name of the container. .. _fixes-3: Fixes ~~~~~ - ``containers-run`` mishandled paths when called from a subdirectory. - ``containers-run`` didn’t provide an informative error message when ``cmdexec`` contained an unknown placeholder. - ``containers-add`` ignores the ``--update`` flag when the container doesn’t yet exist, but it confusingly still used the word β€œupdate” in the commit message. 0.3.1 (Mar 05, 2019) – Upgrayeddd --------------------------------- .. _fixes-4: Fixes ~~~~~ - ``containers-list`` recursion actually does recursion. 0.3.0 (Mar 05, 2019) – Upgrayedd -------------------------------- API changes ~~~~~~~~~~~ - ``containers-list`` no longer lists containers from subdatasets by default. Specify ``--recursive`` to do so. - ``containers-run`` no longer considers subdataset containers in its automatic selection of a container name when no name is specified. If the current dataset has one container, that container is selected. Subdataset containers must always be explicitly specified. .. _new-features-2: New features ~~~~~~~~~~~~ - ``containers-add`` learned to update a previous container when passed ``--update``. - ``containers-add`` now supports Singularity’s β€œdocker://” scheme in the URL. - To avoid unnecessary recursion into subdatasets, ``containers-run`` now decides to look for containers in subdatasets based on whether the name has a slash (which is true of all subdataset containers). 0.2.2 (Dec 19, 2018) – The more the merrier ------------------------------------------- - list/use containers recursively from installed subdatasets - Allow to specify container by path rather than just by name - Adding a container from local filesystem will copy it now 0.2.1 (Jul 14, 2018) – Explicit lyrics -------------------------------------- - Add support ``datalad run --explicit``. 0.2 (Jun 08, 2018) – Docker --------------------------- - Initial support for adding and running Docker containers. - Add support ``datalad run --sidecar``. - Simplify storage of ``call_fmt`` arguments in the Git config, by benefiting from ``datalad run`` being able to work with single-string compound commands. 0.1.2 (May 28, 2018) – The docs ------------------------------- - Basic beginner documentation 0.1.1 (May 22, 2018) – The fixes -------------------------------- .. _new-features-3: New features ~~~~~~~~~~~~ - Add container images straight from singularity-hub, no need to manually specify ``--call-fmt`` arguments. .. _api-changes-1: API changes ~~~~~~~~~~~ - Use β€œname” instead of β€œlabel” for referring to a container (e.g. ``containers-run -n ...`` instead of ``containers-run -l``. .. _fixes-5: Fixes ~~~~~ - Pass relative container path to ``datalad run``. - ``containers-run`` no longer hides ``datalad run`` failures. 0.1 (May 19, 2018) – The Release -------------------------------- - Initial release with basic functionality to add, remove, and list containers in a dataset, plus a ``run`` command wrapper that injects the container image as an input dependency of a command call. datalad-container-1.2.6/docs/source/conf.py000066400000000000000000000113771501235133500206300ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # datalad_container documentation build configuration file, created by # sphinx-quickstart on Tue Oct 13 08:41:19 2015. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import datetime import os import sys from os import pardir from os.path import ( abspath, dirname, exists, join as opj, ) import datalad_container # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) # generate missing pieces for setup_py_path in (opj(pardir, 'setup.py'), # travis opj(pardir, pardir, 'setup.py')): # RTD if exists(setup_py_path): sys.path.insert(0, os.path.abspath(dirname(setup_py_path))) try: for cmd in 'manpage',: #'examples': os.system( '{} build_{} --cmdsuite {} --manpath {} --rstpath {}'.format( setup_py_path, cmd, 'datalad_container:command_suite', abspath(opj(dirname(setup_py_path), 'build', 'man')), opj(dirname(__file__), 'generated', 'man'))) except: # shut up and do your best pass # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.inheritance_diagram', 'sphinx.ext.viewcode', 'sphinx.ext.napoleon', 'sphinx_copybutton', ] # for the module reference autosummary_generate = True # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The master toctree document. master_doc = 'index' # General information about the project. project = u'Datalad for containerized environments' copyright = u'2018-{}, DataLad team'.format(datetime.datetime.now().year) author = u'DataLad team' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. version = datalad_container.__version__ release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = [] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {"python": ('https://docs.python.org/', None)} # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'sphinx_rtd_theme' # The name of an image file (relative to this directory) to place at the top # of the sidebar. html_logo = '_static/datalad_logo.png' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # If true, the index is split into individual pages for each letter. html_split_index = True # If true, links to the reST sources are added to the pages. html_show_sourcelink = False # smart quotes are incompatible with the RST flavor of the generated manpages # but see `smartquotes_action` for more fine-grained control, in case # some of this functionality is needed smartquotes = False datalad-container-1.2.6/docs/source/index.rst000066400000000000000000000024501501235133500211620ustar00rootroot00000000000000DataLad extension for containerized environments ************************************************ This extension equips DataLad's `run/rerun `_ functionality with the ability to transparently execute commands in containerized computational environments. On re-run, DataLad will automatically obtain any required container at the correct version prior execution. Documentation ============= This is the technical documentation of the functionality and commands provided by this DataLad extension package. For an introduction to the general topic and a tutorial, please see the DataLad Handbook at https://handbook.datalad.org/r?containers. * :ref:`Documentation index ` * `API reference`_ .. toctree:: :maxdepth: 1 changelog acknowledgements metadata-extraction API Reference ============= Command manuals --------------- .. toctree:: :maxdepth: 1 generated/man/datalad-containers-add generated/man/datalad-containers-remove generated/man/datalad-containers-list generated/man/datalad-containers-run Python API ---------- .. currentmodule:: datalad_container .. autosummary:: :toctree: generated containers_add containers_remove containers_list containers_run utils .. |---| unicode:: U+02014 .. em dash datalad-container-1.2.6/docs/source/metadata-extraction.rst000066400000000000000000000042241501235133500240120ustar00rootroot00000000000000Metadata Extraction ******************* If `datalad-metalad`_ extension is installed, `datalad-container` can extract metadata from singularity containers images. (It is recommended to use a tool like `jq` if you would like to read the output yourself.) Singularity Inspect ------------------- Adds metadata gathered from `singularity inspect` and the version of `singularity` or `apptainer`. For example: (From the ReproNim/containers repository) `datalad meta-extract -d . container_inspect images/bids/bids-pymvpa--1.0.2.sing | jq` .. code-block:: { "type": "file", "dataset_id": "b02e63c2-62c1-11e9-82b0-52540040489c", "dataset_version": "9ed0a39406e518f0309bb665a99b64dec719fb08", "path": "images/bids/bids-pymvpa--1.0.2.sing", "extractor_name": "container_inspect", "extractor_version": "0.0.1", "extraction_parameter": {}, "extraction_time": 1680097317.7093463, "agent_name": "Austin Macdonald", "agent_email": "austin@dartmouth.edu", "extracted_metadata": { "@id": "datalad:SHA1-s993116191--cc7ac6e6a31e9ac131035a88f699dfcca785b844", "type": "file", "path": "images/bids/bids-pymvpa--1.0.2.sing", "content_byte_size": 0, "comment": "SingularityInspect extractor executed at 1680097317.6012993", "container_system": "apptainer", "container_system_version": "1.1.6-1.fc37", "container_inspect": { "data": { "attributes": { "labels": { "org.label-schema.build-date": "Thu,_19_Dec_2019_14:58:41_+0000", "org.label-schema.build-size": "2442MB", "org.label-schema.schema-version": "1.0", "org.label-schema.usage.singularity.deffile": "Singularity.bids-pymvpa--1.0.2", "org.label-schema.usage.singularity.deffile.bootstrap": "docker", "org.label-schema.usage.singularity.deffile.from": "bids/pymvpa:v1.0.2", "org.label-schema.usage.singularity.version": "2.5.2-feature-squashbuild-secbuild-2.5.6e68f9725" } } }, "type": "container" } } } .. _datalad-metalad: http://docs.datalad.org/projects/metalad/en/latest/ datalad-container-1.2.6/docs/utils/000077500000000000000000000000001501235133500171605ustar00rootroot00000000000000datalad-container-1.2.6/docs/utils/pygments_ansi_color.py000066400000000000000000000140121501235133500236060ustar00rootroot00000000000000# -*- coding: utf-8 -*- """Pygments lexer for text containing ANSI color codes.""" from __future__ import ( absolute_import, unicode_literals, ) import itertools import re import pygments.lexer import pygments.token Color = pygments.token.Token.Color _ansi_code_to_color = { 0: 'Black', 1: 'Red', 2: 'Green', 3: 'Yellow', 4: 'Blue', 5: 'Magenta', 6: 'Cyan', 7: 'White', } def _token_from_lexer_state(bold, fg_color, bg_color): """Construct a token given the current lexer state. We can only emit one token even though we have a multiple-tuple state. To do work around this, we construct tokens like "BoldRed". """ token_name = '' if bold: token_name += 'Bold' if fg_color: token_name += fg_color if bg_color: token_name += 'BG' + bg_color if token_name == '': return pygments.token.Text else: return getattr(Color, token_name) def color_tokens(fg_colors, bg_colors): """Return color tokens for a given set of colors. Pygments doesn't have a generic "color" token; instead everything is contextual (e.g. "comment" or "variable"). That doesn't make sense for us, where the colors actually *are* what we care about. This function will register combinations of tokens (things like "Red" or "BoldRedBGGreen") based on the colors passed in. You can also define the tokens yourself, but note that the token names are *not* currently guaranteed to be stable between releases as I'm not really happy with this approach. Usage: fg_colors = bg_colors = { 'Black': '#000000', 'Red': '#EF2929', 'Green': '#8AE234', 'Yellow': '#FCE94F', 'Blue': '#3465A4', 'Magenta': '#c509c5', 'Cyan': '#34E2E2', 'White': '#ffffff', } class MyStyle(pygments.styles.SomeStyle): styles = dict(pygments.styles.SomeStyle.styles) styles.update(color_tokens(fg_colors, bg_colors)) """ styles = {} for bold, fg_color, bg_color in itertools.product( (False, True), {None} | set(fg_colors), {None} | set(bg_colors), ): token = _token_from_lexer_state(bold, fg_color, bg_color) if token is not pygments.token.Text: value = [] if bold: value.append('bold') if fg_color: value.append(fg_colors[fg_color]) if bg_color: value.append('bg:' + bg_colors[bg_color]) styles[token] = ' '.join(value) return styles class AnsiColorLexer(pygments.lexer.RegexLexer): name = 'ANSI Color' aliases = ('ansi-color', 'ansi', 'ansi-terminal') flags = re.DOTALL | re.MULTILINE def __init__(self, *args, **kwargs): super(AnsiColorLexer, self).__init__(*args, **kwargs) self.reset_state() def reset_state(self): self.bold = False self.fg_color = None self.bg_color = None @property def current_token(self): return _token_from_lexer_state( self.bold, self.fg_color, self.bg_color, ) def process(self, match): """Produce the next token and bit of text. Interprets the ANSI code (which may be a color code or some other code), changing the lexer state and producing a new token. If it's not a color code, we just strip it out and move on. Some useful reference for ANSI codes: * http://ascii-table.com/ansi-escape-sequences.php """ # "after_escape" contains everything after the start of the escape # sequence, up to the next escape sequence. We still need to separate # the content from the end of the escape sequence. after_escape = match.group(1) # TODO: this doesn't handle the case where the values are non-numeric. # This is rare but can happen for keyboard remapping, e.g. # '\x1b[0;59;"A"p' parsed = re.match( r'([0-9;=]*?)?([a-zA-Z])(.*)$', after_escape, re.DOTALL | re.MULTILINE, ) if parsed is None: # This shouldn't ever happen if we're given valid text + ANSI, but # people can provide us with utter junk, and we should tolerate it. text = after_escape else: value, code, text = parsed.groups() if code == 'm': # "m" is "Set Graphics Mode" # Special case \x1b[m is a reset code if value == '': self.reset_state() else: values = value.split(';') for value in values: try: value = int(value) except ValueError: # Shouldn't ever happen, but could with invalid # ANSI. continue else: fg_color = _ansi_code_to_color.get(value - 30) bg_color = _ansi_code_to_color.get(value - 40) if fg_color: self.fg_color = fg_color elif bg_color: self.bg_color = bg_color elif value == 1: self.bold = True elif value == 22: self.bold = False elif value == 39: self.fg_color = None elif value == 49: self.bg_color = None elif value == 0: self.reset_state() yield match.start(), self.current_token, text tokens = { # states have to be native strings str('root'): [ (r'\x1b\[([^\x1b]*)', process), (r'[^\x1b]+', pygments.token.Text), ], } datalad-container-1.2.6/pyproject.toml000066400000000000000000000012211501235133500200000ustar00rootroot00000000000000[build-system] requires = ["setuptools >= 59.0.0", "tomli", "wheel"] [tool.isort] force_grid_wrap = 2 include_trailing_comma = true multi_line_output = 3 combine_as_imports = true [tool.codespell] skip = '.git,*.pdf,*.svg,venvs,versioneer.py,venvs' # DNE - do not exist ignore-words-list = 'dne' [tool.versioneer] # See the docstring in versioneer.py for instructions. Note that you must # re-run 'versioneer.py setup' after changing this section, and commit the # resulting files. VCS = 'git' style = 'pep440' versionfile_source = 'datalad_container/_version.py' versionfile_build = 'datalad_container/_version.py' tag_prefix = '' parentdir_prefix = '' datalad-container-1.2.6/requirements-devel.txt000066400000000000000000000000711501235133500214470ustar00rootroot00000000000000# requirements for a development environment -e .[devel] datalad-container-1.2.6/requirements.txt000066400000000000000000000001421501235133500203510ustar00rootroot00000000000000# If you want to develop, use requirements-devel.txt # git+https://github.com/datalad/datalad.git datalad-container-1.2.6/setup.cfg000066400000000000000000000030511501235133500167100ustar00rootroot00000000000000[metadata] url = https://github.com/datalad/datalad-container author = The DataLad Team and Contributors author_email = team@datalad.org description = DataLad extension package for working with containerized environments long_description = file:README.md long_description_content_type = text/markdown; charset=UTF-8 license = MIT classifiers = Programming Language :: Python License :: OSI Approved :: BSD License Programming Language :: Python :: 3 [options] python_requires = >= 3.7 install_requires = datalad >= 0.18.0 requests>=1.2 # to talk to Singularity-hub packages = find: include_package_data = True [options.extras_require] extras = datalad-metalad # this matches the name used by -core and what is expected by some CI setups devel = %(extras)s pytest pytest-cov coverage sphinx sphinx-rtd-theme sphinx-copybutton [options.packages.find] # do not ship the build helpers exclude= _datalad_buildsupport [options.entry_points] # 'datalad.extensions' is THE entrypoint inspected by the datalad API builders datalad.extensions = # the label in front of '=' is the command suite label # the entrypoint can point to any symbol of any name, as long it is # valid datalad interface specification (see demo in this extensions) container = datalad_container:command_suite datalad.metadata.extractors = container_inspect = datalad_container.extractors.metalad_container:MetaladContainerInspect [coverage:report] show_missing = True omit = # versioneer code datalad_container/_version.py datalad-container-1.2.6/setup.py000077500000000000000000000005461501235133500166120ustar00rootroot00000000000000#!/usr/bin/env python from setuptools import setup import versioneer from _datalad_buildsupport.setup import ( BuildManPage, ) cmdclass = versioneer.get_cmdclass() cmdclass.update(build_manpage=BuildManPage) if __name__ == '__main__': setup(name='datalad_container', version=versioneer.get_version(), cmdclass=cmdclass, ) datalad-container-1.2.6/tools/000077500000000000000000000000001501235133500162305ustar00rootroot00000000000000datalad-container-1.2.6/tools/Singularity.testhelper000066400000000000000000000002031501235133500226360ustar00rootroot00000000000000# # This produces a minimal image that can be used for testing the # extension itself. # Bootstrap:docker From:debian:stable-slim datalad-container-1.2.6/tools/appveyor_env_setup.bat000066400000000000000000000001451501235133500226550ustar00rootroot00000000000000set PY=%1-x64 set TMP=C:\DLTMP set TEMP=C:\DLTMP set PATH=C:\Python%PY%;C:\Python%PY%\Scripts;%PATH% datalad-container-1.2.6/tools/ci/000077500000000000000000000000001501235133500166235ustar00rootroot00000000000000datalad-container-1.2.6/tools/ci/install-singularity.sh000077500000000000000000000010231501235133500231740ustar00rootroot00000000000000#!/bin/bash set -ex -o pipefail release="$(curl -fsSL https://api.github.com/repos/sylabs/singularity/releases/latest | jq -r .tag_name)" codename="$(lsb_release -cs)" arch="$(dpkg --print-architecture)" wget -O /tmp/singularity-ce.deb "https://github.com/sylabs/singularity/releases/download/$release/singularity-ce_${release#v}-${codename}_$arch.deb" set -x sudo DEBIAN_FRONTEND=noninteractive apt-get install -y uidmap libfuse2 fuse2fs sudo dpkg -i /tmp/singularity-ce.deb sudo DEBIAN_FRONTEND=noninteractive apt-get install -f datalad-container-1.2.6/tools/ci/prep-travis-forssh-sudo.sh000077500000000000000000000001411501235133500237040ustar00rootroot00000000000000#!/usr/bin/env bash echo "127.0.0.1 datalad-test" >> /etc/hosts apt-get install openssh-client datalad-container-1.2.6/tools/ci/prep-travis-forssh.sh000077500000000000000000000011051501235133500227350ustar00rootroot00000000000000#!/bin/bash mkdir -p ~/.ssh echo -e "Host localhost\n\tStrictHostKeyChecking no\n\tIdentityFile /tmp/dl-test-ssh-id\n" >> ~/.ssh/config echo -e "Host datalad-test\n\tStrictHostKeyChecking no\n\tIdentityFile /tmp/dl-test-ssh-id\n" >> ~/.ssh/config ssh-keygen -f /tmp/dl-test-ssh-id -N "" cat /tmp/dl-test-ssh-id.pub >> ~/.ssh/authorized_keys eval $(ssh-agent) ssh-add /tmp/dl-test-ssh-id echo "DEBUG: test connection to localhost ..." ssh -v localhost exit echo "DEBUG: test connection to datalad-test ..." ssh -v datalad-test exit # tmp: don't run the actual tests: # exit 1 datalad-container-1.2.6/tools/containers_add_dhub_tags.py000066400000000000000000000332051501235133500236020ustar00rootroot00000000000000"""Feed tagged Docker Hub images to datalad-containers-add. This command takes a set of Docker Hub repositories, looks up the tags, and calls `datalad containers-add ... dhub://REPO:TAG@digest`. The output of datalad-container's Docker adapter is dumped to images/REPO/TAG/ARCH-DATE-SHORTDIGEST/ where SHORTDIGEST is the first 12 characters of .config.digest key of the manifest returned by Docker Hub for the image for the arch which was uploaded on the DATE. In addition, that image record and manifest are written to a satellite to that directory .image.json and .manifest.json files. The step of adding the image is skipped if the path is already present locally. """ import fileinput import json import logging import re from pathlib import Path from pprint import pprint import requests from datalad.api import ( containers_add, save, ) lgr = logging.getLogger("containers_add_dhub_tags") REGISTRY_AUTH_URL = ("https://auth.docker.io/token?service=registry.docker.io" "&scope=repository:{repo}:pull") REGISTRY_ENDPOINT = "https://registry-1.docker.io/v2" DHUB_ENDPOINT = "https://hub.docker.com/v2" # TODO: wrap it up with feeding the repositories to consider # or if we just make it one repository at a time, then could become CLI options target_architectures = '.*' target_tags = '.*' # TODO: forget_tags = 'master' -- those for which we might not want to retain prior versions # or may be exclude them completely since too frequently changing etc? # TEST on busybox on just a few architectures and tags - it is tiny but has too many #target_architectures = '^(amd64|.*86)$' #target_tags = '(latest|1.32.0)' # TODO this could be a CLI option default_architecture = 'amd64' def clean_container_name(name): """Transform `name` for use in datalad-containers-add. Note that, although it probably doesn't matter in practice, this transformation is susceptible to conflicts and ambiguity. """ if name.startswith("_/"): name = name[2:] name = name.replace("_", "-") # TODO: research feasibility to create "hierarchical" organization # by using . as a separator. Then we could have a "default" # one and then various past instances in sublevels of # .version.architecture.date--shortdigest return re.sub(r"[^0-9a-zA-Z-]", "--", name) def add_container(url, name, target): lgr.info("Adding %s as %s", url, name) # TODO: This would result in a commit for each image, which would # be good to avoid. # # This containers_add() call also prevents doing things in # parallel. containers_add( name=name, url=url, image=str(target), # Pass update=True to let the image for an existing entry # (particularly the one for the "latest" tag) be updated. update=True) return name def write_json(target, content): lgr.info("Writing %s", target) target.parent.mkdir(parents=True, exist_ok=True) target.write_text(json.dumps(content)) return target # # Registry -- requires authentication to query # from contextlib import contextmanager class RepoRegistry(object): def __init__(self, repo): resp_auth = requests.get(REGISTRY_AUTH_URL.format(repo=repo)) resp_auth.raise_for_status() self.repo = repo self._headers = { "Authorization": "Bearer " + resp_auth.json()["token"], } def get(self, query, headers=None): headers = headers or {} headers.update(self._headers) resp_man = requests.get(f"{REGISTRY_ENDPOINT}/{self.repo}/{query}", headers=headers) resp_man.raise_for_status() return resp_man.json() def get_manifest(self, reference): lgr.debug("Getting manifest for %s:%s", self.repo, reference) # TODO: Can we check with HEAD first to see if the digest # matches what we have locally? return self.get( f'manifests/{reference}', # return the single (first, if multiple e.g. for a reference being a tag) # manifest headers={"Accept": "application/vnd.docker.distribution.manifest.v2+json"} ) # # HUB -- no authentication required # def walk_pages(url): next_page = url while next_page: lgr.debug("GET %s", next_page) response = requests.get(next_page) response.raise_for_status() data = response.json() next_page = data.get("next") yield from data.get("results", []) def get_repo_tag_images(repo): url = f"{DHUB_ENDPOINT}/repositories/{repo}/tags" for result in walk_pages(url): images = result["images"] # there could be records with images not having been uploaded, # then it seems digest is not there and 'last_pushed' is None for i, image in list(enumerate(images))[::-1]: if 'digest' not in image: assert not image.get('last_pushed') images.pop(i) yield result["name"], sorted(images, key=lambda i: i['digest']) def get_namespace_repos(name): lgr.info("Getting repositories for %s...", name) url = f"{DHUB_ENDPOINT}/repositories/{name}/" for result in walk_pages(url): assert name == result["namespace"] yield f"{name}/{result['name']}" def parse_input(line): line = line.strip() lgr.debug("Processing input: %s", line) if line.endswith("/"): kind = "namespace" name = line[:-1] else: kind = "repository" if "/" in line: name = line else: lgr.debug( "Assuming official image and assigning library/ namespace") name = "library/" + line return name, kind def process_files(files): failed = [] for line in fileinput.input(files): name, kind = parse_input(line) if kind == "namespace": try: repos = list(get_namespace_repos(name)) except requests.HTTPError as exc: lgr.warning( "Failed to list repositories for %s (status %s). Skipping", name, exc.response.status_code) failed.append(name) continue else: repos = [name] target_architectures_re = re.compile(target_architectures) target_tags_re = re.compile(target_tags) for repo in repos: lgr.info("Working on %s", repo) try: registry = RepoRegistry(repo) #pprint(list(zip(sorted(_all_tags['latest'], key=lambda r: r['digest']), sorted(_all_tags['1.32.0'], # key=lambda r: r['digest'])))) tag_images = dict(get_repo_tag_images(repo)) # 'latest' tag is special in docker, it is the default one # which might typically point to some other release/version. # If we find that it is the case, we do not create a dedicated "latest" # image/datalad container -- we just add container entry pointing to that # one. If there is no matching one -- we do get "latest" latest_matching_tag = None # NOTE: "master" is also often used to signal a moving target # it might, or not, correspond to tagged release. I guess we are just # doomed to breed those if target_tags_re.match('latest'): matching_tags = [] for tag, images in tag_images.items(): if tag == 'latest' or not target_tags_re.match(tag): lgr.debug("Skipping tag %(tag)s") continue if images == tag_images['latest']: matching_tags.append(tag) if len(matching_tags) >= 1: if len(matching_tags) > 1: lgr.info( "Multiple tags images match latest, taking the first: %s", ', '.join(matching_tags)) latest_matching_tag = matching_tags[0] lgr.info("Taking %s as the one for 'latest'", latest_matching_tag) else: # TODO: if there is no latest, we should at least establish the # convenient one for each tag pass for tag, images in tag_images.items(): if tag == 'latest' and latest_matching_tag: continue # skip since we will handle it if not target_tags_re.match(tag): lgr.debug("Skipping tag %(tag)s") continue multiarch = len({i['architecture'] for i in images}) > 1 for image in images: architecture = image['architecture'] if not target_architectures_re.match(architecture): lgr.debug("Skipping architecture %(architecture)s", image) continue manifest = registry.get_manifest(image['digest']) digest = manifest["config"]["digest"] # yoh: if I got it right, it is actual image ID we see in docker images assert digest.startswith("sha256:") digest = digest[7:] digest_short = digest[:12] # use short version in name last_pushed = image.get('last_pushed') if last_pushed: assert last_pushed.endswith('Z') # take only date last_pushed = last_pushed[:10].replace('-', '') assert len(last_pushed) == 8 cleaner_repo = repo # this is how it looks on hub.docker.com URL if repo.startswith('library/'): cleaner_repo = "_/" + cleaner_repo[len('library/'):] image_name = f"{cleaner_repo}/{tag}/" if multiarch: image_name += f"{architecture}-" if last_pushed: # apparently not in all, e.g. no for repronim/neurodocker # may be None for those built on the hub? image_name += f"{last_pushed}-" image_name += f"{digest_short}" dl_container_name = clean_container_name(str(image_name)) image_path = Path("images") / image_name url = f"dhub://{repo}:{tag}@{image['digest']}" save_paths = [] if image_path.exists(): lgr.info("%s already exists, skipping adding", str(image_path)) else: save_paths.append(write_json(Path(str(image_path) + '.manifest.json'), manifest)) save_paths.append(write_json(Path(str(image_path) + '.image.json'), image)) add_container(url, dl_container_name, image_path) # TODO: either fix datalad-container for https://github.com/datalad/datalad-container/issues/98 # or here, since we have manifest, we can datalad download-url, and add-archive-content # of the gzipped layers (but without untarring) - that should add datalad-archive # urls to individual layers in the "saved" version # TODO: make it in a single commit with add_container at least, # or one commit for the whole repo sweep save(path=save_paths, message=f"Added manifest and image records for {dl_container_name}") # TODO: ensure .datalad/config to have additional useful fields: # architecture, os, and manually "updateurl" since not added for # dhub:// ATM if tag == latest_matching_tag and architecture == default_architecture: # TODO remove section if exists, copy this one lgr.warning("Tracking of 'latest' is not yet implemented") except requests.HTTPError as exc: lgr.warning( "Failed processing %s. Skipping\n status %s for %s", repo, exc.response.status_code, exc.response.url) failed.append(name) continue return failed def main(args): import argparse parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( "-v", "--verbose", action="store_true") parser.add_argument( "files", metavar="FILE", nargs="*", help=("File with list of names. " "If a name doesn't contain a slash, " "it's treated as an official image by prepending 'library/'. " "A name ending with a slash is taken as a namespace, " "and Docker Hub is queried to obtain a list of repositories " "under that namespace (e.g., all the repositories of a user). " "If not specified, the names are read from stdin.")) namespace = parser.parse_args(args[1:]) logging.basicConfig( level=logging.DEBUG if namespace.verbose else logging.INFO, format="%(message)s") return process_files(namespace.files) if __name__ == "__main__": import sys failed = main(sys.argv) sys.exit(len(failed) > 0) datalad-container-1.2.6/tools/mk_minimal_chroot.sh000077500000000000000000000010431501235133500222600ustar00rootroot00000000000000#!/bin/bash # # bootstrap a tiny chroot (26MB compressed) # # run with sudo set -e -u chrootdir=$(mktemp -d) echo "Working in $chrootdir" debootstrap --variant=minbase --no-check-gpg stretch "$chrootdir" find "$chrootdir"/var/cache/apt/archives -type f -delete find "$chrootdir"/var/lib/apt/lists/ -type f -delete rm -rf "$chrootdir"/usr/share/doc/* rm -rf "$chrootdir"/usr/share/man tar --show-transformed-names --transform=s,^.*$(basename $chrootdir),minichroot, -cvjf minichroot.tar.xz "$chrootdir" echo "chroot tarball at minichroot.tar.xz" datalad-container-1.2.6/versioneer.py000066400000000000000000002512251501235133500176320ustar00rootroot00000000000000 # Version: 0.29 """The Versioneer - like a rocketeer, but for versions. The Versioneer ============== * like a rocketeer, but for versions! * https://github.com/python-versioneer/python-versioneer * Brian Warner * License: Public Domain (Unlicense) * Compatible with: Python 3.7, 3.8, 3.9, 3.10, 3.11 and pypy3 * [![Latest Version][pypi-image]][pypi-url] * [![Build Status][travis-image]][travis-url] This is a tool for managing a recorded version number in setuptools-based python projects. The goal is to remove the tedious and error-prone "update the embedded version string" step from your release process. Making a new release should be as easy as recording a new tag in your version-control system, and maybe making new tarballs. ## Quick Install Versioneer provides two installation modes. The "classic" vendored mode installs a copy of versioneer into your repository. The experimental build-time dependency mode is intended to allow you to skip this step and simplify the process of upgrading. ### Vendored mode * `pip install versioneer` to somewhere in your $PATH * A [conda-forge recipe](https://github.com/conda-forge/versioneer-feedstock) is available, so you can also use `conda install -c conda-forge versioneer` * add a `[tool.versioneer]` section to your `pyproject.toml` or a `[versioneer]` section to your `setup.cfg` (see [Install](INSTALL.md)) * Note that you will need to add `tomli; python_version < "3.11"` to your build-time dependencies if you use `pyproject.toml` * run `versioneer install --vendor` in your source tree, commit the results * verify version information with `python setup.py version` ### Build-time dependency mode * `pip install versioneer` to somewhere in your $PATH * A [conda-forge recipe](https://github.com/conda-forge/versioneer-feedstock) is available, so you can also use `conda install -c conda-forge versioneer` * add a `[tool.versioneer]` section to your `pyproject.toml` or a `[versioneer]` section to your `setup.cfg` (see [Install](INSTALL.md)) * add `versioneer` (with `[toml]` extra, if configuring in `pyproject.toml`) to the `requires` key of the `build-system` table in `pyproject.toml`: ```toml [build-system] requires = ["setuptools", "versioneer[toml]"] build-backend = "setuptools.build_meta" ``` * run `versioneer install --no-vendor` in your source tree, commit the results * verify version information with `python setup.py version` ## Version Identifiers Source trees come from a variety of places: * a version-control system checkout (mostly used by developers) * a nightly tarball, produced by build automation * a snapshot tarball, produced by a web-based VCS browser, like github's "tarball from tag" feature * a release tarball, produced by "setup.py sdist", distributed through PyPI Within each source tree, the version identifier (either a string or a number, this tool is format-agnostic) can come from a variety of places: * ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows about recent "tags" and an absolute revision-id * the name of the directory into which the tarball was unpacked * an expanded VCS keyword ($Id$, etc) * a `_version.py` created by some earlier build step For released software, the version identifier is closely related to a VCS tag. Some projects use tag names that include more than just the version string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool needs to strip the tag prefix to extract the version identifier. For unreleased software (between tags), the version identifier should provide enough information to help developers recreate the same tree, while also giving them an idea of roughly how old the tree is (after version 1.2, before version 1.3). Many VCS systems can report a description that captures this, for example `git describe --tags --dirty --always` reports things like "0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the 0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has uncommitted changes). The version identifier is used for multiple purposes: * to allow the module to self-identify its version: `myproject.__version__` * to choose a name and prefix for a 'setup.py sdist' tarball ## Theory of Operation Versioneer works by adding a special `_version.py` file into your source tree, where your `__init__.py` can import it. This `_version.py` knows how to dynamically ask the VCS tool for version information at import time. `_version.py` also contains `$Revision$` markers, and the installation process marks `_version.py` to have this marker rewritten with a tag name during the `git archive` command. As a result, generated tarballs will contain enough information to get the proper version. To allow `setup.py` to compute a version too, a `versioneer.py` is added to the top level of your source tree, next to `setup.py` and the `setup.cfg` that configures it. This overrides several distutils/setuptools commands to compute the version when invoked, and changes `setup.py build` and `setup.py sdist` to replace `_version.py` with a small static file that contains just the generated version data. ## Installation See [INSTALL.md](./INSTALL.md) for detailed installation instructions. ## Version-String Flavors Code which uses Versioneer can learn about its version string at runtime by importing `_version` from your main `__init__.py` file and running the `get_versions()` function. From the "outside" (e.g. in `setup.py`), you can import the top-level `versioneer.py` and run `get_versions()`. Both functions return a dictionary with different flavors of version information: * `['version']`: A condensed version string, rendered using the selected style. This is the most commonly used value for the project's version string. The default "pep440" style yields strings like `0.11`, `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section below for alternative styles. * `['full-revisionid']`: detailed revision identifier. For Git, this is the full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". * `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the commit date in ISO 8601 format. This will be None if the date is not available. * `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that this is only accurate if run in a VCS checkout, otherwise it is likely to be False or None * `['error']`: if the version string could not be computed, this will be set to a string describing the problem, otherwise it will be None. It may be useful to throw an exception in setup.py if this is set, to avoid e.g. creating tarballs with a version string of "unknown". Some variants are more useful than others. Including `full-revisionid` in a bug report should allow developers to reconstruct the exact code being tested (or indicate the presence of local changes that should be shared with the developers). `version` is suitable for display in an "about" box or a CLI `--version` output: it can be easily compared against release notes and lists of bugs fixed in various releases. The installer adds the following text to your `__init__.py` to place a basic version in `YOURPROJECT.__version__`: from ._version import get_versions __version__ = get_versions()['version'] del get_versions ## Styles The setup.cfg `style=` configuration controls how the VCS information is rendered into a version string. The default style, "pep440", produces a PEP440-compliant string, equal to the un-prefixed tag name for actual releases, and containing an additional "local version" section with more detail for in-between builds. For Git, this is TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags --dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and that this commit is two revisions ("+2") beyond the "0.11" tag. For released software (exactly equal to a known tag), the identifier will only contain the stripped tag, e.g. "0.11". Other styles are available. See [details.md](details.md) in the Versioneer source tree for descriptions. ## Debugging Versioneer tries to avoid fatal errors: if something goes wrong, it will tend to return a version of "0+unknown". To investigate the problem, run `setup.py version`, which will run the version-lookup code in a verbose mode, and will display the full contents of `get_versions()` (including the `error` string, which may help identify what went wrong). ## Known Limitations Some situations are known to cause problems for Versioneer. This details the most significant ones. More can be found on Github [issues page](https://github.com/python-versioneer/python-versioneer/issues). ### Subprojects Versioneer has limited support for source trees in which `setup.py` is not in the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are two common reasons why `setup.py` might not be in the root: * Source trees which contain multiple subprojects, such as [Buildbot](https://github.com/buildbot/buildbot), which contains both "master" and "slave" subprojects, each with their own `setup.py`, `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI distributions (and upload multiple independently-installable tarballs). * Source trees whose main purpose is to contain a C library, but which also provide bindings to Python (and perhaps other languages) in subdirectories. Versioneer will look for `.git` in parent directories, and most operations should get the right version string. However `pip` and `setuptools` have bugs and implementation details which frequently cause `pip install .` from a subproject directory to fail to find a correct version string (so it usually defaults to `0+unknown`). `pip install --editable .` should work correctly. `setup.py install` might work too. Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in some later version. [Bug #38](https://github.com/python-versioneer/python-versioneer/issues/38) is tracking this issue. The discussion in [PR #61](https://github.com/python-versioneer/python-versioneer/pull/61) describes the issue from the Versioneer side in more detail. [pip PR#3176](https://github.com/pypa/pip/pull/3176) and [pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve pip to let Versioneer work correctly. Versioneer-0.16 and earlier only looked for a `.git` directory next to the `setup.cfg`, so subprojects were completely unsupported with those releases. ### Editable installs with setuptools <= 18.5 `setup.py develop` and `pip install --editable .` allow you to install a project into a virtualenv once, then continue editing the source code (and test) without re-installing after every change. "Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a convenient way to specify executable scripts that should be installed along with the python package. These both work as expected when using modern setuptools. When using setuptools-18.5 or earlier, however, certain operations will cause `pkg_resources.DistributionNotFound` errors when running the entrypoint script, which must be resolved by re-installing the package. This happens when the install happens with one version, then the egg_info data is regenerated while a different version is checked out. Many setup.py commands cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into a different virtualenv), so this can be surprising. [Bug #83](https://github.com/python-versioneer/python-versioneer/issues/83) describes this one, but upgrading to a newer version of setuptools should probably resolve it. ## Updating Versioneer To upgrade your project to a new release of Versioneer, do the following: * install the new Versioneer (`pip install -U versioneer` or equivalent) * edit `setup.cfg` and `pyproject.toml`, if necessary, to include any new configuration settings indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. * re-run `versioneer install --[no-]vendor` in your source tree, to replace `SRC/_version.py` * commit any changed files ## Future Directions This tool is designed to make it easily extended to other version-control systems: all VCS-specific components are in separate directories like src/git/ . The top-level `versioneer.py` script is assembled from these components by running make-versioneer.py . In the future, make-versioneer.py will take a VCS name as an argument, and will construct a version of `versioneer.py` that is specific to the given VCS. It might also take the configuration arguments that are currently provided manually during installation by editing setup.py . Alternatively, it might go the other direction and include code from all supported VCS systems, reducing the number of intermediate scripts. ## Similar projects * [setuptools_scm](https://github.com/pypa/setuptools_scm/) - a non-vendored build-time dependency * [minver](https://github.com/jbweston/miniver) - a lightweight reimplementation of versioneer * [versioningit](https://github.com/jwodder/versioningit) - a PEP 518-based setuptools plugin ## License To make Versioneer easier to embed, all its code is dedicated to the public domain. The `_version.py` that it creates is also in the public domain. Specifically, both are released under the "Unlicense", as described in https://unlicense.org/. [pypi-image]: https://img.shields.io/pypi/v/versioneer.svg [pypi-url]: https://pypi.python.org/pypi/versioneer/ [travis-image]: https://img.shields.io/travis/com/python-versioneer/python-versioneer.svg [travis-url]: https://travis-ci.com/github/python-versioneer/python-versioneer """ # pylint:disable=invalid-name,import-outside-toplevel,missing-function-docstring # pylint:disable=missing-class-docstring,too-many-branches,too-many-statements # pylint:disable=raise-missing-from,too-many-lines,too-many-locals,import-error # pylint:disable=too-few-public-methods,redefined-outer-name,consider-using-with # pylint:disable=attribute-defined-outside-init,too-many-arguments import configparser import errno import json import os import re import subprocess import sys from pathlib import Path from typing import Any, Callable, cast, Dict, List, Optional, Tuple, Union from typing import NoReturn import functools have_tomllib = True if sys.version_info >= (3, 11): import tomllib else: try: import tomli as tomllib except ImportError: have_tomllib = False class VersioneerConfig: """Container for Versioneer configuration parameters.""" VCS: str style: str tag_prefix: str versionfile_source: str versionfile_build: Optional[str] parentdir_prefix: Optional[str] verbose: Optional[bool] def get_root() -> str: """Get the project root directory. We require that all commands are run from the project root, i.e. the directory that contains setup.py, setup.cfg, and versioneer.py . """ root = os.path.realpath(os.path.abspath(os.getcwd())) setup_py = os.path.join(root, "setup.py") pyproject_toml = os.path.join(root, "pyproject.toml") versioneer_py = os.path.join(root, "versioneer.py") if not ( os.path.exists(setup_py) or os.path.exists(pyproject_toml) or os.path.exists(versioneer_py) ): # allow 'python path/to/setup.py COMMAND' root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) setup_py = os.path.join(root, "setup.py") pyproject_toml = os.path.join(root, "pyproject.toml") versioneer_py = os.path.join(root, "versioneer.py") if not ( os.path.exists(setup_py) or os.path.exists(pyproject_toml) or os.path.exists(versioneer_py) ): err = ("Versioneer was unable to run the project root directory. " "Versioneer requires setup.py to be executed from " "its immediate directory (like 'python setup.py COMMAND'), " "or in a way that lets it use sys.argv[0] to find the root " "(like 'python path/to/setup.py COMMAND').") raise VersioneerBadRootError(err) try: # Certain runtime workflows (setup.py install/develop in a setuptools # tree) execute all dependencies in a single python process, so # "versioneer" may be imported multiple times, and python's shared # module-import table will cache the first one. So we can't use # os.path.dirname(__file__), as that will find whichever # versioneer.py was first imported, even in later projects. my_path = os.path.realpath(os.path.abspath(__file__)) me_dir = os.path.normcase(os.path.splitext(my_path)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir and "VERSIONEER_PEP518" not in globals(): print("Warning: build in %s is using versioneer.py from %s" % (os.path.dirname(my_path), versioneer_py)) except NameError: pass return root def get_config_from_root(root: str) -> VersioneerConfig: """Read the project setup.cfg file to determine Versioneer config.""" # This might raise OSError (if setup.cfg is missing), or # configparser.NoSectionError (if it lacks a [versioneer] section), or # configparser.NoOptionError (if it lacks "VCS="). See the docstring at # the top of versioneer.py for instructions on writing your setup.cfg . root_pth = Path(root) pyproject_toml = root_pth / "pyproject.toml" setup_cfg = root_pth / "setup.cfg" section: Union[Dict[str, Any], configparser.SectionProxy, None] = None if pyproject_toml.exists() and have_tomllib: try: with open(pyproject_toml, 'rb') as fobj: pp = tomllib.load(fobj) section = pp['tool']['versioneer'] except (tomllib.TOMLDecodeError, KeyError) as e: print(f"Failed to load config from {pyproject_toml}: {e}") print("Try to load it from setup.cfg") if not section: parser = configparser.ConfigParser() with open(setup_cfg) as cfg_file: parser.read_file(cfg_file) parser.get("versioneer", "VCS") # raise error if missing section = parser["versioneer"] # `cast`` really shouldn't be used, but its simplest for the # common VersioneerConfig users at the moment. We verify against # `None` values elsewhere where it matters cfg = VersioneerConfig() cfg.VCS = section['VCS'] cfg.style = section.get("style", "") cfg.versionfile_source = cast(str, section.get("versionfile_source")) cfg.versionfile_build = section.get("versionfile_build") cfg.tag_prefix = cast(str, section.get("tag_prefix")) if cfg.tag_prefix in ("''", '""', None): cfg.tag_prefix = "" cfg.parentdir_prefix = section.get("parentdir_prefix") if isinstance(section, configparser.SectionProxy): # Make sure configparser translates to bool cfg.verbose = section.getboolean("verbose") else: cfg.verbose = section.get("verbose") return cfg class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" # these dictionaries contain VCS-specific tools LONG_VERSION_PY: Dict[str, str] = {} HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs: str, method: str) -> Callable: # decorator """Create decorator to mark a method as the handler of a VCS.""" def decorate(f: Callable) -> Callable: """Store f in HANDLERS[vcs][method].""" HANDLERS.setdefault(vcs, {})[method] = f return f return decorate def run_command( commands: List[str], args: List[str], cwd: Optional[str] = None, verbose: bool = False, hide_stderr: bool = False, env: Optional[Dict[str, str]] = None, ) -> Tuple[Optional[str], Optional[int]]: """Call the given command(s).""" assert isinstance(commands, list) process = None popen_kwargs: Dict[str, Any] = {} if sys.platform == "win32": # This hides the console window if pythonw.exe is used startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW popen_kwargs["startupinfo"] = startupinfo for command in commands: try: dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git process = subprocess.Popen([command] + args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None), **popen_kwargs) break except OSError as e: if e.errno == errno.ENOENT: continue if verbose: print("unable to run %s" % dispcmd) print(e) return None, None else: if verbose: print("unable to find command, tried %s" % (commands,)) return None, None stdout = process.communicate()[0].strip().decode() if process.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) print("stdout was %s" % stdout) return None, process.returncode return stdout, process.returncode LONG_VERSION_PY['git'] = r''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. # Generated by versioneer-0.29 # https://github.com/python-versioneer/python-versioneer """Git implementation of _version.py.""" import errno import os import re import subprocess import sys from typing import Any, Callable, Dict, List, Optional, Tuple import functools def get_keywords() -> Dict[str, str]: """Get the keywords needed to look up the version information.""" # these strings will be replaced by git during git-archive. # setup.py/versioneer.py will grep for the variable names, so they must # each be defined on a line of their own. _version.py will just call # get_keywords(). git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} return keywords class VersioneerConfig: """Container for Versioneer configuration parameters.""" VCS: str style: str tag_prefix: str parentdir_prefix: str versionfile_source: str verbose: bool def get_config() -> VersioneerConfig: """Create, populate and return the VersioneerConfig() object.""" # these strings are filled in when 'setup.py versioneer' creates # _version.py cfg = VersioneerConfig() cfg.VCS = "git" cfg.style = "%(STYLE)s" cfg.tag_prefix = "%(TAG_PREFIX)s" cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" cfg.verbose = False return cfg class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" LONG_VERSION_PY: Dict[str, str] = {} HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs: str, method: str) -> Callable: # decorator """Create decorator to mark a method as the handler of a VCS.""" def decorate(f: Callable) -> Callable: """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f return decorate def run_command( commands: List[str], args: List[str], cwd: Optional[str] = None, verbose: bool = False, hide_stderr: bool = False, env: Optional[Dict[str, str]] = None, ) -> Tuple[Optional[str], Optional[int]]: """Call the given command(s).""" assert isinstance(commands, list) process = None popen_kwargs: Dict[str, Any] = {} if sys.platform == "win32": # This hides the console window if pythonw.exe is used startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW popen_kwargs["startupinfo"] = startupinfo for command in commands: try: dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git process = subprocess.Popen([command] + args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None), **popen_kwargs) break except OSError as e: if e.errno == errno.ENOENT: continue if verbose: print("unable to run %%s" %% dispcmd) print(e) return None, None else: if verbose: print("unable to find command, tried %%s" %% (commands,)) return None, None stdout = process.communicate()[0].strip().decode() if process.returncode != 0: if verbose: print("unable to run %%s (error)" %% dispcmd) print("stdout was %%s" %% stdout) return None, process.returncode return stdout, process.returncode def versions_from_parentdir( parentdir_prefix: str, root: str, verbose: bool, ) -> Dict[str, Any]: """Try to determine the version from the parent directory name. Source tarballs conventionally unpack into a directory that includes both the project name and a version string. We will also support searching up two directory levels for an appropriately named parent directory """ rootdirs = [] for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: print("Tried directories %%s but none started with prefix %%s" %% (str(rootdirs), parentdir_prefix)) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @register_vcs_handler("git", "get_keywords") def git_get_keywords(versionfile_abs: str) -> Dict[str, str]: """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from # _version.py. keywords: Dict[str, str] = {} try: with open(versionfile_abs, "r") as fobj: for line in fobj: if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["full"] = mo.group(1) if line.strip().startswith("git_date ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["date"] = mo.group(1) except OSError: pass return keywords @register_vcs_handler("git", "keywords") def git_versions_from_keywords( keywords: Dict[str, str], tag_prefix: str, verbose: bool, ) -> Dict[str, Any]: """Get version information from git keywords.""" if "refnames" not in keywords: raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because # it's been around since git-1.5.3, and it's too difficult to # discover which version we're using, or to work around using an # older one. date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %%d # expansion behaves like git log --decorate=short and strips out the # refs/heads/ and refs/tags/ prefixes that would let us distinguish # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%%s', no digits" %% ",".join(refs - tags)) if verbose: print("likely tags: %%s" %% ",".join(sorted(tags))) for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] # Filter out refs that exactly match prefix or that don't start # with a number once the prefix is stripped (mostly a concern # when prefix is '') if not re.match(r'\d', r): continue if verbose: print("picking %%s" %% r) return {"version": r, "full-revisionid": keywords["full"].strip(), "dirty": False, "error": None, "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") return {"version": "0+unknown", "full-revisionid": keywords["full"].strip(), "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs( tag_prefix: str, root: str, verbose: bool, runner: Callable = run_command ) -> Dict[str, Any]: """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* expanded, and _version.py hasn't already been rewritten with a short version string, meaning we're inside a checked out source tree. """ GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] # GIT_DIR can interfere with correct operation of Versioneer. # It may be intended to be passed to the Versioneer-versioned project, # but that should not change where we get our version from. env = os.environ.copy() env.pop("GIT_DIR", None) runner = functools.partial(runner, env=env) _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %%s not under git control" %% root) raise NotThisMethod("'git rev-parse --git-dir' returned error") # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) describe_out, rc = runner(GITS, [ "describe", "--tags", "--dirty", "--always", "--long", "--match", f"{tag_prefix}[[:digit:]]*" ], cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() pieces: Dict[str, Any] = {} pieces["long"] = full_out pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) # --abbrev-ref was added in git-1.6.3 if rc != 0 or branch_name is None: raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") branch_name = branch_name.strip() if branch_name == "HEAD": # If we aren't exactly on a branch, pick a branch which represents # the current commit. If all else fails, we are on a branchless # commit. branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) # --contains was added in git-1.5.4 if rc != 0 or branches is None: raise NotThisMethod("'git branch --contains' returned error") branches = branches.split("\n") # Remove the first line if we're running detached if "(" in branches[0]: branches.pop(0) # Strip off the leading "* " from the list of branches. branches = [branch[2:] for branch in branches] if "master" in branches: branch_name = "master" elif not branches: branch_name = None else: # Pick the first branch that is returned. Good or bad. branch_name = branches[0] pieces["branch"] = branch_name # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out # look for -dirty suffix dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: git_describe = git_describe[:git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%%s'" %% describe_out) return pieces # tag full_tag = mo.group(1) if not full_tag.startswith(tag_prefix): if verbose: fmt = "tag '%%s' doesn't start with prefix '%%s'" print(fmt %% (full_tag, tag_prefix)) pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" %% (full_tag, tag_prefix)) return pieces pieces["closest-tag"] = full_tag[len(tag_prefix):] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) # commit: short hex revision ID pieces["short"] = mo.group(3) else: # HEX: no tags pieces["closest-tag"] = None out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() date = runner(GITS, ["show", "-s", "--format=%%ci", "HEAD"], cwd=root)[0].strip() # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces def plus_or_dot(pieces: Dict[str, Any]) -> str: """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" def render_pep440(pieces: Dict[str, Any]) -> str: """Build up version string, with post-release "local version identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty Exceptions: 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += plus_or_dot(pieces) rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_branch(pieces: Dict[str, Any]) -> str: """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . The ".dev0" means not master branch. Note that .dev0 sorts backwards (a feature branch will appear "older" than the master branch). Exceptions: 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0" if pieces["branch"] != "master": rendered += ".dev0" rendered += "+untagged.%%d.g%%s" %% (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def pep440_split_post(ver: str) -> Tuple[str, Optional[int]]: """Split pep440 version string at the post-release segment. Returns the release segments before the post-release and the post-release version number (or -1 if no post-release segment is present). """ vc = str.split(ver, ".post") return vc[0], int(vc[1] or 0) if len(vc) == 2 else None def render_pep440_pre(pieces: Dict[str, Any]) -> str: """TAG[.postN.devDISTANCE] -- No -dirty. Exceptions: 1: no tags. 0.post0.devDISTANCE """ if pieces["closest-tag"]: if pieces["distance"]: # update the post release segment tag_version, post_version = pep440_split_post(pieces["closest-tag"]) rendered = tag_version if post_version is not None: rendered += ".post%%d.dev%%d" %% (post_version + 1, pieces["distance"]) else: rendered += ".post0.dev%%d" %% (pieces["distance"]) else: # no commits, use the tag as the version rendered = pieces["closest-tag"] else: # exception #1 rendered = "0.post0.dev%%d" %% pieces["distance"] return rendered def render_pep440_post(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that .dev0 sorts backwards (a dirty tree will appear "older" than the corresponding clean one), but you shouldn't be releasing software with -dirty anyways. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%%d" %% pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%%s" %% pieces["short"] else: # exception #1 rendered = "0.post%%d" %% pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += "+g%%s" %% pieces["short"] return rendered def render_pep440_post_branch(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . The ".dev0" means not master branch. Exceptions: 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%%d" %% pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%%s" %% pieces["short"] if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0.post%%d" %% pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += "+g%%s" %% pieces["short"] if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_old(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%%d" %% pieces["distance"] if pieces["dirty"]: rendered += ".dev0" else: # exception #1 rendered = "0.post%%d" %% pieces["distance"] if pieces["dirty"]: rendered += ".dev0" return rendered def render_git_describe(pieces: Dict[str, Any]) -> str: """TAG[-DISTANCE-gHEX][-dirty]. Like 'git describe --tags --dirty --always'. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"]: rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render_git_describe_long(pieces: Dict[str, Any]) -> str: """TAG-DISTANCE-gHEX[-dirty]. Like 'git describe --tags --dirty --always -long'. The distance/hash is unconditional. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: """Render the given version pieces into the requested style.""" if pieces["error"]: return {"version": "unknown", "full-revisionid": pieces.get("long"), "dirty": None, "error": pieces["error"], "date": None} if not style or style == "default": style = "pep440" # the default if style == "pep440": rendered = render_pep440(pieces) elif style == "pep440-branch": rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) elif style == "pep440-post-branch": rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": rendered = render_git_describe(pieces) elif style == "git-describe-long": rendered = render_git_describe_long(pieces) else: raise ValueError("unknown style '%%s'" %% style) return {"version": rendered, "full-revisionid": pieces["long"], "dirty": pieces["dirty"], "error": None, "date": pieces.get("date")} def get_versions() -> Dict[str, Any]: """Get version information or return default if unable to do so.""" # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have # __file__, we can work backwards from there to the root. Some # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which # case we can only use expanded keywords. cfg = get_config() verbose = cfg.verbose try: return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) except NotThisMethod: pass try: root = os.path.realpath(__file__) # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. for _ in cfg.versionfile_source.split('/'): root = os.path.dirname(root) except NameError: return {"version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to find root of source tree", "date": None} try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) return render(pieces, cfg.style) except NotThisMethod: pass try: if cfg.parentdir_prefix: return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) except NotThisMethod: pass return {"version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to compute version", "date": None} ''' @register_vcs_handler("git", "get_keywords") def git_get_keywords(versionfile_abs: str) -> Dict[str, str]: """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from # _version.py. keywords: Dict[str, str] = {} try: with open(versionfile_abs, "r") as fobj: for line in fobj: if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["full"] = mo.group(1) if line.strip().startswith("git_date ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["date"] = mo.group(1) except OSError: pass return keywords @register_vcs_handler("git", "keywords") def git_versions_from_keywords( keywords: Dict[str, str], tag_prefix: str, verbose: bool, ) -> Dict[str, Any]: """Get version information from git keywords.""" if "refnames" not in keywords: raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because # it's been around since git-1.5.3, and it's too difficult to # discover which version we're using, or to work around using an # older one. date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d # expansion behaves like git log --decorate=short and strips out the # refs/heads/ and refs/tags/ prefixes that would let us distinguish # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] # Filter out refs that exactly match prefix or that don't start # with a number once the prefix is stripped (mostly a concern # when prefix is '') if not re.match(r'\d', r): continue if verbose: print("picking %s" % r) return {"version": r, "full-revisionid": keywords["full"].strip(), "dirty": False, "error": None, "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") return {"version": "0+unknown", "full-revisionid": keywords["full"].strip(), "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs( tag_prefix: str, root: str, verbose: bool, runner: Callable = run_command ) -> Dict[str, Any]: """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* expanded, and _version.py hasn't already been rewritten with a short version string, meaning we're inside a checked out source tree. """ GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] # GIT_DIR can interfere with correct operation of Versioneer. # It may be intended to be passed to the Versioneer-versioned project, # but that should not change where we get our version from. env = os.environ.copy() env.pop("GIT_DIR", None) runner = functools.partial(runner, env=env) _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) raise NotThisMethod("'git rev-parse --git-dir' returned error") # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) describe_out, rc = runner(GITS, [ "describe", "--tags", "--dirty", "--always", "--long", "--match", f"{tag_prefix}[[:digit:]]*" ], cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() pieces: Dict[str, Any] = {} pieces["long"] = full_out pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) # --abbrev-ref was added in git-1.6.3 if rc != 0 or branch_name is None: raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") branch_name = branch_name.strip() if branch_name == "HEAD": # If we aren't exactly on a branch, pick a branch which represents # the current commit. If all else fails, we are on a branchless # commit. branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) # --contains was added in git-1.5.4 if rc != 0 or branches is None: raise NotThisMethod("'git branch --contains' returned error") branches = branches.split("\n") # Remove the first line if we're running detached if "(" in branches[0]: branches.pop(0) # Strip off the leading "* " from the list of branches. branches = [branch[2:] for branch in branches] if "master" in branches: branch_name = "master" elif not branches: branch_name = None else: # Pick the first branch that is returned. Good or bad. branch_name = branches[0] pieces["branch"] = branch_name # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out # look for -dirty suffix dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: git_describe = git_describe[:git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%s'" % describe_out) return pieces # tag full_tag = mo.group(1) if not full_tag.startswith(tag_prefix): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" % (full_tag, tag_prefix)) return pieces pieces["closest-tag"] = full_tag[len(tag_prefix):] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) # commit: short hex revision ID pieces["short"] = mo.group(3) else: # HEX: no tags pieces["closest-tag"] = None out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces def do_vcs_install(versionfile_source: str, ipy: Optional[str]) -> None: """Git-specific installation logic for Versioneer. For Git, this means creating/changing .gitattributes to mark _version.py for export-subst keyword substitution. """ GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] files = [versionfile_source] if ipy: files.append(ipy) if "VERSIONEER_PEP518" not in globals(): try: my_path = __file__ if my_path.endswith((".pyc", ".pyo")): my_path = os.path.splitext(my_path)[0] + ".py" versioneer_file = os.path.relpath(my_path) except NameError: versioneer_file = "versioneer.py" files.append(versioneer_file) present = False try: with open(".gitattributes", "r") as fobj: for line in fobj: if line.strip().startswith(versionfile_source): if "export-subst" in line.strip().split()[1:]: present = True break except OSError: pass if not present: with open(".gitattributes", "a+") as fobj: fobj.write(f"{versionfile_source} export-subst\n") files.append(".gitattributes") run_command(GITS, ["add", "--"] + files) def versions_from_parentdir( parentdir_prefix: str, root: str, verbose: bool, ) -> Dict[str, Any]: """Try to determine the version from the parent directory name. Source tarballs conventionally unpack into a directory that includes both the project name and a version string. We will also support searching up two directory levels for an appropriately named parent directory """ rootdirs = [] for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: print("Tried directories %s but none started with prefix %s" % (str(rootdirs), parentdir_prefix)) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") SHORT_VERSION_PY = """ # This file was generated by 'versioneer.py' (0.29) from # revision-control system data, or from the parent directory name of an # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. import json version_json = ''' %s ''' # END VERSION_JSON def get_versions(): return json.loads(version_json) """ def versions_from_file(filename: str) -> Dict[str, Any]: """Try to determine the version from _version.py if present.""" try: with open(filename) as f: contents = f.read() except OSError: raise NotThisMethod("unable to read _version.py") mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S) if not mo: mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", contents, re.M | re.S) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) def write_to_version_file(filename: str, versions: Dict[str, Any]) -> None: """Write the given version number to the given _version.py file.""" contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) print("set %s to '%s'" % (filename, versions["version"])) def plus_or_dot(pieces: Dict[str, Any]) -> str: """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" def render_pep440(pieces: Dict[str, Any]) -> str: """Build up version string, with post-release "local version identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty Exceptions: 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += plus_or_dot(pieces) rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_branch(pieces: Dict[str, Any]) -> str: """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . The ".dev0" means not master branch. Note that .dev0 sorts backwards (a feature branch will appear "older" than the master branch). Exceptions: 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0" if pieces["branch"] != "master": rendered += ".dev0" rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def pep440_split_post(ver: str) -> Tuple[str, Optional[int]]: """Split pep440 version string at the post-release segment. Returns the release segments before the post-release and the post-release version number (or -1 if no post-release segment is present). """ vc = str.split(ver, ".post") return vc[0], int(vc[1] or 0) if len(vc) == 2 else None def render_pep440_pre(pieces: Dict[str, Any]) -> str: """TAG[.postN.devDISTANCE] -- No -dirty. Exceptions: 1: no tags. 0.post0.devDISTANCE """ if pieces["closest-tag"]: if pieces["distance"]: # update the post release segment tag_version, post_version = pep440_split_post(pieces["closest-tag"]) rendered = tag_version if post_version is not None: rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) else: rendered += ".post0.dev%d" % (pieces["distance"]) else: # no commits, use the tag as the version rendered = pieces["closest-tag"] else: # exception #1 rendered = "0.post0.dev%d" % pieces["distance"] return rendered def render_pep440_post(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that .dev0 sorts backwards (a dirty tree will appear "older" than the corresponding clean one), but you shouldn't be releasing software with -dirty anyways. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%s" % pieces["short"] else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += "+g%s" % pieces["short"] return rendered def render_pep440_post_branch(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . The ".dev0" means not master branch. Exceptions: 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%s" % pieces["short"] if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += "+g%s" % pieces["short"] if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_old(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" return rendered def render_git_describe(pieces: Dict[str, Any]) -> str: """TAG[-DISTANCE-gHEX][-dirty]. Like 'git describe --tags --dirty --always'. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"]: rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render_git_describe_long(pieces: Dict[str, Any]) -> str: """TAG-DISTANCE-gHEX[-dirty]. Like 'git describe --tags --dirty --always -long'. The distance/hash is unconditional. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: """Render the given version pieces into the requested style.""" if pieces["error"]: return {"version": "unknown", "full-revisionid": pieces.get("long"), "dirty": None, "error": pieces["error"], "date": None} if not style or style == "default": style = "pep440" # the default if style == "pep440": rendered = render_pep440(pieces) elif style == "pep440-branch": rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) elif style == "pep440-post-branch": rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": rendered = render_git_describe(pieces) elif style == "git-describe-long": rendered = render_git_describe_long(pieces) else: raise ValueError("unknown style '%s'" % style) return {"version": rendered, "full-revisionid": pieces["long"], "dirty": pieces["dirty"], "error": None, "date": pieces.get("date")} class VersioneerBadRootError(Exception): """The project root directory is unknown or missing key files.""" def get_versions(verbose: bool = False) -> Dict[str, Any]: """Get the project version from whatever source is available. Returns dict with two keys: 'version' and 'full'. """ if "versioneer" in sys.modules: # see the discussion in cmdclass.py:get_cmdclass() del sys.modules["versioneer"] root = get_root() cfg = get_config_from_root(root) assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS verbose = verbose or bool(cfg.verbose) # `bool()` used to avoid `None` assert cfg.versionfile_source is not None, \ "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" versionfile_abs = os.path.join(root, cfg.versionfile_source) # extract version from first of: _version.py, VCS command (e.g. 'git # describe'), parentdir. This is meant to work for developers using a # source checkout, for users of a tarball created by 'setup.py sdist', # and for users of a tarball/zipball created by 'git archive' or github's # download-from-tag feature or the equivalent in other VCSes. get_keywords_f = handlers.get("get_keywords") from_keywords_f = handlers.get("keywords") if get_keywords_f and from_keywords_f: try: keywords = get_keywords_f(versionfile_abs) ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) if verbose: print("got version from expanded keyword %s" % ver) return ver except NotThisMethod: pass try: ver = versions_from_file(versionfile_abs) if verbose: print("got version from file %s %s" % (versionfile_abs, ver)) return ver except NotThisMethod: pass from_vcs_f = handlers.get("pieces_from_vcs") if from_vcs_f: try: pieces = from_vcs_f(cfg.tag_prefix, root, verbose) ver = render(pieces, cfg.style) if verbose: print("got version from VCS %s" % ver) return ver except NotThisMethod: pass try: if cfg.parentdir_prefix: ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) if verbose: print("got version from parentdir %s" % ver) return ver except NotThisMethod: pass if verbose: print("unable to compute version") return {"version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to compute version", "date": None} def get_version() -> str: """Get the short version string for this project.""" return get_versions()["version"] def get_cmdclass(cmdclass: Optional[Dict[str, Any]] = None): """Get the custom setuptools subclasses used by Versioneer. If the package uses a different cmdclass (e.g. one from numpy), it should be provide as an argument. """ if "versioneer" in sys.modules: del sys.modules["versioneer"] # this fixes the "python setup.py develop" case (also 'install' and # 'easy_install .'), in which subdependencies of the main project are # built (using setup.py bdist_egg) in the same python process. Assume # a main project A and a dependency B, which use different versions # of Versioneer. A's setup.py imports A's Versioneer, leaving it in # sys.modules by the time B's setup.py is executed, causing B to run # with the wrong versioneer. Setuptools wraps the sub-dep builds in a # sandbox that restores sys.modules to it's pre-build state, so the # parent is protected against the child's "import versioneer". By # removing ourselves from sys.modules here, before the child build # happens, we protect the child from the parent's versioneer too. # Also see https://github.com/python-versioneer/python-versioneer/issues/52 cmds = {} if cmdclass is None else cmdclass.copy() # we add "version" to setuptools from setuptools import Command class cmd_version(Command): description = "report generated version string" user_options: List[Tuple[str, str, str]] = [] boolean_options: List[str] = [] def initialize_options(self) -> None: pass def finalize_options(self) -> None: pass def run(self) -> None: vers = get_versions(verbose=True) print("Version: %s" % vers["version"]) print(" full-revisionid: %s" % vers.get("full-revisionid")) print(" dirty: %s" % vers.get("dirty")) print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) cmds["version"] = cmd_version # we override "build_py" in setuptools # # most invocation pathways end up running build_py: # distutils/build -> build_py # distutils/install -> distutils/build ->.. # setuptools/bdist_wheel -> distutils/install ->.. # setuptools/bdist_egg -> distutils/install_lib -> build_py # setuptools/install -> bdist_egg ->.. # setuptools/develop -> ? # pip install: # copies source tree to a tempdir before running egg_info/etc # if .git isn't copied too, 'git describe' will fail # then does setup.py bdist_wheel, or sometimes setup.py install # setup.py egg_info -> ? # pip install -e . and setuptool/editable_wheel will invoke build_py # but the build_py command is not expected to copy any files. # we override different "build_py" commands for both environments if 'build_py' in cmds: _build_py: Any = cmds['build_py'] else: from setuptools.command.build_py import build_py as _build_py class cmd_build_py(_build_py): def run(self) -> None: root = get_root() cfg = get_config_from_root(root) versions = get_versions() _build_py.run(self) if getattr(self, "editable_mode", False): # During editable installs `.py` and data files are # not copied to build_lib return # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) cmds["build_py"] = cmd_build_py if 'build_ext' in cmds: _build_ext: Any = cmds['build_ext'] else: from setuptools.command.build_ext import build_ext as _build_ext class cmd_build_ext(_build_ext): def run(self) -> None: root = get_root() cfg = get_config_from_root(root) versions = get_versions() _build_ext.run(self) if self.inplace: # build_ext --inplace will only build extensions in # build/lib<..> dir with no _version.py to write to. # As in place builds will already have a _version.py # in the module dir, we do not need to write one. return # now locate _version.py in the new build/ directory and replace # it with an updated value if not cfg.versionfile_build: return target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) if not os.path.exists(target_versionfile): print(f"Warning: {target_versionfile} does not exist, skipping " "version update. This can happen if you are running build_ext " "without first running build_py.") return print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) cmds["build_ext"] = cmd_build_ext if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe # type: ignore # nczeczulin reports that py2exe won't like the pep440-style string # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. # setup(console=[{ # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION # "product_version": versioneer.get_version(), # ... class cmd_build_exe(_build_exe): def run(self) -> None: root = get_root() cfg = get_config_from_root(root) versions = get_versions() target_versionfile = cfg.versionfile_source print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) _build_exe.run(self) os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] f.write(LONG % {"DOLLAR": "$", "STYLE": cfg.style, "TAG_PREFIX": cfg.tag_prefix, "PARENTDIR_PREFIX": cfg.parentdir_prefix, "VERSIONFILE_SOURCE": cfg.versionfile_source, }) cmds["build_exe"] = cmd_build_exe del cmds["build_py"] if 'py2exe' in sys.modules: # py2exe enabled? try: from py2exe.setuptools_buildexe import py2exe as _py2exe # type: ignore except ImportError: from py2exe.distutils_buildexe import py2exe as _py2exe # type: ignore class cmd_py2exe(_py2exe): def run(self) -> None: root = get_root() cfg = get_config_from_root(root) versions = get_versions() target_versionfile = cfg.versionfile_source print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) _py2exe.run(self) os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] f.write(LONG % {"DOLLAR": "$", "STYLE": cfg.style, "TAG_PREFIX": cfg.tag_prefix, "PARENTDIR_PREFIX": cfg.parentdir_prefix, "VERSIONFILE_SOURCE": cfg.versionfile_source, }) cmds["py2exe"] = cmd_py2exe # sdist farms its file list building out to egg_info if 'egg_info' in cmds: _egg_info: Any = cmds['egg_info'] else: from setuptools.command.egg_info import egg_info as _egg_info class cmd_egg_info(_egg_info): def find_sources(self) -> None: # egg_info.find_sources builds the manifest list and writes it # in one shot super().find_sources() # Modify the filelist and normalize it root = get_root() cfg = get_config_from_root(root) self.filelist.append('versioneer.py') if cfg.versionfile_source: # There are rare cases where versionfile_source might not be # included by default, so we must be explicit self.filelist.append(cfg.versionfile_source) self.filelist.sort() self.filelist.remove_duplicates() # The write method is hidden in the manifest_maker instance that # generated the filelist and was thrown away # We will instead replicate their final normalization (to unicode, # and POSIX-style paths) from setuptools import unicode_utils normalized = [unicode_utils.filesys_decode(f).replace(os.sep, '/') for f in self.filelist.files] manifest_filename = os.path.join(self.egg_info, 'SOURCES.txt') with open(manifest_filename, 'w') as fobj: fobj.write('\n'.join(normalized)) cmds['egg_info'] = cmd_egg_info # we override different "sdist" commands for both environments if 'sdist' in cmds: _sdist: Any = cmds['sdist'] else: from setuptools.command.sdist import sdist as _sdist class cmd_sdist(_sdist): def run(self) -> None: versions = get_versions() self._versioneer_generated_versions = versions # unless we update this, the command will keep using the old # version self.distribution.metadata.version = versions["version"] return _sdist.run(self) def make_release_tree(self, base_dir: str, files: List[str]) -> None: root = get_root() cfg = get_config_from_root(root) _sdist.make_release_tree(self, base_dir, files) # now locate _version.py in the new base_dir directory # (remembering that it may be a hardlink) and replace it with an # updated value target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, self._versioneer_generated_versions) cmds["sdist"] = cmd_sdist return cmds CONFIG_ERROR = """ setup.cfg is missing the necessary Versioneer configuration. You need a section like: [versioneer] VCS = git style = pep440 versionfile_source = src/myproject/_version.py versionfile_build = myproject/_version.py tag_prefix = parentdir_prefix = myproject- You will also need to edit your setup.py to use the results: import versioneer setup(version=versioneer.get_version(), cmdclass=versioneer.get_cmdclass(), ...) Please read the docstring in ./versioneer.py for configuration instructions, edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. """ SAMPLE_CONFIG = """ # See the docstring in versioneer.py for instructions. Note that you must # re-run 'versioneer.py setup' after changing this section, and commit the # resulting files. [versioneer] #VCS = git #style = pep440 #versionfile_source = #versionfile_build = #tag_prefix = #parentdir_prefix = """ OLD_SNIPPET = """ from ._version import get_versions __version__ = get_versions()['version'] del get_versions """ INIT_PY_SNIPPET = """ from . import {0} __version__ = {0}.get_versions()['version'] """ def do_setup() -> int: """Do main VCS-independent setup function for installing Versioneer.""" root = get_root() try: cfg = get_config_from_root(root) except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e: if isinstance(e, (OSError, configparser.NoSectionError)): print("Adding sample versioneer config to setup.cfg", file=sys.stderr) with open(os.path.join(root, "setup.cfg"), "a") as f: f.write(SAMPLE_CONFIG) print(CONFIG_ERROR, file=sys.stderr) return 1 print(" creating %s" % cfg.versionfile_source) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] f.write(LONG % {"DOLLAR": "$", "STYLE": cfg.style, "TAG_PREFIX": cfg.tag_prefix, "PARENTDIR_PREFIX": cfg.parentdir_prefix, "VERSIONFILE_SOURCE": cfg.versionfile_source, }) ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") maybe_ipy: Optional[str] = ipy if os.path.exists(ipy): try: with open(ipy, "r") as f: old = f.read() except OSError: old = "" module = os.path.splitext(os.path.basename(cfg.versionfile_source))[0] snippet = INIT_PY_SNIPPET.format(module) if OLD_SNIPPET in old: print(" replacing boilerplate in %s" % ipy) with open(ipy, "w") as f: f.write(old.replace(OLD_SNIPPET, snippet)) elif snippet not in old: print(" appending to %s" % ipy) with open(ipy, "a") as f: f.write(snippet) else: print(" %s unmodified" % ipy) else: print(" %s doesn't exist, ok" % ipy) maybe_ipy = None # Make VCS-specific changes. For git, this means creating/changing # .gitattributes to mark _version.py for export-subst keyword # substitution. do_vcs_install(cfg.versionfile_source, maybe_ipy) return 0 def scan_setup_py() -> int: """Validate the contents of setup.py against Versioneer's expectations.""" found = set() setters = False errors = 0 with open("setup.py", "r") as f: for line in f.readlines(): if "import versioneer" in line: found.add("import") if "versioneer.get_cmdclass()" in line: found.add("cmdclass") if "versioneer.get_version()" in line: found.add("get_version") if "versioneer.VCS" in line: setters = True if "versioneer.versionfile_source" in line: setters = True if len(found) != 3: print("") print("Your setup.py appears to be missing some important items") print("(but I might be wrong). Please make sure it has something") print("roughly like the following:") print("") print(" import versioneer") print(" setup( version=versioneer.get_version(),") print(" cmdclass=versioneer.get_cmdclass(), ...)") print("") errors += 1 if setters: print("You should remove lines like 'versioneer.VCS = ' and") print("'versioneer.versionfile_source = ' . This configuration") print("now lives in setup.cfg, and should be removed from setup.py") print("") errors += 1 return errors def setup_command() -> NoReturn: """Set up Versioneer and exit with appropriate error code.""" errors = do_setup() errors += scan_setup_py() sys.exit(1 if errors else 0) if __name__ == "__main__": cmd = sys.argv[1] if cmd == "setup": setup_command()