pax_global_header00006660000000000000000000000064147150565240014523gustar00rootroot0000000000000052 comment=c4f3c7791e4f0ea66be8f59e3f675fa59dc97a83 cads-api-client-1.5.4/000077500000000000000000000000001471505652400144675ustar00rootroot00000000000000cads-api-client-1.5.4/.cruft.json000066400000000000000000000011431471505652400165620ustar00rootroot00000000000000{ "template": "https://github.com/ecmwf-projects/cookiecutter-conda-package", "commit": "3b3ac4ab7d8039431b21fec307a590e995c83dfa", "checkout": null, "context": { "cookiecutter": { "project_name": "cads-api-client", "project_slug": "cads_api_client", "project_short_description": "CADS API Python client", "copyright_holder": "European Union", "copyright_year": "2022", "mypy_strict": "True", "integration_tests": "True", "pypi": true, "_template": "https://github.com/ecmwf-projects/cookiecutter-conda-package" } }, "directory": null } cads-api-client-1.5.4/.github/000077500000000000000000000000001471505652400160275ustar00rootroot00000000000000cads-api-client-1.5.4/.github/workflows/000077500000000000000000000000001471505652400200645ustar00rootroot00000000000000cads-api-client-1.5.4/.github/workflows/on-push.yml000066400000000000000000000153411471505652400222040ustar00rootroot00000000000000name: on-push on: push: branches: - main tags: - '*' pull_request: branches: - main concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true defaults: run: shell: bash -l {0} jobs: pre-commit: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: 3.x - uses: pre-commit/action@v3.0.1 combine-environments: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: 3.x - name: Install conda-merge run: | python -m pip install conda-merge - name: Combine environments run: | for SUFFIX in ci integration; do conda-merge ci/environment-$SUFFIX.yml environment.yml > ci/combined-environment-$SUFFIX.yml || exit done - uses: actions/upload-artifact@v4 with: name: combined-environments path: ci/combined-environment-*.yml unit-tests: name: unit-tests needs: combine-environments runs-on: ubuntu-latest strategy: matrix: python-version: ['3.11'] steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: name: combined-environments path: ci - name: Get current date id: date run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}" - uses: mamba-org/setup-micromamba@v2 with: environment-file: ci/combined-environment-ci.yml environment-name: DEVELOP cache-environment: true cache-environment-key: environment-${{ steps.date.outputs.date }} cache-downloads-key: downloads-${{ steps.date.outputs.date }} create-args: >- python=${{ matrix.python-version }} - name: Install package run: | python -m pip install --no-deps -e . - name: Run tests run: | make unit-tests COV_REPORT=xml type-check: needs: [combine-environments, unit-tests] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: name: combined-environments path: ci - name: Get current date id: date run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}" - uses: mamba-org/setup-micromamba@v2 with: environment-file: ci/combined-environment-ci.yml environment-name: DEVELOP cache-environment: true cache-environment-key: environment-${{ steps.date.outputs.date }} cache-downloads-key: downloads-${{ steps.date.outputs.date }} create-args: >- python=3.11 - name: Install package run: | python -m pip install --no-deps -e . - name: Run code quality checks run: | make type-check docs-build: needs: [combine-environments, unit-tests] runs-on: ubuntu-latest permissions: contents: write steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: name: combined-environments path: ci - name: Get current date id: date run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}" - uses: mamba-org/setup-micromamba@v2 with: environment-file: ci/combined-environment-ci.yml environment-name: DEVELOP cache-environment: true cache-environment-key: environment-${{ steps.date.outputs.date }} cache-downloads-key: downloads-${{ steps.date.outputs.date }} create-args: >- python=3.11 - name: Install package run: | python -m pip install --no-deps -e . - name: Build documentation run: | make docs-build - name: Deploy documentation uses: peaceiris/actions-gh-pages@v4 if: | github.event_name == 'push' && startsWith(github.ref, 'refs/tags') with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./docs/_build/html/ integration-tests: needs: [combine-environments, unit-tests] if: | success() && true runs-on: ubuntu-latest strategy: matrix: python-version: ['3.8', '3.13'] extra: ['-ci'] steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: name: combined-environments path: ci - name: Get current date id: date run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}" - uses: mamba-org/setup-micromamba@v2 with: environment-file: ci/combined-environment${{ matrix.extra }}.yml environment-name: DEVELOP${{ matrix.extra }} cache-environment: true cache-environment-key: environment-${{ steps.date.outputs.date }} cache-downloads-key: downloads-${{ steps.date.outputs.date }} create-args: >- python=${{ matrix.python-version }} - name: Install package run: | python -m pip install --no-deps -e . - name: Run tests env: CADS_API_URL: ${{ secrets.CADS_API_URL }} CADS_API_KEY: ${{ secrets.CADS_API_KEY }} run: | make ci-integration-tests COV_REPORT=xml distribution: runs-on: ubuntu-latest needs: [unit-tests, type-check, docs-build, integration-tests] if: | always() && needs.unit-tests.result == 'success' && needs.type-check.result == 'success' && needs.docs-build.result == 'success' && (needs.integration-tests.result == 'success' || needs.integration-tests.result == 'skipped') steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.11' - name: Install package run: | python -m pip install --upgrade pip python -m pip install build twine - name: Build distribution run: | python -m build - name: Check wheels run: | cd dist || exit python -m pip install cads_api_client*.whl || exit python -m twine check --strict * || exit python -c "import cads_api_client" || exit cd .. - uses: actions/upload-artifact@v4 with: name: distribution path: dist upload-to-pypi: runs-on: ubuntu-latest needs: distribution if: | always() && true && needs.distribution.result == 'success' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags') environment: name: pypi url: https://pypi.org/p/cads-api-client permissions: id-token: write # IMPORTANT: this permission is mandatory for trusted publish steps: - uses: actions/download-artifact@v4 with: name: distribution path: dist - uses: pypa/gh-action-pypi-publish@v1.12.2 with: verbose: true cads-api-client-1.5.4/.gitignore000066400000000000000000000212121471505652400164550ustar00rootroot00000000000000# setuptools-scm version.py # Sphinx automatic generation of API docs/README.md docs/_api/ # Combined environments ci/combined-environment-*.yml # download tests *.grib # Created by https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks,vim,visualstudiocode,pycharm,emacs,linux,macos,windows # Edit at https://www.toptal.com/developers/gitignore?templates=python,jupyternotebooks,vim,visualstudiocode,pycharm,emacs,linux,macos,windows ### Emacs ### # -*- mode: gitignore; -*- *~ \#*\# /.emacs.desktop /.emacs.desktop.lock *.elc auto-save-list tramp .\#* # Org-mode .org-id-locations *_archive # flymake-mode *_flymake.* # eshell files /eshell/history /eshell/lastdir # elpa packages /elpa/ # reftex files *.rel # AUCTeX auto folder /auto/ # cask packages .cask/ dist/ # Flycheck flycheck_*.el # server auth directory /server/ # projectiles files .projectile # directory configuration .dir-locals.el # network security /network-security.data ### JupyterNotebooks ### # gitignore template for Jupyter Notebooks # website: http://jupyter.org/ .ipynb_checkpoints */.ipynb_checkpoints/* # IPython profile_default/ ipython_config.py # Remove previous ipynb_checkpoints # git rm -r .ipynb_checkpoints/ ### Linux ### # temporary files which can be created if a process still has a handle open of a deleted file .fuse_hidden* # KDE directory preferences .directory # Linux trash folder which might appear on any partition or disk .Trash-* # .nfs files are created when an open file is removed but is still being accessed .nfs* ### macOS ### # General .DS_Store .AppleDouble .LSOverride # Icon must end with two \r Icon # Thumbnails ._* # Files that might appear in the root of a volume .DocumentRevisions-V100 .fseventsd .Spotlight-V100 .TemporaryItems .Trashes .VolumeIcon.icns .com.apple.timemachine.donotpresent # Directories potentially created on remote AFP share .AppleDB .AppleDesktop Network Trash Folder Temporary Items .apdisk ### macOS Patch ### # iCloud generated files *.icloud ### PyCharm ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff .idea/**/workspace.xml .idea/**/tasks.xml .idea/**/usage.statistics.xml .idea/**/dictionaries .idea/**/shelf # AWS User-specific .idea/**/aws.xml # Generated files .idea/**/contentModel.xml # Sensitive or high-churn files .idea/**/dataSources/ .idea/**/dataSources.ids .idea/**/dataSources.local.xml .idea/**/sqlDataSources.xml .idea/**/dynamic.xml .idea/**/uiDesigner.xml .idea/**/dbnavigator.xml # Gradle .idea/**/gradle.xml .idea/**/libraries # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake cmake-build-*/ # Mongo Explorer plugin .idea/**/mongoSettings.xml # File-based project format *.iws # IntelliJ out/ # mpeltonen/sbt-idea plugin .idea_modules/ # JIRA plugin atlassian-ide-plugin.xml # Cursive Clojure plugin .idea/replstate.xml # SonarLint plugin .idea/sonarlint/ # Crashlytics plugin (for Android Studio and IntelliJ) com_crashlytics_export_strings.xml crashlytics.properties crashlytics-build.properties fabric.properties # Editor-based Rest Client .idea/httpRequests # Android studio 3.1+ serialized cache file .idea/caches/build_file_checksums.ser ### PyCharm Patch ### # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 # *.iml # modules.xml # .idea/misc.xml # *.ipr # Sonarlint plugin # https://plugins.jetbrains.com/plugin/7973-sonarlint .idea/**/sonarlint/ # SonarQube Plugin # https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin .idea/**/sonarIssues.xml # Markdown Navigator plugin # https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced .idea/**/markdown-navigator.xml .idea/**/markdown-navigator-enh.xml .idea/**/markdown-navigator/ # Cache file creation bug # See https://youtrack.jetbrains.com/issue/JBR-2257 .idea/$CACHE_FILE$ # CodeStream plugin # https://plugins.jetbrains.com/plugin/12206-codestream .idea/codestream.xml # Azure Toolkit for IntelliJ plugin # https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij .idea/**/azureSettings.xml ### Python ### # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ cover/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder .pybuilder/ target/ # Jupyter Notebook # IPython # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: # .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # poetry # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. # This is especially recommended for binary packages to ensure reproducibility, and is more # commonly ignored for libraries. # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control #poetry.lock # pdm # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. #pdm.lock # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it # in version control. # https://pdm.fming.dev/#use-with-ide .pdm.toml # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # Cython debug symbols cython_debug/ # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ ### Python Patch ### # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration poetry.toml # ruff .ruff_cache/ # LSP config files pyrightconfig.json ### Vim ### # Swap [._]*.s[a-v][a-z] !*.svg # comment out if you don't need vector files [._]*.sw[a-p] [._]s[a-rt-v][a-z] [._]ss[a-gi-z] [._]sw[a-p] # Session Session.vim Sessionx.vim # Temporary .netrwhist # Auto-generated tag files tags # Persistent undo [._]*.un~ ### VisualStudioCode ### .vscode/ # .vscode/* # !.vscode/settings.json # !.vscode/tasks.json # !.vscode/launch.json # !.vscode/extensions.json # !.vscode/*.code-snippets # Local History for Visual Studio Code .history/ # Built Visual Studio Code Extensions *.vsix ### VisualStudioCode Patch ### # Ignore all local history of files .history .ionide ### Windows ### # Windows thumbnail cache files Thumbs.db Thumbs.db:encryptable ehthumbs.db ehthumbs_vista.db # Dump file *.stackdump # Folder config file [Dd]esktop.ini # Recycle Bin used on file shares $RECYCLE.BIN/ # Windows Installer files *.cab *.msi *.msix *.msm *.msp # Windows shortcuts *.lnk # End of https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks,vim,visualstudiocode,pycharm,emacs,linux,macos,windows cads-api-client-1.5.4/.pre-commit-config-cruft.yaml000066400000000000000000000002221471505652400220650ustar00rootroot00000000000000repos: - repo: https://github.com/cruft/cruft rev: 2.15.0 hooks: - id: cruft entry: cruft update -y additional_dependencies: [toml] cads-api-client-1.5.4/.pre-commit-config.yaml000066400000000000000000000017151471505652400207540ustar00rootroot00000000000000repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-json - id: check-yaml - id: check-toml - id: check-added-large-files - id: check-merge-conflict - id: debug-statements - id: mixed-line-ending - repo: https://github.com/keewis/blackdoc rev: v0.3.9 hooks: - id: blackdoc additional_dependencies: [black==23.11.0] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.7.3 hooks: - id: ruff args: [--fix, --show-fixes] - id: ruff-format - repo: https://github.com/executablebooks/mdformat rev: 0.7.18 hooks: - id: mdformat - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks rev: v2.14.0 hooks: - id: pretty-format-yaml args: [--autofix, --preserve-quotes] - id: pretty-format-toml args: [--autofix] - repo: https://github.com/gitleaks/gitleaks rev: v8.21.2 hooks: - id: gitleaks cads-api-client-1.5.4/Dockerfile000066400000000000000000000004121471505652400164560ustar00rootroot00000000000000FROM continuumio/miniconda3 WORKDIR /src/cads-api-client COPY environment.yml /src/cads-api-client/ RUN conda install -c conda-forge gcc python=3.11 \ && conda env update -n base -f environment.yml COPY . /src/cads-api-client RUN pip install --no-deps -e . cads-api-client-1.5.4/LICENSE000066400000000000000000000261221471505652400154770ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2022, European Union. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. cads-api-client-1.5.4/Makefile000066400000000000000000000022411471505652400161260ustar00rootroot00000000000000PROJECT := cads_api_client CONDA := conda CONDAFLAGS := COV_REPORT := html default: qa unit-tests type-check qa: pre-commit run --all-files unit-tests: python -m pytest -vv --cov=. --cov-report=$(COV_REPORT) type-check: python -m mypy . conda-env-update: $(CONDA) install -y -c conda-forge conda-merge $(CONDA) run conda-merge environment.yml ci/environment-ci.yml > ci/combined-environment-ci.yml $(CONDA) env update $(CONDAFLAGS) -f ci/combined-environment-ci.yml docker-build: docker build -t $(PROJECT) . docker-run: docker run --rm -ti -v $(PWD):/srv $(PROJECT) template-update: pre-commit run --all-files cruft -c .pre-commit-config-cruft.yaml docs-build: cp README.md docs/. && cd docs && rm -fr _api && make clean && make html # DO NOT EDIT ABOVE THIS LINE, ADD COMMANDS BELOW integration-tests: python -m pytest -vv --cov=. --cov-report=$(COV_REPORT) tests/integration*.py legacy-tests: python -m pytest -vv --cov=. --cov-report=$(COV_REPORT) tests/integration_test_*legacy*.py doc-tests: python -m pytest -vv --doctest-glob='*.md' README.md all-tests: unit-tests integration-tests doc-tests ci-integration-tests: unit-tests legacy-tests doc-tests cads-api-client-1.5.4/README.md000066400000000000000000000066701471505652400157570ustar00rootroot00000000000000# cads-api-client CADS API Python client Technical documentation: https://ecmwf-projects.github.io/cads-api-client/ ## Configuration The `ApiClient` requires the `url` to the API root and a valid API `key`. You can also set the `CADS_API_URL` and `CADS_API_KEY` environment variables, or use a configuration file. The configuration file must be located at `~/.cads-api-client.json`, or at the path specified by the `CADS_API_RC` environment variable. ``` $ cat $HOME/.cads-api-client.json {"url": "https://cds.climate.copernicus.eu/api", "key": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"} ``` It is possible (though not recommended) to use the API key of one of the test users: ``` 00112233-4455-6677-c899-aabbccddeeff ``` This key is used for anonymous tests and is designed to be the least performant option for accessing the system. ## Quick Start Configure the logging level to display INFO messages: ```python >>> import logging >>> logging.basicConfig(level="INFO") ``` Instantiate the API client and verify the authentication: ```python >>> import os >>> from cads_api_client import ApiClient >>> client = ApiClient( ... url=os.getenv("CADS_API_URL"), ... key=os.getenv("CADS_API_KEY"), ... ) >>> client.check_authentication() {...} ``` Explore a collection: ```python >>> collection_id = "reanalysis-era5-pressure-levels" >>> collection = client.get_collection(collection_id) >>> collection.end_datetime datetime.datetime(...) >>> request = { ... "product_type": "reanalysis", ... "variable": "temperature", ... "year": "2022", ... "month": "01", ... "day": "01", ... "pressure_level": "1000", ... "time": "00:00", ... } >>> collection.process.apply_constraints(**request) {...} ``` Retrieve data: ```python >>> client.retrieve(collection_id, target="tmp1-era5.grib", **request) # blocks 'tmp1-era5.grib' >>> remote = client.submit(collection_id, **request) # doesn't block >>> remote.request_uid '...' >>> remote.status '...' >>> remote.download("tmp2-era5.grib") # blocks 'tmp2-era5.grib' ``` Interact with a previously submitted job: ```python >>> remote = client.get_remote(remote.request_uid) >>> remote.delete() {...} ``` ## Workflow for developers/contributors For best experience create a new conda environment (e.g. DEVELOP) with Python 3.11: ``` conda create -n DEVELOP -c conda-forge python=3.11 conda activate DEVELOP ``` Before pushing to GitHub, run the following commands: 1. Update conda environment: `make conda-env-update` 1. Install this package: `pip install -e .` 1. Sync with the latest [template](https://github.com/ecmwf-projects/cookiecutter-conda-package) (optional): `make template-update` 1. Run quality assurance checks: `make qa` 1. Run tests: `make unit-tests` 1. Run the static type checker: `make type-check` 1. Build the documentation (see [Sphinx tutorial](https://www.sphinx-doc.org/en/master/tutorial/)): `make docs-build` ## License ``` Copyright 2022, European Union. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ``` cads-api-client-1.5.4/cads_api_client/000077500000000000000000000000001471505652400175705ustar00rootroot00000000000000cads-api-client-1.5.4/cads_api_client/__init__.py000066400000000000000000000022721471505652400217040ustar00rootroot00000000000000"""CADS API Python client.""" # Copyright 2022, European Union. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. try: # NOTE: the `version.py` file must not be present in the git repository # as it is generated by setuptools at install time from .version import __version__ except ImportError: # pragma: no cover # Local copy or not installed with setuptools __version__ = "999" from .api_client import ApiClient from .catalogue import Collection, Collections from .processing import Jobs, Process, Processes, Remote, Results __all__ = [ "__version__", "ApiClient", "Collection", "Collections", "Jobs", "Process", "Processes", "Remote", "Results", ] cads-api-client-1.5.4/cads_api_client/api_client.py000066400000000000000000000326601471505652400222600ustar00rootroot00000000000000# Copyright 2022, European Union. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import annotations import functools import warnings from typing import Any, Callable, Literal import attrs import multiurl.base import requests import cads_api_client from . import __version__, catalogue, config, processing, profile @attrs.define(slots=False) class ApiClient: """A client to interact with the CADS API. Parameters ---------- url: str or None, default: None API URL. If None, infer from CADS_API_URL or CADS_API_RC. key: str or None, default: None API Key. If None, infer from CADS_API_KEY or CADS_API_RC. verify: bool, default: True Whether to verify the TLS certificate at the remote end. timeout: float or tuple[float,float], default: 60 How many seconds to wait for the server to send data, as a float, or a (connect, read) tuple. progress: bool, default: True Whether to display the progress bar during download. cleanup: bool, default: False Whether to delete requests after completion. sleep_max: float, default: 120 Maximum time to wait (in seconds) while checking for a status change. retry_after: float, default: 120 Time to wait (in seconds) between retries. maximum_tries: int, default: 500 Maximum number of retries. session: requests.Session Requests session. """ url: str | None = None key: str | None = None verify: bool = True timeout: float | tuple[float, float] = 60 progress: bool = True cleanup: bool = False sleep_max: float = 120 retry_after: float = 120 maximum_tries: int = 500 session: requests.Session = attrs.field(factory=requests.Session) _log_callback: Callable[..., None] | None = None def __attrs_post_init__(self) -> None: if self.url is None: self.url = str(config.get_config("url")) if self.key is None: try: self.key = str(config.get_config("key")) except (KeyError, FileNotFoundError): warnings.warn("The API key is missing", UserWarning) try: self._catalogue_api.messages.log_messages() except Exception as exc: warnings.warn(str(exc), UserWarning) def _get_headers(self, key_is_mandatory: bool = True) -> dict[str, str]: headers = {"User-Agent": f"cads-api-client/{__version__}"} if self.key is not None: headers["PRIVATE-TOKEN"] = self.key elif key_is_mandatory: raise ValueError("The API key is needed to access this resource") return headers @property def _retry_options(self) -> dict[str, Any]: return { "maximum_tries": self.maximum_tries, "retry_after": self.retry_after, } @property def _download_options(self) -> dict[str, Any]: progress_bar = ( multiurl.base.progress_bar if self.progress else multiurl.base.NoBar ) return { "progress_bar": progress_bar, } @property def _request_options(self) -> dict[str, Any]: return { "timeout": self.timeout, "verify": self.verify, } def _get_request_kwargs( self, key_is_mandatory: bool = True ) -> processing.RequestKwargs: return processing.RequestKwargs( headers=self._get_headers(key_is_mandatory=key_is_mandatory), session=self.session, retry_options=self._retry_options, request_options=self._request_options, download_options=self._download_options, sleep_max=self.sleep_max, cleanup=self.cleanup, log_callback=self._log_callback, ) @functools.cached_property def _catalogue_api(self) -> catalogue.Catalogue: return catalogue.Catalogue( f"{self.url}/catalogue", **self._get_request_kwargs(key_is_mandatory=False), ) @functools.cached_property def _retrieve_api(self) -> processing.Processing: return processing.Processing( f"{self.url}/retrieve", **self._get_request_kwargs() ) @functools.cached_property def _profile_api(self) -> profile.Profile: return profile.Profile(f"{self.url}/profiles", **self._get_request_kwargs()) def accept_licence(self, licence_id: str, revision: int) -> dict[str, Any]: """Accept a licence. Parameters ---------- licence_id: str Licence ID. revision: int Licence revision number. Returns ------- dict[str,Any] Content of the response. """ return self._profile_api.accept_licence(licence_id, revision=revision) def apply_constraints(self, collection_id: str, **request: Any) -> dict[str, Any]: """Apply constraints to the parameters in a request. Parameters ---------- collection_id: str Collection ID (e.g., ``"projections-cmip6"``). **request: Any Request parameters. Returns ------- dict[str,Any] Dictionary of valid values. """ return self.get_process(collection_id).apply_constraints(**request) def check_authentication(self) -> dict[str, Any]: """Verify authentication. Returns ------- dict[str,Any] Content of the response. Raises ------ requests.HTTPError If the authentication fails. """ return self._profile_api.check_authentication() def download_results(self, request_uid: str, target: str | None = None) -> str: """Download the results of a request. Parameters ---------- request_uid: str Request UID. target: str or None Target path. If None, download to the working directory. Returns ------- str Path to the retrieved file. """ return self.get_remote(request_uid).download(target) def estimate_costs(self, collection_id: str, **request: Any) -> dict[str, Any]: """Estimate costs of the parameters in a request. Parameters ---------- collection_id: str Collection ID (e.g., ``"projections-cmip6"``). **request: Any Request parameters. Returns ------- dict[str,Any] Dictionary of estimated costs. """ return self.get_process(collection_id).estimate_costs(**request) def get_accepted_licences( self, scope: Literal[None, "all", "dataset", "portal"] = None, ) -> list[dict[str, Any]]: """Retrieve accepted licences. Parameters ---------- scope: {None, 'all', 'dataset', 'portal'} Licence scope. Returns ------- list[dict[str,Any]] List of dictionaries with license information. """ params = {k: v for k, v in zip(["scope"], [scope]) if v is not None} licences: list[dict[str, Any]] licences = self._profile_api.accepted_licences(**params).get("licences", []) return licences def get_collection(self, collection_id: str) -> cads_api_client.Collection: """Retrieve a catalogue collection. Parameters ---------- collection_id: str Collection ID (e.g., ``"projections-cmip6"``). Returns ------- cads_api_client.Collection """ return self._catalogue_api.get_collection(collection_id) def get_collections( self, limit: int | None = None, sortby: Literal[None, "id", "relevance", "title", "update"] = None, query: str | None = None, keywords: list[str] | None = None, ) -> cads_api_client.Collections: """Retrieve catalogue collections. Parameters ---------- limit: int | None Number of processes per page. sortby: {None, 'id', 'relevance', 'title', 'update'} Field to sort results by. query: str or None Full-text search query. keywords: list[str] or None Filter by keywords. Returns ------- cads_api_client.Collections """ params = { k: v for k, v in zip( ["limit", "sortby", "q", "kw"], [limit, sortby, query, keywords] ) if v is not None } return self._catalogue_api.get_collections(**params) def get_jobs( self, limit: int | None = None, sortby: Literal[None, "created", "-created"] = None, status: Literal[None, "accepted", "running", "successful", "failed"] = None, ) -> cads_api_client.Jobs: """Retrieve submitted jobs. Parameters ---------- limit: int or None Number of processes per page. sortby: {None, 'created', '-created'} Field to sort results by. status: {None, 'accepted', 'running', 'successful', 'failed'} Status of the results. Returns ------- cads_api_client.Jobs """ params = { k: v for k, v in zip(["limit", "sortby", "status"], [limit, sortby, status]) if v is not None } return self._retrieve_api.get_jobs(**params) def get_licences( self, scope: Literal[None, "all", "dataset", "portal"] = None, ) -> list[dict[str, Any]]: """Retrieve licences. Parameters ---------- scope: {None, 'all', 'dataset', 'portal'} Licence scope. Returns ------- list[dict[str,Any]] List of dictionaries with license information. """ params = {k: v for k, v in zip(["scope"], [scope]) if v is not None} licences: list[dict[str, Any]] licences = self._catalogue_api.get_licenses(**params).get("licences", []) return licences def get_process(self, collection_id: str) -> cads_api_client.Process: """ Retrieve a process. Parameters ---------- collection_id: str Collection ID (e.g., ``"projections-cmip6"``). Returns ------- cads_api_client.Process """ return self._retrieve_api.get_process(collection_id) def get_processes( self, limit: int | None = None, sortby: Literal[None, "id", "-id"] = None, ) -> cads_api_client.Processes: params = { k: v for k, v in zip(["limit", "sortby"], [limit, sortby]) if v is not None } return self._retrieve_api.get_processes(**params) def get_remote(self, request_uid: str) -> cads_api_client.Remote: """ Retrieve the remote object of a request. Parameters ---------- request_uid: str Request UID. Returns ------- cads_api_client.Remote """ return self._retrieve_api.get_job(request_uid).make_remote() def get_results(self, request_uid: str) -> cads_api_client.Results: """ Retrieve the results of a request. Parameters ---------- request_uid: str Request UID. Returns ------- cads_api_client.Results """ return self.get_remote(request_uid).make_results() def retrieve( self, collection_id: str, target: str | None = None, **request: Any, ) -> str: """Submit a request and retrieve the results. Parameters ---------- collection_id: str Collection ID (e.g., ``"projections-cmip6"``). target: str or None Target path. If None, download to the working directory. **request: Any Request parameters. Returns ------- str Path to the retrieved file. """ return self.submit(collection_id, **request).download(target) def submit(self, collection_id: str, **request: Any) -> cads_api_client.Remote: """Submit a request. Parameters ---------- collection_id: str Collection ID (e.g., ``"projections-cmip6"``). **request: Any Request parameters. Returns ------- cads_api_client.Remote """ return self._retrieve_api.submit(collection_id, **request) def submit_and_wait_on_results( self, collection_id: str, **request: Any ) -> cads_api_client.Results: """Submit a request and wait for the results to be ready. Parameters ---------- collection_id: str Collection ID (e.g., ``"projections-cmip6"``). **request: Any Request parameters. Returns ------- cads_api_client.Results """ return self._retrieve_api.submit(collection_id, **request).make_results() cads-api-client-1.5.4/cads_api_client/catalogue.py000066400000000000000000000116531471505652400221140ustar00rootroot00000000000000# Copyright 2022, European Union. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import annotations import datetime import warnings from typing import Any, Callable import attrs import requests import cads_api_client from . import config from .processing import ApiResponse, ApiResponsePaginated, RequestKwargs @attrs.define class Collections(ApiResponsePaginated): """A class to interact with catalogue collections.""" @property def collection_ids(self) -> list[str]: """List of collection IDs.""" return [collection["id"] for collection in self._json_dict["collections"]] @attrs.define class Collection(ApiResponse): """A class to interact with a catalogue collection.""" @property def begin_datetime(self) -> datetime.datetime | None: """Begin datetime of the collection.""" if (value := self._json_dict["extent"]["temporal"]["interval"][0][0]) is None: return value return datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) @property def end_datetime(self) -> datetime.datetime | None: """End datetime of the collection.""" if (value := self._json_dict["extent"]["temporal"]["interval"][0][1]) is None: return value return datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) @property def bbox(self) -> tuple[float, float, float, float]: """Bounding box of the collection (W, S, E, N).""" return tuple(self._json_dict["extent"]["spatial"]["bbox"][0]) @property def id(self) -> str: """Collection ID.""" return str(self._json_dict["id"]) @property def process(self) -> cads_api_client.Process: """Collection process.""" url = self._get_link_href(rel="retrieve") return cads_api_client.Process.from_request("get", url, **self._request_kwargs) @property def form(self) -> list[dict[str, Any]]: """Form JSON.""" url = f"{self.url}/form.json" return ApiResponse.from_request( "get", url, log_messages=False, **self._request_kwargs )._json_list @property def constraints(self) -> list[dict[str, Any]]: """Constraints JSON.""" url = f"{self.url}/constraints.json" return ApiResponse.from_request( "get", url, log_messages=False, **self._request_kwargs )._json_list def submit(self, **request: Any) -> cads_api_client.Remote: warnings.warn( "`.submit` has been deprecated, and in the future will raise an error." " Please use `.process.submit` from now on.", DeprecationWarning, ) return self.process.submit(**request) @attrs.define(slots=False) class Catalogue: url: str headers: dict[str, Any] session: requests.Session retry_options: dict[str, Any] request_options: dict[str, Any] download_options: dict[str, Any] sleep_max: float cleanup: bool log_callback: Callable[..., None] | None force_exact_url: bool = False def __attrs_post_init__(self) -> None: if not self.force_exact_url: self.url += f"/{config.SUPPORTED_API_VERSION}" @property def _request_kwargs(self) -> RequestKwargs: return RequestKwargs( headers=self.headers, session=self.session, retry_options=self.retry_options, request_options=self.request_options, download_options=self.download_options, sleep_max=self.sleep_max, cleanup=self.cleanup, log_callback=self.log_callback, ) def get_collections(self, **params: Any) -> Collections: url = f"{self.url}/datasets" return Collections.from_request( "get", url, params=params, **self._request_kwargs ) def get_collection(self, collection_id: str) -> Collection: url = f"{self.url}/collections/{collection_id}" return Collection.from_request("get", url, **self._request_kwargs) def get_licenses(self, **params: Any) -> dict[str, Any]: url = f"{self.url}/vocabularies/licences" response = ApiResponse.from_request( "get", url, params=params, **self._request_kwargs ) return response._json_dict @property def messages(self) -> ApiResponse: url = f"{self.url}/messages" return ApiResponse.from_request( "get", url, log_messages=False, **self._request_kwargs ) cads-api-client-1.5.4/cads_api_client/config.py000066400000000000000000000025471471505652400214170ustar00rootroot00000000000000# Copyright 2022, European Union. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import annotations import json import os from typing import Any SUPPORTED_API_VERSION = "v1" def read_configuration_file(config_path: str | None = None) -> dict[Any, Any]: if config_path is None: config_path = os.getenv("CADS_API_RC", "~/.cads-api-client.json") config_path = os.path.expanduser(config_path) try: with open(config_path) as fin: config = json.load(fin) assert isinstance(config, dict) except FileNotFoundError: raise except Exception: raise ValueError(f"failed to parse {config_path!r} file") return config def get_config(key: str, config_path: str | None = None) -> Any: return ( os.getenv(f"CADS_API_{key.upper()}") or read_configuration_file(config_path)[key] ) cads-api-client-1.5.4/cads_api_client/legacy_api_client.py000066400000000000000000000224071471505652400236020ustar00rootroot00000000000000# Copyright 2022, European Union. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import annotations import collections import logging import typing import warnings from types import TracebackType from typing import Any, Callable, TypeVar, overload import cdsapi.api import multiurl import requests from . import __version__ as cads_api_client_version from .api_client import ApiClient from .processing import Remote, Results LOGGER = logging.getLogger(__name__) F = TypeVar("F", bound=Callable[..., Any]) class LoggingContext: def __init__(self, logger: logging.Logger, quiet: bool, debug: bool) -> None: self.old_level = logger.level if quiet: logger.setLevel(logging.WARNING) else: logger.setLevel(logging.DEBUG if debug else logging.INFO) self.new_handlers = [] if not logger.handlers: formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") handler = logging.StreamHandler() handler.setFormatter(formatter) logger.addHandler(handler) self.new_handlers.append(handler) self.logger = logger def __enter__(self) -> logging.Logger: return self.logger def __exit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None, ) -> None: self.logger.setLevel(self.old_level) for handler in self.new_handlers: self.logger.removeHandler(handler) class LegacyApiClient(cdsapi.api.Client): # type: ignore[misc] def __init__( self, url: str | None = None, key: str | None = None, quiet: bool = False, debug: bool = False, verify: bool | int | None = None, timeout: int = 60, progress: bool = True, full_stack: None = None, delete: bool = False, retry_max: int = 500, sleep_max: float = 120, wait_until_complete: bool = True, info_callback: Callable[..., None] | None = None, warning_callback: Callable[..., None] | None = None, error_callback: Callable[..., None] | None = None, debug_callback: Callable[..., None] | None = None, metadata: None = None, forget: None = None, session: requests.Session | None = None, ) -> None: self.issue_deprecated_kwargs_warning( full_stack=full_stack, metadata=metadata, forget=forget ) self.url, self.key, verify = cdsapi.api.get_url_key_verify(url, key, verify) self.verify = bool(verify) self.quiet = quiet self._debug = debug self.timeout = timeout self.progress = progress self.delete = delete self.retry_max = retry_max self.sleep_max = sleep_max self.wait_until_complete = wait_until_complete self.info_callback = info_callback self.warning_callback = warning_callback self.error_callback = error_callback self.debug_callback = debug_callback self.session = requests.Session() if session is None else session self.client = ApiClient( url=self.url, key=self.key, verify=self.verify, timeout=self.timeout, progress=self.progress, cleanup=self.delete, sleep_max=self.sleep_max, retry_after=self.sleep_max, maximum_tries=self.retry_max, session=self.session, log_callback=self.log, ) self.debug( "CDSAPI %s", { "url": self.url, "key": self.key, "quiet": self.quiet, "verify": self.verify, "timeout": self.timeout, "progress": self.progress, "sleep_max": self.sleep_max, "retry_max": self.retry_max, "delete": self.delete, "cads_api_client_version": cads_api_client_version, }, ) @classmethod def issue_deprecated_kwargs_warning(self, **kwargs: Any) -> None: if kwargs := {k: v for k, v in kwargs.items() if v is not None}: warnings.warn( f"The following parameters are deprecated: {kwargs}." " Set them to None to silence this warning.", UserWarning, ) @classmethod def raise_toolbox_error(self) -> None: raise NotImplementedError( "Legacy CDS Toolbox is now discontinued." " Watch for announcements/updates on new CDS improved capabilities on our Forum (https://forum.ecmwf.int/)." ) @overload def retrieve(self, name: str, request: dict[str, Any], target: str) -> str: ... @overload def retrieve( self, name: str, request: dict[str, Any], target: None = ... ) -> Results: ... def retrieve( self, name: str, request: dict[str, Any], target: str | None = None ) -> str | Remote | Results: submitted: Remote | Results if self.wait_until_complete: submitted = self.client.submit_and_wait_on_results( collection_id=name, **request, ) else: submitted = self.client.submit( collection_id=name, **request, ) return submitted if target is None else submitted.download(target) def log(self, level: int, *args: Any, **kwargs: Any) -> None: with LoggingContext( logger=LOGGER, quiet=self.quiet, debug=self._debug ) as logger: if level == logging.INFO and self.info_callback is not None: self.info_callback(*args, **kwargs) elif level == logging.WARNING and self.warning_callback is not None: self.warning_callback(*args, **kwargs) elif level == logging.ERROR and self.error_callback is not None: self.error_callback(*args, **kwargs) elif level == logging.DEBUG and self.debug_callback is not None: self.debug_callback(*args, **kwargs) else: logger.log(level, *args, **kwargs) def info(self, *args: Any, **kwargs: Any) -> None: self.log(logging.INFO, *args, **kwargs) def warning(self, *args: Any, **kwargs: Any) -> None: self.log(logging.WARNING, *args, **kwargs) def error(self, *args: Any, **kwargs: Any) -> None: self.log(logging.ERROR, *args, **kwargs) def debug(self, *args: Any, **kwargs: Any) -> None: self.log(logging.DEBUG, *args, **kwargs) @typing.no_type_check def service(self, name, *args, **kwargs): self.raise_toolbox_error() @typing.no_type_check def workflow(self, code, *args, **kwargs): self.raise_toolbox_error() def status(self, context: Any = None) -> dict[str, list[str]]: status = collections.defaultdict(list) messages = self.client._catalogue_api.messages._json_dict.get("messages", []) for message in messages: status[message["severity"]].append(message["content"]) return dict(status) @typing.no_type_check def _download(self, results, targets=None): if isinstance(results, (cdsapi.api.Result, Remote, Results)): if targets: path = targets.pop(0) else: path = None return results.download(path) if isinstance(results, (list, tuple)): return [self._download(x, targets) for x in results] if isinstance(results, dict): if "location" in results and "contentLength" in results: reply = dict( location=results["location"], content_length=results["contentLength"], content_type=results.get("contentType"), ) if targets: path = targets.pop(0) else: path = None return cdsapi.api.Result(self, reply).download(path) r = {} for v in results.values(): r[v] = self._download(v, targets) return r return results @typing.no_type_check def download(self, results, targets=None): if targets: # Make a copy targets = [t for t in targets] return self._download(results, targets) def remote(self, url: str) -> cdsapi.api.Result: r = requests.head(url) reply = dict( location=url, content_length=r.headers["Content-Length"], content_type=r.headers["Content-Type"], ) return cdsapi.api.Result(self, reply) def robust(self, call: F) -> F: robust: F = multiurl.robust(call, **self.client._retry_options) return robust cads-api-client-1.5.4/cads_api_client/processing.py000066400000000000000000000535171471505652400223310ustar00rootroot00000000000000# Copyright 2022, European Union. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import annotations import datetime import functools import logging import os import time import urllib.parse import warnings from typing import Any, Callable, Type, TypedDict, TypeVar try: from typing import Self except ImportError: from typing_extensions import Self import attrs import multiurl import requests import cads_api_client from . import config T_ApiResponse = TypeVar("T_ApiResponse", bound="ApiResponse") LOGGER = logging.getLogger(__name__) LEVEL_NAMES_MAPPING = { "CRITICAL": 50, "FATAL": 50, "ERROR": 40, "WARNING": 30, "WARN": 30, "INFO": 20, "DEBUG": 10, "NOTSET": 0, } class RequestKwargs(TypedDict): headers: dict[str, str] session: requests.Session retry_options: dict[str, Any] request_options: dict[str, Any] download_options: dict[str, Any] sleep_max: float cleanup: bool log_callback: Callable[..., None] | None class ProcessingFailedError(RuntimeError): pass class DownloadError(RuntimeError): pass class LinkError(Exception): pass def error_json_to_message(error_json: dict[str, Any]) -> str: error_messages = [ str(error_json[key]) for key in ("title", "traceback", "detail") if key in error_json ] return "\n".join(error_messages) def cads_raise_for_status(response: requests.Response) -> None: if 400 <= response.status_code < 500: try: error_json = response.json() except Exception: pass else: message = "\n".join( [ f"{response.status_code} Client Error: {response.reason} for url: {response.url}", error_json_to_message(error_json), ] ) raise requests.HTTPError(message, response=response) response.raise_for_status() def get_level_and_message(message: str) -> tuple[int, str]: level = 20 for severity in LEVEL_NAMES_MAPPING: if message.startswith(severity): level = LEVEL_NAMES_MAPPING[severity] message = message.replace(severity, "", 1).lstrip(":").lstrip() break return level, message def log(*args: Any, callback: Callable[..., None] | None = None, **kwargs: Any) -> None: if callback is None: LOGGER.log(*args, **kwargs) else: callback(*args, **kwargs) @attrs.define(slots=False) class ApiResponse: response: requests.Response headers: dict[str, str] session: requests.Session retry_options: dict[str, Any] request_options: dict[str, Any] download_options: dict[str, Any] sleep_max: float cleanup: bool log_callback: Callable[..., None] | None @property def _request_kwargs(self) -> RequestKwargs: return RequestKwargs( headers=self.headers, session=self.session, retry_options=self.retry_options, request_options=self.request_options, download_options=self.download_options, sleep_max=self.sleep_max, cleanup=self.cleanup, log_callback=self.log_callback, ) @classmethod def from_request( cls: Type[T_ApiResponse], method: str, url: str, headers: dict[str, str], session: requests.Session | None, retry_options: dict[str, Any], request_options: dict[str, Any], download_options: dict[str, Any], sleep_max: float, cleanup: bool, log_callback: Callable[..., None] | None, log_messages: bool = True, **kwargs: Any, ) -> T_ApiResponse: if session is None: session = requests.Session() robust_request = multiurl.robust(session.request, **retry_options) inputs = kwargs.get("json", {}).get("inputs", {}) log( logging.DEBUG, f"{method.upper()} {url} {inputs or ''}".strip(), callback=log_callback, ) response = robust_request( method, url, headers=headers, **request_options, **kwargs ) log(logging.DEBUG, f"REPLY {response.text}", callback=log_callback) cads_raise_for_status(response) self = cls( response, headers=headers, session=session, retry_options=retry_options, request_options=request_options, download_options=download_options, sleep_max=sleep_max, cleanup=cleanup, log_callback=log_callback, ) if log_messages: self.log_messages() return self @property def url(self) -> str: """URL.""" return str(self.response.request.url) @property def json(self) -> Any: """Content of the response.""" return self.response.json() @property def _json_dict(self) -> dict[str, Any]: assert isinstance(content := self.json, dict) return content @property def _json_list(self) -> list[dict[str, Any]]: assert isinstance(content := self.json, list) return content def log_messages(self) -> None: if message_str := self._json_dict.get("message"): level, message_str = get_level_and_message(message_str) self.log(level, message_str) messages = self._json_dict.get("messages", []) dataset_messages = ( self._json_dict.get("metadata", {}) .get("datasetMetadata", {}) .get("messages", []) ) for message_dict in messages + dataset_messages: if not (content := message_dict.get("content")): continue if date := message_dict.get("date"): content = f"[{date}] {content}" severity = message_dict.get("severity", "notset").upper() level = LEVEL_NAMES_MAPPING.get(severity, 20) self.log(level, content) def _get_links(self, rel: str | None = None) -> list[dict[str, str]]: links = [] for link in self._json_dict.get("links", []): if rel is not None and link.get("rel") == rel: links.append(link) return links def _get_link_href(self, rel: str | None = None) -> str: links = self._get_links(rel) if len(links) != 1: raise LinkError(f"link not found or not unique {rel=}") return links[0]["href"] def _from_rel_href(self, rel: str) -> Self | None: rels = self._get_links(rel=rel) if len(rels) > 1: raise LinkError(f"link not unique {rel=}") if len(rels) == 1: out = self.from_request("get", rels[0]["href"], **self._request_kwargs) else: out = None return out def log(self, *args: Any, **kwargs: Any) -> None: log(*args, callback=self.log_callback, **kwargs) def info(self, *args: Any, **kwargs: Any) -> None: self.log(logging.INFO, *args, **kwargs) def warning(self, *args: Any, **kwargs: Any) -> None: self.log(logging.WARNING, *args, **kwargs) def error(self, *args: Any, **kwargs: Any) -> None: self.log(logging.ERROR, *args, **kwargs) def debug(self, *args: Any, **kwargs: Any) -> None: self.log(logging.DEBUG, *args, **kwargs) @attrs.define class ApiResponsePaginated(ApiResponse): @property def next(self) -> Self | None: """Next page.""" return self._from_rel_href(rel="next") @property def prev(self) -> Self | None: """Previous page.""" return self._from_rel_href(rel="prev") @attrs.define class Processes(ApiResponsePaginated): @property def collection_ids(self) -> list[str]: """Available collection IDs.""" return [proc["id"] for proc in self._json_dict["processes"]] @property def process_ids(self) -> list[str]: warnings.warn( "`.process_ids` has been deprecated, and in the future will raise an error." " Please use `.collection_ids` from now on.", DeprecationWarning, ) return self.collection_ids @attrs.define class Process(ApiResponse): """A class to interact with a process.""" @property def id(self) -> str: """Process ID.""" process_id: str = self._json_dict["id"] return process_id def submit(self, **request: Any) -> cads_api_client.Remote: """Submit a request. Parameters ---------- **request: Any Request parameters. Returns ------- cads_api_client.Remote """ job = Job.from_request( "post", f"{self.url}/execution", json={"inputs": request}, **self._request_kwargs, ) return job.make_remote() def apply_constraints(self, **request: Any) -> dict[str, Any]: """Apply constraints to the parameters in a request. Parameters ---------- **request: Any Request parameters. Returns ------- dict[str,Any] Dictionary of valid values. """ response = ApiResponse.from_request( "post", f"{self.url}/constraints", json={"inputs": request}, **self._request_kwargs, ) return response._json_dict def estimate_costs(self, **request: Any) -> dict[str, Any]: """Estimate costs of the parameters in a request. Parameters ---------- **request: Any Request parameters. Returns ------- dict[str,Any] Dictionary of estimated costs. """ response = ApiResponse.from_request( "post", f"{self.url}/costing", json={"inputs": request}, **self._request_kwargs, ) return response._json_dict @attrs.define(slots=False) class Remote: """A class to interact with a submitted job.""" url: str headers: dict[str, str] session: requests.Session retry_options: dict[str, Any] request_options: dict[str, Any] download_options: dict[str, Any] sleep_max: float cleanup: bool log_callback: Callable[..., None] | None def __attrs_post_init__(self) -> None: self.log_start_time = None self.last_status = None self.info(f"Request ID is {self.request_uid}") @property def _request_kwargs(self) -> RequestKwargs: return RequestKwargs( headers=self.headers, session=self.session, retry_options=self.retry_options, request_options=self.request_options, download_options=self.download_options, sleep_max=self.sleep_max, cleanup=self.cleanup, log_callback=self.log_callback, ) def _log_metadata(self, metadata: dict[str, Any]) -> None: logs = metadata.get("log", []) for self.log_start_time, message in sorted(logs): level, message = get_level_and_message(message) self.log(level, message) def _get_api_response(self, method: str, **kwargs: Any) -> ApiResponse: return ApiResponse.from_request( method, self.url, **self._request_kwargs, **kwargs ) @property def request_uid(self) -> str: """Request UID.""" return self.url.rpartition("/")[2] @property def json(self) -> dict[str, Any]: """Content of the response.""" params = {"log": True, "request": True} if self.log_start_time: params["logStartTime"] = self.log_start_time return self._get_api_response("get", params=params)._json_dict @property def collection_id(self) -> str: """Collection ID.""" return str(self.json["processID"]) @property def request(self) -> dict[str, Any]: """Request parameters.""" return dict(self.json["metadata"]["request"]["ids"]) @property def status(self) -> str: """Request status.""" reply = self.json self._log_metadata(reply.get("metadata", {})) status = reply["status"] if self.last_status != status: self.info(f"status has been updated to {status}") self.last_status = status return str(status) @property def creation_datetime(self) -> datetime.datetime: """Creation datetime of the job.""" return datetime.datetime.fromisoformat(self.json["created"]) @property def start_datetime(self) -> datetime.datetime | None: """Start datetime of the job. If None, job has not started.""" value = self.json.get("started") return value if value is None else datetime.datetime.fromisoformat(value) @property def end_datetime(self) -> datetime.datetime | None: """End datetime of the job. If None, job has not finished.""" value = self.json.get("finished") return value if value is None else datetime.datetime.fromisoformat(value) def _wait_on_results(self) -> None: sleep = 1.0 while not self.results_ready: self.debug(f"results not ready, waiting for {sleep} seconds") time.sleep(sleep) sleep = min(sleep * 1.5, self.sleep_max) @property def results_ready(self) -> bool: """Check if results are ready.""" status = self.status if status == "successful": return True if status in ("accepted", "running"): return False if status == "failed": results = self.make_results(wait=False) raise ProcessingFailedError(error_json_to_message(results._json_dict)) if status in ("dismissed", "deleted"): raise ProcessingFailedError(f"API state {status!r}") raise ProcessingFailedError(f"Unknown API state {status!r}") def make_results(self, wait: bool = True) -> Results: if wait: self._wait_on_results() response = self._get_api_response("get") try: results_url = response._get_link_href(rel="results") except LinkError: results_url = f"{self.url}/results" results = Results.from_request("get", results_url, **self._request_kwargs) return results def download(self, target: str | None = None) -> str: """Download the results. Parameters ---------- target: str or None Target path. If None, download to the working directory. Returns ------- str Path to the retrieved file. """ results = self.make_results() return results.download(target) def delete(self) -> dict[str, Any]: """Delete job. Returns ------- dict[str,Any] Content of the response. """ response = self._get_api_response("delete") self.cleanup = False return response._json_dict def _warn(self) -> None: message = ( ".update and .reply are available for backward compatibility." " You can now use .download directly without needing to check whether the request is completed." ) warnings.warn(message, DeprecationWarning) def update(self, request_id: str | None = None) -> None: self._warn() if request_id: assert request_id == self.request_uid try: del self.reply except AttributeError: pass self.reply @functools.cached_property def reply(self) -> dict[str, Any]: self._warn() reply = dict(self.json) reply.setdefault("state", reply["status"]) if reply["state"] == "successful": reply["state"] = "completed" elif reply["state"] == "queued": reply["state"] = "accepted" elif reply["state"] == "failed": reply.setdefault("error", {}) try: self.make_results() except Exception as exc: reply["error"].setdefault("message", str(exc)) reply.setdefault("request_id", self.request_uid) return reply def log(self, *args: Any, **kwargs: Any) -> None: log(*args, callback=self.log_callback, **kwargs) def info(self, *args: Any, **kwargs: Any) -> None: self.log(logging.INFO, *args, **kwargs) def warning(self, *args: Any, **kwargs: Any) -> None: self.log(logging.WARNING, *args, **kwargs) def error(self, *args: Any, **kwargs: Any) -> None: self.log(logging.ERROR, *args, **kwargs) def debug(self, *args: Any, **kwargs: Any) -> None: self.log(logging.DEBUG, *args, **kwargs) def __del__(self) -> None: if self.cleanup: try: self.delete() except Exception as exc: warnings.warn(str(exc), UserWarning) @attrs.define class Job(ApiResponse): def make_remote(self) -> Remote: if self.response.request.method == "POST": url = self._get_link_href(rel="monitor") else: url = self._get_link_href(rel="self") return Remote(url, **self._request_kwargs) @attrs.define class Jobs(ApiResponsePaginated): """A class to interact with submitted jobs.""" @property def request_uids(self) -> list[str]: """List of request UIDs.""" return [job["jobID"] for job in self._json_dict["jobs"]] @property def job_ids(self) -> list[str]: warnings.warn( "`.job_ids` has been deprecated, and in the future will raise an error." " Please use `.request_uids` from now on.", DeprecationWarning, ) return self.request_uids @attrs.define class Results(ApiResponse): """A class to interact with the results of a job.""" def _check_size(self, target: str) -> None: if (target_size := os.path.getsize(target)) != (size := self.content_length): raise DownloadError( f"Download failed: downloaded {target_size} byte(s) out of {size}" ) @property def asset(self) -> dict[str, Any]: """Asset dictionary.""" return dict(self._json_dict["asset"]["value"]) def _download(self, url: str, target: str) -> requests.Response: download_options = {"stream": True, "resume_transfers": True} download_options.update(self.download_options) multiurl.download( url, target=target, **self.retry_options, **self.request_options, **download_options, ) return requests.Response() # mutliurl robust needs a response def download( self, target: str | None = None, ) -> str: """Download the results. Parameters ---------- target: str or None Target path. If None, download to the working directory. Returns ------- str Path to the retrieved file. """ url = self.location if target is None: parts = urllib.parse.urlparse(url) target = parts.path.strip("/").split("/")[-1] if os.path.exists(target): os.remove(target) robust_download = multiurl.robust(self._download, **self.retry_options) robust_download(url, target) self._check_size(target) return target @property def location(self) -> str: """File location.""" result_href = self.asset["href"] return urllib.parse.urljoin(self.response.url, result_href) @property def content_length(self) -> int: """File size in Bytes.""" return int(self.asset["file:size"]) @property def content_type(self) -> str: """File MIME type.""" return str(self.asset["type"]) @attrs.define(slots=False) class Processing: url: str headers: dict[str, str] session: requests.Session retry_options: dict[str, Any] request_options: dict[str, Any] download_options: dict[str, Any] sleep_max: float cleanup: bool log_callback: Callable[..., None] | None force_exact_url: bool = False def __attrs_post_init__(self) -> None: if not self.force_exact_url: self.url += f"/{config.SUPPORTED_API_VERSION}" @property def _request_kwargs(self) -> RequestKwargs: return RequestKwargs( headers=self.headers, session=self.session, retry_options=self.retry_options, request_options=self.request_options, download_options=self.download_options, sleep_max=self.sleep_max, cleanup=self.cleanup, log_callback=self.log_callback, ) def get_processes(self, **params: Any) -> Processes: url = f"{self.url}/processes" return Processes.from_request("get", url, params=params, **self._request_kwargs) def get_process(self, process_id: str) -> Process: url = f"{self.url}/processes/{process_id}" return Process.from_request("get", url, **self._request_kwargs) def get_jobs(self, **params: Any) -> Jobs: url = f"{self.url}/jobs" return Jobs.from_request("get", url, params=params, **self._request_kwargs) def get_job(self, job_id: str) -> Job: url = f"{self.url}/jobs/{job_id}" return Job.from_request("get", url, **self._request_kwargs) def submit(self, collection_id: str, **request: Any) -> Remote: return self.get_process(collection_id).submit(**request) cads-api-client-1.5.4/cads_api_client/profile.py000066400000000000000000000047411471505652400216100ustar00rootroot00000000000000# Copyright 2022, European Union. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import annotations from typing import Any, Callable import attrs import requests from . import config, processing @attrs.define(slots=False) class Profile: url: str headers: dict[str, Any] session: requests.Session retry_options: dict[str, Any] request_options: dict[str, Any] download_options: dict[str, Any] sleep_max: float cleanup: bool log_callback: Callable[..., None] | None force_exact_url: bool = False def __attrs_post_init__(self) -> None: if not self.force_exact_url: self.url += f"/{config.SUPPORTED_API_VERSION}" @property def _request_kwargs(self) -> processing.RequestKwargs: return processing.RequestKwargs( headers=self.headers, session=self.session, retry_options=self.retry_options, request_options=self.request_options, download_options=self.download_options, sleep_max=self.sleep_max, cleanup=self.cleanup, log_callback=self.log_callback, ) def _get_api_response( self, method: str, url: str, **kwargs: Any ) -> processing.ApiResponse: return processing.ApiResponse.from_request( method, url, **self._request_kwargs, **kwargs, ) def accept_licence(self, licence_id: str, revision: int) -> dict[str, Any]: url = f"{self.url}/account/licences/{licence_id}" return self._get_api_response( "put", url, json={"revision": revision} )._json_dict def accepted_licences(self, **params: Any) -> dict[str, Any]: url = f"{self.url}/account/licences" return self._get_api_response("get", url, params=params)._json_dict def check_authentication(self) -> dict[str, Any]: url = f"{self.url}/account/verification/pat" return self._get_api_response("post", url)._json_dict cads-api-client-1.5.4/cads_api_client/py.typed000066400000000000000000000000001471505652400212550ustar00rootroot00000000000000cads-api-client-1.5.4/ci/000077500000000000000000000000001471505652400150625ustar00rootroot00000000000000cads-api-client-1.5.4/ci/environment-ci.yml000066400000000000000000000005351471505652400205450ustar00rootroot00000000000000# environment-ci.yml: Additional dependencies to install in the CI environment. channels: - conda-forge - nodefaults dependencies: - make - mypy - myst-parser - pip - pre-commit - pydata-sphinx-theme - pytest - pytest-cov - sphinx - sphinx-autoapi # DO NOT EDIT ABOVE THIS LINE, ADD DEPENDENCIES BELOW - cdsapi - types-requests - pip: - responses cads-api-client-1.5.4/ci/environment-integration.yml000066400000000000000000000004041471505652400224700ustar00rootroot00000000000000# environment-integration.yml: Additional dependencies to install in the integration environment (e.g., pinned dependencies). channels: - conda-forge - nodefaults dependencies: - make - pytest - pytest-cov # DO NOT EDIT ABOVE THIS LINE, ADD DEPENDENCIES BELOW cads-api-client-1.5.4/docs/000077500000000000000000000000001471505652400154175ustar00rootroot00000000000000cads-api-client-1.5.4/docs/Makefile000066400000000000000000000011721471505652400170600ustar00rootroot00000000000000# Minimal makefile for Sphinx documentation # # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = . BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) cads-api-client-1.5.4/docs/_static/000077500000000000000000000000001471505652400170455ustar00rootroot00000000000000cads-api-client-1.5.4/docs/_static/.gitkeep000066400000000000000000000000001471505652400204640ustar00rootroot00000000000000cads-api-client-1.5.4/docs/_templates/000077500000000000000000000000001471505652400175545ustar00rootroot00000000000000cads-api-client-1.5.4/docs/_templates/.gitkeep000066400000000000000000000000001471505652400211730ustar00rootroot00000000000000cads-api-client-1.5.4/docs/conf.py000066400000000000000000000045251471505652400167240ustar00rootroot00000000000000# Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html # -- Import and path setup --------------------------------------------------- import os import sys import cads_api_client sys.path.insert(0, os.path.abspath("../")) # -- Project information ----------------------------------------------------- project = "cads_api_client" copyright = "2022, European Union" author = "European Union" version = cads_api_client.__version__ release = cads_api_client.__version__ # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ "autoapi.extension", "myst_parser", "sphinx.ext.autodoc", "sphinx.ext.napoleon", ] # autodoc configuration autodoc_typehints = "none" # autoapi configuration autoapi_add_toctree_entry = False autoapi_dirs = ["../cads_api_client"] autoapi_ignore = ["*/version.py"] autoapi_member_order = "groupwise" autoapi_options = [ "members", "inherited-members", # "undoc-members", "show-inheritance", "show-module-summary", "imported-members", ] autoapi_own_page_level = "class" autoapi_root = "_api" # napoleon configuration napoleon_google_docstring = False napoleon_numpy_docstring = True napoleon_preprocess_types = True # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = "pydata_sphinx_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] cads-api-client-1.5.4/docs/index.md000066400000000000000000000011121471505652400170430ustar00rootroot00000000000000# Welcome to cads-api-client's documentation! The `cads-api-client` provides programmatic access to the CDS, ADS and EWDS data catalogues. These pages provide technical documentation for advanced users and/or developers. For a more detailed and user friendly documentation please refer to the [Climate Data Store (CDS) User Guide](https://confluence.ecmwf.int/x/vTRtD) in the Copernicus Knowledge Base. ```{toctree} :caption: 'Contents:' :maxdepth: 2 README.md API Reference <_api/cads_api_client/index> ``` # Indices and tables - {ref}`genindex` - {ref}`modindex` - {ref}`search` cads-api-client-1.5.4/docs/make.bat000066400000000000000000000013751471505652400170320ustar00rootroot00000000000000@ECHO OFF pushd %~dp0 REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set SOURCEDIR=. set BUILDDIR=_build %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.https://www.sphinx-doc.org/ exit /b 1 ) if "%1" == "" goto help %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end popd cads-api-client-1.5.4/environment.yml000066400000000000000000000004301471505652400175530ustar00rootroot00000000000000# environment.yml: Mandatory dependencies only. channels: - conda-forge - nodefaults # EXAMPLE: # dependencies: # - package1 # - package2 # DO NOT EDIT ABOVE THIS LINE, ADD DEPENDENCIES BELOW AS SHOWN IN THE EXAMPLE dependencies: - attrs - multiurl - requests - typing-extensions cads-api-client-1.5.4/pyproject.toml000066400000000000000000000034471471505652400174130ustar00rootroot00000000000000[build-system] build-backend = "setuptools.build_meta" requires = ["setuptools>=64", "setuptools_scm>=8"] [project] classifiers = [ "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Science/Research", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering" ] dependencies = [ "attrs", "multiurl >= 0.3.2", "requests", "typing-extensions" ] description = "CADS API Python client" dynamic = ["version"] license = {file = "LICENSE"} name = "cads-api-client" readme = "README.md" requires-python = ">=3.8" [project.optional-dependencies] legacy = ["cdsapi"] [tool.coverage.run] branch = true [tool.mypy] strict = true [[tool.mypy.overrides]] ignore_missing_imports = true module = ["cdsapi.*", "multiurl.*"] [tool.ruff] # Same as Black. indent-width = 4 line-length = 88 target-version = "py38" [tool.ruff.lint] ignore = [ # pydocstyle: Missing Docstrings "D1" ] select = [ # pyflakes "F", # pycodestyle "E", "W", # isort "I", # pydocstyle "D", # flake8-future-annotations "FA" ] [tool.ruff.lint.pycodestyle] max-line-length = 110 [tool.ruff.lint.pydocstyle] convention = "numpy" [tool.setuptools] packages = ["cads_api_client"] [tool.setuptools.package-data] "cads_api_client" = ["py.typed"] [tool.setuptools_scm] write_to = "cads_api_client/version.py" write_to_template = ''' # Do not change! Do not track in version control! __version__ = "{version}" ''' cads-api-client-1.5.4/tests/000077500000000000000000000000001471505652400156315ustar00rootroot00000000000000cads-api-client-1.5.4/tests/conftest.py000066400000000000000000000012651471505652400200340ustar00rootroot00000000000000from __future__ import annotations import os from typing import TYPE_CHECKING import pytest if TYPE_CHECKING: from cads_api_client import ApiClient @pytest.fixture def api_root_url() -> str: from cads_api_client import config try: return str(config.get_config("url")) except Exception: return "http://localhost:8080/api" @pytest.fixture def api_anon_key() -> str: return os.getenv("CADS_API_ANON_KEY", "00112233-4455-6677-c899-aabbccddeeff") @pytest.fixture def api_anon_client(api_root_url: str, api_anon_key: str) -> ApiClient: from cads_api_client import ApiClient return ApiClient(url=api_root_url, key=api_anon_key, maximum_tries=0) cads-api-client-1.5.4/tests/integration_test_10_catalogue.py000066400000000000000000000070071471505652400241150ustar00rootroot00000000000000import pytest from cads_api_client import ApiClient, Collection, processing @pytest.fixture def collection(api_anon_client: ApiClient) -> Collection: return api_anon_client.get_collection("test-adaptor-mars") def test_catalogue_collections(api_anon_client: ApiClient) -> None: collections = api_anon_client.get_collections() assert collections.next is not None collection_ids = list(collections.collection_ids) while len(collection_ids) != collections.json["numberMatched"]: assert (next_collections := collections.next) is not None collections = next_collections collection_ids.extend(collections.collection_ids) assert "reanalysis-era5-single-levels" in collection_ids def test_catalogue_collections_limit(api_anon_client: ApiClient) -> None: collections = api_anon_client.get_collections(limit=1) assert len(collections.collection_ids) == 1 def test_catalogue_collections_sortby(api_anon_client: ApiClient) -> None: collections_title = api_anon_client.get_collections(sortby="title") collections_relevance = api_anon_client.get_collections(sortby="relevance") assert collections_title.collection_ids != collections_relevance.collection_ids def test_catalogue_collections_query(api_anon_client: ApiClient) -> None: collections = api_anon_client.get_collections(query="test") assert collections.collection_ids def test_catalogue_collections_keywords(api_anon_client: ApiClient) -> None: collections = api_anon_client.get_collections(keywords=["Product type: Reanalysis"]) assert collections.collection_ids def test_catalogue_collection_begin_datetime(collection: Collection) -> None: assert collection.begin_datetime is not None assert collection.begin_datetime.isoformat() == "1959-01-01T00:00:00+00:00" def test_catalogue_collection_end_datetime(collection: Collection) -> None: assert collection.end_datetime is not None assert collection.end_datetime.isoformat() == "2023-05-09T00:00:00+00:00" def test_catalogue_collection_bbox(collection: Collection) -> None: assert collection.bbox == (0, -89, 360, 89) def test_catalogue_collection_id(collection: Collection) -> None: assert collection.id == "test-adaptor-mars" def test_catalogue_collection_json(collection: Collection) -> None: assert isinstance(collection.json, dict) def test_catalogue_collection_process(collection: Collection) -> None: process = collection.process assert isinstance(process, processing.Process) assert process.id == "test-adaptor-mars" def test_catalogue_collection_url(collection: Collection) -> None: assert collection.url.endswith("collections/test-adaptor-mars") def test_catalogue_get_licences(api_anon_client: ApiClient) -> None: all_licences = api_anon_client.get_licences(scope="all") assert all("id" in licence and "revision" in licence for licence in all_licences) portal_licences = api_anon_client.get_licences(scope="portal") assert portal_licences for portal_licence in portal_licences: assert portal_licence in all_licences def test_catalogue_form(collection: Collection) -> None: assert set(collection.form[0]) == { "css", "default", "details", "help", "id", "label", "name", "required", "type", } def test_catalogue_constraints(collection: Collection) -> None: assert set(collection.constraints[0]) == { "day", "month", "product_type", "time", "variable", "year", } cads-api-client-1.5.4/tests/integration_test_20_processing.py000066400000000000000000000055171471505652400243320ustar00rootroot00000000000000import logging import pytest from requests import HTTPError from cads_api_client import ApiClient, Process, Processes, Remote def test_processig_processes_limit(api_anon_client: ApiClient) -> None: processes = api_anon_client.get_processes(limit=1) assert isinstance(processes, Processes) assert len(processes.collection_ids) == 1 next_processes = processes.next assert next_processes is not None assert len(next_processes.process_ids) == 1 def test_processing_processes_sortby(api_anon_client: ApiClient) -> None: processes = api_anon_client.get_processes(sortby="id") assert len(processes.collection_ids) > 1 assert processes.collection_ids == sorted(processes.collection_ids) processes = api_anon_client.get_processes(sortby="-id") assert processes.collection_ids == sorted(processes.collection_ids, reverse=True) def test_processing_process( caplog: pytest.LogCaptureFixture, api_anon_client: ApiClient ) -> None: process = api_anon_client.get_process("test-adaptor-dummy") assert isinstance(process, Process) assert process.id == "test-adaptor-dummy" with caplog.at_level(logging.INFO, logger="cads_api_client.processing"): remote = process.submit() assert isinstance(remote, Remote) assert "The job has been submitted as an anonymous user" in caplog.text def test_processing_apply_constraints(api_anon_client: ApiClient) -> None: result = api_anon_client.apply_constraints( "test-adaptor-url", version="deprecated (1.0)" ) assert result["reference_dataset"] == ["cru", "cru_and_gpcc"] with pytest.raises(HTTPError, match="invalid param 'foo'"): api_anon_client.apply_constraints("test-adaptor-url", foo="bar") def test_processing_estimate_costs(api_anon_client: ApiClient) -> None: result = api_anon_client.estimate_costs("test-adaptor-url", variable=["foo", "bar"]) assert result == { "id": "size", "cost": 2.0, "limit": 1000.0, "cost_bar_steps": None, } def test_processing_get_jobs_status(api_anon_client: ApiClient) -> None: remote = api_anon_client.submit("test-adaptor-dummy", format="foo") request_uid = remote.request_uid with pytest.raises(HTTPError, match="400 Client Error: Bad Request"): remote.make_results() assert request_uid in api_anon_client.get_jobs(status="failed").request_uids assert request_uid not in api_anon_client.get_jobs(status="successful").request_uids def test_processing_get_jobs_sortby(api_anon_client: ApiClient) -> None: uid1 = api_anon_client.submit("test-adaptor-dummy").request_uid uid2 = api_anon_client.submit("test-adaptor-dummy").request_uid assert [uid2, uid1] == api_anon_client.get_jobs( sortby="-created", limit=2 ).request_uids assert [uid2] != api_anon_client.get_jobs(sortby="created", limit=1).request_uids cads-api-client-1.5.4/tests/integration_test_30_remote.py000066400000000000000000000054361471505652400234520ustar00rootroot00000000000000import contextlib import datetime import os import pathlib import uuid from typing import Any import pytest from requests import HTTPError from cads_api_client import ApiClient, Remote does_not_raise = contextlib.nullcontext @pytest.fixture def remote(api_anon_client: ApiClient) -> Remote: return api_anon_client.submit("test-adaptor-dummy", size=1) def test_remote_delete(remote: Remote) -> None: result = remote.delete() assert result["status"] == "dismissed" with pytest.raises(HTTPError, match="404 Client Error"): remote.status def test_remote_download(remote: Remote, tmp_path: pathlib.Path) -> None: target = str(tmp_path / "dummy.grib") result = remote.download(target=target) assert result == target assert os.path.getsize(result) == 1 def test_remote_collection_id(remote: Remote) -> None: assert remote.collection_id == "test-adaptor-dummy" def test_remote_json(remote: Remote) -> None: assert isinstance(remote.json, dict) def test_remote_request(remote: Remote) -> None: assert remote.request == { "elapsed": 0.0, "format": "grib", "size": 1, } def test_remote_request_uid(remote: Remote) -> None: assert uuid.UUID(remote.request_uid) def test_remote_status(remote: Remote) -> None: assert remote.status in ("accepted", "running", "successful") def test_remote_failed(api_anon_client: ApiClient) -> None: remote = api_anon_client.submit("test-adaptor-dummy", format="foo") with pytest.raises(HTTPError, match="400 Client Error: Bad Request"): remote.download() assert remote.status == "failed" @pytest.mark.parametrize( "cleanup,raises", [ (True, pytest.raises(HTTPError, match="404 Client Error")), (False, does_not_raise()), ], ) def test_remote_cleanup( api_root_url: str, api_anon_key: str, cleanup: bool, raises: contextlib.nullcontext[Any], ) -> None: client = ApiClient( url=api_root_url, key=api_anon_key, cleanup=cleanup, maximum_tries=0 ) remote = client.submit("test-adaptor-dummy") request_uid = remote.request_uid del remote client = ApiClient(url=api_root_url, key=api_anon_key, maximum_tries=0) with raises: client.get_remote(request_uid) def test_remote_datetimes(api_anon_client: ApiClient) -> None: remote = api_anon_client.submit( "test-adaptor-dummy", elapsed=1, _timestamp=datetime.datetime.now().isoformat(), ) assert remote.results_ready is False assert isinstance(remote.creation_datetime, datetime.datetime) assert remote.end_datetime is None remote.make_results() assert remote.start_datetime is not None assert remote.end_datetime is not None assert remote.creation_datetime < remote.start_datetime < remote.end_datetime cads-api-client-1.5.4/tests/integration_test_40_results.py000066400000000000000000000061441471505652400236560ustar00rootroot00000000000000import contextlib import pathlib import random from typing import Any import pytest import requests from cads_api_client import ApiClient, Results does_not_raise = contextlib.nullcontext @pytest.fixture def results(api_anon_client: ApiClient) -> Results: return api_anon_client.submit_and_wait_on_results("test-adaptor-dummy", size=1) def test_results_asset(results: Results) -> None: assert results.asset["type"] == "application/x-grib" assert results.asset["file:size"] == 1 @pytest.mark.parametrize("progress", [True, False]) def test_results_progress( api_root_url: str, api_anon_key: str, tmp_path: pathlib.Path, progress: bool, capsys: pytest.CaptureFixture[str], ) -> None: with capsys.disabled(): client = ApiClient( url=api_root_url, key=api_anon_key, progress=progress, maximum_tries=0 ) submitted = client.submit("test-adaptor-dummy") submitted.download(target=str(tmp_path / "test.grib")) captured = capsys.readouterr() assert captured.err if progress else not captured.err @pytest.mark.parametrize( "maximum_tries,raises", [ (500, does_not_raise()), (1, pytest.raises(requests.ConnectionError, match="Random error.")), ], ) def test_results_robust_download( api_root_url: str, api_anon_key: str, monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path, maximum_tries: int, raises: contextlib.nullcontext[Any], ) -> None: from multiurl.http import FullHTTPDownloader def patched_iter_content(self, *args, **kwargs): # type: ignore for chunk in self.iter_content(chunk_size=1): if random.choice([True, False]): raise requests.ConnectionError("Random error.") yield chunk def make_stream(self): # type: ignore request = self.issue_request(self.range) return request.patched_iter_content client = ApiClient( url=api_root_url, key=api_anon_key, retry_after=0, maximum_tries=maximum_tries ) results = client.submit_and_wait_on_results("test-adaptor-dummy", size=10) monkeypatch.setattr( requests.Response, "patched_iter_content", patched_iter_content, raising=False ) monkeypatch.setattr(FullHTTPDownloader, "make_stream", make_stream) target = tmp_path / "test.grib" with raises: results.download(str(target)) def test_results_override(api_anon_client: ApiClient, tmp_path: pathlib.Path) -> None: target_1 = tmp_path / "tmp1.grib" api_anon_client.retrieve("test-adaptor-dummy", size=1, target=str(target_1)) target_2 = tmp_path / "tmp2.grib" api_anon_client.retrieve("test-adaptor-dummy", size=2, target=str(target_2)) target = tmp_path / "tmp.grib" api_anon_client.retrieve("test-adaptor-dummy", size=1, target=str(target)) assert target.read_bytes() == target_1.read_bytes() api_anon_client.retrieve("test-adaptor-dummy", size=2, target=str(target)) assert target.read_bytes() == target_2.read_bytes() api_anon_client.retrieve("test-adaptor-dummy", size=1, target=str(target)) assert target.read_bytes() == target_1.read_bytes() cads-api-client-1.5.4/tests/integration_test_50_profile.py000066400000000000000000000025331471505652400236140ustar00rootroot00000000000000from __future__ import annotations from typing import Literal import pytest from requests import HTTPError from cads_api_client import ApiClient, config @pytest.mark.parametrize("scope", [None, "all", "dataset", "portal"]) def test_api_client_accept_licence( scope: Literal["all", "dataset", "portal"] | None, ) -> None: try: # Can not use anonymous user config.get_config("key") except Exception: pytest.skip("The API key is missing") client = ApiClient(maximum_tries=0) licence = client.get_licences(scope=scope)[0] licence_id = licence["id"] licence_revision = licence["revision"] expected = {"id": licence_id, "revision": licence_revision} actual = client.accept_licence(licence_id, licence_revision) assert expected == actual assert any( licence["id"] == licence_id and licence["revision"] == licence_revision for licence in client.get_accepted_licences(scope=scope) ) def test_api_client_check_authentication( api_root_url: str, api_anon_client: ApiClient ) -> None: assert api_anon_client.check_authentication() == { "id": -1, "role": "anonymous", "sub": "anonymous", } bad_client = ApiClient(key="foo", url=api_root_url) with pytest.raises(HTTPError, match="401 Client Error"): bad_client.check_authentication() cads-api-client-1.5.4/tests/integration_test_60_api_client.py000066400000000000000000000053751471505652400242730ustar00rootroot00000000000000from __future__ import annotations import os import pathlib import pytest from urllib3.exceptions import InsecureRequestWarning from cads_api_client import ApiClient, Remote, Results, processing def test_api_client_download_results( api_anon_client: ApiClient, tmp_path: pathlib.Path ) -> None: remote = api_anon_client.submit("test-adaptor-dummy") target = str(tmp_path / "test.grib") result = api_anon_client.download_results(remote.request_uid, target) assert result == target assert os.path.exists(result) def test_api_client_get_process(api_anon_client: ApiClient) -> None: process = api_anon_client.get_process("test-adaptor-dummy") assert isinstance(process, processing.Process) assert process.id == "test-adaptor-dummy" assert set(process.headers) == {"User-Agent", "PRIVATE-TOKEN"} def test_api_client_get_remote(api_anon_client: ApiClient) -> None: request_uid = api_anon_client.submit("test-adaptor-dummy").request_uid remote = api_anon_client.get_remote(request_uid) assert remote.request_uid == request_uid assert set(remote.headers) == {"User-Agent", "PRIVATE-TOKEN"} def test_api_client_get_results(api_anon_client: ApiClient) -> None: request_uid = api_anon_client.submit("test-adaptor-dummy").request_uid results = api_anon_client.get_results(request_uid) assert isinstance(results, Results) def test_api_client_retrieve( api_anon_client: ApiClient, tmp_path: pathlib.Path, ) -> None: expected_target = str(tmp_path / "dummy.grib") actual_target = api_anon_client.retrieve( "test-adaptor-dummy", target=expected_target, size=1 ) assert expected_target == actual_target assert os.path.getsize(actual_target) == 1 def test_api_client_submit(api_anon_client: ApiClient) -> None: remote = api_anon_client.submit("test-adaptor-dummy") assert isinstance(remote, Remote) def test_api_client_submit_and_wait_on_results(api_anon_client: ApiClient) -> None: results = api_anon_client.submit_and_wait_on_results("test-adaptor-dummy") assert isinstance(results, Results) def test_api_client_verify(api_root_url: str, api_anon_key: str) -> None: if not api_root_url.startswith("https"): pytest.skip(f"{api_root_url=} does not use https protocol") with pytest.warns(InsecureRequestWarning): ApiClient(url=api_root_url, key=api_anon_key, verify=False, maximum_tries=0) def test_api_client_timeout( api_root_url: str, api_anon_key: str, tmp_path: pathlib.Path, ) -> None: with pytest.warns(UserWarning, match="timeout"): client = ApiClient(url=api_root_url, key=api_anon_key, timeout=0) with pytest.raises(ValueError, match="timeout"): client.retrieve("test-adaptor-dummy", target=str(tmp_path / "test.grib")) cads-api-client-1.5.4/tests/integration_test_70_legacy_api_client.py000066400000000000000000000223411471505652400256100ustar00rootroot00000000000000from __future__ import annotations import contextlib import logging import os import pathlib import time from typing import Any import pytest import requests from cads_api_client import processing from cads_api_client.legacy_api_client import LegacyApiClient does_not_raise = contextlib.nullcontext @pytest.fixture def legacy_client(api_root_url: str, api_anon_key: str) -> LegacyApiClient: return LegacyApiClient(url=api_root_url, key=api_anon_key, retry_max=1) def legacy_update(remote: processing.Remote) -> None: # See https://github.com/ecmwf/cdsapi/blob/master/examples/example-era5-update.py sleep = 1 while True: with pytest.deprecated_call(): remote.update() reply = remote.reply remote.info("Request ID: %s, state: %s" % (reply["request_id"], reply["state"])) if reply["state"] == "completed": break elif reply["state"] in ("queued", "running"): remote.info("Request ID: %s, sleep: %s", reply["request_id"], sleep) time.sleep(sleep) elif reply["state"] in ("failed",): remote.error("Message: %s", reply["error"].get("message")) remote.error("Reason: %s", reply["error"].get("reason")) for n in ( reply.get("error", {}) .get("context", {}) .get("traceback", "") .split("\n") ): if n.strip() == "": break remote.error(" %s", n) raise Exception( "%s. %s." % (reply["error"].get("message"), reply["error"].get("reason")) ) def test_legacy_api_client_retrieve( tmp_path: pathlib.Path, legacy_client: LegacyApiClient ) -> None: collection_id = "test-adaptor-dummy" request = {"size": 1} target = str(tmp_path / "dummy.grib") actual_target = legacy_client.retrieve(collection_id, request, target) assert target == actual_target assert os.path.getsize(target) == 1 def test_legacy_api_client_result( monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path, legacy_client: LegacyApiClient, ) -> None: monkeypatch.chdir(tmp_path) collection_id = "test-adaptor-dummy" request = {"size": 1} result = legacy_client.retrieve(collection_id, request) target = result.download() assert os.path.basename(target) == os.path.basename(result.location) assert os.path.getsize(target) == 1 target = str(tmp_path / "dummy.grib") actual_target = result.download(target) assert target == actual_target assert os.path.getsize(target) == 1 response = requests.head(result.location) assert response.status_code == 200 assert result.content_length == 1 assert result.content_type == "application/x-grib" @pytest.mark.parametrize("quiet", [True, False]) def test_legacy_api_client_quiet( caplog: pytest.LogCaptureFixture, api_root_url: str, api_anon_key: str, quiet: bool, ) -> None: client = LegacyApiClient( url=api_root_url, key=api_anon_key, quiet=quiet, retry_max=0 ) client.retrieve("test-adaptor-dummy", {}) records = [record for record in caplog.records if record.levelname == "INFO"] assert not records if quiet else records @pytest.mark.parametrize("debug", [True, False]) def test_legacy_api_client_debug( caplog: pytest.LogCaptureFixture, api_root_url: str, api_anon_key: str, debug: bool, ) -> None: LegacyApiClient(url=api_root_url, key=api_anon_key, debug=debug, retry_max=0) records = [record for record in caplog.records if record.levelname == "DEBUG"] assert records if debug else not records @pytest.mark.parametrize( "wait_until_complete,expected_type", [(True, processing.Results), (False, processing.Remote)], ) def test_legacy_api_client_wait_until_complete( tmp_path: pathlib.Path, api_root_url: str, api_anon_key: str, wait_until_complete: bool, expected_type: type, ) -> None: client = LegacyApiClient( url=api_root_url, key=api_anon_key, wait_until_complete=wait_until_complete, retry_max=0, ) collection_id = "test-adaptor-dummy" request = {"size": 1} result = client.retrieve(collection_id, request) assert isinstance(result, expected_type) target = tmp_path / "test.grib" result.download(str(target)) assert target.stat().st_size == 1 @pytest.mark.parametrize( "collection_id,format,raises", [ ("test-adaptor-dummy", "grib", does_not_raise()), ( "test-adaptor-dummy", "foo", pytest.raises(Exception, match="400 Client Error"), ), ], ) def test_legacy_api_client_update( api_root_url: str, api_anon_key: str, collection_id: str, format: str, raises: contextlib.nullcontext[Any], ) -> None: client = LegacyApiClient( url=api_root_url, key=api_anon_key, wait_until_complete=False, retry_max=0 ) remote = client.retrieve(collection_id, {"format": format}) assert isinstance(remote, processing.Remote) with raises: legacy_update(remote) @pytest.mark.filterwarnings("ignore:Unverified HTTPS") def test_legacy_api_client_kwargs(api_root_url: str, api_anon_key: str) -> None: session = requests.Session() client = LegacyApiClient( url=api_root_url, key=api_anon_key, verify=0, timeout=1, progress=False, delete=True, retry_max=2, sleep_max=3, wait_until_complete=False, session=session, ) assert client.client.url == api_root_url assert client.client.key == api_anon_key assert client.client.verify is False assert client.client.timeout == 1 assert client.client.progress is False assert client.client.cleanup is True assert client.client.maximum_tries == 2 assert client.client.sleep_max == 3 assert client.client.session is session def test_legacy_api_client_logging( caplog: pytest.LogCaptureFixture, api_root_url: str, api_anon_key: str, ) -> None: logger = logging.getLogger("foo") client = LegacyApiClient( url=api_root_url, key=api_anon_key, info_callback=logger.info, warning_callback=logger.warning, error_callback=logger.error, debug_callback=logger.debug, ) caplog.clear() with caplog.at_level(logging.DEBUG): client.debug("Debug message") client.info("Info message") client.warning("Warning message") client.error("Error message") assert caplog.record_tuples == [ ("foo", 10, "Debug message"), ("foo", 20, "Info message"), ("foo", 30, "Warning message"), ("foo", 40, "Error message"), ] def test_legacy_api_client_download( tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch, api_root_url: str, api_anon_key: str, ) -> None: monkeypatch.chdir(tmp_path) client = LegacyApiClient( url=api_root_url, key=api_anon_key, retry_max=1, wait_until_complete=False, ) remote = client.retrieve("test-adaptor-dummy", {"size": 1}) assert isinstance(remote, processing.Remote) target = client.download(remote) assert os.path.getsize(target) == 1 results = remote.make_results() results_dict = { "location": results.location, "contentLength": results.content_length, } results_tuple = (remote, remote.make_results(), results_dict) target1 = "remote.grib" target2 = "results.grib" target3 = "dict.grib" targets = [target1, target2, target3] assert client.download(results_tuple, targets) == [target1, target2, target3] assert all(os.path.getsize(target) == 1 for target in targets) def test_legacy_api_client_status(legacy_client: LegacyApiClient) -> None: status = legacy_client.status() assert set(status) <= { "critical", "fatal", "error", "warning", "warn", "info", "debug", "notset", } assert all( isinstance(value, list) and isinstance(string, str) for value in status.values() for string in value ) def test_legacy_api_client_remote( legacy_client: LegacyApiClient, tmp_path: pathlib.Path ) -> None: results = legacy_client.retrieve("test-adaptor-dummy", {"size": 1}) remote = legacy_client.remote(results.location) target = str(tmp_path / "dummy.grib") actual_target = remote.download(target) assert target == actual_target assert os.path.getsize(target) == 1 def test_legacy_api_client_warning( api_root_url: str, api_anon_key: str, ) -> None: with pytest.warns( UserWarning, match="deprecated: {'full_stack': 'a', 'metadata': 'b', 'forget': 'c'}", ): LegacyApiClient( url=api_root_url, key=api_anon_key, full_stack="a", # type: ignore[arg-type] metadata="b", # type: ignore[arg-type] forget="c", # type: ignore[arg-type] ) def test_legacy_api_client_toolbox(legacy_client: LegacyApiClient) -> None: with pytest.raises(NotImplementedError): legacy_client.service("service") with pytest.raises(NotImplementedError): legacy_client.workflow("workflow") cads-api-client-1.5.4/tests/integration_test_80_adaptors.py000066400000000000000000000100211471505652400237630ustar00rootroot00000000000000import datetime import filecmp import os import pathlib import zipfile import pytest from requests import HTTPError from cads_api_client import ApiClient def test_adaptors_dummy(api_anon_client: ApiClient, tmp_path: pathlib.Path) -> None: collection_id = "test-adaptor-dummy" target = str(tmp_path / "dummy.grib") remote = api_anon_client.submit( collection_id, _timestamp=datetime.datetime.now().isoformat(), ) assert remote.download(target) == target assert os.path.exists(target) def test_adaptors_dummy_cached( api_anon_client: ApiClient, tmp_path: pathlib.Path ) -> None: collection_id = "test-adaptor-dummy" target_grib = str(tmp_path / "dummy.grib") request = {"size": 1, "_timestamp": datetime.datetime.now().isoformat()} results = api_anon_client.submit_and_wait_on_results(collection_id, **request) assert results.content_type == "application/x-grib" assert results.download(target_grib) == target_grib assert os.path.getsize(target_grib) == 1 expected_bytes = open(target_grib, "rb").read() target_netcdf = str(tmp_path / "dummy.nc") results = api_anon_client.submit_and_wait_on_results( collection_id, format="netcdf", **request ) assert results.content_type == "application/netcdf" assert results.download(target_netcdf) == target_netcdf assert os.path.getsize(target_netcdf) == 1 assert expected_bytes == open(target_netcdf, "rb").read() target_zip = str(tmp_path / "dummy.zip") results = api_anon_client.submit_and_wait_on_results( collection_id, format="zip", **request ) assert results.content_type == "application/zip" assert results.download(target_zip) == target_zip with zipfile.ZipFile(target_zip, "r") as zip_fp: with zip_fp.open("dummy_0.grib", "r") as zip_grib_fs: assert expected_bytes == zip_grib_fs.read() def test_adaptors_url(api_anon_client: ApiClient, tmp_path: pathlib.Path) -> None: collection_id = "test-adaptor-url" request = { "variable": "grid_point_altitude", "reference_dataset": "cru", "version": "2_1", "format": "zip", "_timestamp": datetime.datetime.now().isoformat(), } target1 = str(tmp_path / "wfde1.zip") remote = api_anon_client.submit(collection_id, **request) assert remote.download(target1) == target1 assert os.path.exists(target1) target2 = str(tmp_path / "wfde2.zip") remote = api_anon_client.submit(collection_id, **request) assert remote.download(target2) == target2 assert filecmp.cmp(target1, target2) def test_adaptors_url_constraints( api_anon_client: ApiClient, tmp_path: pathlib.Path ) -> None: with pytest.raises(HTTPError, match="400 Client Error: Bad Request"): api_anon_client.submit("test-adaptor-url", foo="bar") def test_adaptors_direct_mars( api_anon_client: ApiClient, tmp_path: pathlib.Path ) -> None: collection_id = "test-adaptor-direct-mars" request = { "levelist": "1", "dataset": "reanalysis", "time": "00:00:00", "param": "155", "date": "1940-01-01", "expect": "any", "levtype": "pl", "number": "all", "class": "ea", } target = str(tmp_path / "era5-complete.grib") remote = api_anon_client.submit( collection_id, _timestamp=datetime.datetime.now().isoformat(), **request, ) assert remote.download(target) == target assert os.path.exists(target) def test_adaptors_mars(api_anon_client: ApiClient, tmp_path: pathlib.Path) -> None: collection_id = "test-adaptor-mars" request = { "product_type": "reanalysis", "variable": "2m_temperature", "year": "2016", "month": "01", "day": "02", "time": "00:00", } target = str(tmp_path / "era5.grib") remote = api_anon_client.submit( collection_id, **request, _timestamp=datetime.datetime.now().isoformat(), ) assert remote.download(target) == target assert os.path.exists(target) cads-api-client-1.5.4/tests/integration_test_90_features.py000066400000000000000000000041211471505652400237710ustar00rootroot00000000000000import datetime import os import pathlib from pathlib import Path from typing import Any import pytest from cads_api_client import ApiClient def test_features_url_cds_adaptor_area_selection( tmp_path: Path, api_anon_client: ApiClient, ) -> None: collection_id = "test-adaptor-url" request: dict[str, Any] = { "variable": "grid_point_altitude", "reference_dataset": "cru", "version": "2_1", } result_bigger = api_anon_client.retrieve( collection_id, **request, target=str(tmp_path / "bigger.zip"), ) result_smaller = api_anon_client.retrieve( collection_id, **request, target=str(tmp_path / "smaller.zip"), area=[50, 0, 40, 10], ) assert os.path.getsize(result_bigger) > os.path.getsize(result_smaller) @pytest.mark.parametrize( "format,expected_extension", [ ("grib", ".grib"), ("netcdf", ".nc"), ], ) def test_features_mars_cds_adaptor_format( api_anon_client: ApiClient, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, format: str, expected_extension: str, ) -> None: monkeypatch.chdir(tmp_path) collection_id = "test-adaptor-mars" request: dict[str, Any] = { "product_type": "reanalysis", "variable": "2m_temperature", "year": "2016", "month": "01", "day": "02", "time": "00:00", "target": None, } result = api_anon_client.retrieve( collection_id, **request, format=format, ) _, actual_extension = os.path.splitext(result) assert actual_extension == expected_extension assert os.path.getsize(result) def test_features_upload_big_file( api_anon_client: ApiClient, tmp_path: pathlib.Path ) -> None: # See: https://github.com/fsspec/s3fs/pull/910 target = str(tmp_path / "test.grib") size = 1_048_576_000 + 1 api_anon_client.retrieve( "test-adaptor-dummy", size=size, _timestamp=datetime.datetime.now().isoformat(), target=target, ) assert os.path.getsize(target) == size cads-api-client-1.5.4/tests/test_00_version.py000066400000000000000000000001451471505652400212260ustar00rootroot00000000000000import cads_api_client def test_version() -> None: assert cads_api_client.__version__ != "999" cads-api-client-1.5.4/tests/test_10_config.py000066400000000000000000000043601471505652400210120ustar00rootroot00000000000000import json import pathlib import pytest from cads_api_client import config def test_read_configuration( tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch ) -> None: expected_config = {"url": "dummy-url", "key": "dummy-key"} config_file = tmp_path / ".cads-api-client.json" with config_file.open("w") as fp: json.dump({"url": "dummy-url", "key": "dummy-key"}, fp) res = config.read_configuration_file(str(config_file)) assert res == expected_config monkeypatch.setenv("CADS_API_RC", str(config_file)) res = config.read_configuration_file(None) assert res == expected_config def test_read_configuration_error(tmp_path: pathlib.Path) -> None: config_file = tmp_path / ".cads-api-client.json" config_file.write_text("XXX") with pytest.raises(ValueError): config.read_configuration_file(str(config_file)) with pytest.raises(FileNotFoundError): config.read_configuration_file("non-existent-file") def test_get_config_from_configuration_file( tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.delenv("CADS_API_KEY", raising=False) monkeypatch.delenv("CADS_API_URL", raising=False) expected_config = {"url": "dummy-url", "key": "dummy-key"} config_file = tmp_path / ".cads-api-client.json" with config_file.open("w") as fp: json.dump(expected_config, fp) res = config.get_config("url", str(config_file)) assert res == expected_config["url"] with pytest.raises(KeyError): config.get_config("non-existent-key", str(config_file)) def test_get_config_from_environment_variables( tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch ) -> None: expected_config = {"url": "dummy-url", "key": "dummy-key"} file_config = {"url": "wrong-url", "key": "wrong-key"} config_file = tmp_path / ".cads-api-client.json" with config_file.open("w") as fp: json.dump(file_config, fp) monkeypatch.setenv("CADS_API_URL", expected_config["url"]) monkeypatch.setenv("CADS_API_KEY", expected_config["key"]) res = config.get_config("url", str(config_file)) assert res == expected_config["url"] res = config.get_config("key", str(config_file)) assert res == expected_config["key"] cads-api-client-1.5.4/tests/test_10_processing.py000066400000000000000000000343771471505652400217340ustar00rootroot00000000000000import json import logging import pytest import requests import responses from responses.matchers import json_params_matcher from cads_api_client import catalogue, processing COLLECTION_ID = "reanalysis-era5-pressure-levels" JOB_SUCCESSFUL_ID = "9bfc1362-2832-48e1-a235-359267420bb2" JOB_FAILED_ID = "9bfc1362-2832-48e1-a235-359267420bb3" CATALOGUE_URL = "http://localhost:8080/api/catalogue" COLLECTIONS_URL = "http://localhost:8080/api/catalogue/v1/datasets" COLLECTION_URL = ( "http://localhost:8080/api/catalogue/v1/collections/reanalysis-era5-pressure-levels" ) PROCESS_URL = f"http://localhost:8080/api/retrieve/v1/processes/{COLLECTION_ID}" EXECUTE_URL = f"{PROCESS_URL}/execution" JOB_SUCCESSFUL_URL = f"http://localhost:8080/api/retrieve/v1/jobs/{JOB_SUCCESSFUL_ID}" JOB_FAILED_URL = f"http://localhost:8080/api/retrieve/v1/jobs/{JOB_FAILED_ID}" RESULT_SUCCESSFUL_URL = ( f"http://localhost:8080/api/retrieve/v1/jobs/{JOB_SUCCESSFUL_ID}/results" ) RESULT_FAILED_URL = ( f"http://localhost:8080/api/retrieve/v1/jobs/{JOB_FAILED_ID}/results" ) CATALOGUE_JSON = { "type": "Catalog", "id": "stac-fastapi", "links": [ {"rel": "self", "type": "application/json", "href": f"{CATALOGUE_URL}/v1"}, {"rel": "root", "type": "application/json", "href": f"{CATALOGUE_URL}/v1"}, {"rel": "data", "type": "application/json", "href": f"{COLLECTIONS_URL}"}, { "rel": "child", "type": "application/json", "title": "ERA5 hourly data on pressure levels from 1959 to present", "href": f"{COLLECTION_URL}", }, ], } COLLECTIONS_JSON = { "collections": [ { "type": "Collection", "id": f"{COLLECTION_ID}", "links": [ { "rel": "self", "type": "application/json", "href": f"{COLLECTION_URL}", }, { "rel": "parent", "type": "application/json", "href": f"{CATALOGUE_URL}/v1", }, { "rel": "root", "type": "application/json", "href": f"{CATALOGUE_URL}/v1", }, ], }, ], "links": [ {"rel": "root", "type": "application/json", "href": f"{CATALOGUE_URL}/v1"}, {"rel": "parent", "type": "application/json", "href": f"{CATALOGUE_URL}/v1"}, {"rel": "self", "type": "application/json", "href": f"{COLLECTIONS_URL}"}, ], } COLLECTION_JSON = { "type": "Collection", "id": COLLECTION_ID, "links": [ { "rel": "self", "type": "application/json", "href": "http://localhost:8080/api/catalogue/v1/collections/reanalysis-era5-pressure-levels", }, { "rel": "parent", "type": "application/json", "href": "http://localhost:8080/api/catalogue/v1/", }, { "rel": "root", "type": "application/json", "href": "http://localhost:8080/api/catalogue/v1/", }, { "rel": "retrieve", "href": "http://localhost:8080/api/retrieve/v1/processes/reanalysis-era5-pressure-levels", "type": "application/json", }, { "rel": "related", "href": "http://localhost:8080/api/catalogue/v1/collections/reanalysis-era5-single-levels", }, ], "tmp:variables": { "Temperature": { "units": "K", } }, } PROCESS_JSON = { "id": COLLECTION_ID, "links": [ {"href": COLLECTION_URL, "rel": "self", "type": "application/json"}, {"rel": "retrieve", "href": PROCESS_URL, "type": "application/json"}, { "href": f"{COLLECTION_URL}/execution", "rel": "execute", "type": "application/json", "title": "process execution", }, ], "inputs": { "product_type": { "title": "Product type", "schema": { "type": "array", "items": {"enum": ["ensemble_mean", "reanalysis"], "type": "string"}, }, }, "variable": { "title": "Variable", "schema": { "type": "array", "items": {"enum": ["temperature", "vorticity"], "type": "string"}, }, }, "year": { "title": "Year", "schema": { "type": "array", "items": {"enum": ["2022", "0000"], "type": "string"}, }, }, }, "outputs": { "download_url": { "schema": {"type": "string", "format": "url"}, } }, "message": "WARNING: This is a warning message", "metadata": { "datasetMetadata": { "messages": [ { "date": "2023-12-12T13:00:00", "severity": "warning", "content": "This is a warning dataset message", }, { "date": "2023-12-12T14:00:00", "severity": "success", "content": "This is a success dataset message", }, ] } }, } JOB_SUCCESSFUL_JSON = { "processID": f"{COLLECTION_ID}", "type": "process", "jobID": f"{JOB_SUCCESSFUL_ID}", "status": "successful", "created": "2022-09-02T17:30:48.201213", "started": "2022-09-02T17:32:43.890617", "finished": "2022-09-02T17:32:54.308120", "updated": "2022-09-02T17:32:54.308116", "links": [ {"href": f"{JOB_SUCCESSFUL_ID}", "rel": "self", "type": "application/json"}, { "href": f"http://localhost:8080/api/retrieve/v1/jobs/{JOB_SUCCESSFUL_ID}/results", "rel": "results", }, { "href": f"{JOB_SUCCESSFUL_URL}", "rel": "monitor", "type": "application/json", "title": "job status info", }, ], "metadata": { "log": [ ["2024-02-09T09:14:47.811223", "This is a log"], ["2024-02-09T09:14:50.811223", "WARNING: This is a warning log"], ] }, } JOB_FAILED_JSON = { "processID": f"{COLLECTION_ID}", "type": "process", "jobID": f"{JOB_FAILED_ID}", "status": "failed", "created": "2022-09-02T17:30:48.201213", "started": "2022-09-02T17:32:43.890617", "finished": "2022-09-02T17:32:54.308120", "updated": "2022-09-02T17:32:54.308116", "links": [ {"href": f"{JOB_FAILED_ID}", "rel": "self", "type": "application/json"}, { "href": f"http://localhost:8080/api/retrieve/v1/jobs/{JOB_FAILED_ID}/results", "rel": "results", }, { "href": f"{JOB_FAILED_URL}", "rel": "monitor", "type": "application/json", "title": "job status info", }, ], "metadata": { "log": [ ["2024-02-09T09:14:47.811223", "This is a log"], ["2024-02-09T09:14:50.811223", "WARNING: This is a warning log"], ] }, } RESULT_SUCCESSFUL_JSON = { "asset": { "value": { "type": "application/netcdf", "href": "./e7d452a747061ab880887d88814bfb0c27593a73cb7736d2dc340852", "file:checksum": "e7d452a747061ab880887d88814bfb0c27593a73cb7736d2dc340852", "file:size": 8, "file:local_path": [ "/cache-store/", "e7d452a747061ab880887d88814bfb0c27593a73cb7736d2dc340852.nc", ], "xarray:open_kwargs": {}, "xarray:storage_options": {}, } } } RESULT_FAILED_JSON = { "type": "job results failed", "title": "job failed", "status": 400, "instance": "http://127.0.0.1:8080/api/retrieve/v1/jobs/02135eee-39a8-4d1f-8cd7-87682de5b981/results", "trace_id": "ca3e7170-1ce2-48fc-97f8-bbe64fafce44", "traceback": "This is a traceback", } @pytest.fixture def cat() -> catalogue.Catalogue: return catalogue.Catalogue( CATALOGUE_URL, headers={}, session=requests.Session(), retry_options={}, request_options={}, download_options={}, sleep_max=120, cleanup=False, log_callback=None, ) def responses_add() -> None: responses.add( responses.GET, url=f"{CATALOGUE_URL}/v1/", json=CATALOGUE_JSON, content_type="application/json", ) responses.add( responses.GET, url=f"{COLLECTIONS_URL}", json=COLLECTIONS_JSON, content_type="application/json", ) responses.add( responses.GET, url=f"{COLLECTION_URL}", json=COLLECTION_JSON, content_type="application/json", ) responses.add( responses.GET, url=PROCESS_URL, json=PROCESS_JSON, content_type="application/json", ) responses.add( responses.POST, url=EXECUTE_URL, json=JOB_SUCCESSFUL_JSON, match=[ json_params_matcher( { "inputs": {"variable": "temperature", "year": "2022"}, } ) ], content_type="application/json", ) responses.add( responses.GET, url=JOB_SUCCESSFUL_URL, json=JOB_SUCCESSFUL_JSON, content_type="application/json", ) responses.add( responses.POST, url=EXECUTE_URL, json=JOB_FAILED_JSON, match=[ json_params_matcher( { "inputs": {"variable": "temperature", "year": "0000"}, } ) ], content_type="application/json", ) responses.add( responses.GET, url=JOB_FAILED_URL, json=JOB_FAILED_JSON, content_type="application/json", ) responses.add( responses.GET, url=RESULT_FAILED_URL, json=RESULT_FAILED_JSON, content_type="application/json", ) @responses.activate def test_catalogue_collections(cat: catalogue.Catalogue) -> None: responses_add() assert cat.get_collections().json == COLLECTIONS_JSON collection = cat.get_collection(COLLECTION_ID) assert collection.response.json() == COLLECTION_JSON @responses.activate def test_submit(cat: catalogue.Catalogue) -> None: responses_add() collection = cat.get_collection(COLLECTION_ID) assert collection.process.response.json() == PROCESS_JSON remote = collection.process.submit(variable="temperature", year="2022") assert remote.json == JOB_SUCCESSFUL_JSON remote = collection.process.submit(variable="temperature", year="2022") assert remote.url == JOB_SUCCESSFUL_URL assert remote.status == "successful" assert remote.results_ready is True assert remote.creation_datetime.isoformat() == "2022-09-02T17:30:48.201213" assert remote.start_datetime is not None assert remote.start_datetime.isoformat() == "2022-09-02T17:32:43.890617" assert remote.end_datetime is not None assert remote.end_datetime.isoformat() == "2022-09-02T17:32:54.308120" @responses.activate def test_wait_on_result(cat: catalogue.Catalogue) -> None: responses_add() collection = cat.get_collection(COLLECTION_ID) remote = collection.process.submit(variable="temperature", year="2022") remote._wait_on_results() @responses.activate def test_wait_on_result_failed(cat: catalogue.Catalogue) -> None: responses_add() collection = cat.get_collection(COLLECTION_ID) remote = collection.process.submit(variable="temperature", year="0000") with pytest.raises( processing.ProcessingFailedError, match="job failed\nThis is a traceback", ): remote._wait_on_results() assert remote.creation_datetime.isoformat() == "2022-09-02T17:30:48.201213" assert remote.start_datetime is not None assert remote.start_datetime.isoformat() == "2022-09-02T17:32:43.890617" assert remote.end_datetime is not None assert remote.end_datetime.isoformat() == "2022-09-02T17:32:54.308120" @responses.activate def test_remote_logs( caplog: pytest.LogCaptureFixture, cat: catalogue.Catalogue ) -> None: responses_add() collection = cat.get_collection(COLLECTION_ID) with caplog.at_level(logging.DEBUG, logger="cads_api_client.processing"): remote = collection.process.submit(variable="temperature", year="2022") remote._wait_on_results() assert caplog.record_tuples == [ ( "cads_api_client.processing", 10, "GET http://localhost:8080/api/retrieve/v1/processes/reanalysis-era5-pressure-levels", ), ( "cads_api_client.processing", 10, f"REPLY {json.dumps(PROCESS_JSON)}", ), ( "cads_api_client.processing", 30, "This is a warning message", ), ( "cads_api_client.processing", 30, "[2023-12-12T13:00:00] This is a warning dataset message", ), ( "cads_api_client.processing", 20, "[2023-12-12T14:00:00] This is a success dataset message", ), ( "cads_api_client.processing", 10, ( "POST http://localhost:8080/api/retrieve/v1/processes/" "reanalysis-era5-pressure-levels/execution " "{'variable': 'temperature', 'year': '2022'}" ), ), ( "cads_api_client.processing", 10, f"REPLY {json.dumps(JOB_SUCCESSFUL_JSON)}", ), ( "cads_api_client.processing", 20, "Request ID is 9bfc1362-2832-48e1-a235-359267420bb2", ), ( "cads_api_client.processing", 10, "GET http://localhost:8080/api/retrieve/v1/jobs/9bfc1362-2832-48e1-a235-359267420bb2", ), ( "cads_api_client.processing", 10, f"REPLY {json.dumps(JOB_SUCCESSFUL_JSON)}", ), ("cads_api_client.processing", 20, "This is a log"), ("cads_api_client.processing", 30, "This is a warning log"), ("cads_api_client.processing", 20, "status has been updated to successful"), ] cads-api-client-1.5.4/tests/test_40_results.py000066400000000000000000000040411471505652400212450ustar00rootroot00000000000000from __future__ import annotations import os import pathlib import pytest import responses from cads_api_client import Results RESULTS_URL = "http://localhost:8080/api/retrieve/v1/jobs/9bfc1362-2832-48e1-a235-359267420bb2/results" RESULTS_JSON = { "asset": { "value": { "type": "application/x-grib", "href": "http://httpbin.org/bytes/1", "file:size": 1, } } } @pytest.fixture @responses.activate def results() -> Results: responses.add( responses.GET, RESULTS_URL, json=RESULTS_JSON, status=200, content_type="application/json", ) return Results.from_request( "get", RESULTS_URL, headers={}, session=None, retry_options={"maximum_tries": 1}, request_options={}, download_options={}, sleep_max=120, cleanup=False, log_callback=None, ) @pytest.mark.parametrize( "target,expected", [ ("dummy.grib", "dummy.grib"), (None, "1"), ], ) def test_results_download( monkeypatch: pytest.MonkeyPatch, results: Results, tmp_path: pathlib.Path, target: str | None, expected: str, ) -> None: monkeypatch.chdir(tmp_path) actual = results.download(target=target) assert actual == expected assert os.path.getsize(actual) == 1 def test_results_asset(results: Results) -> None: assert results.asset == { "file:size": 1, "href": "http://httpbin.org/bytes/1", "type": "application/x-grib", } def test_results_content_length(results: Results) -> None: assert results.content_length == 1 def test_results_content_type(results: Results) -> None: assert results.content_type == "application/x-grib" def test_results_json(results: Results) -> None: assert results.json == RESULTS_JSON def test_results_location(results: Results) -> None: assert results.location == "http://httpbin.org/bytes/1" def test_results_url(results: Results) -> None: assert results.url == RESULTS_URL