cylc-flow-8.7.0/0000775000175000017500000000000015264420316013614 5ustar alastairalastaircylc-flow-8.7.0/.github/0000775000175000017500000000000015264420307015154 5ustar alastairalastaircylc-flow-8.7.0/.github/dependabot.yml0000664000175000017500000000040715264420307020005 0ustar alastairalastairversion: 2 updates: - package-ecosystem: "pip" directory: "/" schedule: interval: "daily" - package-ecosystem: "github-actions" directory: "/" schedule: interval: "daily" labels: - 'dependencies' - 'infrastructure' cylc-flow-8.7.0/.github/workflows/0000775000175000017500000000000015264420307017211 5ustar alastairalastaircylc-flow-8.7.0/.github/workflows/shortlog.yml0000664000175000017500000000112315264420307021572 0ustar alastairalastairname: check contributor list on: workflow_dispatch: pull_request: paths: - 'CONTRIBUTING.md' - '.github/workflows/shortlog.yml' - '.mailmap' schedule: - cron: '37 06 * * 1' # 06:37, Monday jobs: test: runs-on: 'ubuntu-latest' timeout-minutes: 10 steps: - name: Checkout uses: actions/checkout@v7 with: fetch-depth: 0 # need to fetch all commits to check contributors ref: ${{ github.event.pull_request.head.sha }} - name: Check CONTRIBUTING.md uses: cylc/release-actions/check-shortlog@v1 cylc-flow-8.7.0/.github/workflows/test_fast.yml0000664000175000017500000001147515264420307021740 0ustar alastairalastairname: fast tests on: pull_request: workflow_dispatch: push: branches: - master - '8.*.x' concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true defaults: run: shell: bash -c "exec $CONDA_PREFIX/bin/bash -elo pipefail {0}" env: PIP_PROGRESS_BAR: off jobs: test: runs-on: ${{ matrix.os }} timeout-minutes: 20 strategy: fail-fast: false # don't stop on first failure matrix: os: ['ubuntu-latest'] python-version: ['3.12', '3'] include: - os: 'macos-latest' python-version: '3.12' # oldest version we support # non-utc timezone test - os: 'ubuntu-latest' python-version: '3' time-zone: 'XXX-09:35' env: TZ: ${{ matrix.time-zone }} PYTEST_ADDOPTS: --cov --cov-append -n 5 --color=yes CYLC_COVERAGE: 1 steps: - name: Checkout uses: actions/checkout@v7 - name: Install System Dependencies uses: mamba-org/setup-micromamba@v3 with: cache-environment: true post-cleanup: 'all' environment-name: cylc-fast-test create-args: >- python=${{ matrix.python-version }} pip bash subversion sqlite graphviz - name: Install Python Dependencies run: | pip install -e ."[all]" - name: Patch DNS uses: cylc/release-actions/patch-dns@v1 - name: Configure git # Needed by the odd test uses: cylc/release-actions/configure-git@v1 # Useful debug info: - name: Pip list run: pip list - name: Unit Tests timeout-minutes: 5 run: | pytest cylc/flow tests/unit - name: Integration Tests timeout-minutes: 5 run: | pytest tests/integration --ignore tests/integration/tui/ - name: Tui Integration Tests timeout-minutes: 5 # Re-run failed Tui tests as they can be flaky run: | pytest tests/integration/tui || pytest tests/integration/tui --last-failed - name: Upload failed tests artifact if: failure() uses: actions/upload-artifact@v7 with: name: cylc-run (${{ matrix.os }} py-${{ matrix.python-version }}) path: ~/cylc-run/ include-hidden-files: true - name: Coverage report run: | coverage xml coverage report - name: Upload coverage artifact uses: actions/upload-artifact@v7 with: name: coverage_${{ matrix.os }}_py-${{ matrix.python-version }} path: coverage.xml retention-days: 7 lint: runs-on: ubuntu-latest timeout-minutes: 5 strategy: matrix: python-version: ['3'] env: FORCE_COLOR: 2 steps: - name: Checkout uses: actions/checkout@v7 - name: Install System Dependencies uses: mamba-org/setup-micromamba@v3 with: cache-environment: true post-cleanup: 'all' environment-name: cylc-lint-test create-args: >- python=${{ matrix.python-version }} pip bash shellcheck - name: Install Python Dependencies run: | pip install -e ."[tests]" - parallel: - name: Flake8 run: flake8 - name: Bandit run: | bandit -r --ini .bandit cylc/flow - name: Shellchecker run: etc/bin/shellchecker - name: MyPy run: mypy - name: Towncrier - draft changelog uses: cylc/release-actions/towncrier-draft@v1 linkcheck: needs: lint runs-on: ubuntu-latest timeout-minutes: 10 steps: - name: Checkout uses: actions/checkout@v7 - name: Setup python uses: actions/setup-python@v7 with: python-version: "3.x" cache: "pip" - name: Install Python Dependencies run: pip install ."[tests]" - name: Linkcheck env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: pytest -m linkcheck -ra --dist=load --color=yes -n 10 tests/unit/test_links.py codecov: needs: test runs-on: ubuntu-slim timeout-minutes: 2 steps: - name: Checkout uses: actions/checkout@v7 - name: Download coverage artifacts uses: actions/download-artifact@v8 - name: Codecov upload uses: codecov/codecov-action@v7 with: name: ${{ github.workflow }} flags: fast-tests fail_ci_if_error: true verbose: true # Token not required for public repos, but avoids upload failure due # to rate-limiting (but not for PRs opened from forks) token: ${{ secrets.CODECOV_TOKEN }} cylc-flow-8.7.0/.github/workflows/1_create_release_pr.yml0000664000175000017500000000360715264420307023626 0ustar alastairalastairname: Release stage 1 - create release PR (Cylc 8+ only) on: workflow_dispatch: inputs: version: description: Version number (PEP 440 compliant e.g., 8.12.4, 8.0a2 etc) required: true branch: description: The branch to open the PR against required: false default: 'master' concurrency: # Only let this run 1 at a time group: ${{ github.workflow }} cancel-in-progress: false defaults: run: shell: bash env: FORCE_COLOR: 2 PIP_PROGRESS_BAR: off jobs: create-release-pr: runs-on: ubuntu-latest timeout-minutes: 10 steps: - name: Sanitise workflow inputs # Should be 1st step uses: cylc/release-actions/stage-1/sanitize-inputs@v1 - name: Checkout repo uses: actions/checkout@v7 with: ref: ${{ env.BASE_REF }} fetch-depth: 0 # need to fetch all commits to check contributors - name: Check CONTRIBUTING.md uses: cylc/release-actions/check-shortlog@v1 - name: Setup Python uses: actions/setup-python@v7 with: python-version: '3.x' - name: Create & checkout PR branch uses: cylc/release-actions/stage-1/checkout-pr-branch@v1 - name: Set the package version uses: cylc/release-actions/stage-1/set-python-package-version@v1 with: init-file: 'cylc/flow/__init__.py' pypi-package-name: 'cylc-flow' - name: Test build uses: cylc/release-actions/build-python-package@v1 - name: Generate changelog run: | python3 -m pip install -q towncrier towncrier build --yes - name: Create pull request uses: cylc/release-actions/stage-1/create-release-pr@v1 with: test-workflows: test_fast.yml, test_functional.yml, bash.yml, test_conda-build.yml env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} cylc-flow-8.7.0/.github/workflows/update_copyright.yml0000664000175000017500000000242515264420307023311 0ustar alastairalastairname: Update copyright year on: schedule: - cron: '0 4 1 1 *' # Every Jan 1st @ 04:00 UTC jobs: update-copyright: if: github.repository_owner == 'cylc' runs-on: ubuntu-latest timeout-minutes: 10 steps: - name: Checkout repo uses: actions/checkout@v7 - name: Configure git uses: cylc/release-actions/configure-git@v1 - name: Checkout PR branch uses: cylc/release-actions/checkout-copyright-branch@v1 - name: Update copyright year env: README_FILE: 'README.md' SCRIPTS_INIT_FILE: 'cylc/flow/scripts/__init__.py' run: | pattern="().*(<\/span>)" sed -i -E "s/${pattern}/\1${YEAR}\2/" "$README_FILE" pattern="(_copyright_year ?= ?)[^ #]*(.*)" sed -i -E "s/${pattern}/\1${YEAR}\2/" "$SCRIPTS_INIT_FILE" - name: Commit & push run : | git commit -a -m "Update copyright year" -m "Workflow: ${{ github.workflow }}, run: ${{ github.run_number }}" git push - name: Create pull request uses: cylc/release-actions/create-pr@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: head: ${{ env.BRANCH_NAME }} title: 'Auto PR: update copyright year' body: 'Happy new year!' cylc-flow-8.7.0/.github/workflows/test_tutorial_workflow.yml0000664000175000017500000000261015264420307024567 0ustar alastairalastairname: test-tutorial-workflow on: push: branches: - master - '8.*.x' pull_request: paths-ignore: - '.github/workflows/*.ya?ml' - '!.github/workflows/test_tutorial_workflow.yml' - 'tests/**' - '**.md' workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true defaults: run: shell: bash env: FORCE_COLOR: 2 PIP_PROGRESS_BAR: off jobs: test: strategy: fail-fast: false matrix: include: - os: 'ubuntu-latest' python-version: '3' - os: 'ubuntu-22.04' python-version: '3.12' runs-on: ${{ matrix.os }} timeout-minutes: 20 steps: - name: configure python uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} - name: checkout uses: actions/checkout@v7 - name: Install docs/tutorial dependencies uses: cylc/cylc-doc/.github/actions/install-dependencies@master - name: install cylc-flow run: pip install .[all] # Useful debug info: - run: pip list - name: run tutorial workflow run: | mkdir -p "${HOME}/cylc-run" etc/bin/run-validate-tutorials - name: debug if: failure() run: | find ~/cylc-run -name job.err -exec cat {} + # cylc error files cylc-flow-8.7.0/.github/workflows/branch_sync.yml0000664000175000017500000000073515264420307022232 0ustar alastairalastairname: Sync PR on: push: branches: - '8.*.x' schedule: - cron: '33 04 * * 1-5' # 04:33 UTC Mon-Fri workflow_dispatch: inputs: head_branch: description: Branch to merge into master required: true jobs: sync: if: github.repository_owner == 'cylc' || github.event_name != 'schedule' uses: cylc/release-actions/.github/workflows/branch-sync.yml@v1 with: head_branch: ${{ inputs.head_branch }} secrets: inherit cylc-flow-8.7.0/.github/workflows/bash.yml0000664000175000017500000000565015264420307020657 0ustar alastairalastair# Workflow for testing multiple bash versions with Docker name: bash on: workflow_dispatch: pull_request: paths-ignore: - '.github/workflows/*.ya?ml' - '!.github/workflows/bash.yml' - 'cylc/flow/etc/syntax/**' - 'etc/syntax/**' - 'tests/unit/**' - 'tests/integration/**' - '**.md' - '**/README*/**' push: branches: - master - '8.*.x' paths-ignore: - '.github/workflows/*.ya?ml' - '!.github/workflows/bash.yml' - 'cylc/flow/etc/syntax/**' - 'etc/syntax/**' - 'tests/unit/**' - 'tests/integration/**' - '**.md' - '**/README*/**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true defaults: run: shell: bash env: FORCE_COLOR: 2 PIP_PROGRESS_BAR: off jobs: bash-docker: runs-on: ubuntu-latest timeout-minutes: 15 strategy: fail-fast: false matrix: bash-version: - '3.2' - '4.2' # if you use a value like 5.0, YAML will round it to 5, which will cause an error later on - '5.0' env: IMAGE: 'ghcr.io/cylc/cylc-bash-testing-2:master' steps: - name: Checkout uses: actions/checkout@v7 - name: Run Docker container run: | docker pull "$IMAGE" docker run --name bash -v $(pwd -P):/root/cylc-flow --rm -t -d -m 1G --memory-swap 1G "$IMAGE" /bin/bash docker ps -a - name: Install Cylc dependencies in the container run: | docker exec -w /root/cylc-flow bash python3.12 -m pip install .[all] --break-system-packages - name: Set the container bash version run: docker exec bash update-alternatives --set bash /bash/bash-${{ matrix.bash-version }} - name: Run functional tests that validate or are related to how Cylc uses bash run: | docker exec -w /root/cylc-flow bash \ ./etc/bin/run-functional-tests -v \ tests/functional/broadcast/00-simple.t \ tests/functional/cylc-poll/11-event-time.t \ tests/functional/cylc-poll/15-job-st-file-no-batch.t \ tests/functional/events/28-inactivity.t \ tests/functional/events/34-task-abort.t \ tests/functional/job-file-trap/00-sigusr1.t \ tests/functional/job-file-trap/02-pipefail.t \ tests/functional/job-file-trap/03-user-trap.t \ tests/functional/jobscript/00-torture.t \ tests/functional/pause-resume/12-pause-then-retry.t \ tests/functional/shutdown/09-now2.t - name: Copy cylc-run out of container if: failure() run: | docker cp bash:/root/cylc-run . - name: Upload artifact if: failure() uses: actions/upload-artifact@v7 with: name: 'cylc-run (bash-${{ matrix.bash-version }})' path: cylc-run include-hidden-files: true cylc-flow-8.7.0/.github/workflows/2_auto_publish_release.yml0000664000175000017500000000561715264420307024364 0ustar alastairalastairname: Release stage 2 - auto publish on: pull_request: # types AND paths types: [closed] paths: ['cylc/flow/__init__.py'] # NOTE: While this is too generic, we use the `if` condition of the job to narrow it down # NOTE: Don't use `branches` as we might create release on any branch concurrency: # Only let this run 1 at a time group: ${{ github.workflow }} cancel-in-progress: false defaults: run: shell: bash env: FORCE_COLOR: 2 PIP_PROGRESS_BAR: off # Best not to include the GH token here, only do it for the steps that need it MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} CHANGELOG_FILE: CHANGES.md jobs: publish: if: >- # NOTE: Can't use top-level env here unfortunately github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release') runs-on: ubuntu-latest timeout-minutes: 10 steps: - name: Checkout repo uses: actions/checkout@v7 with: ref: ${{ env.MERGE_SHA }} - name: Setup Python uses: actions/setup-python@v7 with: python-version: '3.x' - name: Get the version number uses: cylc/release-actions/stage-2/get-version-from-pr@v1 - name: Build uses: cylc/release-actions/build-python-package@v1 - name: Publish distribution to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ # uses the API token feature of PyPI - least permissions possible password: ${{ secrets.PYPI_TOKEN }} # # Can try using this for testing: # repository_url: https://test.pypi.org/legacy/ - name: Publish GitHub release id: create-release uses: cylc/release-actions/create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: commitish: ${{ env.MERGE_SHA }} tag_name: ${{ env.VERSION }} release_name: cylc-flow-${{ env.VERSION }} prerelease: ${{ env.PRERELEASE }} body: | See [${{ env.CHANGELOG_FILE }}](https://github.com/${{ github.repository }}/blob/master/${{ env.CHANGELOG_FILE }}) for detail. Cylc 8 can be installed via pypi or Conda - you don't need to download this release directly. See the [installation](https://cylc.github.io/cylc-doc/latest/html/installation.html) section of the documentation. # TODO: Get topmost changelog section somehow and use that as the body? - name: Comment on the release PR with the results & next steps if: always() uses: cylc/release-actions/stage-2/comment-on-pr@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: release-url: ${{ steps.create-release.outputs.html_url }} - name: Bump dev version uses: cylc/release-actions/stage-2/bump-dev-version@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: init-file: 'cylc/flow/__init__.py' cylc-flow-8.7.0/.github/workflows/test_functional.yml0000664000175000017500000002615515264420307023146 0ustar alastairalastairname: functional tests on: workflow_dispatch: pull_request: paths-ignore: - '.github/workflows/*.ya?ml' - '!.github/workflows/test_functional.yml' - 'cylc/flow/etc/syntax/**' - 'etc/syntax/**' - 'tests/conftest.py' - 'tests/unit/**' - 'tests/integration/**' - '**.md' - '**/README*/**' push: branches: - master - '8.*.x' paths-ignore: - '.github/workflows/*.ya?ml' - '!.github/workflows/test_functional.yml' - 'cylc/flow/etc/syntax/**' - 'etc/syntax/**' - 'tests/conftest.py' - 'tests/unit/**' - 'tests/integration/**' - '**.md' - '**/README*/**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true defaults: run: shell: bash -c "exec $CONDA_PREFIX/bin/bash -elo pipefail {0}" env: FORCE_COLOR: 2 PIP_PROGRESS_BAR: off jobs: test: runs-on: ${{ matrix.os }} name: ${{ matrix.name || matrix.chunk }} timeout-minutes: 45 strategy: fail-fast: false matrix: os: ['ubuntu-22.04'] python-version: ['3.12'] # oldest version we support test-base: ['tests/f'] chunk: ['1/4', '2/4', '3/4', '4/4'] platform: ['_local_background* _local_at*'] # NOTE: includes must define ALL of the matrix values include: # latest python - name: 'py-3-latest' os: 'ubuntu-latest' python-version: '3' test-base: 'tests/f' chunk: '1/4' platform: '_local_background*' # tests/k - name: 'flaky' os: 'ubuntu-latest' python-version: '3.12' test-base: 'tests/k' chunk: '1/1' platform: '_local_background* _local_at*' # remote platforms - name: '_remote_background_indep_poll' os: 'ubuntu-latest' python-version: '3.12' test-base: 'tests/f tests/k' chunk: '1/1' platform: '_remote_background_indep_poll _remote_at_indep_poll' - name: '_remote_background_indep_tcp' os: 'ubuntu-latest' test-base: 'tests/f tests/k' python-version: '3.12' chunk: '1/1' platform: '_remote_background_indep_tcp _remote_at_indep_tcp' # macos - name: 'macos 1/5' os: 'macos-latest' python-version: '3.12' # oldest version we support test-base: 'tests/f' chunk: '1/5' platform: '_local_background*' - name: 'macos 2/5' os: 'macos-latest' python-version: '3' # newest available test-base: 'tests/f' chunk: '2/5' platform: '_local_background*' env: # Use non-UTC time zone TZ: XXX-05:30 # these vars are used by etc/bin/run-functional-tests CYLC_TEST_PLATFORMS: ${{ matrix.platform }} CYLC_COVERAGE: 1 REMOTE_PLATFORM: ${{ contains(matrix.platform, '_remote') }} steps: - name: Checkout uses: actions/checkout@v7 - name: Configure Python uses: mamba-org/setup-micromamba@v3 with: cache-environment: true post-cleanup: 'all' environment-name: cylc-functional-test create-args: >- python=${{ matrix.python-version }} pip bash coreutils grep sed sqlite subversion tree - name: Create global config run: | CONF_PATH="$HOME/.cylc/flow/8" mkdir -p "$CONF_PATH" touch "$CONF_PATH/global.cylc" ln -s "$CONF_PATH/global.cylc" "$CONF_PATH/global-tests.cylc" echo "GLOBAL_CFG_PATH=${CONF_PATH}/global.cylc" >> "$GITHUB_ENV" - name: Patch DNS uses: cylc/release-actions/patch-dns@v1 - name: Add localhost entries to global config if: startsWith(runner.os, 'macos') run: | cat >> "$GLOBAL_CFG_PATH" <<__HERE__ [platforms] [[localhost, $(hostname -f), $(hostname -s)]] hosts = localhost install target = localhost ssh command = ssh -oBatchMode=yes -oConnectTimeout=8 -oStrictHostKeyChecking=no __HERE__ cat "$GLOBAL_CFG_PATH" - name: Add .github/bin/ to PATH # Sets up mocked mail command & any other custom executables # Adding to $GITHUB_PATH does not work when using setup-micromamba and/or login shell run: | echo "export PATH=\"${{ github.workspace }}/.github/bin:$PATH\"" >> ~/.bash_profile - name: Install run: | pip install -e ."[all]" mkdir "$HOME/cylc-run" - name: Configure Atrun if: contains(matrix.platform, '_local_at') run: | sudo apt-get update sudo apt-get install -y at cat >> "$GLOBAL_CFG_PATH" << __HERE__ [platforms] [[_local_at_indep_tcp]] hosts = localhost install target = localhost job runner = at __HERE__ - name: Swarm Configure run: | etc/bin/swarm --yes --debug configure - name: Swarm Build if: env.REMOTE_PLATFORM == 'true' run: | # `swarm configure` seems to get ignored so override the user config cp etc/conf/ssh_config $HOME/.ssh/config # build and run the swarm etc/bin/swarm --yes --debug build etc/bin/swarm --yes --debug run # test that it's up and running before proceeding sleep 1 ssh -vv _remote_background_indep_poll hostname - name: Configure git # Needed by the odd test uses: cylc/release-actions/configure-git@v1 # Useful debug info: - name: Pip list run: pip list - name: Inspect versions run: | python --version echo rsync --version - name: Filter Tests env: # NOTE: we only want the CHUNK set in this step else we will # re-chunk tests later when they run CHUNK: ${{ matrix.chunk }} run: | etc/bin/run-functional-tests \ --dry \ ${{ matrix.test-base }} \ > test-file if [[ $REMOTE_PLATFORM == 'true' ]]; then # skip tests that don't configure platform requirements grep -l --color=never REQUIRE_PLATFORM $(cat test-file) > test-file fi echo "----- Filtered Tests -----" sort test-file - name: Test id: test timeout-minutes: 35 continue-on-error: true run: | echo "finished=false" >> $GITHUB_OUTPUT if [[ '${{ matrix.test-base }}' == 'tests/k' ]]; then NPROC=4 else NPROC=8 fi # NOTE: test base is purposefully un-quoted etc/bin/run-functional-tests \ -j "${NPROC}" \ --state=save \ $(cat test-file) \ || (echo "finished=true" >> $GITHUB_OUTPUT && false) - name: Time Out if: steps.test.outcome == 'failure' && steps.test.outputs.finished != 'true' run: | echo '::error:: tests timed-out' # help to identify the tests that were running at the time cylc scan --state=all --format=rich --color-blind # fail the workflow false - name: Re-run failed tests timeout-minutes: 10 if: steps.test.outcome == 'failure' && steps.test.outputs.finished == 'true' run: | # re-run failed tests providing that they didn't time out first time # TODO: make the tests deterministic so we don't need to do this etc/bin/run-functional-tests \ -j 1 \ -v \ --state=save,failed $(cat test-file) - name: Copy cylc-run out of container if: failure() && steps.test.outcome == 'failure' && env.REMOTE_PLATFORM == 'true' run: | # pick the first host in the list host="$(cut -d ' ' -f 1 <<< "${{ matrix.platform }}")" # copy back the remote cylc-run dir rsync -av \ "${host}:/root/cylc-run/" \ "${HOME}/cylc-run/${host}/" - name: Debug if: failure() && steps.test.outcome == 'failure' timeout-minutes: 1 run: | find "$HOME/cylc-run" -name '*.err' -type f \ -exec echo \; -exec echo '====== {} ======' \; -exec cat '{}' \; find "$HOME/cylc-run" -name '*.log' -type f \ -exec echo \; -exec echo '====== {} ======' \; -exec cat '{}' \; find "${TMPDIR:-/tmp}/${USER}/cylctb-"* -type f \ -exec echo \; -exec echo '====== {} ======' \; -exec cat '{}' \; - name: Set artifact upload name if: always() id: uploadname run: | # artifact name cannot contain '/' characters CID="$(sed 's|/|-|g' <<< "${{ matrix.name || matrix.chunk }}")" echo "uploadname=$CID" >> $GITHUB_OUTPUT - name: Upload failed tests artifact if: failure() && steps.test.outcome == 'failure' uses: actions/upload-artifact@v7 with: name: cylc-run (${{ steps.uploadname.outputs.uploadname }}) path: ~/cylc-run/ include-hidden-files: true - name: Fetch Remote Coverage if: env.REMOTE_PLATFORM == 'true' run: | # pick the first host in the list host="$(cut -d ' ' -f 1 <<< "${{ matrix.platform }}")" # copy back the remote coverage files rsync -av \ "${host}:/cylc/" \ '.' \ --include='.coverage*' \ --exclude='*' \ >rsyncout cat rsyncout # fiddle the python source location to match the local system for db in $(grep --color=never '.coverage\.' rsyncout); do sqlite3 "$db" " UPDATE file SET path = REPLACE(path, '/cylc/cylc/', '$PWD/cylc/') " done - name: Shutdown if: always() run: | etc/bin/swarm kill - name: Combine coverage & report run: | coverage combine -a coverage xml coverage report - name: Upload coverage artifact uses: actions/upload-artifact@v7 with: name: coverage_${{ steps.uploadname.outputs.uploadname }} path: coverage.xml retention-days: 7 codecov: needs: test runs-on: ubuntu-latest timeout-minutes: 2 steps: - name: Checkout uses: actions/checkout@v7 - name: Download coverage artifacts uses: actions/download-artifact@v8 - name: Codecov upload uses: codecov/codecov-action@v7 with: name: ${{ github.workflow }} flags: functional-tests fail_ci_if_error: true verbose: true # Token not required for public repos, but avoids upload failure due # to rate-limiting (but not for PRs opened from forks) token: ${{ secrets.CODECOV_TOKEN }} cylc-flow-8.7.0/.github/workflows/test_conda-build.yml0000664000175000017500000000471315264420307023161 0ustar alastairalastairname: conda builds on: pull_request: paths: - 'conda-environment.yml' - '.github/workflows/test_conda-build.yml' - 'pyproject.toml' schedule: - cron: '17 22 * * 6' # Every Saturday at 22:17 workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true defaults: run: shell: bash env: FORCE_COLOR: 2 jobs: test_conda_install: if: github.repository_owner == 'cylc' || github.event_name != 'schedule' runs-on: ubuntu-latest timeout-minutes: 10 strategy: fail-fast: false matrix: python-version: ['3.12', '3'] env: ENV_FILE: conda-environment.yml # prevent pip from meddling with the conda installed dependencies (during the conda build step) PIP_NO_DEPS: True steps: - name: checkout cylc-flow uses: actions/checkout@v7 - name: modify conda env file run: | # write environment file echo " - python = ${{ matrix.python-version }}" >> "$ENV_FILE" echo " - pip" >> "$ENV_FILE" # list pip as a dependency echo " - pip:" >> "$ENV_FILE" # add a pip section echo " - ." >> "$ENV_FILE" # install cylc-flow (pip install .) cat "$ENV_FILE" - name: build conda env uses: conda-incubator/setup-miniconda@v4 with: environment-file: ${{ env.ENV_FILE }} - name: check cylc installation shell: bash -el {0} run: | # make sure Cylc is functioning conda run -n test cylc version --long # make sure pip is happy with the packages installed # (checks pip deps are compatible with conda deps) conda run -n test pip check - name: check for activate scripts shell : bash -el {0} run: | # https://github.com/cylc/cylc-flow/issues/3704#issuecomment-897442365 # locate all activate scripts find "$CONDA_PREFIX" -name "activate.d" | tee > activates.txt # ignore the conda activate script itself sed -i "\|$CONDA_PREFIX/etc/conda/activate.d|d" activates.txt # check to make sure no packages have contributed new activate scripts # (we rely on having a conda activate-less environment) if [[ $(wc -l < activates.txt) -ne 0 ]]; then echo '::error::Found activate scripts in installation.' cat activates.txt >&2 exit 1 fi cylc-flow-8.7.0/.github/workflows/build.yml0000664000175000017500000000255615264420307021043 0ustar alastairalastairname: Build # build the project whenever the configuration is changed on: workflow_dispatch: pull_request: paths: - 'README.md' # check markdown is valid - 'MANIFEST.in' # check packaging - 'pyproject.toml' # check build config - '.gitignore' - '.github/workflows/build.yml' concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true defaults: run: shell: bash -leo pipefail {0} env: FORCE_COLOR: 2 PIP_PROGRESS_BAR: off jobs: test: runs-on: ${{ matrix.os }} timeout-minutes: 10 strategy: fail-fast: false matrix: os: ['ubuntu-latest', 'macos-latest'] python: ['3.12', '3'] steps: - name: Checkout uses: actions/checkout@v7 - name: Setup Python uses: mamba-org/setup-micromamba@v3 with: cache-environment: true post-cleanup: 'all' environment-name: cylc-build create-args: >- python=${{ matrix.python }} - name: Build uses: cylc/release-actions/build-python-package@v1 - name: Inspect run: | unzip -l dist/*.whl | tee files grep -E 'cylc_flow.*.dist-info/.*COPYING' files grep 'cylc/flow/py.typed' files grep 'cylc/flow/etc' files grep 'cylc/flow/etc/cylc-completion.bash' files cylc-flow-8.7.0/.github/bin/0000775000175000017500000000000015264420307015724 5ustar alastairalastaircylc-flow-8.7.0/.github/bin/mail0000775000175000017500000000236215264420307016577 0ustar alastairalastair#!/usr/bin/env python3 """Mock the mailx / mailutils `mail` command""" import argparse import smtplib import sys import os parser = argparse.ArgumentParser( description='Cylc functional tests mail command.') parser.add_argument( '-s', metavar='subject', dest='subject', type=str, help='e-mail subject') parser.add_argument( '-r', metavar='reply_to', dest='sender', type=str, help='e-mail reply-to address') parser.add_argument( 'to', metavar='to', type=str, help='e-mail destination address') parser.add_argument( 'body', metavar='body', nargs='?', type=argparse.FileType('r'), default=sys.stdin, help='e-mail body') smtp_server = os.getenv('smtp') try: host, port = smtp_server.split(':') port = int(port) except (AttributeError, ValueError): raise OSError( 'The environment variable "smtp" must be set to a string of the form ' '"host:port" (e.g. "localhost:8025") for the mocked mail command ' 'to work.') args = parser.parse_args() sender_email = args.sender receiver_email = args.to message = f"""\ Subject: {args.subject} {args.body.read()}""" # https://realpython.com/python-send-email/ with smtplib.SMTP(host, port) as server: server.sendmail(sender_email, receiver_email, message) cylc-flow-8.7.0/.wci.yml0000664000175000017500000000346415264420307015206 0ustar alastairalastair# name: # will use GitHub repository name if not provided name: Cylc # icon: # will use GitHub organization icon if not provided # headline: # will use GitHub repository description if not provided headline: A general purpose workflow engine with a particular gift for cycling. # description: # will use GitHub repository description if not provided description: Cylc is a decentralised, distributed, DAG/DCG workflow scheduler. # language: # will attempt to fetch GitHub repository main language if not provided language: Python # release: # will attempt to fetch latest release from GitHub repository if not provided # version: # date: # format: YYYY-MM-DD # url: # documentation: # nothing will be used if not provided # general: # installation: # tutorial: documentation: general: https://cylc.github.io/cylc-doc/stable/html/index.html installation: https://cylc.github.io/cylc-doc/stable/html/installation.html tutorial: https://cylc.github.io/cylc-doc/stable/html/tutorial/index.html # social: # nothing will be used if not provided # twitter: # youtube: # execution_environment: # nothing will be used if not provided # interfaces: # list of interfaces # - # - # resource_managers: # list of supported resource managers # - # - resource_managers: # list of supported resource managers - PBS - Slurm - Moab - LSF - atd - background # transfer_protocols: # list of supported transfer protocols # - # - cylc-flow-8.7.0/pyproject.toml0000664000175000017500000002077215264420307016540 0ustar alastairalastair[build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [project] name = "cylc-flow" authors = [ {name = "Earth Sciences New Zealand & British Crown (Met Office) & Contributors"}, ] description = "A workflow engine for cycling systems" readme = "README.md" requires-python = ">=3.12" keywords = [ "cycling-workflows", "hpc", "job-scheduler", "metascheduler", "workflow-automation", "workflow-engine", "workflow-management", "scheduling", ] license = "GPL-3.0-or-later" license-files = ["COPYING", "CONTRIBUTING.md"] classifiers = [ "Environment :: Console", "Environment :: Web Environment", "Intended Audience :: Developers", "Intended Audience :: System Administrators", "Intended Audience :: Science/Research", "Operating System :: POSIX :: Linux", "Programming Language :: Python", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Scientific/Engineering :: Atmospheric Science", ] dependencies = [ "ansimarkup>=1.0.0", "colorama>=0.4,<1", "graphql-core>=3.2,<3.3", "graphene>=3.4.0,<3.5", "jinja2>=3.1", "metomi-isodatetime>=1!3.2.0,<1!3.3.0", # Constrain protobuf version for compatible Scheduler-UIS comms across hosts "packaging", "protobuf>=5,<8", "psutil>=5.6.0", "pyzmq>=22", # NOTE: exclude two urwid versions that were not compatible with Tui "urwid>=2.2,!=2.6.2,!=2.6.3,<5", ] dynamic = ["version"] [project.optional-dependencies] graph = [ "pillow", ] main_loop-log_data_store = [ "pympler", "matplotlib", ] main_loop-log_main_loop = [ "matplotlib", ] main_loop-log_memory = [ "pympler", "matplotlib", ] main_loop-log_db = [ "sqlparse", ] report-timings = [ "pandas>=2,<4", "matplotlib", ] tests = [ "aiosmtpd", "async_generator", "bandit>=1.7.0", "coverage>=7.9.0", "flake8-broken-line>=0.3.0", "flake8-bugbear>=21.0.0", "flake8-builtins>=1.5.0", "flake8-comprehensions>=3.5.0", "flake8-debugger>=4.0.0", # https://github.com/flake8-implicit-str-concat/flake8-implicit-str-concat/issues/82 "flake8-implicit-str-concat>=0.4,!=0.7.0", "flake8-mutable>=1.2.0", "flake8-simplify>=0.14.0; python_version<'3.14'", "flake8-type-checking", "flake8>=3.0.0", "mypy>=0.910", # https://github.com/pytest-dev/pytest-asyncio/issues/706 "pytest-asyncio>=0.21.2,!=0.23.*", "pytest-cov>=2.8.0", "pytest-xdist>=2", "pytest-mock>=3.7", "pytest>=6", "testfixtures>=6.11.0", "towncrier>=24.7.0", # Type annotation stubs # http://mypy-lang.blogspot.com/2021/05/the-upcoming-switch-to-modular-typeshed.html "types-Jinja2>=0.1.3", "types-protobuf>=0.1.10,!=5.29.1.20250402", ] tutorials = [ "h5py", "requests", ] all = ["cylc-flow[graph,main_loop-log_data_store,main_loop-log_db,main_loop-log_main_loop,main_loop-log_memory,tests,tutorials,report-timings]"] [project.scripts] # top level shell commands clyc = "cylc.flow.scripts.cylc:main" cylc = "cylc.flow.scripts.cylc:main" [project.entry-points."cylc.command"] # cylc subcommands broadcast = "cylc.flow.scripts.broadcast:main" cat-log = "cylc.flow.scripts.cat_log:main" check-versions = "cylc.flow.scripts.check_versions:main" clean = "cylc.flow.scripts.clean:main" client = "cylc.flow.scripts.client:main" completion-server = "cylc.flow.scripts.completion_server:main" config = "cylc.flow.scripts.config:main" cycle-point = "cylc.flow.scripts.cycle_point:main" diff = "cylc.flow.scripts.diff:main" dump = "cylc.flow.scripts.dump:main" ext-trigger = "cylc.flow.scripts.ext_trigger:main" get-resources = "cylc.flow.scripts.get_resources:main" function-run = "cylc.flow.scripts.function_run:main" get-workflow-contact = "cylc.flow.scripts.get_workflow_contact:main" get-workflow-version = "cylc.flow.scripts.get_workflow_version:main" graph = "cylc.flow.scripts.graph:main" hold = "cylc.flow.scripts.hold:main" install = "cylc.flow.scripts.install:main" jobs-kill = "cylc.flow.scripts.jobs_kill:main" jobs-poll = "cylc.flow.scripts.jobs_poll:main" jobs-submit = "cylc.flow.scripts.jobs_submit:main" kill = "cylc.flow.scripts.kill:main" lint = "cylc.flow.scripts.lint:main" list = "cylc.flow.scripts.list:main" message = "cylc.flow.scripts.message:main" pause = "cylc.flow.scripts.pause:main" ping = "cylc.flow.scripts.ping:main" play = "cylc.flow.scripts.play:main" poll = "cylc.flow.scripts.poll:main" profiler = "cylc.flow.scripts.profiler:main" psutils = "cylc.flow.scripts.psutil:main" reinstall = "cylc.flow.scripts.reinstall:main" release = "cylc.flow.scripts.release:main" reload = "cylc.flow.scripts.reload:main" remote-init = "cylc.flow.scripts.remote_init:main" remote-tidy = "cylc.flow.scripts.remote_tidy:main" remove = "cylc.flow.scripts.remove:main" report-timings = "cylc.flow.scripts.report_timings:main" # report-timings scan = "cylc.flow.scripts.scan:cli" show = "cylc.flow.scripts.show:main" set = "cylc.flow.scripts.set:main" stop = "cylc.flow.scripts.stop:main" subscribe = "cylc.flow.scripts.subscribe:main" verbosity = "cylc.flow.scripts.verbosity:main" workflow-state = "cylc.flow.scripts.workflow_state:main" tui = "cylc.flow.scripts.tui:main" trigger = "cylc.flow.scripts.trigger:main" validate = "cylc.flow.scripts.validate:main" view = "cylc.flow.scripts.view:main" vip = "cylc.flow.scripts.validate_install_play:main" vr = "cylc.flow.scripts.validate_reinstall:main" [project.entry-points."cylc.main_loop"] # async functions to run within the scheduler main loop health_check = "cylc.flow.main_loop.health_check" auto_restart = "cylc.flow.main_loop.auto_restart" log_data_store = "cylc.flow.main_loop.log_data_store" # main_loop-log_data_store log_db = "cylc.flow.main_loop.log_db" # main_loop-log_db log_main_loop = "cylc.flow.main_loop.log_main_loop" # main_loop-log_main_loop log_memory = "cylc.flow.main_loop.log_memory" # main_loop-log_memory reset_bad_hosts = "cylc.flow.main_loop.reset_bad_hosts" log_tracemalloc = "cylc.flow.main_loop.log_tracemalloc" # NOTE: all entry points should be listed here even if Cylc Flow does not # provide any implementations, to make entry point scraping easier [project.entry-points."cylc.pre_configure"] global_template_variables = "cylc.flow.plugins.global_template_variables:pre_configure" [project.entry-points."cylc.post_install"] log_vc_info = "cylc.flow.plugins.log_vc_info:main" [project.entry-points."cylc.xtriggers"] # NOTE: Built-in xtrigger modules # - must contain a function (the xtrigger) with the same name as the module # - and may contain a "validate" function to check arguments echo = "cylc.flow.xtriggers.echo" wall_clock = "cylc.flow.xtriggers.wall_clock" workflow_state = "cylc.flow.xtriggers.workflow_state" suite_state = "cylc.flow.xtriggers.suite_state" xrandom = "cylc.flow.xtriggers.xrandom" [project.urls] Home = "https://cylc.org/" Documentation = "https://cylc.github.io/cylc-doc/" Source = "https://github.com/cylc/cylc-flow" Tracker = "https://github.com/cylc/cylc-flow/issues" [tool.setuptools] include-package-data = true [tool.setuptools.dynamic] version = {attr = "cylc.flow.__version__"} [tool.setuptools.packages.find] include = ["cylc*"] namespaces = true [tool.towncrier] directory = "changes.d" name = "Cylc" package = "cylc.flow" filename = "CHANGES.md" template = "changes.d/changelog-template.jinja" underlines = ["", "", ""] title_format = "## __cylc-{version} (Released {project_date})__" issue_format = "[#{issue}](https://github.com/cylc/cylc-flow/pull/{issue})" ignore = ["changelog-template.jinja"] # These changelog sections will be shown in the defined order: [[tool.towncrier.type]] directory = "break" # NB this is just the filename not directory e.g. 123.break.md name = "⚠ Breaking Changes" showcontent = true [[tool.towncrier.type]] directory = "feat" name = "🚀 Enhancements" showcontent = true [[tool.towncrier.type]] directory = "fix" name = "🔧 Fixes" showcontent = true # Not mandated to use these tools, but if you do: [tool.ruff] line-length = 79 target-version = "py312" [tool.ruff.format] quote-style = "preserve" [tool.black] line-length = 79 target-version = ['py312'] skip-string-normalization = true [tool.isort] profile = "black" line_length = 79 force_grid_wrap = 2 lines_after_imports = 2 combine_as_imports = true force_sort_within_sections = true [tool.ruff.lint.isort] # force-grid-wrap = 2 # astral-sh/ruff#2601 lines-after-imports = 2 combine-as-imports = true force-sort-within-sections = true cylc-flow-8.7.0/tests/0000775000175000017500000000000015264420307014756 5ustar alastairalastaircylc-flow-8.7.0/tests/integration/0000775000175000017500000000000015264420307017301 5ustar alastairalastaircylc-flow-8.7.0/tests/integration/test_get_old_tvars.py0000664000175000017500000000520215264420307023545 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from pytest import param from cylc.flow.option_parsers import Options from cylc.flow.scripts.validate import ( run as validate, get_option_parser as validate_gop ) from cylc.flow.scripts.view import ( _main as view, get_option_parser as view_gop ) from cylc.flow.scripts.graph import ( _main as graph, get_option_parser as graph_gop ) from cylc.flow.scripts.config import ( _main as config, get_option_parser as config_gop ) from cylc.flow.scripts.list import ( _main as cylclist, get_option_parser as list_gop ) @pytest.fixture(scope='module') def _setup(mod_scheduler, mod_flow): """Provide an installed flow with a database to try assorted simple Cylc scripts against. """ conf = { '#!jinja2': '', 'scheduler': { 'allow implicit tasks': True }, 'scheduling': { 'graph': { 'R1': r'{{FOO}}' } } } schd = mod_scheduler(mod_flow(conf), templatevars=['FOO="bar"']) yield schd @pytest.mark.parametrize( 'function, parser, expect', ( param(validate, validate_gop, 'Valid for', id="validate"), param(view, view_gop, 'FOO', id="view"), param(graph, graph_gop, '1/bar', id='graph'), param(config, config_gop, 'R1 = bar', id='config'), param(cylclist, list_gop, 'bar', id='list') ) ) async def test_validate_with_old_tvars( _setup, mod_start, capsys, function, parser, expect, ): """It (A Cylc CLI Command - see parameters) can get template vars stored in db. Else the jinja2 in the config would cause these tasks to fail. """ parser = parser() opts = Options(parser)() if function == graph: opts.reference = True async with mod_start(_setup): await function(opts, _setup.workflow_name) assert expect in capsys.readouterr().out cylc-flow-8.7.0/tests/integration/utils/0000775000175000017500000000000015264420307020441 5ustar alastairalastaircylc-flow-8.7.0/tests/integration/utils/entry_points.py0000664000175000017500000000205315264420307023550 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Utilities for working with entry points.""" class EntryPointWrapper: """Wraps a method to make it look like an entry point.""" def __init__(self, fcn): self.name = fcn.__name__ self.fcn = fcn def load(self): return self.fcn cylc-flow-8.7.0/tests/integration/utils/test_flow_tools.py0000664000175000017500000000242715264420307024246 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests to ensure the tests are working - very meta. https://github.com/cylc/cylc-flow/pull/2740#discussion_r206086008 """ from pathlib import Path # test _make_flow via the conftest fixture def test_flow(run_dir, flow, one_conf): """It should create a flow in the run directory.""" id_ = flow(one_conf) assert Path(run_dir / id_).exists() assert Path(run_dir / id_ / 'flow.cylc').exists() with open(Path(run_dir / id_ / 'flow.cylc'), 'r') as flow_file: assert 'scheduling' in flow_file.read() cylc-flow-8.7.0/tests/integration/utils/__init__.py0000664000175000017500000000221515264420307022552 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """General utilities for the integration test infrastructure. These utilities are not intended for direct use by tests (hence the underscore function names). Use the fixtures provided in the conftest instead. """ def _rm_if_empty(path): """Convenience wrapper for removing empty directories.""" try: path.rmdir() except OSError: return False return True cylc-flow-8.7.0/tests/integration/utils/flow_tools.py0000664000175000017500000001531115264420307023203 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Wrappers for creating and launching flows. These utilities are not intended for direct use by tests (hence the underscore function names). Use the fixtures provided in the conftest instead. """ import asyncio from contextlib import ( asynccontextmanager, contextmanager, ) import logging from pathlib import Path from secrets import token_hex from typing import ( Any, Optional, Union, ) import pytest from cylc.flow import CYLC_LOG from cylc.flow.scheduler import ( Scheduler, SchedulerStop, ) from cylc.flow.scheduler_cli import RunOptions from cylc.flow.workflow_files import WorkflowFiles from cylc.flow.workflow_status import StopMode from .flow_writer import flow_config_str def _make_src_flow(src_path, conf, filename=WorkflowFiles.FLOW_FILE): """Construct a workflow on the filesystem""" flow_src_dir = (src_path / token_hex(4)) flow_src_dir.mkdir(parents=True, exist_ok=True) if isinstance(conf, dict): conf = flow_config_str(conf) with open((flow_src_dir / filename), 'w+') as flow_file: flow_file.write(conf) return flow_src_dir def _make_flow( cylc_run_dir: Union[Path, str], test_dir: Path, conf: Union[dict, str], name: Optional[str] = None, workflow_id: Optional[str] = None, defaults: Optional[bool] = True, filename: str = WorkflowFiles.FLOW_FILE, ) -> str: """Construct a workflow on the filesystem. Args: conf: Either a workflow config dictionary, or a graph string to be used as the R1 graph in the workflow config. defaults: Set up a common defaults. * [scheduling]allow implicit tasks = true Set false for Cylc 7 upgrader tests. """ if workflow_id: flow_run_dir = (cylc_run_dir / workflow_id) else: if name is None: name = token_hex(4) flow_run_dir = (test_dir / name) flow_run_dir.mkdir(parents=True, exist_ok=True) workflow_id = str(flow_run_dir.relative_to(cylc_run_dir)) if isinstance(conf, str): conf = { 'scheduling': { 'graph': { 'R1': conf } } } if defaults: # set the default simulation runtime to zero (can be overridden) ( conf.setdefault('runtime', {}) .setdefault('root', {}) .setdefault('simulation', {}) .setdefault('default run length', 'PT0S') ) # allow implicit tasks by default: conf.setdefault('scheduler', {}).setdefault( 'allow implicit tasks', 'True') with open((flow_run_dir / filename), 'w+') as flow_file: flow_file.write(flow_config_str(conf)) return workflow_id @contextmanager def _make_scheduler(): """Return a scheduler object for a flow registration.""" schd: Scheduler = None # type: ignore[assignment] def __make_scheduler(id_: str, **opts: Any) -> Scheduler: opts = { # safe n sane defaults for integration tests 'paused_start': True, 'run_mode': 'simulation', **opts, } options = RunOptions(**opts) # create workflow nonlocal schd schd = Scheduler(id_, options) return schd yield __make_scheduler # Teardown if hasattr(schd, 'workflow_db_mgr'): schd.workflow_db_mgr.on_workflow_shutdown() @asynccontextmanager async def _start_flow( caplog: Optional[pytest.LogCaptureFixture], schd: Scheduler, level: int = logging.INFO ): """Start a scheduler but don't set the main loop running.""" if caplog: caplog.set_level(level, CYLC_LOG) await schd.install() # Speed up main loop schd.INTERVAL_MAIN_LOOP = schd.INTERVAL_MAIN_LOOP_QUICK = 1e-4 try: # Nested `try...finally` to ensure caplog always yielded even if # exception occurs in Scheduler try: await schd.start() finally: # After this `yield`, the `with` block of the context manager # is executed: yield caplog finally: # Cleanup - this always runs after the `with` block of the # context manager. # Need to shut down Scheduler, but time out in case something # goes wrong: async with asyncio.timeout(5): await schd.shutdown(SchedulerStop("integration test teardown")) @asynccontextmanager async def _run_flow( caplog: Optional[pytest.LogCaptureFixture], schd: Scheduler, level: int = logging.INFO ): """Start a scheduler and set the main loop running.""" if caplog: caplog.set_level(level, CYLC_LOG) await schd.install() # Speed up main loop schd.INTERVAL_MAIN_LOOP = schd.INTERVAL_MAIN_LOOP_QUICK = 1e-4 task: Optional[asyncio.Task] = None try: # Nested `try...finally` to ensure caplog always yielded even if # exception occurs in Scheduler try: await schd.start() # Do not await as we need to yield control to the main loop: task = asyncio.create_task(schd.run_scheduler()) finally: # After this `yield`, the `with` block of the context manager # is executed: yield caplog finally: # Cleanup - this always runs after the `with` block of the # context manager. # Need to shut down Scheduler, but time out in case something # goes wrong: async with asyncio.timeout(5): if task: # ask the scheduler to shut down nicely, # let main loop handle it: schd._set_stop(StopMode.REQUEST_NOW_NOW) await task if schd.contact_data: async with asyncio.timeout(5): # Scheduler still running... try more forceful tear down: await schd.shutdown(SchedulerStop("integration test teardown")) if task: # Brute force cleanup if something went wrong: task.cancel() cylc-flow-8.7.0/tests/integration/utils/flow_writer.py0000664000175000017500000000560315264420307023362 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Utility for writing Cylc Flow configuration files. These utilities are not intended for direct use by tests (hence the underscore function names). Use the fixtures provided in the conftest instead. """ from textwrap import dedent from typing import List def _write_header(name: str, level: int) -> str: """Write a cylc section definition.""" indent = ' ' * (level - 1) return f'{indent}{"[" * level}{name}{"]" * level}' def _write_setting(key: str, value: str, level: int) -> List[str]: """Write a cylc setting definition.""" indent = ' ' * (level - 1) value = str(value) if '\n' in value: value = dedent(value).strip() ret = [f'{indent}{key} = """'] if 'script' in key: ret.extend(value.splitlines()) else: ret.extend([ f'{indent} {line}' for line in value.splitlines() ]) ret += [f'{indent}"""'] else: ret = [f'{" " * (level - 1)}{key} = {value}'] return ret def _write_section(name: str, section: dict, level: int) -> List[str]: """Write an entire cylc section including headings and settings.""" ret = [] ret.append(_write_header(name, level)) ret.extend(_write_conf(section, level)) return ret def _write_conf(conf: dict, level: int) -> List[str]: ret = [] for key, value in conf.items(): # write out settings first if key.lower() == '#!jinja2': ret.append('#!jinja2') elif not isinstance(value, dict): ret.extend( _write_setting(key, value, level + 1) ) for key, value in conf.items(): # then sections after if isinstance(value, dict): ret.extend( _write_section(key, value, level + 1) ) return ret def flow_config_str(conf: dict) -> str: """Convert a configuration dictionary into cylc/parsec format. Args: conf (dict): A [nested] dictionary of configurations. Returns: str - Multiline string in cylc/parsec format. """ return '\n'.join(_write_conf(conf, 0)) + '\n' cylc-flow-8.7.0/tests/integration/utils/test_utils.py0000664000175000017500000000257215264420307023220 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests to ensure the tests are working - very meta. https://github.com/cylc/cylc-flow/pull/2740#discussion_r206086008 And yes, these are unit-tests inside a functional test framework thinggy. """ from pathlib import Path from . import ( _rm_if_empty, ) def test_rm_if_empty(tmp_path): """It should remove dirs if empty and suppress exceptions otherwise.""" path1 = Path(tmp_path, 'foo') path2 = Path(path1, 'bar') path2.mkdir(parents=True) _rm_if_empty(path1) assert path2.exists() _rm_if_empty(path2) assert not path2.exists() _rm_if_empty(path1) assert not path1.exists() cylc-flow-8.7.0/tests/integration/utils/test_flow_writer.py0000664000175000017500000000575115264420307024425 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests to ensure the tests are working - very meta. https://github.com/cylc/cylc-flow/pull/2740#discussion_r206086008 And yes, these are unit-tests inside a functional test framework thinggy. """ from textwrap import dedent from .flow_writer import ( _write_header, _write_setting, _write_section, flow_config_str ) def test_write_header(): """It should write out cylc configuration headings.""" assert _write_header('foo', 1) == '[foo]' assert _write_header('foo', 2) == ' [[foo]]' def test_write_setting_singleline(): """It should write out cylc configuration settings.""" assert _write_setting('key', 'value', 1) == [ 'key = value' ] assert _write_setting('key', 'value', 2) == [ ' key = value' ] def test_write_setting_script(): """It should preserve indentation for script items.""" assert _write_setting('script', 'a\nb\nc', 2) == [ ' script = """', 'a', 'b', 'c', ' """' ] def test_write_setting_multiline(): """It should write out cylc configuration settings over multiple lines.""" assert _write_setting('key', 'foo\nbar', 1) == [ 'key = """', ' foo', ' bar', '"""' ] assert _write_setting('key', 'foo\nbar', 2) == [ ' key = """', ' foo', ' bar', ' """' ] def test_write_section(): """It should write out entire cylc configurations.""" assert _write_section( 'foo', { 'bar': { 'pub': 'beer' }, 'baz': 42 }, 1 ) == [ '[foo]', ' baz = 42', ' [[bar]]', ' pub = beer' ] def test_flow_config_str(): """It should write out entire cylc configuration files.""" assert flow_config_str( { '#!JiNjA2': None, 'foo': { 'bar': { 'pub': 'beer' }, 'baz': 42 }, 'qux': 'asdf' } ) == dedent(''' #!jinja2 qux = asdf [foo] baz = 42 [[bar]] pub = beer ''').strip() + '\n' cylc-flow-8.7.0/tests/integration/scripts/0000775000175000017500000000000015264420307020770 5ustar alastairalastaircylc-flow-8.7.0/tests/integration/scripts/test_validate_reinstall.py0000664000175000017500000001011415264420307026244 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.option_parsers import Options from cylc.flow.scripts.validate_reinstall import ( get_option_parser as vr_gop, vr_cli, ) ValidateReinstallOptions = Options(vr_gop()) def answer_prompts(monkeypatch, *responses): """Hardcode responses to "cylc vr" interactive prompts.""" # make it look like we are running this command in a terminal monkeypatch.setattr( 'cylc.flow.scripts.validate_reinstall.is_terminal', lambda: True ) monkeypatch.setattr( 'cylc.flow.scripts.reinstall.is_terminal', lambda: True ) # patch user input count = -1 def _input(prompt): nonlocal count, responses responses = responses count += 1 print(prompt) # send the prompt to stdout for testing return responses[count] monkeypatch.setattr( 'cylc.flow.scripts.validate_reinstall._input', _input, ) monkeypatch.setattr( 'cylc.flow.scripts.reinstall._input', _input, ) async def test_prompt_for_running_workflow_with_no_changes( monkeypatch, capsys, one_run, capcall, ): """It should reinstall and restart the workflow with no changes. See: https://github.com/cylc/cylc-flow/issues/6261 We hope to get users into the habit of "cylc vip" to create a new run, and "cylc vr" to continue an old one (picking up any new changes in the process). If there are no changes to reinstall (or if the user chooses not to reinstall) the "cylc vr" prompts whether to continue or do nothing. The "nothing to reinstall" situation can be interpreted two ways: 1. Unexpected error, the user expected there to be something to reinstall, but there wasn't. E.g, they forgot to press save. 2. Unexpected annoyance, I wanted to restart the workflow, just do it. To handle this we explain that there are no changes to reinstall and prompt the user to see if they want to press save or restart the workflow. """ # disable the clean_sysargv logic (this interferes with other tests) cleanup_sysargv_calls = capcall( 'cylc.flow.scripts.validate_reinstall.cleanup_sysargv' ) # answer "y" to prompt answer_prompts(monkeypatch, 'n', 'y') # attempt to restart it with "cylc vr" ret = await vr_cli( vr_gop(), ValidateReinstallOptions(), one_run ) # the workflow should reinstall assert ret # the user should have been warned that there were no changes to reinstall outerr = capsys.readouterr()[0] assert 'Reinstall would make the above changes' in outerr # they should have been presented with a prompt # (to which we have hardcoded the response "y") assert 'Restart anyway?' in outerr # the workflow should have restarted assert len(cleanup_sysargv_calls) == 1 async def test_reinstall_abort( monkeypatch, capsys, one_run, ): """It should abort reinstallation according to user prompt.""" # answer 'n' to prompt answer_prompts(monkeypatch, 'n', 'n') # attempt to restart it with "cylc vr" ret = await vr_cli( vr_gop(), ValidateReinstallOptions(), one_run ) assert ret is False # they should have been presented with a prompt # (to which we have hardcoded the response "n") assert 'Continue' in capsys.readouterr()[0] cylc-flow-8.7.0/tests/integration/scripts/test_back_compat_flow_all.py0000664000175000017500000000674315264420307026535 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test set, trigger, and remove command back-compat with --flow=all. This option is now just the default (flows = ['all'] -> []). BACK COMPAT: handle --flow=all from pre-8.5 clients """ import pytest from cylc.flow.commands import ( force_trigger_tasks, remove_tasks, run_cmd, set_prereqs_and_outputs, ) from cylc.flow.exceptions import InputError from cylc.flow.scheduler import Scheduler from cylc.flow.task_outputs import TASK_OUTPUT_SUCCEEDED async def test_back_compat_flow_all(flow, scheduler, start): """Handle --flow=all from old clients. The trigger, set, and remove commands no longer take --flow=all, but for a while we need to handle that option coming in from older clients. (Prior to 8.5 it was the schema default for remove, and was documented as an option for set and trigger). """ conf = { 'scheduling': { 'graph': { 'R1': 'a & b' }, }, } schd: Scheduler = scheduler(flow(conf)) async with start(schd): foo, bar = schd.pool.get_tasks() # For comparison, set should fail with an illegal flow value (allx) with pytest.raises( InputError, match="Flow values must be integers, or 'new', or 'none'" ): await run_cmd( set_prereqs_and_outputs(schd, [foo.identity], ['allx']) ) # but the old --flow=all is OK - it converts to the default. await run_cmd( set_prereqs_and_outputs(schd, [foo.identity], ['all']) ) assert ( TASK_OUTPUT_SUCCEEDED in foo.state.outputs.get_completed_outputs() ) # For comparison, trigger should fail with an illegal flow value (allx) with pytest.raises( InputError, match="Flow values must be integers, or 'new', or 'none'" ): await run_cmd( force_trigger_tasks(schd, [bar.identity], ['allx']) ) # but the old --flow=all is OK - it converts to the default. await run_cmd( force_trigger_tasks(schd, [bar.identity], ['all']) ) assert bar in schd.pool.tasks_to_trigger_now # For comparison, remove should fail with an illegal flow value (allx) with pytest.raises( InputError, match="Flow values must be integers" ): await run_cmd( remove_tasks(schd, [bar.identity], ['allx']) ) # but the old --flow=all is OK - it converts to the default. await run_cmd( remove_tasks(schd, [bar.identity], ['all']) ) assert bar not in schd.pool.get_tasks() assert bar not in schd.pool.tasks_to_trigger_now cylc-flow-8.7.0/tests/integration/scripts/test_set.py0000664000175000017500000004062515264420307023203 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test "cylc set" functionality. Note: see also functional tests """ from secrets import token_hex from typing import Callable import pytest from cylc.flow.commands import ( run_cmd, set_prereqs_and_outputs, ) from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.data_messages_pb2 import ( PbJob, PbTaskProxy, ) from cylc.flow.data_store_mgr import ( JOBS, TASK_PROXIES, task_mean_elapsed_time, ) from cylc.flow.id import TaskTokens from cylc.flow.scheduler import Scheduler from cylc.flow.task_outputs import ( TASK_OUTPUT_FAILED, TASK_OUTPUT_STARTED, TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_SUCCEEDED, ) from cylc.flow.task_proxy import TaskProxy from cylc.flow.task_state import ( TASK_STATUS_FAILED, TASK_STATUS_PREPARING, TASK_STATUS_SUCCEEDED, TASK_STATUS_WAITING, ) def outputs_section(*names: str) -> dict: """Create outputs section with random messages for the given output names. """ return { 'outputs': { name: token_hex() for name in names } } async def test_set_parentless_spawning( flow, scheduler, run, complete, ): """Ensure that setting outputs does not interfere with parentless spawning. Setting outputs manually causes the logic to follow a different code pathway to "natural" output satisfaction. If we're not careful this could lead to "premature shutdown" (i.e. the scheduler thinks it's finished when it isn't), this test makes sure that's not the case. """ id_ = flow({ 'scheduling': { 'initial cycle point': '1', 'cycling mode': 'integer', 'runahead limit': 'P0', 'graph': {'P1': 'a => z'}, }, }) schd: Scheduler = scheduler(id_, paused_start=False) async with run(schd): # mark cycle 1 as succeeded schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'a'), TaskTokens('1', 'z')}, ['succeeded'], [], ['1'], ) # the parentless task "a" should be spawned out to the next cycle assert schd.pool.get_task_ids() == {'2/a'} # the workflow should run on to the next cycle await complete(schd, '2/a', timeout=5) assert schd.pool.get_task_ids() == {'2/z', '3/a'} async def test_rerun_incomplete( flow, scheduler, run, complete, reflog, ): """Incomplete tasks should be re-run.""" id_ = flow({ 'scheduling': { 'graph': {'R1': 'a => z'}, }, 'runtime': { # register a custom output 'a': {'outputs': {'x': 'xyz'}}, }, }) schd = scheduler(id_, paused_start=False) async with run(schd): # generate 1/a:x but do not complete 1/a schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'a')}, ['x'], [], ['1'] ) triggers = reflog(schd) await complete(schd) assert triggers == { # the task 1/a should have been run despite the earlier # setting of the "x" output ('1/a', None), ('1/z', ('1/a',)), } async def test_data_store( flow, scheduler, start, ): """Test that manually set prereqs/outputs are applied to the data store.""" id_ = flow({ 'scheduling': { 'graph': {'R1': 'a => z'}, }, 'runtime': { # register a custom output 'a': {'outputs': {'x': 'xyz'}}, }, }) schd: Scheduler = scheduler(id_) async with start(schd): await schd.update_data_structure() data = schd.data_store_mgr.data[schd.tokens.id] # 1/a not the same data-store object post set and not in pool # post succeeded. task_a_id = schd.pool.get_task(IntegerPoint('1'), 'a').tokens.id # set the 1/a:succeeded prereq of 1/z schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'z')}, [], ['1/a:succeeded'], ['1']) task_z = data[TASK_PROXIES][ schd.pool.get_task(IntegerPoint('1'), 'z').tokens.id ] await schd.update_data_structure() assert task_z.prerequisites[0].satisfied is True # set 1/a:x the task should be waiting with output x satisfied schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'a')}, ['x'], [], ['1'] ) await schd.update_data_structure() task_a: PbTaskProxy = data[TASK_PROXIES][task_a_id] assert task_a.state == TASK_STATUS_WAITING assert task_a.outputs['x'].satisfied is True assert task_a.outputs['succeeded'].satisfied is False # set 1/a:succeeded the task should be succeeded with output x sat schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'a')}, ['succeeded'], [], ['1'] ) await schd.update_data_structure() task_a: PbTaskProxy = data[TASK_PROXIES][task_a_id] assert task_a.state == TASK_STATUS_SUCCEEDED assert task_a.outputs['x'].satisfied is True assert task_a.outputs['succeeded'].satisfied is True async def test_incomplete_detection( one_conf, flow, scheduler, start, log_filter, ): """It should detect and log finished tasks left with incomplete outputs.""" schd = scheduler(flow(one_conf)) async with start(schd): schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'one')}, ['failed'], [], ['1'] ) assert log_filter(contains='1/one did not complete') async def test_pre_all(flow, scheduler, run): """Ensure that --pre=all is interpreted as a special case and _not_ tokenized. """ id_ = flow({'scheduling': {'graph': {'R1': 'a => z'}}}) schd = scheduler(id_, paused_start=False) async with run(schd) as log: schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'z')}, [], ['all'], [] ) warn_or_higher = [i for i in log.records if i.levelno > 30] assert warn_or_higher == [] async def test_bad_prereq( flow: 'Callable', scheduler: 'Callable', run: 'Callable', complete: 'Callable', caplog: 'pytest.LogCaptureFixture' ): """Attempting to set an invalid prerequisite should not leave a trace in the DB that prevents the target task from spawning. later on. """ id_ = flow({ 'scheduling': { 'graph': {'R1': 'a => b => c'}, }, }) schd = scheduler(id_, paused_start=False) async with run(schd): schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'c')}, [], ['1/a'], [] ) assert schd.pool.get_task_ids() == {'1/a'} assert '1/c does not depend on "1/a:succeeded"' in caplog.text schd.workflow_db_mgr.process_queued_ops() # This will fail if the previous set left 1/c in the DB: schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'c')}, [], ['1/b'], [] ) assert schd.pool.get_task_ids() == {'1/a', '1/c'} await complete(schd, '1/c', timeout=5) async def test_no_outputs_given(flow, scheduler, start): """Test `cylc set` without providing any outputs. It should set the "success pathway" outputs. """ schd: Scheduler = scheduler( flow({ 'scheduling': { 'graph': { 'R1': r""" foo? => alpha foo:submitted? => bravo foo:started? => charlie foo:x => xray # Optional custom outputs not emitted: foo:y? => yankee # Non-success-pathway outputs not emitted: foo:submit-failed? => delta """, }, }, 'runtime': { 'foo': outputs_section('x', 'y'), }, }) ) async with start(schd): foo = schd.pool.get_tasks()[0] await run_cmd( set_prereqs_and_outputs(schd, {foo.tokens}, []) ) assert set(foo.state.outputs.get_completed_outputs()) == { TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_STARTED, TASK_OUTPUT_SUCCEEDED, 'x' } assert schd.pool.get_task_ids() == { '1/alpha', '1/bravo', '1/charlie', '1/xray', } async def test_completion_expr(flow, scheduler, start): """Test `cylc set` without providing any outputs on a task that has a custom completion expression.""" conf = { 'scheduling': { 'graph': { 'R1': 'foo? | foo:x? => bar' }, }, 'runtime': { 'foo': { **outputs_section('x'), 'completion': '(succeeded or x) or failed' }, }, } schd: Scheduler = scheduler(flow(conf)) async with start(schd): foo = schd.pool.get_tasks()[0] await run_cmd( set_prereqs_and_outputs(schd, {foo.tokens}, []) ) assert set(foo.state.outputs.get_completed_outputs()) == { TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_STARTED, TASK_OUTPUT_SUCCEEDED, } async def test_set_submitted(flow, scheduler, start): """`cylc set --out submitted` should spawn children that depend on the output, but not affect the task's state.""" schd: Scheduler = scheduler(flow('foo:submitted? => bar')) async with start(schd): foo = schd.pool.get_tasks()[0] await run_cmd( set_prereqs_and_outputs( schd, [foo.identity], [], ['submitted'] ) ) assert set(foo.state.outputs.get_completed_outputs()) == { TASK_OUTPUT_SUBMITTED, } assert schd.pool.get_task_ids() == {'1/foo', '1/bar'} assert foo.state(TASK_STATUS_WAITING) async def test_job_state(flow, scheduler, start, db_select): """Test that setting outputs does not change the job state.""" schd: Scheduler = scheduler(flow('foo => bar')) fail_time = '1950-01-01T00:00:00Z' def db_task_states(itask: TaskProxy): return db_select( schd, True, 'task_states', 'status', name=itask.tdef.name ) def assert_job_failed(itask: TaskProxy): """...in the DB and data store.""" assert db_select( schd, True, 'task_jobs', 'run_status', 'time_run_exit', name=itask.tdef.name, ) == [(1, fail_time)] pb_job: PbJob = schd.data_store_mgr.data[schd.tokens.id][JOBS][ itask.job_tokens.id ] assert pb_job.state == TASK_STATUS_FAILED assert pb_job.finished_time == fail_time async with start(schd): foo = schd.pool.get_tasks()[0] schd.submit_task_jobs([foo]) schd.task_events_mgr.process_message( foo, 'INFO', TASK_OUTPUT_FAILED, event_time=fail_time ) await schd.update_data_structure() assert foo.state(TASK_STATUS_FAILED) assert db_task_states(foo) == [(TASK_STATUS_FAILED,)] assert_job_failed(foo) await run_cmd( set_prereqs_and_outputs(schd, [foo.identity], []) ) await schd.update_data_structure() # Task is now succeeded: assert foo.state(TASK_STATUS_SUCCEEDED) assert db_task_states(foo) == [(TASK_STATUS_SUCCEEDED,)] # But job is still failed: assert_job_failed(foo) async def test_set_already_succeeded( flow, scheduler, run, complete, db_select ): """Doing `cylc set` on a task that has already succeeded should not change anything.""" schd: Scheduler = scheduler( flow('foo => bar'), paused_start=False, ) def db_task_states(itask: TaskProxy): return db_select( schd, True, 'task_states', 'submit_num', 'status', 'time_updated', name=itask.tdef.name, ) def data_store_task_state(itask: TaskProxy): return schd.data_store_mgr.data[schd.tokens.id][TASK_PROXIES][ itask.tokens.id ].state async with run(schd): foo = schd.pool.get_tasks()[0] await complete(schd, foo.identity) time_updated = db_task_states(foo)[0][2] expected = [(1, TASK_STATUS_SUCCEEDED, time_updated)] assert db_task_states(foo) == expected assert data_store_task_state(foo) == TASK_STATUS_SUCCEEDED await run_cmd( set_prereqs_and_outputs(schd, [foo.identity], []) ) assert foo.state(TASK_STATUS_SUCCEEDED) await schd.update_data_structure() assert db_task_states(foo) == expected assert data_store_task_state(foo) == TASK_STATUS_SUCCEEDED async def test_timings(one_conf, flow, scheduler, start, caplog): """Test that setting outputs does not change the task timings.""" wid = flow({ **one_conf, 'runtime': { 'one': { 'execution time limit': 'PT100S', }, }, }) schd: Scheduler = scheduler(wid) async with start(schd): itask = schd.pool.get_tasks()[0] def check_times(): assert not itask.tdef.elapsed_times assert task_mean_elapsed_time(itask.tdef) == 100 itask.state_reset(TASK_STATUS_PREPARING) schd.task_events_mgr.process_message( itask, 'INFO', TASK_OUTPUT_STARTED ) check_times() await run_cmd(set_prereqs_and_outputs(schd, [itask.identity], [])) check_times() async def test_set_clears_modifiers(flow, scheduler, start): """Manually setting a final output on a task should clear some states. The is_held and is_retry task states should be cleared if a task is manually set to a final state as these states become moot and confusing. See https://github.com/cylc/cylc-flow/issues/7065 """ id_ = flow( { 'scheduling': { 'graph': {'R1': 'a => b'}, }, 'runtime': { 'a': { 'outputs': { 'x': 'xxx', }, }, }, }, ) schd = scheduler(id_) async with start(schd): task_a = schd.pool.get_task(IntegerPoint('1'), 'a') assert task_a # line up both submission and execution retries schd.task_events_mgr._retry_task(task_a, 0, True) schd.task_events_mgr._retry_task(task_a, 0, False) # mark the task as held schd.pool.hold_tasks({task_a.tokens}) # update the data store await schd.update_data_structure() task_a_data_store = schd.data_store_mgr.data[schd.id]['task_proxies'][ task_a.tokens.id ] # the task should have the states is_held and is_retry assert task_a.state.is_held is True assert task_a_data_store.is_retry is True assert task_a_data_store.is_retry is True # set a custom output await run_cmd( set_prereqs_and_outputs(schd, [task_a.identity], [], ['x']) ) # the is_held and is_retry statuses should be unchanged assert task_a.state.is_held is True assert task_a_data_store.is_retry is True assert task_a_data_store.is_retry is True # manually complete the task await run_cmd( set_prereqs_and_outputs( schd, [task_a.identity], [], [TASK_OUTPUT_SUCCEEDED] ) ) await schd.update_data_structure() # the is_held and is_retry states should have been cleared assert task_a.state.is_held is False assert task_a_data_store.is_retry is False assert task_a_data_store.is_retry is False cylc-flow-8.7.0/tests/integration/scripts/test_reinstall.py0000664000175000017500000003445615264420307024412 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio from contextlib import asynccontextmanager from pathlib import Path import re from secrets import token_hex from types import SimpleNamespace from ansimarkup import strip as cstrip import pytest from cylc.flow.exceptions import WorkflowFilesError from cylc.flow.install import reinstall_workflow from cylc.flow.option_parsers import Options from cylc.flow.scripts.reinstall import ( get_option_parser as reinstall_gop, reinstall_cli, ) from cylc.flow.workflow_files import WorkflowFiles from ..utils.entry_points import EntryPointWrapper ReInstallOptions = Options(reinstall_gop()) # cli opts # interactive: yes no # rose: yes no # workflow_running: yes no @pytest.fixture(autouse=True) def color_strip(monkeypatch: pytest.MonkeyPatch): """Strip colour as the normal colour stripping doesn't apply to tests.""" monkeypatch.setattr('cylc.flow.scripts.reinstall.cparse', cstrip) @pytest.fixture def interactive(monkeypatch): monkeypatch.setattr( 'cylc.flow.scripts.reinstall.is_terminal', lambda: True, ) @pytest.fixture def non_interactive(monkeypatch): monkeypatch.setattr( 'cylc.flow.scripts.reinstall.is_terminal', lambda: False, ) @pytest.fixture def answer_prompt(monkeypatch: pytest.MonkeyPatch): """Answer reinstall prompt.""" def inner(answer: str): monkeypatch.setattr( 'cylc.flow.scripts.reinstall._input', lambda x: answer ) return inner @pytest.fixture def one_src(tmp_path): src_dir = tmp_path / 'src' src_dir.mkdir() (src_dir / 'flow.cylc').touch() (src_dir / 'rose-suite.conf').touch() return SimpleNamespace(path=src_dir) @pytest.fixture def one_run(one_src, test_dir, run_dir): w_run_dir = test_dir / token_hex(4) w_run_dir.mkdir() (w_run_dir / 'flow.cylc').touch() (w_run_dir / 'rose-suite.conf').touch() install_dir = (w_run_dir / WorkflowFiles.Install.DIRNAME) install_dir.mkdir(parents=True) (install_dir / WorkflowFiles.Install.SOURCE).symlink_to( one_src.path, target_is_directory=True, ) return SimpleNamespace( path=w_run_dir, id=str(w_run_dir.relative_to(run_dir)), ) async def test_rejects_random_workflows(one, one_run): """It should only work with workflows installed by cylc install.""" with pytest.raises(WorkflowFilesError) as exc_ctx: await reinstall_cli(opts=ReInstallOptions(), workflow_id=one.workflow) assert 'was not installed with cylc install' in str(exc_ctx.value) async def test_invalid_source_dir(one_src, one_run): """It should detect & fail for an invalid source symlink""" source_link = Path( one_run.path, WorkflowFiles.Install.DIRNAME, WorkflowFiles.Install.SOURCE, ) source_link.unlink() source_link.symlink_to(one_src.path / 'flow.cylc') with pytest.raises(WorkflowFilesError) as exc_ctx: await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) assert 'Workflow source dir is not accessible' in str(exc_ctx.value) async def test_no_changes_needed(one_src, one_run, capsys, interactive): """It should not reinstall if no changes are needed. This is not a hard requirement, in practice rsync output may differ from expectation so this is a nice-to-have, not expected to work 100% of the time. """ assert not await reinstall_cli( opts=ReInstallOptions(), workflow_id=one_run.id ) assert 'up to date with' in capsys.readouterr().out async def test_non_interactive( one_src, one_run, capsys, capcall, non_interactive ): """It should not perform a dry-run or prompt in non-interactive mode.""" # capture reinstall calls reinstall_calls = capcall( 'cylc.flow.scripts.reinstall.reinstall_workflow', reinstall_workflow, ) # give it something to reinstall (one_src.path / 'a').touch() # reinstall assert await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) # only one rsync call should have been made (i.e. no --dry-run) assert len(reinstall_calls) == 1 assert 'Successfully reinstalled' in capsys.readouterr().out async def test_non_prompt_yes( one_src, one_run, capsys, capcall, interactive, answer_prompt ): """It should not prompt in interactive shell if --yes is supplied.""" # capture reinstall calls (one_src.path / 'b').touch() async with asyncio.timeout(2): assert await reinstall_cli( opts=ReInstallOptions(skip_interactive=True), workflow_id=one_run.id, ) assert 'Successfully reinstalled' in capsys.readouterr().out async def test_interactive( one_src, one_run, capsys, capcall, interactive, answer_prompt ): """It should perform a dry-run and prompt in interactive mode.""" # capture reinstall calls reinstall_calls = capcall( 'cylc.flow.scripts.reinstall.reinstall_workflow', reinstall_workflow, ) # give it something to reinstall (one_src.path / 'a').touch() answer_prompt('n') assert ( await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) is False ) # only one rsync call should have been made (i.e. the --dry-run) assert [call[1].get('dry_run') for call in reinstall_calls] == [True] assert 'reinstall cancelled' in capsys.readouterr().out reinstall_calls.clear() answer_prompt('y') assert await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) # two rsync calls should have been made (i.e. the --dry-run and the real) assert [call[1].get('dry_run') for call in reinstall_calls] == [ True, False ] assert 'Successfully reinstalled' in capsys.readouterr().out async def test_workflow_running( one_src, one_run, monkeypatch, capsys, non_interactive, ): """It should advise running "cylc reload" where applicable.""" # the message we are expecting reload_message = f'Run "cylc reload {one_run.id}"' # reinstall with a stopped workflow (reload message shouldn't show) assert await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) assert reload_message not in capsys.readouterr().out # reinstall with a running workflow (reload message should show) monkeypatch.setattr( # make it look like the workflow is running 'cylc.flow.scripts.reinstall.load_contact_file', lambda x: None, ) assert await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) assert reload_message in capsys.readouterr().out async def test_rsync_stuff(one_src, one_run, capsys, non_interactive): """Make sure rsync is working correctly.""" # src contains files: a, b (one_src.path / 'a').touch() with open(one_src.path / 'b', 'w+') as b_file: b_file.write('x') (one_src.path / 'b').touch() # run contains files: b, c (where b is different to the source copy) (one_run.path / 'b').touch() (one_run.path / 'c').touch() await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) # a should have been left assert (one_run.path / 'a').exists() # b should have been updated assert (one_run.path / 'b').exists() with open(one_run.path / 'b', 'r') as b_file: assert b_file.read() == 'x' # c should have been removed assert not (one_run.path / 'c').exists() async def test_rose_warning( one_src, one_run, capsys, interactive, answer_prompt ): """It should warn that Rose installed files will be deleted. See https://github.com/cylc/cylc-rose/issues/149 """ # fragment of the message we expect rose_message = ( 'Files created by Rose file installation will show as deleted' ) answer_prompt('n') (one_src.path / 'a').touch() # give it something to install # reinstall (with rose-suite.conf file) await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) assert rose_message in capsys.readouterr().err # reinstall (no rose-suite.conf file) (one_src.path / 'rose-suite.conf').unlink() await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) assert rose_message not in capsys.readouterr().err async def test_keyboard_interrupt( one_src, one_run, interactive, monkeypatch, capsys ): """It should handle a KeyboardInterrupt during dry-run elegantly. E.G. A user may ctrl+c rather than answering "n" (for no). To make it clear a canceled message should show. """ def raise_keyboard_interrupt(): raise KeyboardInterrupt() # currently the first call in the dry-run branch monkeypatch.setattr( 'cylc.flow.scripts.reinstall.is_terminal', raise_keyboard_interrupt, ) await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) assert 'reinstall cancelled' in capsys.readouterr().out async def test_rsync_fail(one_src, one_run, mock_glbl_cfg, non_interactive): """It should raise an error on rsync failure.""" mock_glbl_cfg( 'cylc.flow.install.glbl_cfg', ''' [platforms] [[localhost]] rsync command = false ''', ) (one_src.path / 'a').touch() # give it something to install with pytest.raises(WorkflowFilesError) as exc_ctx: await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) assert 'An error occurred reinstalling' in str(exc_ctx.value) async def test_permissions_change( one_src, one_run, interactive, answer_prompt, capsys: pytest.CaptureFixture, ): """It detects permissions changes.""" # Add script file: script_path: Path = one_src.path / 'myscript' script_path.touch() await reinstall_cli( opts=ReInstallOptions(skip_interactive=True), workflow_id=one_run.id ) assert (one_run.path / 'myscript').is_file() capsys.readouterr() # clears capsys # Change permissions (e.g. user forgot to make it executable before) script_path.chmod(0o777) # Answer "no" to reinstall prompt (we just want to test dry run) answer_prompt('n') await reinstall_cli( opts=ReInstallOptions(), workflow_id=one_run.id ) out, _ = capsys.readouterr() # On some systems may get "recv" instead of "send" assert re.search(r'(send|recv) myscript\n', out) @pytest.fixture def my_install_plugin(monkeypatch): """This configures a single post_install plugin. The plugin starts an async task, then returns. """ progress = [] @EntryPointWrapper def post_install_basic(*_, **__): """Simple plugin that returns one env var and one template var.""" async def my_async(): # the async task await asyncio.sleep(2) progress.append('end') # start the async task progress.append('start') asyncio.get_event_loop().create_task(my_async()) progress.append('return') # return a blank result return { 'env': {}, 'template_variables': {}, } monkeypatch.setattr( 'cylc.flow.plugins.iter_entry_points', lambda namespace: ( [post_install_basic] if namespace == 'cylc.post_install' else [] ) ) return progress async def test_async_block( one_src, one_run, my_install_plugin, monkeypatch, ): """Ensure async tasks created by post_install plugins are awaited. The cylc-rose plugin may create asyncio tasks when run but cannot await them (because it isn't async itself). To get around this we have "cylc reinstall" use "async_block" which detects tasks created in the background and awaits them. This test ensures that the async_block mechanism is correctly plugged in to "cylc reinstall". See https://github.com/cylc/cylc-rose/issues/274 """ # this is what it should do (one_src.path / 'a').touch() # give it something to install assert my_install_plugin == [] await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) # the presence of "end" means that the task was awaited assert my_install_plugin == ['start', 'return', 'end'] # substitute the "async_block" (which waits for asyncio tasks started in # the background) for a fake implementation (which doesn't) @asynccontextmanager async def async_block(): yield monkeypatch.setattr( 'cylc.flow.plugins._async_block', async_block, ) # this is what it would have done without async block (one_src.path / 'b').touch() # give it something else to install my_install_plugin.clear() assert my_install_plugin == [] await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) # the absence of "end" means that the task was not awaited assert my_install_plugin == ['start', 'return'] async def test_suite_rc(tmp_path, install): """It should reject workflows with suite.rc files.""" (tmp_path / WorkflowFiles.FLOW_FILE).touch() workflow_id = await install(tmp_path) # flow.cylc present await reinstall_cli(opts=ReInstallOptions(), workflow_id=workflow_id) # flow.cylc and suite.rc present (tmp_path / WorkflowFiles.SUITE_RC).touch() with pytest.raises( WorkflowFilesError, match=f'Both flow.cylc and suite.rc.*{tmp_path}' ): await reinstall_cli(opts=ReInstallOptions(), workflow_id=workflow_id) # suite.rc (tmp_path / WorkflowFiles.FLOW_FILE).unlink() with pytest.raises( WorkflowFilesError, match=f'suite.rc found in {tmp_path}' ): await reinstall_cli(opts=ReInstallOptions(), workflow_id=workflow_id) cylc-flow-8.7.0/tests/integration/scripts/test_completion_server.py0000664000175000017500000001353715264420307026151 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Integration tests for the "cylc completion-server command. See also the more extensive unit tests for this module. """ from cylc.flow.scripts.completion_server import complete_cylc def setify(coro): """Cast returned lists to sets for coroutines. Convenience function to use when you want to test output not order. """ async def _coro(*args, **kwargs): ret = await coro(*args, **kwargs) if isinstance(ret, list): return set(ret) return ret return _coro async def test_list_prereqs_and_outputs(flow, scheduler, start): """Test the success cases for listing task prereqs/outputs. The error cases are tested in a unit test (doesn't require a running scheduler). """ _complete_cylc = setify(complete_cylc) # Note: results are un-ordered id_ = flow({ 'scheduler': { 'allow implicit tasks': 'True', }, 'scheduling': { 'initial cycle point': '1', 'cycling mode': 'integer', 'graph': { 'P1': ''' a => b c => d b[-P1] => b ''' }, }, 'runtime': { 'a': {}, 'b': { 'outputs': { 'foo': 'abc def ghi', } } } }) schd = scheduler(id_) async with start(schd): await schd.update_data_structure() b1 = schd.tokens.duplicate(cycle='1', task='b') d1 = schd.tokens.duplicate(cycle='1', task='d') e1 = schd.tokens.duplicate(cycle='1', task='e') # does not exist # list prereqs (b1) assert await _complete_cylc('cylc', 'set', b1.id, '--pre', '') == { # keywords 'all', # intra-cycle dependency '1/a:succeeded', # inter-cycle dependency '0/b:succeeded', } # list outputs (b1) assert await _complete_cylc('cylc', 'set', b1.id, '--out', '') == { # regular task outputs 'expired', 'failed', 'started', 'submit-failed', 'submitted', 'succeeded', # custom task outputs 'foo', } # list prereqs (d1) assert await _complete_cylc('cylc', 'set', d1.id, '--pre', '') == { # keywords 'all', # d1 prereqs '1/c:succeeded', } # list prereqs for multiple (b1, d1) assert await _complete_cylc( 'cylc', 'set', b1.id, d1.id, '--pre', '', ) == { # keywords 'all', # b1 prereqs '1/a:succeeded', '0/b:succeeded', # d1 prereqs '1/c:succeeded', } # list prereqs for multiple (b1, d1) - alternative format assert await _complete_cylc( 'cylc', 'set', f'{schd.id}//', f'//{b1.relative_id}', f'//{d1.relative_id}', '--pre', '', ) == { # keywords 'all', # b1 prereqs '1/a:succeeded', '0/b:succeeded', # d1 prereqs '1/c:succeeded', } # list outputs for a non-existent task assert await _complete_cylc('cylc', 'set', e1.id, '--out', '') == set() # list outputs for a non-existent workflow assert await _complete_cylc( 'cylc', 'set', # this invalid workflow shouldn't prevent it from returning values # for the valid one 'no-such-workflow//', f'{schd.id}//', f'//{b1.relative_id}', f'//{d1.relative_id}', '--pre', '', ) == { # keywords 'all', # b1 prereqs '1/a:succeeded', '0/b:succeeded', # d1 prereqs '1/c:succeeded', } # start a second workflow to test multi-workflow functionality id2 = flow({ 'scheduling': { 'graph': { 'R1': ''' x => z ''' } }, 'runtime': {'x': {}, 'z': {}}, }) schd2 = scheduler(id2) async with start(schd2): await schd2.update_data_structure() z1 = schd2.tokens.duplicate(cycle='1', task='z') # list prereqs for multiple tasks in multiple workflows # (it should combine the results from both workflows) assert await _complete_cylc( 'cylc', 'set', b1.id, z1.id, '--pre', '', ) == { # keywords 'all', # workflow1//1/b prereqs '0/b:succeeded', '1/a:succeeded', # workflow2//1/z prereqs '1/x:succeeded' } cylc-flow-8.7.0/tests/integration/scripts/__init__.py0000664000175000017500000000140715264420307023103 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . cylc-flow-8.7.0/tests/integration/scripts/test_list.py0000664000175000017500000002161615264420307023362 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test the "cylc list" command.""" import pytest from cylc.flow.exceptions import InputError from cylc.flow.option_parsers import Options from cylc.flow.scripts.list import ( get_option_parser, _main, ) ListOptions = Options(get_option_parser()) @pytest.fixture(scope='module') async def cylc_list(mod_flow, mod_scheduler, mod_start): id_ = mod_flow( { 'scheduling': { # NOTE: all "a*" tasks are in the graph, but not "b*" tasks 'initial cycle point': 1, 'cycling mode': 'integer', 'graph': {'P1': 'a12 & a111 & a112 & a121 & a2'}, }, 'runtime': { 'A': {'meta': {'title': 'Title For A'}}, 'A1': {'inherit': 'A'}, 'A11': {'inherit': 'A1', 'meta': {'title': 'Title For A11'}}, 'A12': {'inherit': 'A1'}, 'a2': {'inherit': 'A'}, 'a12': {'inherit': 'A1'}, 'a111': {'inherit': 'A11'}, 'a112': {'inherit': 'A11'}, 'a121': {'inherit': 'A12'}, 'B': {}, 'b1': {'inherit': 'B'}, }, } ) schd = mod_scheduler(id_) async def _list(capsys, **kwargs): capsys.readouterr() await _main(ListOptions(**kwargs), schd.workflow) out, err = capsys.readouterr() return out.splitlines() async with mod_start(schd): yield _list @pytest.fixture def supports_utf8(monkeypatch): monkeypatch.setenv('LANG', 'C.utf-8') @pytest.fixture def does_not_support_utf8(monkeypatch): monkeypatch.setenv('LANG', 'C') async def test_plain(cylc_list, supports_utf8, capsys): """Test the default output format.""" assert await cylc_list(capsys) == [ 'a111', 'a112', 'a12', 'a121', 'a2', ] assert await cylc_list(capsys, all_tasks=True) == [ 'a111', 'a112', 'a12', 'a121', 'a2', 'b1', # <= in the runtime but not in the graph ] assert await cylc_list(capsys, all_namespaces=True) == [ 'A', 'A1', 'A11', 'A12', 'B', 'a111', 'a112', 'a12', 'a121', 'a2', 'b1', 'root', ] with pytest.raises(InputError): await cylc_list(capsys, all_tasks=True, all_namespaces=True) async def test_mro(cylc_list, supports_utf8, capsys): """Test the --mro (method resolution order) option.""" assert await cylc_list(capsys, mro=True) == [ 'a111 a111 A11 A1 A root', 'a112 a112 A11 A1 A root', 'a12 a12 A1 A root', 'a121 a121 A12 A1 A root', 'a2 a2 A root', ] assert await cylc_list(capsys, mro=True, all_tasks=True) == [ 'a111 a111 A11 A1 A root', 'a112 a112 A11 A1 A root', 'a12 a12 A1 A root', 'a121 a121 A12 A1 A root', 'a2 a2 A root', 'b1 b1 B root', ] assert await cylc_list(capsys, mro=True, all_namespaces=True) == [ 'A A root', 'A1 A1 A root', 'A11 A11 A1 A root', 'A12 A12 A1 A root', 'B B root', 'a111 a111 A11 A1 A root', 'a112 a112 A11 A1 A root', 'a12 a12 A1 A root', 'a121 a121 A12 A1 A root', 'a2 a2 A root', 'b1 b1 B root', 'root root', ] with pytest.raises(InputError): await cylc_list(capsys, titles=True, mro=True) async def test_tree(cylc_list, supports_utf8, capsys): """Test the --tree option.""" assert ( await cylc_list(capsys, tree=True) # NOTE: the --all-tasks and --all-namespaces opts should be ignored == await cylc_list(capsys, tree=True, all_tasks=True) == await cylc_list(capsys, tree=True, all_namespaces=True) == [ 'root ', ' `-A ', ' |-A1 ', ' | |-A11 ', ' | | |-a111 ', ' | | `-a112 ', ' | |-A12 ', ' | | `-a121 ', ' | `-a12 ', ' `-a2 ', ] ) assert ( await cylc_list(capsys, tree=True, box=True) == await cylc_list(capsys, box=True) # --tree implicit with --box == [ 'root ', ' └─A ', ' ├─A1 ', ' │ ├─A11 ', ' │ │ ├─a111 ', ' │ │ └─a112 ', ' │ ├─A12 ', ' │ │ └─a121 ', ' │ └─a12 ', ' └─a2 ', ] ) assert await cylc_list(capsys, tree=True, titles=True) == [ 'root ', ' `-A ', ' |-A1 ', ' | |-A11 ', ' | | |-a111 Title For A11', ' | | `-a112 Title For A11', ' | |-A12 ', ' | | `-a121 Title For A', ' | `-a12 Title For A', ' `-a2 Title For A', ] assert await cylc_list(capsys, tree=True, box=True, titles=True) == [ 'root ', ' └─A ', ' ├─A1 ', ' │ ├─A11 ', ' │ │ ├─a111 Title For A11', ' │ │ └─a112 Title For A11', ' │ ├─A12 ', ' │ │ └─a121 Title For A', ' │ └─a12 Title For A', ' └─a2 Title For A', ] async def test_box_with_lang_c(cylc_list, does_not_support_utf8, capsys): """It falls back to plain output if unicode support isn't there.""" assert await cylc_list(capsys, tree=True, box=True) == [ 'a111', 'a112', 'a12', 'a121', 'a2', ] async def test_titles(cylc_list, supports_utf8, capsys): """Test the --titles option.""" assert await cylc_list(capsys, titles=True) == [ 'a111 Title For A11', 'a112 Title For A11', 'a12 Title For A', 'a121 Title For A', 'a2 Title For A', ] assert await cylc_list(capsys, titles=True, all_tasks=True) == [ 'a111 Title For A11', 'a112 Title For A11', 'a12 Title For A', 'a121 Title For A', 'a2 Title For A', 'b1 ', ] assert await cylc_list(capsys, titles=True, all_namespaces=True) == [ 'A Title For A', 'A1 Title For A', 'A11 Title For A11', 'A12 Title For A', 'B ', 'a111 Title For A11', 'a112 Title For A11', 'a12 Title For A', 'a121 Title For A', 'a2 Title For A', 'b1 ', 'root ', ] async def test_points(cylc_list, supports_utf8, capsys): """Test the --points option.""" # specify start and stop points assert await cylc_list(capsys, prange='1,2') == [ '1/a111', '1/a112', '1/a12', '1/a121', '1/a2', '2/a111', '2/a112', '2/a12', '2/a121', '2/a2', ] # leave start and stop points implicit assert await cylc_list(capsys, prange=',') == [ '1/a111', '1/a112', '1/a12', '1/a121', '1/a2', '2/a111', '2/a112', '2/a12', '2/a121', '2/a2', '3/a111', '3/a112', '3/a12', '3/a121', '3/a2', ] # leave start point implicit assert await cylc_list(capsys, prange=',2') == [ '1/a111', '1/a112', '1/a12', '1/a121', '1/a2', '2/a111', '2/a112', '2/a12', '2/a121', '2/a2', ] # leave stop point implicit assert await cylc_list(capsys, prange='4,') == [ '4/a111', '4/a112', '4/a12', '4/a121', '4/a2', '5/a111', '5/a112', '5/a12', '5/a121', '5/a2', '6/a111', '6/a112', '6/a12', '6/a121', '6/a2', ] with pytest.raises(InputError): await cylc_list(capsys, prange='1,2', all_tasks=True) with pytest.raises(InputError): await cylc_list(capsys, prange='1,2', all_namespaces=True) cylc-flow-8.7.0/tests/integration/scripts/conftest.py0000664000175000017500000000327115264420307023172 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from uuid import uuid1 import pytest from cylc.flow.workflow_files import WorkflowFiles from ..utils.flow_writer import flow_config_str @pytest.fixture def one_src(tmp_path, one_conf): """Basic source workflow containing the config of the "one" test flow.""" src_dir = tmp_path (src_dir / 'flow.cylc').write_text(flow_config_str(one_conf)) return src_dir @pytest.fixture def one_run(one_src, test_dir, run_dir): """Basic source workflow with a _cylc-install/source symlink.""" w_run_dir = test_dir / str(uuid1()) w_run_dir.mkdir() (w_run_dir / 'flow.cylc').write_text( (one_src / 'flow.cylc').read_text() ) install_dir = (w_run_dir / WorkflowFiles.Install.DIRNAME) install_dir.mkdir(parents=True) (install_dir / WorkflowFiles.Install.SOURCE).symlink_to( one_src, target_is_directory=True, ) return str(w_run_dir.relative_to(run_dir)) cylc-flow-8.7.0/tests/integration/scripts/test_report_timings.py0000664000175000017500000000754415264420307025460 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests for the "cylc report-timings" command.""" import pytest from cylc.flow.commands import force_trigger_tasks, run_cmd from cylc.flow.option_parsers import Options from cylc.flow.scripts.report_timings import ( FORMATS, _main, get_option_parser, ) RTOptions = Options(get_option_parser()) @pytest.fixture(scope='module') async def workflow(mod_flow, mod_scheduler, mod_run, mod_complete): """Simple workflow to test report-timings against. Uses real cycles which run in order to ensure averaging is performed over something "real". """ id_ = mod_flow({ 'scheduling': { 'initial cycle point': '1', 'final cycle point': '3', 'cycling mode': 'integer', 'graph': { 'P1': ''' a => b => c b[-P1] => b ''' }, }, }) schd = mod_scheduler(id_, paused_start=False) async with mod_run(schd): await mod_complete(schd, '2/b') # re-run 1/b (activates _reshape_timings logic) await run_cmd(force_trigger_tasks(schd, ['1/b'], ['1'])) await mod_complete(schd) yield schd def test_raw(workflow, capsys): """Test --format=raw""" _main(RTOptions(format='raw'), workflow.tokens.id) out, err = capsys.readouterr() # nothing written to sterrr assert not err # something written to stdout lines = out.splitlines() assert len(lines) == 11 # (3 tasks x 3 cycles) + 1 re-run + 1 header row assert lines[1].split()[:4] == ['a', '1', 'simulation', 'simulation'] def test_summary(workflow, capsys): """Test --format=summary""" _main(RTOptions(format='summary'), workflow.tokens.id) out, err = capsys.readouterr() # nothing written to sterrr assert not err lines = out.splitlines() # the header assert 'Host: simulation' in lines[1] assert 'Job Runner: simulation' in lines[1] # the first data row assert lines[6].split() == [ 'a', '3.0', '0.0', '0.0', '0.0', '0.0', '0.0', '0.0', '0.0', ] def test_html(workflow, capsys): """Test --format=html""" _main(RTOptions(format='html'), workflow.tokens.id) out, err = capsys.readouterr() # nothing written to sterrr assert not err # headings present assert '

Queue Time

' in out # plots present assert ' 3 cylc-flow-8.7.0/tests/integration/scripts/test_dump.py0000664000175000017500000000676715264420307023366 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test the "cylc dump" command.""" import pytest from cylc.flow.option_parsers import ( Options, ) from cylc.flow.scripts.dump import ( dump, get_option_parser, ) DumpOptions = Options(get_option_parser()) async def test_dump_tasks(flow, scheduler, start): """It should show n=0 tasks. See: https://github.com/cylc/cylc-flow/pull/5600 """ id_ = flow({ 'scheduler': { 'allow implicit tasks': 'true', }, 'scheduling': { 'graph': { 'R1': 'a => b => c', }, }, }) schd = scheduler(id_) async with start(schd): # schd.release_tasks_to_run() await schd.update_data_structure() ret = [] await dump( id_, DumpOptions(disp_form='tasks', legacy_format=True), write=ret.append ) assert ret == ['a, 1, waiting, not-held, queued, not-runahead'] @pytest.mark.parametrize( 'attributes_bool, flow_nums, full_ids, dump_str', [ pytest.param( True, [1, 2], False, '1/a:waiting (held,queued,runahead) flows=[1,2]', id='1' ), pytest.param( False, [1, 2], False, '1/a:waiting', id='2' ), pytest.param( True, [1, 2], True, '{workflow_id}//1/a', id='3' ) ] ) async def test_dump_format( flow, scheduler, start, attributes_bool, flow_nums, full_ids, dump_str ): """Check the new "cylc dump" output format, i.e. task IDs. See: https://github.com/cylc/cylc-flow/pull/6440 """ id_ = flow({ 'scheduler': { 'allow implicit tasks': 'true', }, 'scheduling': { 'graph': { 'R1': 'a', }, }, }) schd = scheduler(id_) async with start(schd): [itask] = schd.pool.get_tasks() itask.state_reset( is_held=attributes_bool, is_runahead=attributes_bool, is_queued=attributes_bool ) itask.flow_nums = set(flow_nums) schd.pool.data_store_mgr.delta_task_held( itask.tdef.name, itask.point, itask.state.is_held) schd.pool.data_store_mgr.delta_task_state(itask) schd.pool.data_store_mgr.delta_task_flow_nums(itask) await schd.update_data_structure() ret = [] await dump( id_, DumpOptions( disp_form='tasks', full_ids=full_ids, show_flows=attributes_bool ), write=ret.append ) assert ret == [dump_str.format(**{"workflow_id": id_})] cylc-flow-8.7.0/tests/integration/scripts/test_graph.py0000664000175000017500000000741015264420307023504 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.option_parsers import Options from cylc.flow.scripts.graph import _main, get_option_parser Opts = Options(get_option_parser()) @pytest.fixture def disable_graph_open(monkeypatch): """Prevent "cylc graph" from trying to pop open the image.""" monkeypatch.setattr( 'cylc.flow.scripts.graph.open_image', lambda *_a, **_k: None, ) async def test_blank_graph(one, disable_graph_open, capsys): """It should inform the user if there are no tasks to display.""" # graph with one task await _main(Opts(color='never'), one.tokens.id) out, err = capsys.readouterr() assert 'Graph rendered to' in out # graph with no tasks (the only task is in cycle "1") await _main(Opts(color='never'), one.tokens.id, '5') out, err = capsys.readouterr() assert 'No tasks to display' in err assert 'Try changing the start and stop values' in err async def test_flatten_icp(flow, tmp_path): """It should flatten out absolute dependencies in the ICP. Tests the "--flatten-icp" flag. """ id_ = flow({ 'scheduler': { 'cycle point format': 'CCYY', }, 'scheduling': { 'initial cycle point': '2000', 'final cycle point': '2004', 'graph': { 'R1': 'start', 'P1Y': 'start[^] => foo => bar', # ICP dep (^) 'R1/2003': 'foo[2001] => pub', # non-ICP ABS dep (2001) }, }, }) graph_file = tmp_path / 'graph.dot' # test default behaviour (expanded ICP deps): await _main( Opts(output=str(graph_file), flatten_icp_dependence=False), id_, '2000', '2003', ) with open(graph_file, 'r') as graph_file_: edges = {line.strip() for line in graph_file_ if '->' in line} assert edges == { '"2000/foo" -> "2000/bar"', '"2000/start" -> "2000/foo"', # NOTE: inter-cycle ICP dep '"2000/start" -> "2001/foo"', # NOTE: ICP dep '"2000/start" -> "2002/foo"', # NOTE: ICP dep '"2000/start" -> "2003/foo"', # NOTE: ICP dep '"2001/foo" -> "2001/bar"', '"2001/foo" -> "2003/pub"', '"2002/foo" -> "2002/bar"', '"2003/foo" -> "2003/bar"', } # test "--flatten-icp" behaviour: await _main( Opts(output=str(graph_file), flatten_icp_dependence=True), id_, '2000', '2003', ) with open(graph_file, 'r') as graph_file_: edges = {line.strip() for line in graph_file_ if '->' in line} assert edges == { '"2000/foo" -> "2000/bar"', '"2000/start" -> "2000/foo"', '"2001/foo" -> "2001/bar"', '"2001/foo" -> "2003/pub"', # NOTE: not flattened (non-ICP abs dep) '"2002/foo" -> "2002/bar"', '"2003/foo" -> "2003/bar"', '"R1.2001/start" -> "2001/foo"', # NOTE: ICP dep flattened '"R1.2002/start" -> "2002/foo"', # NOTE: ICP dep flattened '"R1.2003/start" -> "2003/foo"', # NOTE: ICP dep flattened } cylc-flow-8.7.0/tests/integration/scripts/test_validate_integration.py0000664000175000017500000002060215264420307026575 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Integration tests for Cylc Validate CLI script.""" import logging import pytest from cylc.flow.exceptions import WorkflowConfigError from cylc.flow.parsec.exceptions import IllegalItemError, Jinja2Error async def test_validate_against_source_checks_source( capsys, validate, workflow_source, install, one_conf ): """Validation fails if validating against source with broken config. """ src_dir = workflow_source(one_conf) workflow_id = await install(src_dir) # Check that the original installation validates OK: validate(workflow_id, against_source=True) # Break the source config: with open(src_dir / 'flow.cylc', 'a') as handle: handle.write('\n[runtime]\n[[foo]]\nAgrajag = bowl of petunias') # # Check that Validate now fails: with pytest.raises(IllegalItemError, match='Agrajag'): validate(workflow_id, against_source=True) async def test_validate_against_source_gets_old_tvars( workflow_source, capsys, validate, scheduler, run, install, run_dir ): """Validation will retrieve template variables from a previously played workflow. """ src_dir = workflow_source({ '#!jinja2': None, 'scheduler': { 'allow implicit tasks': True }, 'scheduling': { 'initial cycle point': '1854', 'graph': { 'P1Y': 'foo' }, }, 'runtime': { 'foo': { 'script': 'cylc pause ${CYLC_WORKFLOW_ID}' } } }) wf_id = await install(src_dir) installed_dir = run_dir / wf_id # Check that the original installation validates OK: validate(installed_dir) # # Start a scheduler with tvars option: schd = scheduler( wf_id, templatevars=['FOO="foo"'] ) async with run(schd): pass # Replace foo in graph with {{FOO}} and check that this still # Validates (using db value for FOO): flow_file = (installed_dir / 'flow.cylc') flow_file.write_text( flow_file.read_text().replace('P1Y = foo', 'P1Y = {{FOO}}')) validate(wf_id, against_source=True) # Check that the source will not validate alone: flow_file = (src_dir / 'flow.cylc') flow_file.write_text( flow_file.read_text().replace('P1Y = foo', 'P1Y = {{FOO}}')) with pytest.raises(Jinja2Error): validate(src_dir) def test_validate_simple_graph(flow, validate, caplog): """Test deprecation notice for Cylc 7 simple graph (no recurrence section) """ id_ = flow({ 'scheduler': {'allow implicit tasks': True}, 'scheduling': {'dependencies': {'graph': 'foo'}} }) validate(id_) expect = ( 'graph items were automatically upgraded' ' in "workflow definition" - support for the old item will be ' 'removed at Cylc 8.9:' '\n * (8.0.0) [scheduling][dependencies]graph -> [scheduling][graph]R1' ) assert expect in caplog.messages def test_pre_cylc8(flow, validate, caplog): """Test all current non-silent workflow obsoletions and deprecations. """ id_ = flow({ 'cylc': { 'events': { 'reset timer': 10, 'reset inactivity timer': 15, } }, "scheduling": { "initial cycle point": "20150808T00", "final cycle point": "20150808T00", "graph": { "P1D": "foo => cat & dog" }, "special tasks": { "external-trigger": 'cat("meow available")' } }, 'runtime': { 'foo, cat, dog': { 'suite state polling': {'template': ''}, 'events': {'reset timer': 20} } } }, defaults=False) validate(id_) for warning in ( ( ' * (7.8.0) [runtime][foo, cat, dog][suite state polling]template' ' - DELETED (OBSOLETE)' ' - this will fail validation in Cylc 8.9'), (' * (7.8.1) [cylc][events]reset timer - DELETED (OBSOLETE)' ' - this will fail validation in Cylc 8.9'), (' * (7.8.1) [cylc][events]reset inactivity timer - DELETED (OBSOLETE)' ' - this will fail validation in Cylc 8.9'), ( ' * (7.8.1) [runtime][foo, cat, dog][events]reset timer' ' - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9'), ( ' * (8.0.0) [runtime][foo, cat, dog][suite state polling]' ' -> [runtime][foo, cat, dog][workflow state polling]' ' - value unchanged - support for the old item will be ' 'removed at Cylc 8.9'), (' * (8.0.0) [cylc] -> [scheduler] - value unchanged' ' - support for the old item will be removed at Cylc 8.9') ): assert warning in caplog.messages def test_graph_upgrade_msg_default(flow, validate, caplog, log_filter): """It lists Cycling definitions which need upgrading.""" id_ = flow({ 'scheduler': {'allow implicit tasks': True}, 'scheduling': { 'initial cycle point': 1042, 'dependencies': { 'R1': {'graph': 'foo'}, 'P1Y': {'graph': 'bar & baz'} } }, }) validate(id_) assert log_filter(contains='[scheduling][dependencies][X]graph') assert log_filter(contains='for X in:\n P1Y, R1') def test_graph_upgrade_msg_graph_equals(flow, validate, caplog, log_filter): """It gives a more useful message in special case where graph is key rather than section: [scheduling] [[dependencies]] graph = foo => bar """ id_ = flow({ 'scheduler': {'allow implicit tasks': True}, 'scheduling': {'dependencies': {'graph': 'foo => bar'}}, }) validate(id_) assert log_filter( contains='[scheduling][dependencies]graph -> [scheduling][graph]R1' ) def test_graph_upgrade_msg_graph_equals2(flow, validate, caplog, log_filter): """Both an implicit R1 and explicit reccurance exist: It appends a note. """ id_ = flow({ 'scheduler': {'allow implicit tasks': True}, 'scheduling': { 'initial cycle point': '1000', 'dependencies': { 'graph': 'foo => bar', 'P1Y': {'graph': 'a => b'}}}, }) validate(id_) expect = ( 'graph items were automatically upgraded in' ' "workflow definition" - support for the old item will be ' 'removed at Cylc 8.9:' '\n * (8.0.0) [scheduling][dependencies][X]graph' ' -> [scheduling][graph]X - for X in:' '\n P1Y, graph' '\n ([scheduling][dependencies]graph moves to [scheduling][graph]R1)' ) assert log_filter(contains=expect) def test_undefined_parent(flow, validate): """It should catch tasks which inherit from implicit families.""" id_ = flow({ 'scheduling': {'graph': {'R1': 'foo'}}, 'runtime': {'foo': {'inherit': 'FOO'}} }) with pytest.raises(WorkflowConfigError, match='undefined parent for foo'): validate(id_) def test_log_parent_demoted(flow, validate, monkeypatch, caplog, log_filter): """It should log family "demotion" in verbose mode.""" monkeypatch.setattr( 'cylc.flow.flags.verbosity', 10, ) caplog.set_level(logging.DEBUG) id_ = flow({ 'scheduling': { 'graph': { 'R1': 'foo' } }, 'runtime': { 'foo': {'inherit': 'None, FOO'}, 'FOO': {}, } }) validate(id_) assert log_filter(contains='First parent(s) demoted to secondary') assert log_filter(contains="FOO as parent of 'foo'") cylc-flow-8.7.0/tests/integration/scripts/test_kill.py0000664000175000017500000002121715264420307023337 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio import logging from secrets import token_hex from unittest.mock import Mock import pytest from cylc.flow.commands import ( kill_tasks, remove_tasks, run_cmd, set_prereqs_and_outputs, ) from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.scheduler import Scheduler from cylc.flow.task_proxy import TaskProxy from cylc.flow.task_remote_mgr import ( REMOTE_FILE_INSTALL_DONE, REMOTE_FILE_INSTALL_IN_PROGRESS, REMOTE_INIT_DONE, REMOTE_INIT_IN_PROGRESS, ) from cylc.flow.task_state import ( TASK_STATUS_FAILED, TASK_STATUS_PREPARING, TASK_STATUS_RUNNING, TASK_STATUS_SUBMIT_FAILED, TASK_STATUS_WAITING, ) LOCALHOST = 'localhost' async def task_state(itask: TaskProxy, state: str, timeout=4, **kwargs): """Await task state.""" async with asyncio.timeout(timeout): while not itask.state(state, **kwargs): await asyncio.sleep(0.1) def patch_remote_init(schd: Scheduler, value: str): """Set remote init state.""" schd.task_job_mgr.task_remote_mgr.remote_init_map[LOCALHOST] = value async def test_simulation(flow, scheduler, run): """Test killing a running task in simulation mode.""" conf = { 'scheduling': { 'graph': { 'R1': 'foo', }, }, 'runtime': { 'root': { 'simulation': { 'default run length': 'PT30S', }, }, }, } schd: Scheduler = scheduler(flow(conf), paused_start=False) async with run(schd): itask = schd.pool.get_tasks()[0] await task_state(itask, TASK_STATUS_RUNNING) await run_cmd(kill_tasks(schd, [itask.identity])) await task_state(itask, TASK_STATUS_FAILED, is_held=True) assert schd.check_workflow_stalled() async def test_kill_preparing( flow, scheduler, run, monkeypatch: pytest.MonkeyPatch, log_filter ): """Test killing a preparing task.""" schd: Scheduler = scheduler( flow('foo'), run_mode='live', paused_start=False ) async with run(schd): # Make the task indefinitely preparing: monkeypatch.setattr( schd.task_job_mgr, '_prep_submit_task_job', Mock(return_value=None) ) itask = schd.pool.get_tasks()[0] await task_state(itask, TASK_STATUS_PREPARING, is_held=False) await run_cmd(kill_tasks(schd, [itask.identity])) await task_state(itask, TASK_STATUS_SUBMIT_FAILED, is_held=True) assert log_filter(logging.ERROR, 'killed in job prep') async def test_kill_preparing_pipeline( flow, scheduler, start, monkeypatch: pytest.MonkeyPatch ): """Test killing a preparing task through various stages of the preparing pipeline that involve submitting subprocesses and waiting for them to complete.""" # Make localhost look like a remote target so we can test # remote init/file install stages: monkeypatch.setattr( 'cylc.flow.task_job_mgr.get_localhost_install_target', Mock(return_value=token_hex()), ) schd: Scheduler = scheduler( flow('one'), run_mode='live', paused_start=False ) async with start(schd): remote_mgr = schd.task_job_mgr.task_remote_mgr mock_eval_platform = Mock(return_value=None) monkeypatch.setattr(remote_mgr, 'eval_platform', mock_eval_platform) mock_remote_init = Mock() monkeypatch.setattr(remote_mgr, 'remote_init', mock_remote_init) mock_file_install = Mock() monkeypatch.setattr(remote_mgr, 'file_install', mock_file_install) itask = schd.pool.get_tasks()[0] # Platform eval: schd.submit_task_jobs([itask]) assert itask.state(TASK_STATUS_PREPARING) assert schd.release_tasks_to_run() is False await run_cmd(kill_tasks(schd, [itask.identity])) assert itask.state(TASK_STATUS_SUBMIT_FAILED) assert schd.release_tasks_to_run() is False # Set to finished: mock_eval_platform.return_value = LOCALHOST # Should not submit after finish because it was killed: assert schd.release_tasks_to_run() is False # Remote init: patch_remote_init(schd, REMOTE_INIT_IN_PROGRESS) schd.submit_task_jobs([itask]) assert itask.state(TASK_STATUS_PREPARING) assert schd.release_tasks_to_run() is False await run_cmd(kill_tasks(schd, [itask.identity])) assert itask.state(TASK_STATUS_SUBMIT_FAILED) assert schd.release_tasks_to_run() is False # Set to finished: patch_remote_init(schd, REMOTE_INIT_DONE) # Should not submit after finish because it was killed: assert schd.release_tasks_to_run() is False assert not mock_remote_init.called # Remote file install: patch_remote_init(schd, REMOTE_FILE_INSTALL_IN_PROGRESS) schd.submit_task_jobs([itask]) assert itask.state(TASK_STATUS_PREPARING) assert schd.release_tasks_to_run() is False await run_cmd(kill_tasks(schd, [itask.identity])) assert itask.state(TASK_STATUS_SUBMIT_FAILED) assert schd.release_tasks_to_run() is False # Set to finished: patch_remote_init(schd, REMOTE_FILE_INSTALL_DONE) # Should not submit after finish because it was killed: assert schd.release_tasks_to_run() is False assert not mock_file_install.called @pytest.mark.parametrize('method', (remove_tasks, set_prereqs_and_outputs)) async def test_kill_hold_retry(method, flow, scheduler, start): """Test the interaction between kill, hold and automated retry. When killed, tasks should be put into the held state to suppress automatic retries. Once the task has been completed or been removed, the held and retrying states no longer serve a purpose and should be cleared (see https://github.com/cylc/cylc-flow/pull/7100). """ id_ = flow({ 'scheduling': { 'graph': { 'R1': 'execution & submission' }, }, 'runtime': { 'execution, submission': { 'execution retry delays': 'PT0S', 'submission retry delays': 'PT0S', } } }) schd: Scheduler = scheduler(id_) async with start(schd): # a task we will make execution-fail execution = schd.pool.get_task(IntegerPoint('1'), 'execution') # a task we will make submission-fail submission = schd.pool.get_task(IntegerPoint('1'), 'submission') assert execution and submission # the tasks should not be held (doesn't hurt to check!) assert execution.state(is_held=False) assert submission.state(is_held=False) # fake job submissions execution.state_reset(TASK_STATUS_RUNNING) submission.state_reset(TASK_STATUS_PREPARING) schd.task_job_mgr._set_retry_timers(execution, execution.tdef.rtconfig) schd.task_job_mgr._set_retry_timers( submission, submission.tdef.rtconfig ) # kill the tasks await run_cmd( kill_tasks(schd, [execution.identity, submission.identity]) ) # the tasks should be in the held state with a retry lined up assert execution.state(TASK_STATUS_WAITING, is_held=True) # held assert submission.state(TASK_STATUS_WAITING, is_held=True) assert list(execution.state.xtriggers.values()) == [False] # retry assert list(submission.state.xtriggers.values()) == [False] # either remove or complete the tasks await run_cmd( method(schd, [execution.identity, submission.identity], []) ) # the held state should be cleared assert execution.state(is_held=False) assert submission.state(is_held=False) # the retry state should be cleared assert list(execution.state.xtriggers.values()) == [True] assert list(submission.state.xtriggers.values()) == [True] cylc-flow-8.7.0/tests/integration/scripts/test_show.py0000664000175000017500000002133715264420307023367 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import json import pytest import re from types import SimpleNamespace from colorama import init as colour_init from cylc.flow.id import Tokens from cylc.flow.scripts.show import ( ShowOptions, show, ) RE_STATE = re.compile('state:.*') @pytest.fixture(scope='module') def mod_my_conf(): """A workflow configuration with some workflow metadata.""" return { 'meta': { 'title': 'Workflow Title', 'description': """ My multiline description. """, 'URL': 'http://ismycomputerturnedon.com/', 'answer': '42', }, 'scheduling': { 'graph': { 'R1': 'foo' } }, 'runtime': { 'foo': { 'meta': { 'title': 'Task Title', 'description': ''' Task multiline description ''', 'URL': ( 'http://hasthelargehadroncollider' 'destroyedtheworldyet.com/' ), 'question': 'mutually exclusive', }, }, }, } @pytest.fixture(scope='module') async def mod_my_schd(mod_flow, mod_scheduler, mod_start, mod_my_conf): """A "started" workflow.""" id_ = mod_flow(mod_my_conf) schd = mod_scheduler(id_) async with mod_start(schd): yield schd async def test_workflow_meta_query(mod_my_schd, capsys): """It should fetch workflow metadata.""" colour_init(strip=True, autoreset=True) opts = SimpleNamespace( comms_timeout=5, json=False, list_prereqs=False, task_defs=None, ) # plain output ret = await show(mod_my_schd.workflow, [], opts) assert ret == 0 out, err = capsys.readouterr() assert out.splitlines() == [ 'title: Workflow Title', 'description: My', 'multiline', 'description.', 'answer: 42', 'URL: http://ismycomputerturnedon.com/', ] # json output opts.json = True ret = await show(mod_my_schd.workflow, [], opts) assert ret == 0 out, err = capsys.readouterr() assert json.loads(out) == { 'title': 'Workflow Title', 'description': 'My\nmultiline\ndescription.', 'answer': '42', 'URL': 'http://ismycomputerturnedon.com/', } async def test_task_meta_query(mod_my_schd, capsys): """It should fetch task metadata.""" colour_init(strip=True, autoreset=True) opts = SimpleNamespace( comms_timeout=5, json=False, list_prereqs=False, task_defs=['foo'], ) # plain output ret = await show( mod_my_schd.workflow, None, opts, ) assert ret == 0 out, err = capsys.readouterr() assert out.splitlines() == [ 'title: Task Title', 'question: mutually exclusive', 'description: Task', 'multiline', 'description', 'URL: http://hasthelargehadroncolliderdestroyedtheworldyet.com/', ] # json output opts.json = True ret = await show(mod_my_schd.workflow, [], opts) assert ret == 0 out, err = capsys.readouterr() assert json.loads(out) == { 'foo': { 'title': 'Task Title', 'question': 'mutually exclusive', 'description': 'Task\nmultiline\ndescription', 'URL': 'http://hasthelargehadroncolliderdestroyedtheworldyet.com/', } } async def test_task_instance_query( flow, scheduler, start, capsys ): """It should fetch task instance data, sorted by task name.""" colour_init(strip=True, autoreset=True) opts = SimpleNamespace( comms_timeout=5, json=False, task_defs=None, list_prereqs=False, ) schd = scheduler( flow( { 'scheduling': { 'graph': {'R1': 'zed & dog & cat & ant'}, }, }, ), paused_start=False, ) async with start(schd): await schd.update_data_structure() ret = await show( schd.workflow, [Tokens('//1/*')], opts, ) assert ret == 0 out, _ = capsys.readouterr() assert [ line for line in out.splitlines() if line.startswith("Task ID") ] == [ # results should be sorted 'Task ID: 1/ant', 'Task ID: 1/cat', 'Task ID: 1/dog', 'Task ID: 1/zed', ] @pytest.mark.parametrize( 'workflow_run_mode, run_mode_info', ( ('live', 'Skip'), ('dummy', 'Dummy'), ('simulation', 'Simulation'), ) ) @pytest.mark.parametrize( 'attributes_bool, flow_nums, expected_state, expected_flows', [ pytest.param( False, [1], 'state: waiting (run mode={})', None, ), pytest.param( True, [1, 2], 'state: waiting (held,queued,runahead,run mode={})', 'flows: [1,2]', ) ] ) async def test_task_instance_state_flows( flow, scheduler, start, capsys, workflow_run_mode, run_mode_info, attributes_bool, flow_nums, expected_state, expected_flows ): """It should print task instance state, attributes, and flows.""" colour_init(strip=True, autoreset=True) opts = SimpleNamespace( comms_timeout=5, json=False, task_defs=None, list_prereqs=False, ) schd = scheduler( flow( { 'scheduling': { 'graph': {'R1': 'a'}, }, 'runtime': { 'a': {'run mode': 'skip'} } }, ), paused_start=True, run_mode=workflow_run_mode, ) async with start(schd): [itask] = schd.pool.get_tasks() itask.state_reset( is_held=attributes_bool, is_queued=attributes_bool, is_runahead=attributes_bool ) itask.flow_nums = set(flow_nums) schd.pool.data_store_mgr.delta_task_held( itask.tdef.name, itask.point, itask.state.is_held) schd.pool.data_store_mgr.delta_task_state(itask) schd.pool.data_store_mgr.delta_task_flow_nums(itask) await schd.update_data_structure() ret = await show( schd.workflow, [Tokens('//1/*')], opts, ) assert ret == 0 out, _ = capsys.readouterr() assert [ line for line in out.splitlines() if line.startswith("state:") ] == [ expected_state.format(run_mode_info), ] if expected_flows is not None: assert [ line for line in out.splitlines() if line.startswith("flows:") ] == [ expected_flows, ] async def test_task_run_mode_changes(flow, scheduler, start, capsys): """Broadcasting a change of run mode changes run mode shown by cylc show. """ opts = ShowOptions() schd = scheduler( flow({'scheduling': {'graph': {'R1': 'a'}}}), run_mode='live' ) async with start(schd): # Control: No mode set, the Run Mode setting is not shown: await schd.update_data_structure() ret = await show( schd.workflow, [Tokens('//1/a')], opts, ) assert ret == 0 out, _ = capsys.readouterr() state, = RE_STATE.findall(out) assert 'waiting' in state # Broadcast change task to skip mode: schd.broadcast_mgr.put_broadcast(['1'], ['a'], [{'run mode': 'skip'}]) await schd.update_data_structure() # show now shows skip mode: ret = await show( schd.workflow, [Tokens('//1/a')], opts, ) assert ret == 0 out, _ = capsys.readouterr() state, = RE_STATE.findall(out) assert 'run mode=Skip' in state cylc-flow-8.7.0/tests/integration/scripts/test_cat_log.py0000664000175000017500000000671215264420307024017 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Integration test for the cat log script. """ import pytest import re import shutil from cylc.flow.exceptions import InputError from cylc.flow.option_parsers import Options from cylc.flow.scripts.cat_log import ( _main as cat_log, get_option_parser as cat_log_gop, ) BAD_NAME = "NONEXISTENTWORKFLOWNAME" @pytest.fixture def brokendir(run_dir): brokendir = (run_dir / BAD_NAME) brokendir.mkdir(exist_ok=True) yield brokendir shutil.rmtree(brokendir) async def test_fail_no_file(flow): """It produces a helpful error if there is no workflow log file. """ parser = cat_log_gop() id_ = flow({}) with pytest.raises(InputError, match='Log file not found.'): await cat_log(parser, Options(parser)(), id_) async def test_fail_rotation_out_of_range(flow): """It produces a helpful error if rotation number > number of log files. """ parser = cat_log_gop() id_ = flow({}) path = flow.args[1] name = id_.split('/')[-1] logpath = (path / name / 'log/scheduler') logpath.mkdir(parents=True) (logpath / '01-start-01.log').touch() with pytest.raises(SystemExit): await cat_log(parser, Options(parser)(rotation_num=0), id_) msg = r'--rotation 1 invalid \(max value is 0\)' with pytest.raises(InputError, match=msg): await cat_log(parser, Options(parser)(rotation_num=1), id_) async def test_bad_workflow(run_dir): """Test "cylc cat-log" with bad workflow name.""" parser = cat_log_gop() msg = re.compile( fr'^Workflow ID not found: {BAD_NAME}' fr'\n\(Directory not found: {run_dir}/{BAD_NAME}\)$', re.MULTILINE ) with pytest.raises(InputError, match=msg): await cat_log(parser, Options(parser)(filename='l'), BAD_NAME) async def test_bad_workflow2(run_dir, brokendir, capsys): """Check a non existent file in a valid workflow results in error. """ parser = cat_log_gop() with pytest.raises(SystemExit, match='1'): await cat_log( parser, Options(parser)(filename='j'), BAD_NAME ) msg = ( f'File not found: {run_dir}' '/NONEXISTENTWORKFLOWNAME/log/j\n') assert capsys.readouterr().err == msg async def test_bad_task_dir(run_dir, brokendir, capsys): """Check a non-existent job log dir in a valid workflow results in error. """ parser = cat_log_gop() with pytest.raises(SystemExit, match='1'): await cat_log( parser, Options(parser)(mode='list-dir'), BAD_NAME + "//1/foo" ) msg = ( f'Directory not found: {run_dir}' '/NONEXISTENTWORKFLOWNAME/log/job/1/foo/NN\n') assert capsys.readouterr().err == msg cylc-flow-8.7.0/tests/integration/scripts/test_broadcast.py0000664000175000017500000002747315264420307024360 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging from ansimarkup import strip as cstrip import pytest from cylc.flow.network.client import WorkflowRuntimeClient from cylc.flow.option_parsers import Options from cylc.flow.rundb import CylcWorkflowDAO from cylc.flow.scheduler import Scheduler from cylc.flow.scripts.broadcast import ( _main, get_option_parser, ) from cylc.flow.util import sstrip BroadcastOptions = Options(get_option_parser()) async def test_broadcast_multi_workflow( one_conf, flow, scheduler, start, run_dir, test_dir, capsys, ): """Test a multi-workflow broadcast command.""" # create three workflows one = scheduler(flow(one_conf)) two = scheduler(flow(one_conf)) thr = scheduler(flow(one_conf)) # the ID under which all three are installed id_base = test_dir.relative_to(run_dir) async with start(one): async with start(two): async with start(thr): capsys.readouterr() # test a successful broadcast command rets = await _main( BroadcastOptions(settings=['script=true']), f'{id_base}*' ) # all three broadcasts should have succeeded assert list(rets.values()) == [True, True, True] out, err = capsys.readouterr() assert '[*/root] script=true' in out assert err == '' # test an unsuccessful broadcast command rets = await _main( BroadcastOptions( namespaces=['*'], settings=['script=true'], ), f'{id_base}*', ) # all three broadcasts should have failed assert list(rets.values()) == [False, False, False] out, err = capsys.readouterr() assert '[*/root] script=true' not in out assert ( # NOTE: for historical reasons this message goes to stdout # not stderr 'Rejected broadcast:' ' settings are not compatible with the workflow' ) in out assert err == '' async def test_broadcast_multi_namespace( flow, scheduler, start, db_select, ): """Test a multi-namespace broadcast command. See https://github.com/cylc/cylc-flow/issues/6334 """ id_ = flow( { 'scheduling': { 'graph': {'R1': 'a & b & c & fin'}, }, 'runtime': { 'root': {'execution time limit': 'PT1S'}, 'VOWELS': {'execution time limit': 'PT2S'}, 'CONSONANTS': {'execution time limit': 'PT3S'}, 'a': {'inherit': 'VOWELS'}, 'b': {'inherit': 'CONSONANTS'}, 'c': {'inherit': 'CONSONANTS'}, }, } ) schd = scheduler(id_) async with start(schd): # issue a broadcast to multiple namespaces rets = await _main( BroadcastOptions( settings=['execution time limit = PT5S'], namespaces=['root', 'VOWELS', 'CONSONANTS'], ), schd.workflow, ) # the broadcast should succeed assert list(rets.values()) == [True] # the broadcast manager should store the "coerced" setting for task in ['a', 'b', 'c', 'fin']: assert schd.broadcast_mgr.get_broadcast( schd.tokens.duplicate(cycle='1', task=task) ) == {'execution time limit': 5.0} # the database should store the "raw" setting assert sorted( db_select(schd, True, CylcWorkflowDAO.TABLE_BROADCAST_STATES) ) == [ ('*', 'CONSONANTS', 'execution time limit', 'PT5S'), ('*', 'VOWELS', 'execution time limit', 'PT5S'), ('*', 'root', 'execution time limit', 'PT5S'), ] async def test_broadcast_truncated_datetime(flow, scheduler, start, capsys): """It should reject truncated datetime cycle points. See https://github.com/cylc/cylc-flow/issues/6407 """ id_ = flow({ 'scheduling': { 'initial cycle point': '2000', 'graph': { 'R1': 'foo', }, } }) schd = scheduler(id_) async with start(schd): # attempt an invalid broadcast rets = await _main( BroadcastOptions( settings=['[environment]FOO=bar'], point_strings=['050101T0000Z'], # <== truncated ), schd.workflow, ) # the broadcast should fail assert list(rets.values()) == [False] # an error should be recorded _out, err = capsys.readouterr() assert ( 'Rejected broadcast:' ' settings are not compatible with the workflow' ) in err async def test_invalid(one, start, capsys, monkeypatch): """It should reject invalid broadcasts. * Broadcast operations may contain a mix of valid and invalid cycles/namespaces/settings. * Valid settings are applied and reported. * Invalid settings are rejected and reported. """ # disable client side config validation so that we can see invalid # broadcast settings in the CLI output monkeypatch.setattr( 'cylc.flow.scripts.broadcast.cylc_config_validate', lambda *args, **kwargs: True, ) # 1) disable CLI colour output monkeypatch.setattr('cylc.flow.scripts.broadcast.cparse', cstrip) async with start(one): # test the GraphQL API client = WorkflowRuntimeClient(one.workflow) response = await client.async_request( 'graphql', { 'request_string': ''' mutation( $workflows: [WorkflowID]!, $cycles: [BroadcastCyclePoint], $namespaces: [NamespaceName], $settings: [BroadcastSetting] ) { broadcast( mode: Set workflows: $workflows cyclePoints: $cycles namespaces: $namespaces settings: $settings ) { result } } ''', 'variables': { 'workflows': [one.workflow], # valid, invalid 'cycles': ['*', 'invalid'], 'namespaces': ['root', 'invalid'], 'settings': [{'script': 'true'}, {'invalid': 'false'}], } } ) modified, rejected = response['broadcast']['result'][0]['response'] assert modified == [ [ # cycle '*', # namespace 'root', # settings {'script': 'true'}, ] ] assert rejected == { 'point_strings': ['invalid'], 'namespaces': ['invalid'], 'settings': [{'invalid': 'false'}], } # 2) test the CLI capsys.readouterr() # flush out any stdout/err await _main( BroadcastOptions( point_strings=['*', 'invalid'], namespaces=['root', 'invalid'], settings=['script=true', 'invalid=false'], ), one.workflow, ) out, err = capsys.readouterr() assert out == 'Broadcast set:\n+ [*/root] script=true\n' assert err == sstrip(''' ERROR: Rejected broadcast: settings are not compatible with the workflow --namespace=invalid --point=invalid --set=invalid=false ''') + '\n' async def test_internal_error(one, start, capsys, monkeypatch, log_filter): """It should handle internal code errors elegantly.""" # simulate an internal error # Note: The broadcast "lock" mechanism is an example of how such an error # could arise def put_broadcast(*args, **kwargs): raise Exception('FooBar') async with start(one): # issue a broadcast one.broadcast_mgr.put_broadcast = put_broadcast client = WorkflowRuntimeClient(one.workflow) response = await client.async_request( 'graphql', { 'request_string': ''' mutation( $workflows: [WorkflowID]!, $cycles: [BroadcastCyclePoint], $namespaces: [NamespaceName], $settings: [BroadcastSetting] ) { broadcast( mode: Set workflows: $workflows cyclePoints: $cycles namespaces: $namespaces settings: $settings ) { result } } ''', 'variables': { 'workflows': [one.workflow], 'cycles': ['*'], 'namespaces': ['root'], 'settings': [{'script': 'true'}], } } ) # check it comes back with an error code assert ( response['broadcast']['result'][0] )['response']['error']['message'] == 'FooBar' # the error should be logged server-side too assert log_filter(contains='FooBar') async def test_broadcast_auto_upgraded_settings( one: Scheduler, start, log_filter, caplog: pytest.LogCaptureFixture, capsys: pytest.CaptureFixture, ): """Test broadcast of deprecated/obsolete settings.""" opts = { 'point_strings': ['1'], 'namespaces': ['root'], } async with start(one): # Test deprecated setting that is moved to a different section by # the auto-upgrader should still work. rets = await _main( BroadcastOptions( settings=['[job]execution time limit=PT1H'], **opts ), one.workflow, ) assert set(rets.values()) == {True} assert log_filter( logging.WARNING, regex=r"Deprecated config.* upgraded.* broadcast" ) stdout, stderr = capsys.readouterr() assert "Traceback" not in stdout + stderr caplog.clear() # Test obsolete setting that is removed by the auto-upgrader should # be rejected. rets = await _main( BroadcastOptions( settings=['extra log files=whatever'], **opts ), one.workflow, ) assert set(rets.values()) == {False} assert log_filter( logging.WARNING, regex=r"Obsolete config.* rejected by the broadcast", ) stdout, stderr = capsys.readouterr() assert "InputError: No valid settings to broadcast" in stderr assert "Traceback" not in stdout + stderr cylc-flow-8.7.0/tests/integration/scripts/test_install.py0000664000175000017500000000302015264420307024042 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.exceptions import WorkflowFilesError import pytest from cylc.flow.workflow_files import WorkflowFiles async def test_suite_rc(tmp_path, install): """It should reject workflows with suite.rc files.""" # suite.rc present (tmp_path / WorkflowFiles.SUITE_RC).touch() with pytest.raises( WorkflowFilesError, match=f'suite.rc found in {tmp_path}' ): await install(tmp_path, run_name='1') # suite.rc and flow.cylc present (tmp_path / WorkflowFiles.FLOW_FILE).touch() with pytest.raises( WorkflowFilesError, match=f'Both flow.cylc and suite.rc.*{tmp_path}' ): await install(tmp_path, run_name='2') # flow.cylc present (tmp_path / WorkflowFiles.SUITE_RC).unlink() await install(tmp_path, run_name='3') cylc-flow-8.7.0/tests/integration/test_stop_after_cycle_point.py0000664000175000017500000001132115264420307025446 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test logic pertaining to the stop after cycle points. This may be defined in different ways: * In the workflow configuration. * On the command line. * Or loaded from the database. When the workflow hits the "stop after" point, it should be wiped (i.e. set to None). """ from typing import Optional from cylc.flow import commands from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.id import Tokens from cylc.flow.workflow_status import StopMode async def test_stop_after_cycle_point( flow, scheduler, run, reflog, complete, ): """Test the stop after cycle point. This ensures: * The stop after point gets loaded from the config. * The workflow stops when it hits this point. * The point gets wiped when the workflow hits this point. * The point is stored/retrieved from the DB as appropriate. """ async def stops_after_cycle(schd) -> Optional[str]: """Run the workflow until it stops and return the cycle point.""" triggers = reflog(schd) await complete(schd, timeout=2) assert len(triggers) == 1 # only one task (i.e. cycle) should be run return Tokens(list(triggers)[0][0], relative=True)['cycle'] def get_db_value(schd) -> Optional[str]: """Return the cycle point value stored in the DB.""" with schd.workflow_db_mgr.get_pri_dao() as pri_dao: return dict(pri_dao.select_workflow_params())['stopcp'] config = { 'scheduling': { 'cycling mode': 'integer', 'initial cycle point': '1', 'stop after cycle point': '1', 'graph': { 'P1': 'a[-P1] => a', }, }, } id_ = flow(config) schd = scheduler(id_, paused_start=False) async with run(schd): # the cycle point should be loaded from the workflow configuration assert schd.config.stop_point == IntegerPoint('1') # this value should *not* be written to the database assert get_db_value(schd) is None # the workflow should stop after cycle 1 assert await stops_after_cycle(schd) == '1' # change the configured cycle point to "2" config['scheduling']['stop after cycle point'] = '2' id_ = flow(config, workflow_id=id_) schd = scheduler(id_, paused_start=False) async with run(schd): # the cycle point should be reloaded from the workflow configuration assert schd.config.stop_point == IntegerPoint('2') # this value should not be written to the database assert get_db_value(schd) is None # the workflow should stop after cycle 2 assert await stops_after_cycle(schd) == '2' # override the configured value via the CLI option schd = scheduler(id_, paused_start=False, **{'stopcp': '3'}) async with run(schd): # the CLI should take precedence over the config assert schd.config.stop_point == IntegerPoint('3') # this value *should* be written to the database assert get_db_value(schd) == '3' # the workflow should stop after cycle 3 assert await stops_after_cycle(schd) == '3' # once the workflow hits this point, it should get cleared assert get_db_value(schd) is None schd = scheduler(id_, paused_start=False) async with run(schd): # the workflow should fall back to the configured value assert schd.config.stop_point == IntegerPoint('2') # override this value whilst the workflow is running await commands.run_cmd( commands.stop( schd, cycle_point=IntegerPoint('4'), mode=StopMode.REQUEST_CLEAN, ) ) assert schd.config.stop_point == IntegerPoint('4') # the new *should* be written to the database assert get_db_value(schd) == '4' schd = scheduler(id_, paused_start=False) async with run(schd): # the workflow should stop after cycle 4 assert await stops_after_cycle(schd) == '4' cylc-flow-8.7.0/tests/integration/test_config_change.py0000664000175000017500000003603615264420307023474 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) NIWA & British Crown (Met Office) & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test workflow configuration changes. The tests in this module cover configuration changes made via reload/restart. """ import re from typing import TYPE_CHECKING, Literal import pytest from pytest import param from cylc.flow import commands from cylc.flow.exceptions import PlatformLookupError from cylc.flow.task_state import ( TASK_STATUS_FAILED, TASK_STATUS_RUNNING, TASK_STATUS_WAITING, ) if TYPE_CHECKING: from cylc.flow.scheduler import Scheduler def list_tasks(schd, attr='relative_id'): return { getattr(itask.tokens.duplicate(task_sel=itask.state.status), attr) for itask in schd.pool.get_tasks() } async def _test_config_change( flow, scheduler, start_or_run, change_type: Literal['reload'] | Literal['restart'], paused_start=False, ): """Coroutine for testing config changes via either reload or restart. This makes it easy to test behaviours which should be the same for both reload and restart without duplicating the tests and risking functionality drift. Usage: * Initialise the async coroutine. * Yield into it the initial workflow configuration. * It will generate a Scheduler object. * Yield into it the new config. * It will either: * Restart the workflow and generate a new Scheduler object, * Or reload the workflow and yield the original one again. """ config: dict = yield id_ = flow(config) schd: 'Scheduler' = scheduler(id_, paused_start=paused_start) async with start_or_run(schd): config: dict = yield schd flow(config, workflow_id=id_) if change_type == 'reload': await commands.run_cmd(commands.reload_workflow(schd)) yield schd return schd = scheduler(id_, paused_start=paused_start) async with start_or_run(schd): yield schd @pytest.mark.parametrize('change_type', ('reload', 'restart')) async def test_graph_change_handling( flow, scheduler, start, db_select, caplog, change_type: Literal['reload'] | Literal['restart'], ): """It should hadle graph changes made by reloads or restarts. * If tasks are added by graph change, the added tasks should be logged (if reloading, the list cannot be easily determined if restarting). * If tasks are removed by graph change, they should be logged, killed and removed. """ test_generator = _test_config_change(flow, scheduler, start, change_type) config = { 'scheduling': { 'graph': {'R1': 'waiting & running & failed & foo'}, }, # prevent "running" from completing on the first main loop iteration 'runtime': { 'running': {'simulation': {'default run length': 'PT1H'}}, }, } # start the workflow await anext(test_generator) schd = await test_generator.asend(config) # all four tasks should be in the pool assert list_tasks(schd) == { '1/waiting', '1/running', '1/failed', '1/foo', } # put them into various states schd.pool._get_task_by_id('1/waiting').state_reset(is_held=True) schd.pool._get_task_by_id('1/foo').state_reset(is_held=True) schd.pool._get_task_by_id('1/running').state_reset(TASK_STATUS_RUNNING) schd.pool._get_task_by_id('1/failed').state_reset( TASK_STATUS_FAILED, is_held=True ) # and ensure they are written to the database await schd._main_loop() assert set( db_select(schd, True, 'task_pool', 'cycle', 'name', 'status') ) == { ('1', 'foo', TASK_STATUS_WAITING), ('1', 'waiting', TASK_STATUS_WAITING), ('1', 'running', TASK_STATUS_RUNNING), ('1', 'failed', TASK_STATUS_FAILED), } # remove "waiting", "running", "failed"; add "bar": config['scheduling']['graph']['R1'] = 'foo & bar' schd = await test_generator.asend(config) # the "orphaned" tasks should not be in the pool assert list_tasks(schd) == {'1/foo'} # the "orphaned" tasks should not be in the database await schd._main_loop() assert db_select(schd, True, 'task_pool', 'cycle', 'name', 'status') == [ ('1', 'foo', TASK_STATUS_WAITING) ] # the graph changes should have been logged record = None for record in caplog.records: if f'Graph changed due to {change_type}' in record.message: break assert record assert re.search('Removed tasks', record.message) assert re.search(r'\* 1/failed.*:failed', record.message) assert re.search(r'\* 1/running.*:running', record.message) assert re.search(r'\* 1/waiting.*:waiting', record.message) if change_type == 'reload': # NOTE: we cannot easily determine added tasks when restarting assert re.search('Added task definitions', record.message) assert re.search(r'\* bar', record.message) @pytest.mark.parametrize('change_type', ('reload', 'restart')) @pytest.mark.parametrize('psx_type', ('xtrigger', 'parentless')) async def test_psx_removal( flow, scheduler, start, db_select, caplog, change_type: Literal['reload'] | Literal['restart'], psx_type: Literal['xtrigger'] | Literal['parentless'], ): """Orphaned PSX tasks should be removed. Parentless/sequential-xtriggered tasks (PSX) should be removed if orphaned. This is testing the same logic as covered by test_graph_change_handling, however, but making sure the PSX removal (which works differently) still works. """ prefix = '' if psx_type == 'xtrigger': prefix = '@a =>' test_generator = _test_config_change( flow, scheduler, start, change_type, paused_start=True, ) config = { 'scheduling': { 'xtriggers': {'a': 'xrandom(0, sequential=True)'}, 'graph': {'R1': f''' {prefix} x {prefix} y '''}, }, } # start the workflow await anext(test_generator) schd = await test_generator.asend(config) assert list_tasks(schd) == { '1/x', '1/y', } # ensure they are written to the database await schd._main_loop() assert set( db_select(schd, True, 'task_pool', 'cycle', 'name', 'status') ) == { ('1', 'x', TASK_STATUS_WAITING), ('1', 'y', TASK_STATUS_WAITING), } # remove "x" add "z" config['scheduling']['graph']['R1'] = f''' # {prefix} x {prefix} y {prefix} z ''' schd = await test_generator.asend(config) # the "orphaned" tasks should not be in the pool assert list_tasks(schd) == {'1/y'} # the "orphaned" tasks should not be in the database await schd._main_loop() assert db_select(schd, True, 'task_pool', 'cycle', 'name', 'status') == [ ('1', 'y', TASK_STATUS_WAITING) ] # the graph changes should have been logged record = None for record in caplog.records: if f'Graph changed due to {change_type}' in record.message: break assert record assert re.search('Removed tasks', record.message) assert re.search(r'\* 1\/x.*:waiting', record.message) if change_type == 'reload': # NOTE: we cannot easily determine added tasks when restarting assert re.search('Added task definitions', record.message) assert re.search(r'\* z', record.message) @pytest.mark.parametrize('change_type', ('reload', 'restart')) async def test_task_pool_logic(flow, scheduler, run, change_type, complete): """Test the evolution of the task pool across graph changes.""" test_generator = _test_config_change(flow, scheduler, run, change_type) await anext(test_generator) # start the workflow schd = await test_generator.asend( {'scheduling': {'graph': {'R1': 'a => b'}}} ) # run up to 1/a:succeeded await complete(schd, '1/a', timeout=3) assert list_tasks(schd) == {'1/b'} # change the graph # - b # + c # + d schd = await test_generator.asend( {'scheduling': {'graph': {'R1': 'c => a => d'}}} ) # b should be removed - https://github.com/cylc/cylc-flow/issues/7198 # a & b should not be added (SoD status quo) assert list_tasks(schd) == set() @pytest.mark.parametrize('orphan', (True, False)) async def test_missing_platform_config( flow, scheduler, start, one_conf, monkeypatch, log_filter, orphan, ): """It handles PlatformLookupErrors raised during restart. Cylc needs to associate any jobs - which were submitted or running when the workflow was shut down with - the platforms recorded in the DB in order to poll them for status updates, and permit any future operations. If the platform has been removed from the global config since the workflow was shut down, the Scheduler should bail on startup as this is not a situation we can currently manage. However, if the task is an orphan (a task removed by reload) i.e, one we are going to remove anyway, this isn't a critical error. """ one_conf.update({ # prevent the scheduler shutting down on the first main loop iteration 'runtime': { 'root': {'simulation': {'default run length': 'PT1H'}}, }, }) test_generator = _test_config_change(flow, scheduler, start, 'restart') # start the workflow await anext(test_generator) schd = await test_generator.asend(one_conf) await schd._main_loop() # ensure the task pool is written to the DB if orphan: # make the sole task in this workflow an orphan one_conf['scheduling']['graph']['R1'] = 'new-task' # force a PlatformLookupError upon restart def _get_platform(*_args, **_kwargs): raise PlatformLookupError('FOO') monkeypatch.setattr('cylc.flow.task_pool.get_platform', _get_platform) # restart the workflow schd = await test_generator.asend(one_conf) # a PlatformLookupError should have been raised msg = r'platforms are not defined.*\n.*simulation' if orphan: with pytest.raises(StopAsyncIteration): # this error is not critical # NOTE: StopAsyncIteration just means the test generator has shut # down cleanly await anext(test_generator) # the undefined platform should still be logged though assert log_filter(regex=msg) else: with pytest.raises(PlatformLookupError, match=msg): # this error is criticla: ensure the scheduler bails on startup await anext(test_generator) # the workflow should refuse to restart, unless the task is orphaned in # which case the error should be tolerated (just means we can't issue the # kill command but not critical to the workflow) assert ( log_filter(regex='Workflow shutting down - PlatformLookupError') is not orphan ) @pytest.mark.parametrize('change_type', ('reload', 'restart')) @pytest.mark.parametrize( ''' graph_1, graph_2, task_pool_at_startup, task_pool_post_spawn, task_pool_after_reload_or_restart, task_z_prereqs ''', [ param( # Restart after adding a prerequisite to task z '''a => z b => z''', '''a => z b => z c => z''', { '1/a:running', '1/b:running', }, { '1/b:running', '1/z:waiting', }, { '1/b:running', '1/z:waiting', }, [ {('1', 'a', 'succeeded'): 'satisfied naturally'}, {('1', 'b', 'succeeded'): False}, {('1', 'c', 'succeeded'): False}, ], id='added', ), param( # Restart after removing a prerequisite from task z '''a => z b => z c => z''', '''a => z b => z''', { '1/a:running', '1/b:running', '1/c:running', }, { '1/b:running', '1/c:running', '1/z:waiting', }, { '1/b:running', '1/z:waiting', }, [ {('1', 'a', 'succeeded'): 'satisfied naturally'}, {('1', 'b', 'succeeded'): False}, ], id='removed', ), ], ) async def test_prereq_change( flow, scheduler, start, change_type, graph_1, graph_2, task_pool_at_startup, task_pool_post_spawn, task_pool_after_reload_or_restart, task_z_prereqs, ): """It should handle graph prerequisites change on reload/restart. Prerequisite changes must be applied to tasks already in the pool. See https://github.com/cylc/cylc-flow/pull/5334 """ test_generator = _test_config_change(flow, scheduler, start, change_type) # start the workflow await anext(test_generator) schd = await test_generator.asend(graph_1) # Release tasks 1/a and 1/b schd.pool.release_runahead_tasks() schd.release_tasks_to_run() assert ( list_tasks(schd, 'relative_id_with_selectors') == task_pool_at_startup ) # Mark 1/a as succeeded and spawn 1/z task_a = schd.pool.get_tasks()[0] schd.pool.task_events_mgr.process_message(task_a, 1, 'succeeded') assert ( list_tasks(schd, 'relative_id_with_selectors') == task_pool_post_spawn ) # Save our progress schd.workflow_db_mgr.put_task_pool(schd.pool) # Edit the workflow to add a new dependency on "z" then reload/restart schd = await test_generator.asend(graph_2) # Load jobs from db schd.workflow_db_mgr.pri_dao.select_jobs_for_restart( schd.data_store_mgr.insert_db_job ) assert ( list_tasks(schd, 'relative_id_with_selectors') == task_pool_after_reload_or_restart ) # To cover some code for loading prereqs from the DB at restart: schd.data_store_mgr.update_data_structure() # Check resulting dependencies of task z task_z = [ t for t in schd.pool.get_tasks() if t.tdef.name == "z" ][0] assert sorted( ( p._satisfied for p in task_z.state.prerequisites ), key=lambda d: tuple(d.keys())[0], ) == task_z_prereqs cylc-flow-8.7.0/tests/integration/test_queues.py0000664000175000017500000001023415264420307022221 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow import commands from cylc.flow.scheduler import Scheduler @pytest.fixture def param_workflow(flow, scheduler): def _schd(paused_start, queue_limit): id_ = flow({ 'task parameters': { 'x': '1..10', }, 'scheduling': { 'queues': { 'default': { 'limit': queue_limit, }, }, 'graph': { 'R1': ''' ''', }, }, }) return scheduler(id_, paused_start=paused_start) return _schd @pytest.mark.parametrize( 'start_paused,queue_limit', [ (False, 1), (False, 5), (True, 1), (True, 5), ] ) async def test_queue_release( param_workflow, start, capture_submission, start_paused, queue_limit, ): """Tasks should be released up to the limit if the scheduler is not paused. When the scheudler is paused the scheduler should not release tasks from queues because tasks may subsequently be held by the user and we don't want them clogging up the queue limit. https://github.com/cylc/cylc-flow/issues/4627 """ expected_submissions = queue_limit if not start_paused else 0 # start the scheduler (but don't set the main loop running) schd: Scheduler = param_workflow(start_paused, queue_limit) async with start(schd): # capture task submissions (prevents real submissions) submitted_tasks = capture_submission(schd) # release runahead/queued tasks # (if scheduler is paused we should not have any submissions) # (otherwise a number of tasks up to the limit should be released) schd.pool.release_runahead_tasks() schd.release_tasks_to_run() assert len(submitted_tasks) == expected_submissions for _ in range(3): # release runahead/queued tasks # (no further tasks should be released) schd.release_tasks_to_run() assert len(submitted_tasks) == expected_submissions async def test_queue_held_tasks( param_workflow, start, capture_submission ): """Held tasks should not be released from queues. Users can hold tasks whilst they are queued. These held tasks should not be released from their queues. https://github.com/cylc/cylc-flow/issues/4628 """ schd: Scheduler = param_workflow(paused_start=True, queue_limit=1) async with start(schd): # capture task submissions (prevents real submissions) submitted_tasks = capture_submission(schd) # release runahead tasks to their queues schd.pool.release_runahead_tasks() # hold all tasks and resume the workflow # (nothing should have run yet because the workflow started paused) await commands.run_cmd(commands.hold(schd, ['*/*'])) schd.resume_workflow() # release queued tasks # (no tasks should be released from the queues because they are held) schd.release_tasks_to_run() assert len(submitted_tasks) == 0 # un-hold tasks await commands.run_cmd(commands.release(schd, ['*/*'])) # release queued tasks # (tasks should now be released from the queues) schd.release_tasks_to_run() assert len(submitted_tasks) == 1 cylc-flow-8.7.0/tests/integration/test_increment_graph_window.py0000664000175000017500000003234015264420307025450 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from contextlib import suppress from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.data_store_mgr import ( TASK_PROXIES, ) from cylc.flow.id import Tokens def increment_graph_window(schd, task): """Increment the graph window about the active task.""" tokens = schd.tokens.duplicate(cycle='1', task=task) schd.data_store_mgr.increment_graph_window( tokens, IntegerPoint('1'), is_manual_submit=False, ) def get_deltas(schd): """Return the ids and graph-window values in the delta store. Note, call before get_n_window as this clears the delta store. Returns: (added, updated, pruned) """ # populate added deltas schd.data_store_mgr.gather_delta_elements( schd.data_store_mgr.added, 'added', ) # populate pruned deltas schd.data_store_mgr.prune_data_store() # Run depth finder schd.data_store_mgr.window_depth_finder() # populate updated deltas schd.data_store_mgr.gather_delta_elements( schd.data_store_mgr.updated, 'updated', ) return ( { # added Tokens(tb_task_proxy.id)['task']: tb_task_proxy.graph_depth for tb_task_proxy in schd.data_store_mgr.deltas[TASK_PROXIES].added }, { # updated Tokens(tb_task_proxy.id)['task']: tb_task_proxy.graph_depth for tb_task_proxy in schd.data_store_mgr.deltas[ TASK_PROXIES ].updated # only include those updated nodes whose depths have been set if 'graph_depth' in {sub_field.name for sub_field, _ in tb_task_proxy.ListFields()} }, { # pruned Tokens(id_)['task'] for id_ in schd.data_store_mgr.deltas[TASK_PROXIES].pruned }, ) async def get_n_window(schd): """Read out the graph window of the workflow.""" await schd.update_data_structure() data = schd.data_store_mgr.data[schd.data_store_mgr.workflow_id] return { t.name: t.graph_depth for t in data[TASK_PROXIES].values() } async def complete_task(schd, task): """Mark a task as completed.""" schd.data_store_mgr.remove_pool_node(task, IntegerPoint('1')) def add_task(schd, task): """Add a waiting task to the pool.""" schd.data_store_mgr.add_pool_node(task, IntegerPoint('1')) def get_graph_walk_cache(schd): """Return the head task names of cached graph walks.""" # prune graph walk cache schd.data_store_mgr.prune_data_store() # fetch the cached walks n_window_node_walks = sorted( Tokens(task_id)['task'] for task_id in schd.data_store_mgr.n_window_node_walks ) n_window_completed_walks = sorted( Tokens(task_id)['task'] for task_id in schd.data_store_mgr.n_window_completed_walks ) # the IDs in set and keys of dict are only the same at n<2 window. assert n_window_node_walks == n_window_completed_walks return n_window_completed_walks async def test_increment_graph_window_blink(flow, scheduler, start): """Test with a task which drifts in and out of the n-window. This workflow presents a fiendish challenge for the graph window algorithm. The test runs in the n=3 window and simulates running each task in the chain a - s one by one. The task "blink" is dependent on multiple tasks in the chain awkwardly spaced so that the "blink" task routinely disappears from the n-window, only to re-appear again later. The expansion of the window around the "blink" task is difficult to get right as it can be influenced by caches from previous graph walks. """ id_ = flow({ 'scheduler': { 'allow implicit tasks': 'True', }, 'scheduling': { 'cycling mode': 'integer', 'initial cycle point': '1', 'graph': { 'R1': ''' # the "abdef" chain of tasks which run one after another a => b => c => d => e => f => g => h => i => j => k => l => m => n => o => p => q => r => s # these dependencies cause "blink" to disappear and # reappear at set intervals a => blink g => blink m => blink s => blink ''', } } }) schd = scheduler(id_) # the tasks traversed via the "blink" task when... blink = { 1: { # the blink task is n=1 'blink': 1, 'a': 2, 'g': 2, 'm': 2, 's': 2, 'b': 3, 'f': 3, 'h': 3, 'l': 3, 'n': 3, 'r': 3, }, 2: { # the blink task is n=2 'blink': 2, 'a': 3, 'g': 3, 'm': 3, 's': 3, }, 3: { # the blink task is n=3 'blink': 3, }, 4: { # the blink task is n=4 }, } def advance(): """Advance to the next task in the workflow. This works its way down the chain of tasks between "a" and "s" inclusive, yielding what the n-window should look like for this workflow at each step. Yields: tuple - (previous_task, active_task, n_window) previous_task: The task which has just "succeeded". active_task: The task which is about to run. n_window: Dictionary of {task_name: graph_depth} for the n=3 window. """ # the initial window on startup (minus the nodes traversed via "blink") window = { 'a': 0, 'b': 1, 'c': 2, 'd': 3, } # the tasks we will run in order letters = 'abcdefghijklmnopqrs' # the graph-depth of the "blink" task at each stage of the workflow blink_distances = [1] + [*range(2, 5), *range(3, 0, -1)] * 3 for ind, blink_distance in zip(range(len(letters)), blink_distances): previous_task = letters[ind - 1] if ind > 0 else None active_task = letters[ind] yield ( previous_task, active_task, { # the tasks traversed via the "blink" task **blink[blink_distance], # the tasks in the main "abcdefg" chain **{key: abs(value) for key, value in window.items()}, } ) # move each task in the "abcdef" chain down one window = {key: value - 1 for key, value in window.items()} # add the n=3 task in the "abcdef" chain into the window with suppress(IndexError): window[letters[ind + 4]] = 3 # pull out anything which is not supposed to be in the n=3 window window = { key: value for key, value in window.items() if abs(value) < 4 } async with start(schd): schd.data_store_mgr.set_graph_window_extent(3) await schd.update_data_structure() previous_n_window = {} for previous_task, active_task, expected_n_window in advance(): # mark the previous task as completed await complete_task(schd, previous_task) # add the next task to the pool add_task(schd, active_task) # run the graph window algorithm increment_graph_window(schd, active_task) # get the deltas which increment_graph_window created added, updated, pruned = get_deltas(schd) # compare the n-window in the store to what we were expecting n_window = await get_n_window(schd) assert n_window == expected_n_window # compare the deltas to what we were expecting if active_task != 'a': # skip the first task as this is complicated by startup logic assert added == { key: value for key, value in expected_n_window.items() if key not in previous_n_window } # Skip added as depth isn't updated # (the manager only updates those that need it) assert updated == { key: value for key, value in expected_n_window.items() if key not in added } assert pruned == { key for key in previous_n_window if key not in expected_n_window } previous_n_window = n_window async def test_window_resize_rewalk(flow, scheduler, start): """The window resize method should wipe and rebuild the n-window.""" id_ = flow({ 'scheduler': { 'allow implicit tasks': 'true', }, 'scheduling': { 'graph': { 'R1': 'a => b => c => d => e => f => g' } }, }) schd = scheduler(id_) async with start(schd): # start with an empty pool schd.pool.remove(schd.pool.get_tasks()[0]) # the n-window should be empty assert await get_n_window(schd) == {} # expand the window around 1/d add_task(schd, 'd') increment_graph_window(schd, 'd') # set the graph window to n=3 schd.data_store_mgr.set_graph_window_extent(3) assert set(await get_n_window(schd)) == { 'a', 'b', 'c', 'd', 'e', 'f', 'g' } # set the graph window to n=1 schd.data_store_mgr.set_graph_window_extent(1) schd.data_store_mgr.window_resize_rewalk() assert set(await get_n_window(schd)) == { 'c', 'd', 'e' } # set the graph window to n=2 schd.data_store_mgr.set_graph_window_extent(2) schd.data_store_mgr.window_resize_rewalk() assert set(await get_n_window(schd)) == { 'b', 'c', 'd', 'e', 'f' } async def test_cache_pruning(flow, scheduler, start): """It should remove graph walks from the cache when no longer needed. The algorithm caches graph walks for efficiency. This test is designed to ensure we don't introduce a memory leak by failing to clear cached walks at the correct point. """ id_ = flow({ 'scheduler': { 'allow implicit tasks': 'True', }, 'scheduling': { 'graph': { 'R1': ''' # a chain of tasks a => b1 & b2 => c => d1 & d2 => e => f # force "a" to drift into an out of the window a => c a => e ''' } }, }) schd = scheduler(id_) async with start(schd): schd.data_store_mgr.set_graph_window_extent(1) # work through this workflow, step by step checking the cached items... # active: a add_task(schd, 'a') increment_graph_window(schd, 'a') assert get_graph_walk_cache(schd) == ['a'] # active: b1, b2 await complete_task(schd, 'a') add_task(schd, 'b1') add_task(schd, 'b2') increment_graph_window(schd, 'b1') increment_graph_window(schd, 'b2') assert get_graph_walk_cache(schd) == ['a', 'b1', 'b2'] # active: c await complete_task(schd, 'b1') await complete_task(schd, 'b2') add_task(schd, 'c') increment_graph_window(schd, 'c') assert get_graph_walk_cache(schd) == ['a', 'b1', 'b2', 'c'] # active: d1, d2 await complete_task(schd, 'c') add_task(schd, 'd1') add_task(schd, 'd2') increment_graph_window(schd, 'd1') increment_graph_window(schd, 'd2') assert get_graph_walk_cache(schd) == ['c', 'd1', 'd2'] # active: e await complete_task(schd, 'd1') await complete_task(schd, 'd2') add_task(schd, 'e') increment_graph_window(schd, 'e') assert get_graph_walk_cache(schd) == ['d1', 'd2', 'e'] # active: f await complete_task(schd, 'e') add_task(schd, 'f') increment_graph_window(schd, 'f') assert get_graph_walk_cache(schd) == ['e', 'f'] # active: None await complete_task(schd, 'f') increment_graph_window(schd, 'f') assert get_graph_walk_cache(schd) == [] cylc-flow-8.7.0/tests/integration/test_dbstatecheck.py0000664000175000017500000001125015264420307023335 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests for the backend method of workflow_state""" import pytest from cylc.flow.commands import ( run_cmd, force_trigger_tasks ) from cylc.flow.dbstatecheck import CylcWorkflowDBChecker from cylc.flow.scheduler import Scheduler @pytest.fixture(scope='module') async def checker( mod_flow, mod_scheduler, mod_run, mod_complete ): """Make a real world database. We could just write the database manually but this is a better test of the overall working of the function under test. """ wid = mod_flow({ 'scheduling': { 'initial cycle point': '1000', 'final cycle point': '1001', 'graph': { 'P1Y': ''' good:succeeded bad:failed? output:custom_output ''' }, }, 'runtime': { 'bad': {'simulation': {'fail cycle points': '1000'}}, 'output': { 'outputs': {'trigger': 'message', 'custom_output': 'foo'} } } }) schd: Scheduler = mod_scheduler(wid, paused_start=False) async with mod_run(schd): # allow a cycle of the main loop to pass so that flow 2 can be # added to db await mod_complete(schd) # trigger a new task in flow 2 await run_cmd(force_trigger_tasks(schd, ['1000/good'], ['2'])) # update the database schd.process_workflow_db_queue() # yield a DB checker with CylcWorkflowDBChecker( 'somestring', 'utterbunkum', schd.workflow_db_mgr.pub_path ) as _checker: yield _checker def test_basic(checker): """Pass no args, get unfiltered output""" result = checker.workflow_state_query() expect = [ ['bad', '10000101T0000Z', 'failed'], ['bad', '10010101T0000Z', 'succeeded'], ['good', '10000101T0000Z', 'succeeded'], ['good', '10010101T0000Z', 'succeeded'], ['output', '10000101T0000Z', 'succeeded'], ['output', '10010101T0000Z', 'succeeded'], ['good', '10000101T0000Z', 'waiting', '(flows=2)'], ['good', '10010101T0000Z', 'waiting', '(flows=2)'], ] assert sorted(result) == sorted(expect) def test_task(checker): """Filter by task name""" result = checker.workflow_state_query(task='bad') assert sorted(result) == ([ ['bad', '10000101T0000Z', 'failed'], ['bad', '10010101T0000Z', 'succeeded'] ]) def test_point(checker): """Filter by point""" result = checker.workflow_state_query(cycle='10000101T0000Z') assert sorted(result) == sorted([ ['bad', '10000101T0000Z', 'failed'], ['good', '10000101T0000Z', 'succeeded'], ['output', '10000101T0000Z', 'succeeded'], ['good', '10000101T0000Z', 'waiting', '(flows=2)'], ]) def test_status(checker): """Filter by status""" result = checker.workflow_state_query(selector='failed') expect = [ ['bad', '10000101T0000Z', 'failed'], ] assert result == expect def test_output(checker): """Filter by flow number""" result = checker.workflow_state_query(selector='message', is_message=True) expect = [ [ 'output', '10000101T0000Z', "{'submitted': 'submitted', 'started': 'started', 'succeeded': " "'succeeded', 'trigger': 'message', 'custom_output': 'foo'}", ], [ 'output', '10010101T0000Z', "{'submitted': 'submitted', 'started': 'started', 'succeeded': " "'succeeded', 'trigger': 'message', 'custom_output': 'foo'}", ], ] assert result == expect def test_flownum(checker): """Pass no args, get unfiltered output""" result = checker.workflow_state_query(flow_num=2) expect = [ ['good', '10000101T0000Z', 'waiting', '(flows=2)'], ['good', '10010101T0000Z', 'waiting', '(flows=2)'], ] assert result == expect cylc-flow-8.7.0/tests/integration/test_data_store_mgr.py0000664000175000017500000010367215264420307023715 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging from logging import INFO from typing import ( Iterable, List, cast, ) from unittest.mock import Mock import pytest from cylc.flow import LOG from cylc.flow.commands import ( force_trigger_tasks, run_cmd, ) from cylc.flow.data_messages_pb2 import ( PbJob, PbPrerequisite, PbTaskProxy, ) from cylc.flow.data_store_mgr import ( EDGES, FAMILY_PROXIES, JOBS, TASK_PROXIES, TASKS, WORKFLOW, ) from cylc.flow.id import ( TaskTokens, Tokens, ) from cylc.flow.network.log_stream_handler import ProtobufStreamHandler from cylc.flow.scheduler import Scheduler from cylc.flow.task_events_mgr import TaskEventsManager from cylc.flow.task_outputs import ( TASK_OUTPUT_FAILED, TASK_OUTPUT_STARTED, TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_SUCCEEDED, ) from cylc.flow.task_proxy import TaskProxy from cylc.flow.task_state import ( TASK_STATUS_FAILED, TASK_STATUS_PREPARING, TASK_STATUS_RUNNING, TASK_STATUS_SUCCEEDED, TASK_STATUS_WAITING, ) from cylc.flow.wallclock import get_current_time_string # NOTE: Some of these tests mutate the data store, so running them in # isolation may see failures when they actually pass if you run the # whole file def job_config(schd): return { 'owner': schd.owner, 'submit_num': 3, 'task_id': '1/foo', 'job_runner_name': 'background', 'env-script': None, 'err-script': None, 'exit-script': None, 'execution_time_limit': None, 'init-script': None, 'post-script': None, 'pre-script': None, 'script': 'sleep 5; echo "I come in peace"', 'work_d': None, 'directives': {}, 'environment': {"FOO": "foo"}, 'param_var': {}, 'platform': {'name': 'platform'}, 'execution retry delays': [10.0, 20.0], 'execution time limit': 30.0, } @pytest.fixture def job_db_row(): return [ '1', 'foo', 4, '2020-04-03T13:40:18+13:00', 0, '2020-04-03T13:40:20+13:00', None, None, 'background', '20542', 'localhost', ] def int_id(_): return '1/foo/03' def ext_id(schd): return f'~{schd.owner}/{schd.workflow}//{int_id(None)}' def get_pb_prereqs(schd: 'Scheduler') -> 'List[PbPrerequisite]': """Get all protobuf prerequisites from the data store task proxies.""" return [ p for t in cast( 'Iterable[PbTaskProxy]', schd.data_store_mgr.updated[TASK_PROXIES].values() ) for p in t.prerequisites ] def get_pb_job(schd: Scheduler, itask: TaskProxy) -> PbJob: """Get the protobuf job for a given task from the data store.""" return schd.data_store_mgr.data[schd.id][JOBS][itask.job_tokens.id] @pytest.fixture(scope='module') async def mod_harness(mod_flow, mod_scheduler, mod_start): flow_def = { 'scheduler': { 'allow implicit tasks': True }, 'scheduling': { 'graph': { 'R1': 'foo => bar' } } } id_: str = mod_flow(flow_def) schd: 'Scheduler' = mod_scheduler(id_) async with mod_start(schd): await schd.update_data_structure() data = schd.data_store_mgr.data[schd.data_store_mgr.workflow_id] yield schd, data @pytest.fixture(scope='module') async def edgeharness(mod_flow, mod_scheduler, mod_start): """Graph with > n in window edge at n=1.""" flow_def = { 'scheduler': { 'allow implicit tasks': True }, 'scheduling': { 'graph': { 'R1': """ b1 & b2 => c1 & c2 c1 => c2 """ } } } id_: str = mod_flow(flow_def) schd: 'Scheduler' = mod_scheduler(id_) async with mod_start(schd): await schd.update_data_structure() data = schd.data_store_mgr.data[schd.data_store_mgr.workflow_id] yield schd, data @pytest.fixture(scope='module') async def xharness(mod_flow, mod_scheduler, mod_start): """Like harness, but add xtriggers.""" flow_def = { 'scheduler': { 'allow implicit tasks': True }, 'scheduling': { 'xtriggers': { 'x': 'xrandom(0)', 'x2': 'xrandom(0)', 'y': 'xrandom(0, _=1)' }, 'graph': { 'R1': """ @x => foo @x2 => foo @y => foo @x => bar """ } } } id_: str = mod_flow(flow_def) schd: 'Scheduler' = mod_scheduler(id_) async with mod_start(schd): await schd.update_data_structure() data = schd.data_store_mgr.data[schd.data_store_mgr.workflow_id] yield schd, data def collect_states(data, node_type): return [ t.state for t in data[node_type].values() if t.state != '' ] def test_generate_definition_elements(mod_harness): """Test method that generates all definition elements.""" schd, data = mod_harness task_defs = schd.config.taskdefs.keys() assert len(data[TASKS]) == len(task_defs) assert len(data[TASK_PROXIES]) == len(task_defs) def test_generate_graph_elements(mod_harness): schd, data = mod_harness task_defs = schd.config.taskdefs.keys() assert len(data[TASK_PROXIES]) == len(task_defs) def test_get_data_elements(mod_harness): schd, data = mod_harness flow_msg = schd.data_store_mgr.get_data_elements(TASK_PROXIES) assert len(flow_msg.added) == len(data[TASK_PROXIES]) flow_msg = schd.data_store_mgr.get_data_elements(WORKFLOW) assert flow_msg.added.last_updated == data[WORKFLOW].last_updated none_msg = schd.data_store_mgr.get_data_elements('fraggle') assert len(none_msg.ListFields()) == 0 def test_get_entire_workflow(mod_harness): """Test method that populates the entire workflow protobuf message.""" schd, data = mod_harness flow_msg = schd.data_store_mgr.get_entire_workflow() assert len(flow_msg.task_proxies) == len(data[TASK_PROXIES]) def test_increment_graph_window(mod_harness): """Test method that adds and removes elements window boundary.""" schd, data = mod_harness assert schd.data_store_mgr.prune_trigger_nodes assert len(data[TASK_PROXIES]) == 2 def test_in_window_extra_edges(edgeharness): """Test edges beyond walk but within window are generated.""" schd, data = edgeharness w_id = schd.data_store_mgr.workflow_id assert f'{w_id}//$edge|1/c1|1/c2' in data[EDGES] def test_initiate_data_model(mod_harness): """Test method that generates all data elements in order.""" schd: Scheduler schd, data = mod_harness assert len(data[WORKFLOW].task_proxies) == 2 schd.data_store_mgr.initiate_data_model(reloaded=True) assert len(data[WORKFLOW].task_proxies) == 2 # Check n-window preserved on reload: schd.data_store_mgr.set_graph_window_extent(2) schd.data_store_mgr.update_data_structure() assert schd.data_store_mgr.n_edge_distance == 2 schd.data_store_mgr.initiate_data_model(reloaded=True) assert schd.data_store_mgr.n_edge_distance == 2 async def test_delta_task_state(mod_harness): """Test update_data_structure. This method will generate and apply adeltas/updates given.""" schd, data = mod_harness # follow only needs to happen once .. tests working on the same object? w_id = schd.data_store_mgr.workflow_id schd.data_store_mgr.data[w_id] = data assert TASK_STATUS_FAILED not in set(collect_states(data, TASK_PROXIES)) for itask in schd.pool.get_tasks(): itask.state.reset(TASK_STATUS_FAILED) schd.data_store_mgr.delta_task_state(itask) assert TASK_STATUS_FAILED in set(collect_states( schd.data_store_mgr.updated, TASK_PROXIES)) # put things back the way we found them for itask in schd.pool.get_tasks(): itask.state.reset(TASK_STATUS_WAITING) schd.data_store_mgr.delta_task_state(itask) await schd.update_data_structure() async def test_delta_task_held(mod_harness): """Test update_data_structure. This method will generate and apply adeltas/updates given.""" schd: Scheduler schd, data = mod_harness schd.pool.hold_tasks({TaskTokens('*', 'root')}) await schd.update_data_structure() assert True in {t.is_held for t in data[TASK_PROXIES].values()} for itask in schd.pool.get_tasks(): itask.state.reset(is_held=False) schd.data_store_mgr.delta_task_held( itask.tdef.name, itask.point, False ) assert True not in { t.is_held for t in schd.data_store_mgr.updated[TASK_PROXIES].values() } # put things back the way we found them schd.pool.release_held_tasks({TaskTokens('*', 'root')}) await schd.update_data_structure() def test_insert_job(mod_harness): """Test method that adds a new job to the store.""" schd: Scheduler schd, data = mod_harness assert len(schd.data_store_mgr.added[JOBS]) == 0 itask = schd.pool.get_tasks()[0] schd.data_store_mgr.insert_job(itask, 'submitted', job_config(schd)) assert len(schd.data_store_mgr.added[JOBS]) == 1 assert ext_id(schd) in schd.data_store_mgr.added[JOBS] def test_insert_db_job(mod_harness, job_db_row): """Test method that adds a new job from the db to the store.""" schd: Scheduler schd, data = mod_harness assert len(schd.data_store_mgr.added[JOBS]) == 1 schd.data_store_mgr.insert_db_job(0, job_db_row) assert len(schd.data_store_mgr.added[JOBS]) == 2 assert ext_id(schd) in schd.data_store_mgr.added[JOBS] def test_delta_job_msg(mod_harness): """Test method adding messages to job element.""" schd: Scheduler schd, data = mod_harness j_id = ext_id(schd) tokens = Tokens(j_id) # First update creation assert schd.data_store_mgr.updated[JOBS].get('j_id') is None schd.data_store_mgr.delta_job_msg(tokens, 'The Atomic Age') assert schd.data_store_mgr.updated[JOBS][j_id].messages def test_delta_job_attr(mod_harness): """Test method modifying job fields to job element.""" schd: Scheduler schd, data = mod_harness schd.data_store_mgr.delta_job_attr( Mock(job_tokens=Tokens(ext_id(schd))), 'job_runner_name', 'at' ) assert schd.data_store_mgr.updated[JOBS][ext_id(schd)].messages != ( schd.data_store_mgr.added[JOBS][ext_id(schd)].job_runner_name ) def test_delta_job_time(mod_harness): """Test method setting job state change time.""" schd: Scheduler schd, data = mod_harness event_time = get_current_time_string() schd.data_store_mgr.delta_job_time( Mock(job_tokens=Tokens(ext_id(schd))), 'submitted', event_time ) job_updated = schd.data_store_mgr.updated[JOBS][ext_id(schd)] with pytest.raises(ValueError): job_updated.HasField('jumped_time') assert job_updated.submitted_time != ( schd.data_store_mgr.added[JOBS][ext_id(schd)].submitted_time ) async def test_update_data_structure(mod_harness): """Test update_data_structure. This method will generate and apply adeltas/updates given.""" schd, data = mod_harness w_id = schd.data_store_mgr.workflow_id schd.data_store_mgr.data[w_id] = data schd.pool.hold_tasks({TaskTokens('*', 'root')}) await schd.update_data_structure() assert TASK_STATUS_FAILED not in set(collect_states(data, TASK_PROXIES)) assert TASK_STATUS_FAILED not in set(collect_states(data, FAMILY_PROXIES)) assert TASK_STATUS_FAILED not in data[WORKFLOW].state_totals assert len({t.id for t in data[TASK_PROXIES].values() if t.is_held}) == 2 for itask in schd.pool.get_tasks(): itask.state.reset(TASK_STATUS_FAILED) schd.data_store_mgr.delta_task_state(itask) schd.data_store_mgr.update_data_structure() # State change applied assert TASK_STATUS_FAILED in set(collect_states(data, TASK_PROXIES)) # family state changed and applied assert TASK_STATUS_FAILED in set(collect_states(data, FAMILY_PROXIES)) async def test_prune_data_store(flow, scheduler, start): """Test prune_data_store. This method will expand and reduce the data-store to invoke pruning. Also test rapid addition and removal of families (as happens with suicide triggers): https://github.com/cylc/cylc-ui/issues/1999 """ id_ = flow({ 'scheduling': { 'graph': { 'R1': 'foo => bar' } }, 'runtime': { 'FOOBAR': {}, 'FOO': { 'inherit': 'FOOBAR' }, 'foo': { 'inherit': 'FOO' }, 'BAR': { 'inherit': 'FOOBAR' }, 'bar': { 'inherit': 'BAR' } } }) schd = scheduler(id_) async with start(schd): # initialise the data store await schd.update_data_structure() w_id = schd.data_store_mgr.workflow_id data = schd.data_store_mgr.data[w_id] schd.pool.hold_tasks({TaskTokens('*', 'root')}) await schd.update_data_structure() assert ( len({t.id for t in data[TASK_PROXIES].values() if t.is_held}) == 2 ) # Window size reduction to invoke pruning schd.data_store_mgr.set_graph_window_extent(0) schd.data_store_mgr.update_data_structure() assert ( len({t.id for t in data[TASK_PROXIES].values() if t.is_held}) == 1 ) # Test rapid addition and removal # bar/BAR task/family proxies not in .added assert len({ t.name for t in schd.data_store_mgr.added[TASK_PROXIES].values() if t.name == 'bar' }) == 0 assert len({ f.name for f in schd.data_store_mgr.added[FAMILY_PROXIES].values() if f.name == 'BAR' }) == 0 # Add bar/BAR on set output of foo for itask in schd.pool.get_tasks(): schd.pool.spawn_on_output(itask, TASK_OUTPUT_SUCCEEDED) # bar/BAR now found. assert len({ t.name for t in schd.data_store_mgr.added[TASK_PROXIES].values() if t.name == 'bar' }) == 1 assert len({ f.name for f in schd.data_store_mgr.added[FAMILY_PROXIES].values() if f.name == 'BAR' }) == 1 # Before updating the data-store, remove bar/BAR. schd.pool.remove(schd.pool._get_task_by_id('1/bar'), 'Test removal') schd.data_store_mgr.update_data_structure() # bar/BAR not found in data or added stores. assert len({ t.name for t in data[TASK_PROXIES].values() if t.name == 'bar' }) == 0 assert len({ t.name for t in schd.data_store_mgr.added[TASK_PROXIES].values() if t.name == 'bar' }) == 0 assert len({ f.name for f in data[FAMILY_PROXIES].values() if f.name == 'BAR' }) == 0 assert len({ f.name for f in schd.data_store_mgr.added[FAMILY_PROXIES].values() if f.name == 'BAR' }) == 0 async def test_family_ascent_point_prune(mod_harness): """Test _family_ascent_point_prune. This method tries to remove non-existent family.""" schd, data = mod_harness fp_id = 'NotAFamilyProxy' parent_ids = {fp_id} checked_ids = set() node_ids = set() schd.data_store_mgr._family_ascent_point_prune( next(iter(parent_ids)), node_ids, parent_ids, checked_ids, schd.data_store_mgr.family_pruned_ids ) assert len(checked_ids) == 1 assert len(parent_ids) == 0 def test_delta_task_prerequisite(mod_harness): """Test delta_task_prerequisites.""" schd: Scheduler schd, data = mod_harness schd.pool.set_prereqs_and_outputs( {itask.tokens for itask in schd.pool.get_tasks()}, [TASK_STATUS_SUCCEEDED], [], flow=[] ) assert all(p.satisfied for p in get_pb_prereqs(schd)) for itask in schd.pool.get_tasks(): # set prereqs as not-satisfied for prereq in itask.state.prerequisites: for key in prereq: prereq[key] = False schd.data_store_mgr.delta_task_prerequisite(itask) assert not any(p.satisfied for p in get_pb_prereqs(schd)) def test_delta_task_xtrigger(xharness): """Test delta_task_xtrigger.""" schd: Scheduler schd, _ = xharness foo = schd.pool._get_task_by_id('1/foo') bar = schd.pool._get_task_by_id('1/bar') assert not foo.state.xtriggers['x'] # not satisfied assert not foo.state.xtriggers['x2'] # not satisfied assert not foo.state.xtriggers['y'] # not satisfied assert not bar.state.xtriggers['x'] # not satisfied # satisfy foo's dependence on x schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'foo')}, [], ['xtrigger/x:succeeded'], flow=[] ) # check the task pool assert foo.state.xtriggers['x'] # satisfied assert not foo.state.xtriggers['y'] # satisfied assert not bar.state.xtriggers['x'] # not satisfied # data store should have one updated task proxy with satisfied xtrigger x [pbfoo] = schd.data_store_mgr.updated[TASK_PROXIES].values() assert pbfoo.id.endswith('foo') xtrig = pbfoo.xtriggers['x=xrandom(0)'] assert xtrig.label == 'x' assert xtrig.satisfied # unsatisfy it again schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'foo')}, [], ['xtrigger/x:unsatisfied'], flow=[] ) # check the task pool assert not foo.state.xtriggers['x'] # not satisfied # data store should have one updated task proxy with unsatisfied xtrigger x [pbfoo] = schd.data_store_mgr.updated[TASK_PROXIES].values() assert pbfoo.id.endswith('foo') xtrig = pbfoo.xtriggers['x=xrandom(0)'] assert xtrig.label == 'x' assert not xtrig.satisfied # satisfy both of foo's xtriggers at once schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'foo')}, [], ['xtrigger/all:succeeded'], flow=[] ) # check the task pool assert foo.state.xtriggers['x'] # satisfied assert foo.state.xtriggers['y'] # satisfied # data store should have one updated task proxy with satisfied xtrigger x [pbfoo] = schd.data_store_mgr.updated[TASK_PROXIES].values() assert pbfoo.id.endswith('foo') xtrig_x = pbfoo.xtriggers['x=xrandom(0)'] assert xtrig_x.label == 'x' assert xtrig_x.satisfied # updated task proxy should also contain duplicate xtrigger labels xtrig_x2 = pbfoo.xtriggers['x2=xrandom(0)'] assert xtrig_x2.label == 'x2' assert xtrig_x2.satisfied xtrig_y = pbfoo.xtriggers['y=xrandom(0, _=1)'] assert xtrig_y.label == 'y' assert xtrig_y.satisfied async def test_absolute_graph_edges(flow, scheduler, start): """It should add absolute graph edges to the store. See: https://github.com/cylc/cylc-flow/issues/5845 """ runahead_cycles = 1 id_ = flow({ 'scheduling': { 'initial cycle point': '1', 'cycling mode': 'integer', 'runahead limit': f'P{runahead_cycles}', 'graph': { 'R1': 'build', 'P1': 'build[^] => run', }, }, }) schd = scheduler(id_) async with start(schd): await schd.update_data_structure() assert { (Tokens(edge.source).relative_id, Tokens(edge.target).relative_id) for edge in schd.data_store_mgr.data[schd.id][EDGES].values() } == { ('1/build', f'{cycle}/run') # +1 for Python's range() # +2 for Cylc's runahead for cycle in range(1, runahead_cycles + 3) } async def test_group_state_on_window_resize(flow, scheduler, start): """Test group state change on n-window resize. This method will expand and reduce the data-store to change the family/root states. See: https://github.com/cylc/cylc-flow/issues/7115 """ id_ = flow({ 'scheduling': { 'graph': { 'R1': 'foo:failed => bar' } }, 'runtime': { 'FOOBAR': {}, 'FOO': { 'inherit': 'FOOBAR' }, 'foo': { 'inherit': 'FOO', }, 'BAR': { 'inherit': 'FOOBAR' }, 'bar': { 'inherit': 'BAR' } } }) schd = scheduler(id_) async with start(schd): # initialise the data store await schd.update_data_structure() w_id = schd.data_store_mgr.workflow_id data = schd.data_store_mgr.data[w_id] schd.pool.hold_tasks({TaskTokens('1', 'bar')}) await schd.update_data_structure() # At startup foo/FOO/FOOBAR/root should be waiting assert { t.state for t in data[TASK_PROXIES].values() } == {TASK_STATUS_WAITING} assert { data[FAMILY_PROXIES][f_id].state for f_id in [ schd.data_store_mgr.id_.duplicate(cycle='1', task=t).id for t in ['FOO', 'FOOBAR', 'root'] ] } == {TASK_STATUS_WAITING} # Set foo to failed, spawn in bar, remove foo itask = schd.pool._get_task_by_id('1/foo') itask.state.reset(TASK_STATUS_FAILED) schd.pool.spawn_on_output(itask, TASK_OUTPUT_FAILED) schd.data_store_mgr.delta_task_state(itask) schd.pool.remove(itask, 'Test removal') schd.data_store_mgr.update_data_structure() # With both tasks there should be a mix failed and waiting assert { t.state for t in data[TASK_PROXIES].values() } == {TASK_STATUS_FAILED, TASK_STATUS_WAITING} assert [ data[FAMILY_PROXIES][f_id].state for f_id in [ schd.data_store_mgr.id_.duplicate(cycle='1', task=t).id for t in ['FOO', 'BAR', 'FOOBAR', 'root'] ] ] == [ TASK_STATUS_FAILED, TASK_STATUS_WAITING, TASK_STATUS_FAILED, TASK_STATUS_FAILED, ] # Window size reduction to remove failed state task schd.data_store_mgr.set_graph_window_extent(0) schd.data_store_mgr.update_data_structure() # Now bar/BAR/FOOBAR/root should be waiting assert { t.state for t in data[TASK_PROXIES].values() } == {TASK_STATUS_WAITING} assert { data[FAMILY_PROXIES][f_id].state for f_id in [ schd.data_store_mgr.id_.duplicate(cycle='1', task=t).id for t in ['BAR', 'FOOBAR', 'root'] ] } == {TASK_STATUS_WAITING} # Window size increase to add failed state task again schd.data_store_mgr.set_graph_window_extent(1) schd.data_store_mgr.update_data_structure() # Again there should be a mix failed and waiting states assert { t.state for t in data[TASK_PROXIES].values() } == {TASK_STATUS_FAILED, TASK_STATUS_WAITING} assert [ data[FAMILY_PROXIES][f_id].state for f_id in [ schd.data_store_mgr.id_.duplicate(cycle='1', task=t).id for t in ['FOO', 'BAR', 'FOOBAR', 'root'] ] ] == [ TASK_STATUS_FAILED, TASK_STATUS_WAITING, TASK_STATUS_FAILED, TASK_STATUS_FAILED, ] async def test_flow_numbers(flow, scheduler, start): """It should update flow numbers when a task is triggered. See https://github.com/cylc/cylc-flow/issues/6114 """ id_ = flow({ 'scheduling': { 'graph': { 'R1': 'a => b' } } }) schd = scheduler(id_) async with start(schd): # initialise the data store await schd.update_data_structure() # the task should not have a flow number as it is n>0 ds_task = schd.data_store_mgr.get_data_elements(TASK_PROXIES).added[1] assert ds_task.name == 'b' assert ds_task.flow_nums == '[]' # force trigger the task in a new flow await run_cmd(force_trigger_tasks(schd, ['1/b'], ['2'])) # update the data store await schd.update_data_structure() # the task should now exist in the new flow ds_task = schd.data_store_mgr.get_data_elements(TASK_PROXIES).added[1] assert ds_task.name == 'b' assert ds_task.flow_nums == '[2]' async def test_delta_task_outputs(one: 'Scheduler', start): """Ensure task outputs are inserted into the store. Note: Task outputs should *not* be updated incrementally until we have a protocol for doing so, see https://github.com/cylc/cylc-flow/pull/6403 """ def get_data_outputs(): """Return satisfied outputs from the *data* store.""" return { output.label for output in one.data_store_mgr.data[one.id][TASK_PROXIES][ itask.tokens.id ].outputs.values() if output.satisfied } def get_delta_outputs(): """Return satisfied outputs from the *delta* store. Or return None if there's nothing there. """ try: return { output.label for output in one.data_store_mgr.updated[TASK_PROXIES][ itask.tokens.id ].outputs.values() if output.satisfied } except KeyError: return None def _patch_remove(*args, **kwargs): """Prevent the task/workflow from completing.""" pass async with start(one): one.pool.remove = _patch_remove # create a job submission itask = one.pool.get_tasks()[0] assert itask itask.submit_num += 1 one.data_store_mgr.insert_job( itask, itask.state.status, {'submit_num': 1} ) await one.update_data_structure() # satisfy the submitted & started outputs # (note started implies submitted) one.task_events_mgr.process_message( itask, 'INFO', TaskEventsManager.EVENT_STARTED ) # the delta should be populated with the newly satisfied outputs assert get_data_outputs() == set() assert get_delta_outputs() == { TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_STARTED, } # the delta should be applied to the store await one.update_data_structure() assert get_data_outputs() == { TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_STARTED, } assert get_delta_outputs() is None # satisfy the succeeded output one.task_events_mgr.process_message( itask, 'INFO', TaskEventsManager.EVENT_SUCCEEDED ) # the delta should be populated with ALL satisfied outputs # (not just the newly satisfied output) assert get_data_outputs() == { TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_STARTED, } assert get_delta_outputs() == { TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_STARTED, TASK_OUTPUT_SUCCEEDED, } # the delta should be applied to the store await one.update_data_structure() assert get_data_outputs() == { TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_STARTED, TASK_OUTPUT_SUCCEEDED, } assert get_delta_outputs() is None async def test_remove_added_jobs_of_pruned_task(one: Scheduler, start): """When a task is pruned, any of its jobs added in the same batch must be removed from the batch. See https://github.com/cylc/cylc-flow/pull/6656 """ async with start(one): itask = one.pool.get_tasks()[0] itask.state_reset(TASK_STATUS_PREPARING) one.task_events_mgr.process_message(itask, INFO, TASK_OUTPUT_SUCCEEDED) assert not one.data_store_mgr.data[one.id][JOBS] assert len(one.data_store_mgr.added[JOBS]) == 1 one.data_store_mgr.update_data_structure() assert not one.data_store_mgr.data[one.id][JOBS] assert not one.data_store_mgr.added[JOBS] async def test_log_events(one: Scheduler, start): """It should record log events and strip and ANSI formatting.""" async with start(one): handler = ProtobufStreamHandler( one, level=logging.INFO, ) LOG.addHandler(handler) try: # log a message with some ANSIMARKUP formatting LOG.warning( 'here hare here' ) await one.update_data_structure() log_records = one.data_store_mgr.data[one.id][WORKFLOW].log_records assert len(log_records) == 1 log_record = log_records[0] # the message should be in the store, the ANSI formatting should be # stripped assert log_record.level == 'WARNING' assert log_record.message == 'here hare here' finally: LOG.removeHandler(handler) async def test_no_backwards_job_state_change(one: Scheduler, start): """It should not allow backwards job state changes.""" async with start(one): itask = one.pool.get_tasks()[0] itask.state_reset(TASK_STATUS_PREPARING) itask.submit_num += 1 await one.update_data_structure() one.task_events_mgr.process_message(itask, INFO, TASK_OUTPUT_STARTED) await one.update_data_structure() assert get_pb_job(one, itask).state == TASK_STATUS_RUNNING # Simulate late arrival of "submitted" message one.task_events_mgr.process_message(itask, INFO, TASK_OUTPUT_SUBMITTED) await one.update_data_structure() assert get_pb_job(one, itask).state == TASK_STATUS_RUNNING async def test_job_estimated_finish_time(one_conf, flow, scheduler, start): """It should set estimated_finish_time on job elements along with started_time.""" wid = flow({ **one_conf, 'scheduler': {'UTC mode': True}, 'runtime': { 'one': {'execution time limit': 'PT2M'}, }, }) schd: Scheduler = scheduler(wid) date = '2081-07-02T' async def start_job(itask: TaskProxy, start_time: str): if not schd.pool.get_task(itask.point, itask.tdef.name): schd.pool.add_to_pool(itask) await schd.update_data_structure() itask.state_reset(TASK_STATUS_PREPARING) itask.submit_num += 1 itask.jobs = [] schd.task_events_mgr.process_message( itask, INFO, TASK_OUTPUT_SUBMITTED # submit time irrelevant ) await schd.update_data_structure() schd.task_events_mgr.process_message( itask, INFO, TASK_OUTPUT_STARTED, f'{date}{start_time}' ) await schd.update_data_structure() async with start(schd): itask = schd.pool.get_tasks()[0] await start_job(itask, '06:00:00Z') # 1st job: estimate based on execution time limit: assert ( get_pb_job(schd, itask).estimated_finish_time == f'{date}06:02:00Z' ) # Finish this job and start a new one: schd.task_events_mgr.process_message( itask, INFO, TASK_OUTPUT_SUCCEEDED, f'{date}06:00:40Z' ) await start_job(itask, '06:01:00Z') # >=2nd job: estimate based on mean of previous jobs: assert ( get_pb_job(schd, itask).estimated_finish_time == f'{date}06:01:40Z' ) async def test__family_ascent_point_update(flow, scheduler, run, validate): """Check that task states are cascaded up the family tree to the root family and the workflow "containsRetry" flag. https://github.com/cylc/cylc-flow/issues/7174 """ wid = flow({ 'scheduling': { 'initial cycle point': '3333', 'graph': { 'R1': ( 'FAM' '\n@wallclock => is_wallclock' '\n@echo_false => is_xt' ) }, 'xtriggers': { 'wallclock': 'wall_clock()', 'echo_false': 'echo(succeed=False)' } }, 'runtime': { 'FAM': {}, 'is_retry': { 'inherit': 'FAM', 'execution retry delays': 'PT10M', 'simulation': { 'fail cycle points': 'all', 'default run length': 'PT0S' } }, 'normal': {'inherit': 'FAM'}, 'is_wallclock': {'inherit': 'FAM'}, 'is_xt': {'inherit': 'FAM'}, } }) is_flags = ['is_retry', 'is_wallclock', 'is_xtriggered'] validate(wid) schd = scheduler(wid, paused_start=False) async with run(schd): data = schd.data_store_mgr.data assert data[schd.id]['workflow'].contains_retry is False await schd._main_loop() assert data[schd.id]['workflow'].contains_retry is True for family_proxy in data[schd.id]['family_proxies'].values(): for attribute in is_flags: assert getattr(family_proxy, attribute) is True cylc-flow-8.7.0/tests/integration/test_workflow_db_mgr.py0000664000175000017500000001646715264420307024114 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from datetime import datetime, timedelta import pytest import sqlite3 from typing import TYPE_CHECKING from cylc.flow import commands if TYPE_CHECKING: from cylc.flow.scheduler import Scheduler async def test_restart_number( flow, one_conf, start, scheduler, log_filter, db_select ): """The restart number should increment correctly.""" id_ = flow(one_conf) async def test(expected_restart_num: int, do_reload: bool = False): """(Re)start the workflow and check the restart number is as expected. """ schd: 'Scheduler' = scheduler(id_, paused_start=True) async with start(schd): if do_reload: await commands.run_cmd(commands.reload_workflow(schd)) assert schd.workflow_db_mgr.n_restart == expected_restart_num assert log_filter( contains=f"(re)start number={expected_restart_num + 1}" # (In the log, it's 1 higher than backend value) ) assert ('n_restart', f'{expected_restart_num}') in db_select( schd, False, 'workflow_params' ) # First start await test(expected_restart_num=0) # Restart await test(expected_restart_num=1) # Restart + reload - https://github.com/cylc/cylc-flow/issues/4918 await test(expected_restart_num=2, do_reload=True) # Final restart await test(expected_restart_num=3) def db_remove_column(schd: 'Scheduler', table: str, column: str) -> None: """Remove a column from a scheduler DB table. ALTER TABLE DROP COLUMN is not supported by sqlite yet, so we have to copy the table (without the column) and rename it back to the original. """ with schd.workflow_db_mgr.get_pri_dao() as pri_dao: conn = pri_dao.connect() # Get current column names, minus column cursor = conn.execute(f'PRAGMA table_info({table})') desc = cursor.fetchall() c_names = ','.join( [fields[1] for fields in desc if fields[1] != column] ) # Copy table data to a temporary table, and rename it back. conn.execute(rf"CREATE TABLE 'tmp'({c_names})") conn.execute( rf"INSERT INTO 'tmp'({c_names}) SELECT {c_names} FROM {table}") conn.execute(rf"DROP TABLE '{table}'") conn.execute(rf"ALTER TABLE 'tmp' RENAME TO '{table}'") conn.commit() async def test_db_upgrade_pre_803( flow, one_conf, start, scheduler, log_filter, db_select ): """Test scheduler restart with upgrade of pre-8.0.3 DB.""" id_ = flow(one_conf) # Run a scheduler to create a DB. schd: 'Scheduler' = scheduler(id_, paused_start=True) async with start(schd): assert ('n_restart', '0') in db_select(schd, False, 'workflow_params') # Remove task_states:is_manual_submit to fake a pre-8.0.3 DB. db_remove_column(schd, "task_states", "is_manual_submit") db_remove_column(schd, "task_jobs", "flow_nums") schd: 'Scheduler' = scheduler(id_, paused_start=True) # Restart should fail due to the missing column. with pytest.raises(sqlite3.OperationalError): async with start(schd): pass assert ('n_restart', '1') in db_select(schd, False, 'workflow_params') schd: 'Scheduler' = scheduler(id_, paused_start=True) # Run the DB upgrader for version 8.0.2 # (8.0.2 requires upgrade) with schd.workflow_db_mgr.get_pri_dao() as pri_dao: schd.workflow_db_mgr.upgrade_pre_803(pri_dao) # Restart should now succeed. async with start(schd): assert ('n_restart', '2') in db_select(schd, False, 'workflow_params') async def test_workflow_param_rapid_toggle( one_conf, flow, scheduler, run ): """Check that queuing a workflow param toggle operation twice before processing does not cause any problems. https://github.com/cylc/cylc-flow/issues/5593 """ schd: 'Scheduler' = scheduler(flow(one_conf), paused_start=False) async with run(schd): assert schd.is_paused is False schd.pause_workflow() schd.resume_workflow() schd.process_workflow_db_queue() assert schd.is_paused is False w_params = dict(schd.workflow_db_mgr.pri_dao.select_workflow_params()) assert w_params['is_paused'] == '0' async def test_record_only_non_clock_triggers( flow, run, scheduler, complete, db_select ): """Database does not record wall_clock xtriggers. https://github.com/cylc/cylc-flow/issues/5911 Includes: - Not in DB: A normal wall clock xtrigger (wall_clock). - In DB: An xrandom mis-labelled as wall_clock trigger DB). - Not in DB: An execution retry xtrigger. @TODO: Refactor to use simulation mode to speedup after Simulation mode upgrade bugfixes: This should speed this test up considerably. """ rawpoint = '1348' id_ = flow({ "scheduler": { 'cycle point format': '%Y', 'allow implicit tasks': True }, "scheduling": { "initial cycle point": rawpoint, "xtriggers": { "another": "xrandom(100)", "wall_clock": "xrandom(100, _=Not a real wall clock trigger)", "real_wall_clock": "wall_clock()" }, "graph": { "R1": """ @another & @wall_clock & @real_wall_clock => foo @real_wall_clock => bar """ } }, }) schd = scheduler(id_, paused_start=False, run_mode='simulation') async with run(schd): await complete(schd, timeout=20) # Assert that (only) the real clock trigger is not in the db: assert db_select(schd, False, 'xtriggers', 'signature') == [ ('xrandom(100)',), ('xrandom(100, _=Not a real wall clock trigger)',)] async def test_time_zone_writing( one_conf, flow, scheduler, start, db_select, set_timezone ): """Don't store scheduler startup timezone forever. https://github.com/cylc/cylc-flow/issues/6701 """ set_timezone('XXX-19:00') schd = scheduler(flow(one_conf), paused_start=False, run_mode='live') async with start(schd): itask = schd.pool.get_tasks()[0] now = datetime.now().astimezone() set_timezone('XXX-19:17') schd.submit_task_jobs([itask]) # Check the db time_submit: (time_submit,) = db_select(schd, False, 'task_jobs', 'time_submit')[0] time_submit = datetime.strptime(time_submit, '%Y-%m-%dT%H:%M:%S%z') # The submit time should be approx correct: assert ( abs(time_submit - now) < timedelta(seconds=10) ), f"{time_submit} ~= {now}" cylc-flow-8.7.0/tests/integration/test_remove.py0000664000175000017500000004750015264420307022215 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from functools import partial import logging import pytest from cylc.flow.commands import ( force_trigger_tasks, remove_tasks, run_cmd, ) from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.id import TaskTokens from cylc.flow.network.multi import call_multi_async from cylc.flow.network.client_factory import get_client as org_get_client from cylc.flow.scheduler import Scheduler from cylc.flow.scripts import remove from cylc.flow.task_outputs import TASK_OUTPUT_SUCCEEDED from cylc.flow.task_proxy import TaskProxy from cylc.flow.task_state import TASK_STATUS_FAILED @pytest.fixture async def cylc_show_prereqs(cylc_show): """Fixture that returns the prereq info from `cylc show` in an easy-to-use format.""" async def inner(schd: Scheduler, task: str): prerequisites = (await cylc_show(schd, task))[task]['prerequisites'] return [ ( p['satisfied'], {c['taskId']: c['satisfied'] for c in p['conditions']}, ) for p in prerequisites ] return inner @pytest.fixture def example_workflow(flow): return flow({ 'scheduling': { 'graph': { # Note: test both `&` and separate arrows for combining # dependencies 'R1': ''' a1 & a2 => b a3 => b ''', }, }, }) def get_data_store_flow_nums(schd: Scheduler, itask: TaskProxy): _, ds_tproxy = schd.data_store_mgr.store_node_fetcher(itask.tokens) if ds_tproxy: return ds_tproxy.flow_nums async def test_basic( example_workflow, scheduler, start, db_select ): """Test removing a task from all flows.""" schd: Scheduler = scheduler(example_workflow) async with start(schd): a1 = schd.pool._get_task_by_id('1/a1') a3 = schd.pool._get_task_by_id('1/a3') schd.pool.spawn_on_output(a1, TASK_OUTPUT_SUCCEEDED) schd.pool.spawn_on_output(a3, TASK_OUTPUT_SUCCEEDED) await schd.update_data_structure() assert a1 in schd.pool.get_tasks() for table in ('task_states', 'task_outputs'): assert db_select(schd, True, table, 'flow_nums', name='a1') == [ ('[1]',), ] assert db_select( schd, True, 'task_prerequisites', 'satisfied', prereq_name='a1' ) == [ ('satisfied naturally',), ] assert get_data_store_flow_nums(schd, a1) == '[1]' await run_cmd(remove_tasks(schd, ['1/a1'], [])) await schd.update_data_structure() assert a1 not in schd.pool.get_tasks() # removed from pool for table in ('task_states', 'task_outputs'): assert db_select(schd, True, table, 'flow_nums', name='a1') == [ ('[]',), # removed from all flows ] assert db_select( schd, True, 'task_prerequisites', 'satisfied', prereq_name='a1' ) == [ ('0',), # prereq is now unsatisfied ] assert get_data_store_flow_nums(schd, a1) == '[]' async def test_specific_flow( example_workflow, scheduler, start, db_select ): """Test removing a task from a specific flow.""" schd: Scheduler = scheduler(example_workflow) def select_prereqs(): return db_select( schd, True, 'task_prerequisites', 'flow_nums', 'satisfied', prereq_name='a1', ) async with start(schd): a1 = schd.pool._get_task_by_id('1/a1') await run_cmd(force_trigger_tasks(schd, ['1/a1'], ['1', '2'])) schd.pool.spawn_on_output(a1, TASK_OUTPUT_SUCCEEDED) await schd.update_data_structure() assert a1 in schd.pool.get_tasks() assert a1.flow_nums == {1, 2} for table in ('task_states', 'task_outputs'): assert sorted( db_select(schd, True, table, 'flow_nums', name='a1') ) == [ ('[1, 2]',), # triggered task ('[1]',), # original spawned task ] assert select_prereqs() == [ ('[1, 2]', 'satisfied naturally'), ] assert get_data_store_flow_nums(schd, a1) == '[1, 2]' await run_cmd(remove_tasks(schd, ['1/a1'], ['1'])) await schd.update_data_structure() assert a1 in schd.pool.get_tasks() # still in pool assert a1.flow_nums == {2} for table in ('task_states', 'task_outputs'): assert sorted( db_select(schd, True, table, 'flow_nums', name='a1') ) == [ ('[2]',), ('[]',), ] assert select_prereqs() == [ ('[1, 2]', '0'), ] assert get_data_store_flow_nums(schd, a1) == '[2]' async def test_unset_prereq(example_workflow, scheduler, start): """Test removing a task unsets any prerequisites it satisfied.""" schd: Scheduler = scheduler(example_workflow) async with start(schd): for task in ('a1', 'a2', 'a3'): schd.pool.spawn_on_output( schd.pool.get_task(IntegerPoint('1'), task), TASK_OUTPUT_SUCCEEDED, ) b = schd.pool.get_task(IntegerPoint('1'), 'b') assert b.prereqs_are_satisfied() await run_cmd(remove_tasks(schd, ['1/a1'], [])) assert not b.prereqs_are_satisfied() async def test_not_unset_prereq( example_workflow, scheduler, start, db_select ): """Test removing a task does not unset a force-satisfied prerequisite (one that was satisfied by `cylc set --pre`).""" schd: Scheduler = scheduler(example_workflow) async with start(schd): # This set prereq should not be unset by removing a1: schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'b')}, outputs=[], prereqs=['1/a1'], flow=[] ) # Whereas the prereq satisfied by this set output *should* be unset # by removing a2: schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'a2')}, outputs=['succeeded'], prereqs=[], flow=[] ) await schd.update_data_structure() assert sorted( db_select( schd, True, 'task_prerequisites', 'prereq_name', 'satisfied' ) ) == [ ('a1', 'force satisfied'), ('a2', 'satisfied naturally'), ('a3', '0'), ] await run_cmd(remove_tasks(schd, ['1/a1', '1/a2'], [])) await schd.update_data_structure() assert sorted( db_select( schd, True, 'task_prerequisites', 'prereq_name', 'satisfied' ) ) == [ ('a1', 'force satisfied'), ('a2', '0'), ('a3', '0'), ] async def test_nothing_to_do( example_workflow, scheduler, start, log_filter ): """Test removing an invalid task.""" schd: Scheduler = scheduler(example_workflow) async with start(schd): await run_cmd(remove_tasks(schd, ['1/doh'], [])) assert log_filter(logging.WARNING, 'No tasks match "1/doh"') async def test_logging( flow, scheduler, start, log_filter, caplog: pytest.LogCaptureFixture ): """Test logging of a mixture of valid and invalid task removals.""" schd: Scheduler = scheduler( flow({ 'scheduler': { 'cycle point format': 'CCYY', }, 'scheduling': { 'initial cycle point': '2000', 'graph': { 'R3//P1Y': 'b[-P1Y] => a & b', }, }, }) ) tasks_to_remove = [ # Active, removable tasks: '2000/*', # Future, non-removable tasks: '2001/a', '2001/b', # Glob that doesn't match any active tasks: '2002/*', # Invalid tasks: '2005/a', '2000/doh', ] async with start(schd): await run_cmd(remove_tasks(schd, tasks_to_remove, [])) assert log_filter( logging.INFO, "Removed tasks: 2000/a (flows=1), 2000/b (flows=1)" ) assert log_filter(logging.WARNING, "Task(s) not removable: 2001/a, 2001/b") assert log_filter(logging.WARNING, "Invalid cycle point for task: a, 2005") assert log_filter( logging.WARNING, "No tasks match the IDs:\n* 2000/doh\n* 2005/a" ) # No tasks were submitted/running so none should have been killed: assert "job killed" not in caplog.text async def test_logging_flow_nums( example_workflow, scheduler, start, log_filter ): """Test logging of task removals involving flow numbers.""" schd: Scheduler = scheduler(example_workflow) async with start(schd): await run_cmd(force_trigger_tasks(schd, ['1/a1'], ['1', '2'])) # Removing from flow that doesn't exist doesn't work: await run_cmd(remove_tasks(schd, ['1/a1'], ['3'])) assert log_filter( logging.WARNING, "Task(s) not removable: 1/a1 (flows=3)" ) # But if a valid flow is included, it will be removed from that flow: await run_cmd(remove_tasks(schd, ['1/a1'], ['2', '3'])) assert log_filter(logging.INFO, "Removed tasks: 1/a1 (flows=2)") assert schd.pool._get_task_by_id('1/a1').flow_nums == {1} async def test_retrigger(flow, scheduler, run, reflog, complete): """Test prereqs & re-run behaviour when removing tasks.""" schd: Scheduler = scheduler( flow('a => b => c'), paused_start=False, ) async with run(schd): reflog_triggers: set = reflog(schd) await complete(schd, '1/b') await run_cmd(remove_tasks(schd, ['1/a', '1/b'], [])) schd.process_workflow_db_queue() # Removing 1/b should un-queue 1/c: assert len(schd.pool.task_queue_mgr.queues['default'].deque) == 0 assert reflog_triggers == { ('1/a', None), ('1/b', ('1/a',)), } reflog_triggers.clear() await run_cmd(force_trigger_tasks(schd, ['1/a'], [])) await complete(schd) assert reflog_triggers == { ('1/a', None), # 1/b should have run again after 1/a on the re-trigger in flow 1: ('1/b', ('1/a',)), ('1/c', ('1/b',)), } async def test_prereqs( flow, scheduler, run, complete, cylc_show_prereqs, log_filter ): """Test prereqs & stall behaviour when removing tasks.""" schd: Scheduler = scheduler( flow('(a1 | a2) & b => x'), paused_start=False, ) async with run(schd): await complete(schd, '1/a1', '1/a2', '1/b') await run_cmd(remove_tasks(schd, ['1/a1'], [])) assert not schd.pool.is_stalled() # `cylc show` should reflect the now-unsatisfied condition: assert await cylc_show_prereqs(schd, '1/x') == [ (True, {'1/a1': False, '1/a2': True, '1/b': True}) ] await run_cmd(remove_tasks(schd, ['1/b'], [])) # Should cause stall now because 1/c prereq is unsatisfied: assert schd.pool.is_stalled() assert log_filter( logging.WARNING, "1/x is waiting on ['1/a1:succeeded', '1/b:succeeded']", ) assert await cylc_show_prereqs(schd, '1/x') == [ (False, {'1/a1': False, '1/a2': True, '1/b': False}) ] assert schd.pool._get_task_by_id('1/x') await run_cmd(remove_tasks(schd, ['1/a2'], [])) # Should cause 1/x to be removed from the pool as it no longer has # any satisfied prerequisite tasks: assert not schd.pool._get_task_by_id('1/x') assert log_filter( logging.INFO, regex=r"1/x.* removed .* prerequisite task\(s\) removed", ) async def test_downstream_preparing(flow, scheduler, start): """Downstream dependents should not be removed if they are already preparing.""" schd: Scheduler = scheduler( flow(''' a => x a => y '''), ) async with start(schd): a = schd.pool._get_task_by_id('1/a') schd.pool.spawn_on_output(a, TASK_OUTPUT_SUCCEEDED) assert schd.pool.get_task_ids() == {'1/a', '1/x', '1/y'} schd.pool._get_task_by_id('1/y').state_reset('preparing') await run_cmd(remove_tasks(schd, ['1/a'], [])) assert schd.pool.get_task_ids() == {'1/y'} async def test_downstream_other_flows(flow, scheduler, run, complete): """Downstream dependents should not be removed if they exist in other flows.""" schd: Scheduler = scheduler( flow(''' a => b => c => x a => x '''), paused_start=False, ) async with run(schd): await complete(schd, '1/a') await run_cmd(force_trigger_tasks(schd, ['1/c'], ['2'])) c = schd.pool._get_task_by_id('1/c') schd.pool.spawn_on_output(c, TASK_OUTPUT_SUCCEEDED) assert schd.pool._get_task_by_id('1/x').flow_nums == {1, 2} await run_cmd(remove_tasks(schd, ['1/c'], ['2'])) assert schd.pool.get_task_ids() == {'1/b', '1/x'} # Note: in future we might want to remove 1/x from flow 2 as well, to # maintain flow continuity. However it is tricky at the moment because # other prerequisite tasks could exist in flow 2 (we don't know as # prereqs do not hold flow info other than in the DB). assert schd.pool._get_task_by_id('1/x').flow_nums == {1, 2} async def test_suicide(flow, scheduler, run, reflog, complete): """Test that suicide prereqs are unset by `cylc remove`.""" schd: Scheduler = scheduler( flow(''' a => b => c => d => x a & c => !x '''), paused_start=False, ) async with run(schd): reflog_triggers: set = reflog(schd) await complete(schd, '1/b') await run_cmd(remove_tasks(schd, ['1/a'], [])) await complete(schd) assert reflog_triggers == { ('1/a', None), ('1/b', ('1/a',)), ('1/c', ('1/b',)), ('1/d', ('1/c',)), # 1/x not suicided as 1/a was removed: ('1/x', ('1/d',)), } async def test_kill_running(flow, scheduler, run, complete, reflog): """Test removing a running task should kill it. Note this only tests simulation mode and a separate test for live mode exists in tests/functional/cylc-remove. """ schd: Scheduler = scheduler( flow({ 'scheduling': { 'graph': { 'R1': ''' a:started => b => c a:failed => q ''' }, }, 'runtime': { 'a': { 'simulation': { 'default run length': 'PT30S' }, }, }, }), paused_start=False, ) async with run(schd): reflog_triggers = reflog(schd) await complete(schd, '1/b') a = schd.pool._get_task_by_id('1/a') await run_cmd(remove_tasks(schd, ['1/a'], [])) assert a.state(TASK_STATUS_FAILED, is_held=True) await complete(schd) assert reflog_triggers == { ('1/a', None), ('1/b', ('1/a',)), ('1/c', ('1/b',)), # The a:failed output should not cause 1/q to run } async def test_remove_triggered(flow, scheduler, start): """It should remove tasks from pool and from to_trigger set.""" conf = { 'scheduling': { 'graph': { 'R1': 'a & b' }, }, } schd: Scheduler = scheduler(flow(conf)) async with start(schd): foo, bar = schd.pool.get_tasks() # trigger foo (now) await run_cmd( force_trigger_tasks(schd, [foo.identity], []) ) assert foo in schd.pool.tasks_to_trigger_now # trigger bar await run_cmd( force_trigger_tasks(schd, [bar.identity], []) ) assert bar in schd.pool.tasks_to_trigger_now await run_cmd( remove_tasks(schd, [foo.identity, bar.identity], []) ) assert not schd.pool.get_tasks() assert not schd.pool.tasks_to_trigger_now async def test_remove_spawn(flow, scheduler, start): """Test the no-spawn removal of parentless tasks.""" schd: Scheduler = scheduler( flow({ 'scheduler': { 'cycle point format': 'CCYY', }, 'scheduling': { 'initial cycle point': '2000', 'runahead limit': 'P0', 'graph': { 'R3//P1Y': ''' @wall_clock => a b c ''', }, }, }) ) async with start(schd): assert schd.pool.get_task_ids() == { '2000/a', '2000/b', '2000/c', '2001/b', '2001/c', } # Normal removal of parentless runahead and sequential xtrigger (PSX) # spawned. await run_cmd(remove_tasks(schd, ['2000/a', '2000/b', '2001/b'], [])) assert schd.pool.get_task_ids() == { '2000/c', '2001/a', '2001/c', '2002/b', } # no spawn removal await run_cmd(remove_tasks(schd, ['2001/a', '2001/c'], [], True)) assert schd.pool.get_task_ids() == {'2000/c', '2002/b'} # empty the workflow await run_cmd(remove_tasks(schd, ['2002/b', '2000/c'], [], True)) assert schd.pool.get_task_ids() == set() async def test_remove_script(monkeypatch, flow, scheduler, start): """Test script and back-compat handling.""" parser = remove.get_option_parser() conf = { 'scheduling': { 'graph': { 'R1': 'a & b' }, }, } schd: Scheduler = scheduler(flow(conf)) async with start(schd): a, b = schd.pool.get_tasks() # test script works opts, args = parser.parse_args( ['--no-spawn', a.tokens.id] ) rets = await call_multi_async( partial(remove.run, opts), *args, ) assert True in rets.values() await schd._main_loop() assert a not in schd.pool.get_tasks() # test back-compat def mod_get_client(*args, **kwargs): client = org_get_client(*args, **kwargs) client.scheduler_version = '8.6.6' return client monkeypatch.setattr( 'cylc.flow.scripts.remove.get_client', mod_get_client ) # test old version with --no-spawn opts, args = parser.parse_args( ['--no-spawn', b.tokens.id] ) rets = await call_multi_async( partial(remove.run, opts), *args, ) assert False in rets.values() await schd._main_loop() assert b in schd.pool.get_tasks() # test old version without --no-spawn opts, args = parser.parse_args( [b.tokens.id] ) rets = await call_multi_async( partial(remove.run, opts), *args, ) assert True in rets.values() await schd._main_loop() assert not schd.pool.get_tasks() cylc-flow-8.7.0/tests/integration/test_examples.py0000664000175000017500000002157015264420307022535 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Working documentation for the integration test framework. Here are some examples which cover a range of uses (and also provide some useful testing in the process 😀.) """ import asyncio import logging from pathlib import Path import pytest from cylc.flow import __version__ from cylc.flow.scheduler import Scheduler async def test_create_flow(flow, run_dir): """Use the flow fixture to create workflows on the file system.""" # Ensure a flow.cylc file gets written out id_ = flow({ 'scheduler': { 'allow implicit tasks': True }, 'scheduling': { 'graph': { 'R1': 'foo' } } }) workflow_dir = run_dir / id_ flow_file = workflow_dir / 'flow.cylc' assert workflow_dir.exists() assert flow_file.exists() async def test_run(flow, scheduler, run, one_conf): """Create a workflow, initialise the scheduler and run it.""" # Ensure the scheduler can survive for at least one second without crashing id_ = flow(one_conf) schd = scheduler(id_) async with run(schd): await asyncio.sleep(1) # this yields control to the main loop async def test_logging(flow, scheduler, start, one_conf, log_filter): """We can capture log records when we run a scheduler.""" # Ensure that the cylc version is logged on startup. id_ = flow(one_conf) schd = scheduler(id_) async with start(schd): # this returns a list of log records containing __version__ assert log_filter(contains=__version__) async def test_scheduler_arguments(flow, scheduler, start, one_conf): """We can provide options to the scheduler when we __init__ it. These options match their command line equivalents. Use the `dest` value specified in the option parser. """ # Ensure the paused_start option is obeyed by the scheduler. id_ = flow(one_conf) schd = scheduler(id_, paused_start=True) async with start(schd): assert schd.is_paused id_ = flow(one_conf) schd = scheduler(id_, paused_start=False) async with start(schd): assert not schd.is_paused async def test_shutdown(flow, scheduler, start, one_conf): """Shut down a workflow. The scheduler automatically shuts down once you exit the `async with` block, however you can manually shut it down within this block if you like. """ # Ensure the TCP server shuts down with the scheduler. id_ = flow(one_conf) schd = scheduler(id_) async with start(schd): pass assert schd.server.replier.socket.closed async def test_install(flow, scheduler, one_conf, run_dir): """You don't have to run workflows, it's usually best not to! You can take the scheduler through the startup sequence as far as needed for your test. """ # Ensure the installation of the job script is completed. id_ = flow(one_conf) schd = scheduler(id_) await schd.install() assert Path( run_dir, schd.workflow, '.service', 'etc', 'job.sh' ).exists() async def test_task_pool(one, start): """You don't have to run the scheduler to play with the task pool. There are two fixtures to start a scheduler: `start` Takes a scheduler through the startup sequence. `run` Takes a scheduler through the startup sequence, then sets the main loop going. Unless you need the Scheduler main loop running, use `start`. This test uses a pre-prepared Scheduler called "one". """ # Ensure that the correct number of tasks get added to the task pool. async with start(one): # pump the scheduler's heart manually one.pool.release_runahead_tasks() assert len(one.pool.active_tasks) == 1 async def test_exception(one, run, log_filter): """Through an exception into the scheduler to see how it will react. You have to do this from within the scheduler itself. The easy way is to patch the object. """ class MyException(Exception): pass # replace the main loop with something that raises an exception def killer(): raise MyException('mess') one._main_loop = killer # make sure that this error causes the flow to shutdown with pytest.raises(MyException): async with run(one): # The `run` fixture's shutdown logic waits for the main loop to run pass # make sure the exception was logged assert len(log_filter(logging.CRITICAL, contains='mess')) == 1 # make sure the server socket has closed - a good indication of a # successful clean shutdown assert one.server.replier.socket.closed @pytest.fixture(scope='module') async def myflow(mod_flow, mod_scheduler, mod_one_conf): """You can save setup/teardown time by reusing fixtures Write a module-scoped fixture and it can be shared by all tests in the current module. The standard fixtures all have `mod_` alternatives to allow you to do this. Pytest has been configured to run all tests from the same module in the same xdist worker, in other words, module scoped fixtures only get created once per module, even when distributing tests. Obviously this goes with the usual warnings about not mutating the object you are testing in the tests. """ id_ = mod_flow(mod_one_conf) schd = mod_scheduler(id_) return schd def test_module_scoped_fixture(myflow): """Ensure the host is set on __init__. The myflow fixture will be shared between all test functions within this Python module. """ assert myflow.host async def test_db_select(one, start, db_select): """Demonstrate and test querying the workflow database.""" # run a workflow schd: Scheduler = one async with start(schd): # Select all from workflow_params table: assert ('UTC_mode', '0') in db_select(schd, False, 'workflow_params') # Select name & status columns from task_states table: results = db_select(schd, False, 'task_states', 'name', 'status') assert results[0] == ('one', 'waiting') # Select all columns where name==one & status==waiting from # task_states table: results = db_select( schd, False, 'task_states', name='one', status='waiting') assert len(results) == 1 async def test_reflog(flow, scheduler, run, reflog, complete): """Test the triggering of tasks. This is the integration test version of "reftest" in the functional tests. It works by capturing the triggers which caused each submission so that they can be compared with the expected outcome. """ id_ = flow({ 'scheduling': { 'initial cycle point': '1', 'final cycle point': '1', 'cycling mode': 'integer', 'graph': { 'P1': ''' a => b => c x => b => z b[-P1] => b ''' } } }) schd = scheduler(id_, paused_start=False) async with run(schd): triggers = reflog(schd) # Note: add flow_nums=True to capture flows await complete(schd) assert triggers == { # 1/a was triggered by nothing (i.e. it's parentless) ('1/a', None), # 1/b was triggered by three tasks (note the pre-initial dependency) ('1/b', ('0/b', '1/a', '1/x')), ('1/c', ('1/b',)), ('1/x', None), ('1/z', ('1/b',)), } async def test_reftest(flow, scheduler, reftest): """Test the triggering of tasks. This uses the reftest fixture which combines the reflog and complete fixtures. Suitable for use when you just want to do a simple reftest. """ id_ = flow({ 'scheduling': { 'graph': { 'R1': 'a => b' } } }) schd = scheduler(id_, paused_start=False) assert await reftest(schd) == { ('1/a', None), ('1/b', ('1/a',)), } async def test_show(one: Scheduler, start, cylc_show): """Demonstrate the `cylc_show` fixture""" async with start(one): out = await cylc_show(one, '1/one') assert list(out.keys()) == ['1/one'] assert out['1/one']['state'] == 'waiting' cylc-flow-8.7.0/tests/integration/test_force_trigger.py0000664000175000017500000007025415264420307023543 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio import logging from typing import ( Any as Fixture, Callable, ) import pytest from cylc.flow.commands import ( force_trigger_tasks, hold, reload_workflow, remove_tasks, resume, run_cmd, set_prereqs_and_outputs, ) from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.flow_mgr import FLOW_NEW from cylc.flow.scheduler import Scheduler from cylc.flow.task_state import ( TASK_STATUS_FAILED, TASK_STATUS_RUNNING, TASK_STATUS_SUBMITTED, TASK_STATUS_SUCCEEDED, TASK_STATUS_WAITING, ) from cylc.flow.workflow_status import AutoRestartMode async def test_workflow_paused_simple( one_conf, flow, scheduler, run, complete ): """It should run triggered tasks even if the workflow is paused.""" schd: Scheduler = scheduler(flow(one_conf), paused_start=True) async with run(schd): await run_cmd(force_trigger_tasks(schd, ['1/one'], ['1'])) await complete(schd, '1/one', allow_paused=True, timeout=1) async def test_workflow_paused_queues( flow: 'Fixture', scheduler: 'Fixture', start: 'Fixture', capture_submission: 'Fixture', log_filter: Callable ): """ Test manual triggering when the workflow is paused. The usual queue limiting behaviour is expected. https://github.com/cylc/cylc-flow/issues/6192 """ id_ = flow({ 'scheduling': { 'queues': { 'default': { 'limit': 1, }, }, 'graph': { 'R1': ''' a => x & y & z ''', }, }, }) schd = scheduler(id_, paused_start=True) # start the scheduler (but don't set the main loop running) async with start(schd): # capture task submissions (prevents real submissions) submitted_tasks = capture_submission(schd) # paused at start-up so no tasks should be submitted assert len(submitted_tasks) == 0 # manually trigger 1/x - it should be submitted await run_cmd(force_trigger_tasks(schd, ['1/x'], ["1"])) schd.release_tasks_to_run() assert len(submitted_tasks) == 1 # manually trigger 1/y - it should be queued but not submitted # (queue limit reached) await run_cmd(force_trigger_tasks(schd, ['1/y'], ["1"])) schd.release_tasks_to_run() assert len(submitted_tasks) == 1 # manually trigger 1/y again - it should be submitted # (triggering a queued task runs it) await run_cmd(force_trigger_tasks(schd, ['1/y'], ["1"])) schd.release_tasks_to_run() assert len(submitted_tasks) == 2 # manually trigger 1/y yet again - the trigger should be ignored # (task already active) await run_cmd(force_trigger_tasks(schd, ['1/y'], ["1"])) schd.release_tasks_to_run() assert len(submitted_tasks) == 2 assert log_filter( level=logging.WARNING, contains="Job already in process - ignoring" ) async def test_trigger_group_whilst_paused(flow, scheduler, run, complete): """Only group start tasks should run if the group is triggered whilst the scheduler is paused. Group start tasks have only off-group dependencies. Others (with in-group dependencies) should run as normal when their prerequisites are satisfied once the workflow is resumed. """ id_ = flow('a => b => c => d') schd = scheduler(id_, paused_start=True) async with run(schd): # trigger the chain await run_cmd(force_trigger_tasks(schd, ['1/a'], [])) # 1/a should run whilst the workflow is paused (group start-task) await complete(schd, '1/a', allow_paused=True, timeout=1) # 1/b should *not* run whilst the workflow is paused with pytest.raises(AssertionError): await complete(schd, '1/b', allow_paused=True, timeout=2) b = schd.pool._get_task_by_id('1/b') assert b.state.status == TASK_STATUS_WAITING # 1/b and 1/c should run once the workflow is resumed await run_cmd(resume(schd)) await complete(schd, '1/c') async def test_trigger_group( flow, scheduler, run, complete, log_filter ): """Test trigger of a sub-graph of future, then past, tasks. It should satisfy off-group task and xtrigger prerequisites automatically. """ cfg = { 'scheduling': { 'xtriggers': { 'xr': 'xrandom(0)' # never satisfied }, 'special tasks': { 'external-trigger': 'xt(cheese)' }, 'graph': { 'R1': """ # upstream: x => a # sub-graph for group trigger: a => b & c & xt => d # downstream: d => e => y # off-group prerequisites: @xr => x # stop the flow starting @xr => c # off-group xtrigger prerequisite @xr => off => b # off-group task prerequisite # (plus task xt has a push external trigger) # stop the flow from ending without intervention: @xr => y """ }, }, } id_ = flow(cfg) schd = scheduler(id_, paused_start=False) async with run(schd, level=logging.INFO) as log: # Trigger the group ahead of the flow. # It should run the group and flow on to downstream task e. await run_cmd( force_trigger_tasks(schd, ['1/a', '1/b', '1/c', '1/d', '1/xt'], []) ) await complete(schd, '1/e') # It should satisfy off-group prerequisites. assert log_filter( regex="1/c:waiting.*prerequisite force-satisfied: xr = xrandom") assert log_filter( regex="1/b:waiting.*prerequisite force-satisfied: 1/off:succeeded") assert log_filter( regex="1/a:waiting.*prerequisite force-satisfied: 1/x:succeeded") assert log_filter( regex='1/xt:waiting.*external trigger force-satisfied: "cheese"') log.clear() # Trigger the group again, now as past tasks, in the same flow. # It should erase flow 1 history to allow the rerun. # It should not flow on to e, which already ran in flow 1. # Create an active task that needs removing, to test that. await run_cmd( set_prereqs_and_outputs(schd, ['1/c'], [], [], ['all']) ) await run_cmd( force_trigger_tasks(schd, ['1/a', '1/b', '1/c', '1/d', '1/xt'], []) ) await complete(schd, '1/d') assert log_filter( contains=( "Removed tasks: 1/a (flows=1), 1/b (flows=1)," " 1/c (flows=1), 1/d (flows=1), 1/xt (flows=1)" ) ) assert log_filter( regex="1/c:waiting.*prerequisite force-satisfied: xr = xrandom") assert log_filter( regex="1/b:waiting.*prerequisite force-satisfied: 1/off:succeeded") assert log_filter( regex="1/a:waiting.*prerequisite force-satisfied: 1/x:succeeded") assert log_filter( regex='1/xt:waiting.*external trigger force-satisfied: "cheese"') log.clear() # Trigger the group again, as past tasks, in a new flow. # It should flow on to task e again, in flow 2. await run_cmd( force_trigger_tasks( schd, ['1/a', '1/b', '1/c', '1/d', '1/xt'], ['new']) ) await complete(schd, '1/e') assert log_filter( regex=( r"1/c\(flows=2\):waiting.*prerequisite" r" force-satisfied: xr = xrandom" ) ) assert log_filter( regex=( r"1/b\(flows=2\):waiting.*prerequisite" r" force-satisfied: 1/off:succeeded" ) ) assert log_filter( regex=( r"1/a\(flows=2\):waiting.*prerequisite" r" force-satisfied: 1/x:succeeded" ) ) assert log_filter( regex=( r"1/xt\(flows=2\):waiting.*external trigger" r' force-satisfied: "cheese"' ) ) # Task d (in the group) should have run 3 times. assert log_filter( contains="[1/d/03(flows=2):running] => succeeded" ) # Task e (downstream of the group) only twice (once in each flow). assert log_filter( contains="[1/e/02(flows=2):running] => succeeded" ) async def test_trigger_active_task_in_group( flow, scheduler, run, complete, log_filter, reflog, ): """It should remove (and kill) active tasks that are not group start tasks. The workflow `a => b => c` starts out like this: * a (succeeded, n=1) * b (running, n=0) * c (waiting, n=1) Then we reload to add the dependency `d => b` and trigger a, b & c. * a - should be removed and re-spawned. * b - should be removed and re-spawned with `a => b` unsatisfied but `d => b` force-satisfied. * c - should be left alone. See point (4): https://github.com/cylc/cylc-admin/blob/master/docs/proposal-group-trigger.md#details """ conf = { 'scheduling': { 'graph': {'R1': 'a => b => c'}, }, } id_ = flow(conf) schd = scheduler(id_, paused_start=False) async with run(schd): # capture triggering information triggers = reflog(schd) # run until 1/a:succeeded await complete(schd, '1/a') # check 1/b prereqs b_1 = schd.pool.get_task(IntegerPoint('1'), 'b') assert [ (prereq.task, is_satisfied) for condition in b_1.state.prerequisites for prereq, is_satisfied in condition.items() ] == [ # 1/b has a single prereq, it has been satisfied normally ('a', 'satisfied naturally'), ] # submit 1/b schd.submit_task_jobs([b_1]) # reload the workflow adding the dependency "d => b" conf['scheduling']['graph']['R1'] += '\nd => b' flow(conf, workflow_id=id_) await run_cmd(reload_workflow(schd)) # trigger the chain a => b => c await run_cmd(force_trigger_tasks(schd, ['1/a', '1/b', '1/c'], [])) # active task 1/b should be killed assert log_filter( contains=( '[1/b/01:running] removed from the n=0 window: request' ' - active job orphaned' ) ) # check 1/b prereqs b_1 = schd.pool.get_task(IntegerPoint('1'), 'b') # ret reloaded task assert [ (prereq.task, is_satisfied) for condition in b_1.state.prerequisites for prereq, is_satisfied in condition.items() ] == [ # in-group prereq has been reset ('a', False), # off-group prereq has been "force satisfied" ('d', 'force satisfied'), ] # the workflow should run the chain a => b => c as instructed await complete(schd, '1/c') assert triggers == { ('1/a', None), ('1/b', ('1/a',)), # original run ('1/b', ('1/a', '1/d')), # force-triggered run ('1/c', ('1/b',)), } async def test_trigger_group_in_flow( flow, scheduler, run, complete, reflog, db_select, ): """It should remove tasks from the triggered flow(s). Tests the following statement from the proposal: > Use the same flow numbers, as determined by the trigger command in the > usual way, throughout the operation > > -- https://cylc.github.io/cylc-admin/proposal-group-trigger.html#details """ id_ = flow({ 'scheduling': { 'graph': { 'R1': 'a => b => c => d' } } }) schd = scheduler(id_, paused_start=False) async with run(schd): # prevent shutdown after 1/c completes await run_cmd(hold(schd, ['1/d'])) # run the chain, merge in flow "2" part way through triggers = reflog(schd, flow_nums=True) await complete(schd, '1/a') await run_cmd(force_trigger_tasks(schd, ['1/b'], ['2'])) await complete(schd, '1/c') assert triggers == { # (task, flow_nums, triggered_off_of) ('1/a', '[1]', None), ('1/b', '[1, 2]', ('1/a',)), # flow "2" merged in ('1/c', '[1, 2]', ('1/b',)), # flow "2" merged in } # re-run the chain in flow "2" triggers = reflog(schd, flow_nums=True) await run_cmd(force_trigger_tasks(schd, ['1/a', '1/b', '1/c'], ['2'])) await complete(schd, '1/c', timeout=10) assert triggers == { # (task, flow_nums, triggered_off_of) ('1/a', '[2]', None), ('1/b', '[2]', ('1/a',)), ('1/c', '[2]', ('1/b',)), } # ensure that flow "2" was removed from the tasks in the original run # by the group-trigger assert set(db_select( schd, True, 'task_outputs', 'name', 'flow_nums', )) == { # original run ('a', '[1]'), ('b', '[1]'), # flow "2" has been removed ('c', '[1]'), # flow "2" has been removed ('d', '[1, 2]'), # subsequent run ('a', '[2]'), ('b', '[2]'), ('c', '[2]'), } async def test_trigger_n0_tasks( flow, scheduler, run, complete, db_select, ): """It should trigger tasks within their flow if available, else all flows. * N=0 tasks already have a flow assigned. * N!=0 tasks do not yet have a flow assigned. When we are triggering n!=0 tasks, there is no appropriate flow to run them in (this would involve flow merge prediction), so we default to all active flows as the most/only sensible default. Before group trigger, we triggered tasks independently, i.e. we assumed there were no dependencies between the tasks and ran them all simultaneously. With the group trigger extension, we enhanced trigger to make it aware of interdependent tasks. Triggering independent tasks (pre group-trigger behaviour): * If we trigger a n=0 task, we leave it in the flow it is already in. * If we trigger a n!=0 task, we default to all active flows. Triggering interdependent tasks (group trigger extension): If the list of tasks being triggered contains any interdependent tasks, we treat these interdependent tasks as a group. * If we trigger a group which contains n=0 tasks, the whole group should be triggered using the set of flows possessed by these n=0 tasks. * If we trigger a group which does not contain n=0 tasks, we default to all active flows. """ id_ = flow({ 'scheduling': { 'graph': { 'R1': ''' # group 1 (we will trigger a, b & c) a => b => c => z # group 2 (we will trigger e, f & g) e => f => g => z # group 3 (we will trigger y) x => y => z ''' } } }) schd = scheduler(id_, paused_start=False) async with run(schd): # cylc hold 1/x await run_cmd(hold(schd, ['1/x'])) # group 1: spawn n>0 tasks into flows 2 & 3 await run_cmd( set_prereqs_and_outputs(schd, ['1/b'], ['2'], None, ['all']) ) await run_cmd( set_prereqs_and_outputs(schd, ['1/c'], ['3'], None, ['all']) ) # group 2: spawn n>0 tasks into flows 4 & 5 await run_cmd( set_prereqs_and_outputs(schd, ['1/f'], ['4'], None, ['all']) ) await run_cmd( set_prereqs_and_outputs(schd, ['1/g'], ['5'], None, ['all']) ) # trigger all three groups of tasks await run_cmd( force_trigger_tasks( schd, ['1/a', '1/b', '1/c', '1/e', '1/f', '1/g', '1/y'], [] ) ) await complete( schd, '1/a', '1/b', '1/c', '1/e', '1/f', '1/g', '1/y', '1/z' ) assert set(db_select( schd, True, 'task_outputs', 'name', 'flow_nums', )) == { # junk entries inserted on spawn/set ('a', '[1]'), # initial flow spawned on startup ('b', '[]'), # created by "cylc set" ('c', '[]'), # created by "cylc set" ('e', '[1]'), # initial flow spawned on startup ('f', '[]'), # created by "cylc set" ('g', '[]'), # created by "cylc set" ('x', '[1]'), # initial flow spawned on startup # group 1: contained tasks in flows 1, 2 & 3 ('a', '[1, 2, 3]'), ('b', '[1, 2, 3]'), ('c', '[1, 2, 3]'), # group 2: contained tasks in flows 1, 4 & 5 ('e', '[1, 4, 5]'), ('f', '[1, 4, 5]'), ('g', '[1, 4, 5]'), # group 3: contained tasks in flows None ('y', '[1, 2, 3, 4, 5]'), # downstream task ('z', '[1, 2, 3, 4, 5]'), } async def test_replay_outputs(flow, scheduler, start, complete, log_filter): """Triggered group start tasks re-emit (kind of) earlier outputs. https://github.com/cylc/cylc-flow/issues/6858 Example graph: a:started => b => end k:kustom => l => end k:kustom => offg If I trigger a, b, k, l AFTER a:started and k:kustom have completed: cylc trigger workflow //1/a //1/b //1/k //1/l Then I should expect outputs `k:kustom` and `a:started` to be re-used to satisfy b and l in the triggered flow, but NOT off-group task offg. """ msg_prereq = '[1/{}:waiting(runahead)] prerequisite force-satisfied: 1/{}' msg_spawned = "[1/{}:waiting(runahead)] => waiting" msg_removed = "Removed tasks: 1/{}" wid = flow({ 'scheduling': { 'graph': { 'R1': """ a:started => b => end k:kustom => l => end k:kustom => offg """ } }, 'runtime': { 'a': {}, 'k': { 'outputs': {'kustom': 'custom message'} } } }) schd = scheduler(wid, paused_start=True) async with start(schd): # Set initial tasks a and k to "running" so they are recognized as # live during the forthcoming trigger operation. # Complete the a:started and k:kustom outputs. await run_cmd( set_prereqs_and_outputs(schd, ['1/a'], ['1'], ['started'], None) ) await run_cmd( set_prereqs_and_outputs(schd, ['1/k'], ['1'], ['kustom'], None) ) await schd._main_loop() # It should spawn b, l, and offg. for task in ['b', 'l', 'offg']: assert log_filter(contains=msg_spawned.format(task)) # Set a and k as running so they're recognized as live start tasks # by the trigger operation. for itask in schd.pool.get_tasks(): itask.state_reset(TASK_STATUS_RUNNING) # Now trigger the group. await run_cmd( force_trigger_tasks(schd, ['1/a', '1/b', '1/k', '1/l'], []) ) # It should remove b and l (in-group tasks) for task in ['b', 'l']: assert log_filter(contains=msg_removed.format(task)) # But they will be respawned immediately by re-satisfying dependence # on the earlier outputs of a and k for in-group tasks: assert log_filter(contains=msg_prereq.format('b', 'a:started')) assert log_filter(contains=msg_prereq.format('l', 'k:custom message')) # But not for the off-group task offg: assert not log_filter( contains=msg_prereq.format('offg', 'k:custom message')) async def test_trigger_with_sequential_task(flow, scheduler, run, log_filter): """It should trigger a failed sequential task. See https://github.com/cylc/cylc-flow/issues/6911 """ id_ = flow({ 'scheduling': { 'initial cycle point': '1', 'final cycle point': '2', 'cycling mode': 'integer', 'special tasks': { 'sequential': 'foo', }, 'graph': { 'R1': 'install => foo', 'P1': 'foo', }, }, 'runtime': { 'foo': { 'simulation': { 'fail cycle points': '2', }, }, }, }) schd = scheduler(id_, paused_start=False) async with run(schd): # wait for 2/foo:failed async with asyncio.timeout(5): while True: itask = schd.pool._get_task_by_id('2/foo') if itask and itask.state.outputs.is_message_complete('failed'): break await asyncio.sleep(0) # re-trigger 2/foo await run_cmd( force_trigger_tasks( schd, ['2/foo'], [] ) ) # it should re-run async with asyncio.timeout(5): while True: if log_filter(contains='[2/foo/02:running] (received)failed'): break await asyncio.sleep(0) async def test_trigger_with_task_selector(flow, scheduler, start, monkeypatch): """Test task matching with the trigger command. This test is intended to extend the other integration tests for ID matching with a real use case to ensure the code in cylc.flow.commands (which parses and standardises IDs) is working correctly. """ id_ = flow({ 'scheduling': { 'graph': { 'R1': 'a & b & c & d & e & f & g' } } }) schd: Scheduler = scheduler(id_) async with start(schd): trigger_calls = [] def _force_trigger_tasks(_schd, ids, *_, **__): trigger_calls.append( {id_.relative_id_with_selectors for id_ in ids} ) monkeypatch.setattr( 'cylc.flow.commands._force_trigger_tasks', _force_trigger_tasks ) schd.pool._get_task_by_id('1/a').state_reset(TASK_STATUS_SUBMITTED) schd.pool._get_task_by_id('1/b').state_reset(TASK_STATUS_RUNNING) schd.pool._get_task_by_id('1/c').state_reset(TASK_STATUS_SUCCEEDED) schd.pool._get_task_by_id('1/d').state_reset(TASK_STATUS_FAILED) await run_cmd(force_trigger_tasks(schd, ['*:submitted'], [])) assert trigger_calls == [{'1/a'}] trigger_calls.clear() await run_cmd(force_trigger_tasks(schd, ['*:running'], [])) assert trigger_calls == [{'1/b'}] trigger_calls.clear() await run_cmd(force_trigger_tasks(schd, ['*:succeeded'], [])) assert trigger_calls == [{'1/c'}] trigger_calls.clear() await run_cmd(force_trigger_tasks(schd, ['*:failed'], [])) assert trigger_calls == [{'1/d'}] trigger_calls.clear() async def test_pre_warm_start_group_trigger(flow, scheduler, run, complete): """Group-triggered tasks that are before the start point should run in order. This is designed to handle re-running a family of one-time setup tasks in a warm-started workflow. https://github.com/cylc/cylc-flow/pull/7101 """ schd: Scheduler = scheduler( flow({ 'scheduling': { 'cycling mode': 'integer', 'runahead limit': 'P2', 'graph': { 'R1': 'start => c1 => c2 => c3 => foo', 'P1': 'foo[-P1] => foo', }, }, 'runtime': { 'COLD': {}, **{f'c{n}': {'inherit': 'COLD'} for n in (1, 2, 3)}, }, }), paused_start=False, startcp='5' ) async with run(schd): schd.pool.set_hold_point(IntegerPoint('4')) await run_cmd(force_trigger_tasks(schd, ['1/COLD'], [])) assert schd.pool.get_task_ids() == {'1/c1', '5/foo'} await complete(schd, '1/c1', timeout=10) assert schd.pool.get_task_ids() == {'1/c2', '5/foo'} assert schd.pool._get_task_by_id('1/c2').state(TASK_STATUS_WAITING) await complete(schd, '1/c2', '1/c3', timeout=10) assert schd.pool.get_task_ids() == {'5/foo'} # Check list of pre-start tasks to trigger has been cleared: assert not schd.pool.pre_start_tasks_to_trigger async def test_pre_warm_start_trigger_flow_new( flow, scheduler, run, complete ): """In a warm-started workflow, triggering tasks that are < startcp: * Should not flow on in flow=1 * Should flow on in flow=new As we don't have any history for pre-startcp tasks in a warm start, we treat them as complete in flow=1. https://github.com/cylc/cylc-flow/pull/7148 """ schd: Scheduler = scheduler( flow({ 'scheduling': { 'cycling mode': 'integer', 'graph': { 'P1': 'foo[-P1] => foo', }, }, }), paused_start=False, startcp='10' ) async with run(schd): schd.pool.set_hold_point(IntegerPoint('9')) # flow=1 - don't flow on: await run_cmd(force_trigger_tasks(schd, ['1/foo'], [])) assert schd.pool.get_task_ids() == {'1/foo', '10/foo'} await complete(schd, '1/foo', timeout=10) assert schd.pool.get_task_ids() == {'10/foo'} # flow=new - flow on: await run_cmd(force_trigger_tasks(schd, ['3/foo'], [FLOW_NEW])) assert schd.pool.get_task_ids() == {'3/foo', '10/foo'} await complete(schd, '5/foo', timeout=10) assert schd.pool.get_task_ids() == {'6/foo', '10/foo'} async def test_pre_warm_start_interraction_with_auto_restart( flow, scheduler, start, log_filter, ): """Test interaction between warm-start pre-startcp tasks and auto-restart. The list of pre-startcp tasks to run is held in memory only, no DB backup, so get wiped out by restart. """ schd: Scheduler = scheduler( flow( { 'scheduling': { 'cycling mode': 'integer', 'graph': { 'P1': 'foo[-P1] => foo', }, }, } ), paused_start=False, startcp='10', ) async with start(schd): # trigger a pre-initial task (pre start-cycle point) await run_cmd(force_trigger_tasks(schd, ['^/foo'], [])) assert schd.pool.pre_start_tasks_to_trigger # configure the workflow to auto-retsart schd.auto_restart_mode = AutoRestartMode.RESTART_NORMAL schd.auto_restart_time = 0 # the workflow should not auto-restart whilst there are pre-initial # tasks in play await schd.workflow_shutdown() assert log_filter( contains='Waiting for pre start-cycle tasks to complete' ) assert schd.stop_mode is None # remove the pre-initial task await run_cmd(remove_tasks(schd, ['^/foo'], [])) assert schd.pool.pre_start_tasks_to_trigger == set() # the workflow should now auto-restart as normal await schd.workflow_shutdown() assert schd.stop_mode is not None cylc-flow-8.7.0/tests/integration/__init__.py0000664000175000017500000000000015264420307021400 0ustar alastairalastaircylc-flow-8.7.0/tests/integration/test_config.py0000664000175000017500000005724415264420307022173 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging from pathlib import Path import sqlite3 from textwrap import dedent from typing import Any import pytest from cylc.flow import commands from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.cfgspec.globalcfg import GlobalConfig from cylc.flow.exceptions import ( InputError, PointParsingError, ServiceFileError, WorkflowConfigError, XtriggerConfigError, ) from cylc.flow.parsec.exceptions import ListValueError from cylc.flow.parsec.fileparse import read_and_proc from cylc.flow.pathutil import get_workflow_run_pub_db_path from cylc.flow.scheduler import Scheduler Fixture = Any param = pytest.param @pytest.mark.parametrize( 'task_name,valid', [ # valid task names ('a', True), ('a-b', True), ('a_b', True), ('foo', True), ('0aA-+%', True), # invalid task names ('a b', False), ('a£b', False), ('+ab', False), ('@ab', False), # not valid in [runtime] ('_cylc', False), ('_cylcy', False), ] ) def test_validate_task_name( flow, one_conf, validate, task_name: str, valid: bool ): """It should raise errors for invalid task names in the runtime section.""" id_ = flow({ **one_conf, 'runtime': { task_name: {} } }) if valid: validate(id_) else: with pytest.raises(WorkflowConfigError) as exc_ctx: validate(id_) assert task_name in str(exc_ctx.value) @pytest.mark.parametrize( 'task_name', [ 'root', '_cylc', '_cylcy', ] ) def test_validate_implicit_task_name( flow, validate, task_name: str, ): """It should validate implicit task names in the graph. Note that most invalid task names get caught during graph parsing. Here we ensure that names which look like valid graph node names but which are blacklisted get caught and raise errors. """ id_ = flow({ 'scheduling': { 'graph': { 'R1': task_name } }, 'runtime': { # having one item in the runtime allows "root" to be expanded # which makes this test more thorough 'whatever': {} } }) with pytest.raises(WorkflowConfigError) as exc_ctx: validate(id_) assert str(exc_ctx.value).splitlines()[0] == ( f'invalid task name "{task_name}"' ) @pytest.mark.parametrize( 'env_var,valid', [ ('foo', True), ('FOO', True), ('+foo', False), ] ) def test_validate_env_vars(flow, one_conf, validate, env_var, valid): """It should validate environment variable names.""" id_ = flow({ **one_conf, 'runtime': { 'foo': { 'environment': { env_var: 'value' } } } }) if valid: validate(id_) else: with pytest.raises(WorkflowConfigError) as exc_ctx: validate(id_) assert env_var in str(exc_ctx.value) @pytest.mark.parametrize( 'env_val', [ '%(x)s', # valid template but no such parameter x '%(a)123', # invalid template ] ) def test_validate_param_env_templ( flow, one_conf, validate, env_val, log_filter, ): """It should validate parameter environment templates.""" id_ = flow({ **one_conf, 'runtime': { 'foo': { 'environment': { 'foo': env_val } } } }) validate(id_) assert log_filter(contains='bad parameter environment template') assert log_filter(contains=env_val) def test_no_graph(flow, validate): """It should fail for missing graph sections.""" id_ = flow({ 'scheduling': {}, }) with pytest.raises(WorkflowConfigError) as exc_ctx: validate(id_) assert 'missing [scheduling][[graph]] section.' in str(exc_ctx.value) @pytest.mark.parametrize( 'section', [ 'external-trigger', 'clock-trigger', 'clock-expire', ] ) def test_parse_special_tasks_invalid(flow, validate, section): """It should fail for invalid "special tasks".""" id_ = flow({ 'scheduling': { 'initial cycle point': 'now', 'special tasks': { section: 'foo (', # missing closing bracket }, 'graph': { 'R1': 'foo', }, } }) with pytest.raises(WorkflowConfigError) as exc_ctx: validate(id_) assert f'Illegal {section} spec' in str(exc_ctx.value) assert 'foo' in str(exc_ctx.value) def test_parse_special_tasks_interval(flow, validate): """It should fail for invalid durations in clock-triggers.""" id_ = flow({ 'scheduling': { 'initial cycle point': 'now', 'special tasks': { 'clock-trigger': 'foo(PT1Y)', # invalid ISO8601 duration }, 'graph': { 'R1': 'foo' } } }) with pytest.raises(WorkflowConfigError) as exc_ctx: validate(id_) assert 'Illegal clock-trigger spec' in str(exc_ctx.value) assert 'PT1Y' in str(exc_ctx.value) @pytest.mark.parametrize( 'section', [ 'external-trigger', 'clock-trigger', 'clock-expire', ] ) def test_parse_special_tasks_families(flow, scheduler, validate, section): """It should expand families in special tasks.""" id_ = flow({ 'scheduling': { 'initial cycle point': 'now', 'special tasks': { section: 'FOO(P1D)', }, 'graph': { 'R1': 'foo & foot', } }, 'runtime': { # family 'FOO': {}, # nested family 'FOOT': { 'inherit': 'FOO', }, 'foo': { 'inherit': 'FOO', }, 'foot': { 'inherit': 'FOOT', }, } }) if section == 'external-trigger': # external triggers cannot be used for multiple tasks so if family # expansion is completed correctly, validation should fail with pytest.raises(WorkflowConfigError) as exc_ctx: config = validate(id_) assert 'external triggers must be used only once' in str(exc_ctx.value) else: config = validate(id_) assert set(config.cfg['scheduling']['special tasks'][section]) == { # the family FOO has been expanded to the tasks foo, foot 'foo(P1D)', 'foot(P1D)' } def test_queue_treated_as_implicit(flow, validate, caplog, log_filter): """Tasks in queues but not in runtime generate a warning. https://github.com/cylc/cylc-flow/issues/5260 """ id_ = flow( { "scheduling": { "queues": {"my_queue": {"members": "task1, task2"}}, "graph": {"R1": "task2"}, }, "runtime": {"task2": {}}, } ) validate(id_) assert log_filter(contains='Queues contain tasks not defined in runtime') def test_queue_treated_as_comma_separated(flow, validate): """Tasks listed in queue should be separated with commas, not spaces. https://github.com/cylc/cylc-flow/issues/5260 """ id_ = flow( { "scheduling": { "queues": {"my_queue": {"members": "task1 task2"}}, "graph": {"R1": "task2"}, }, "runtime": {"task1": {}, "task2": {}}, } ) with pytest.raises(ListValueError, match="cannot contain a space"): validate(id_) def test_validate_incompatible_db(one_conf, flow, validate): """Validation should fail for an incompatible DB due to not being able to load template vars.""" wid = flow(one_conf) # Create fake outdated DB db_file = Path(get_workflow_run_pub_db_path(wid)) db_file.parent.mkdir(parents=True, exist_ok=True) db_file.touch() conn = sqlite3.connect(db_file) try: conn.execute( 'CREATE TABLE suite_params(key TEXT, value TEXT, PRIMARY KEY(key))' ) conn.commit() finally: conn.close() with pytest.raises( ServiceFileError, match="Workflow database is incompatible" ): validate(wid) # No tables should have been created stmt = "SELECT name FROM sqlite_master WHERE type='table'" conn = sqlite3.connect(db_file) try: tables = [i[0] for i in conn.execute(stmt)] finally: conn.close() assert tables == ['suite_params'] def test_xtrig_validation_wall_clock( flow: 'Fixture', validate: 'Fixture', ): """If an xtrigger module has a `validate()` function is called. https://github.com/cylc/cylc-flow/issues/5448 """ id_ = flow({ 'scheduling': { 'initial cycle point': '1012', 'xtriggers': {'myxt': 'wall_clock(offset=PT7MH)'}, 'graph': {'R1': '@myxt => foo'}, } }) with pytest.raises(WorkflowConfigError, match=( r'\[@myxt\] wall_clock\(offset=PT7MH\)\n' r'Invalid offset: PT7MH' )): validate(id_) def test_xtrig_implicit_wall_clock(flow: Fixture, validate: Fixture): """Test @wall_clock is allowed in graph without explicit xtrigger definition. """ wid = flow({ 'scheduling': { 'initial cycle point': '2024', 'graph': {'R1': '@wall_clock => foo'}, } }) validate(wid) def test_xtrig_validation_echo( flow: 'Fixture', validate: 'Fixture', ): """If an xtrigger module has a `validate()` function is called. https://github.com/cylc/cylc-flow/issues/5448 """ id_ = flow({ 'scheduling': { 'xtriggers': {'myxt': 'echo()'}, 'graph': {'R1': '@myxt => foo'}, } }) with pytest.raises( WorkflowConfigError, match=r'Requires \'succeed=True/False\' arg' ): validate(id_) def test_xtrig_validation_xrandom( flow: 'Fixture', validate: 'Fixture', ): """If an xtrigger module has a `validate()` function it is called. https://github.com/cylc/cylc-flow/issues/5448 """ id_ = flow({ 'scheduling': { 'xtriggers': {'myxt': 'xrandom(200)'}, 'graph': {'R1': '@myxt => foo'}, } }) with pytest.raises( XtriggerConfigError, match=r"'percent' should be a float between 0 and 100" ): validate(id_) def test_xtrig_validation_custom( flow: 'Fixture', validate: 'Fixture', monkeypatch: 'Fixture', ): """If an xtrigger module has a `validate()` function an exception is raised if that validate function fails. https://github.com/cylc/cylc-flow/issues/5448 """ # Rather than create our own xtrigger module on disk # and attempt to trigger a validation failure we # mock our own exception, xtrigger and xtrigger # validation functions and inject these into the # appropriate locations: def kustom_xt(feature): return True, {} def kustom_validate(args): raise Exception('This is only a test.') # Patch xtrigger func & its validate func monkeypatch.setattr( 'cylc.flow.xtrigger_mgr.get_xtrig_func', lambda *args: kustom_validate if "validate" in args else kustom_xt ) id_ = flow({ 'scheduling': { 'initial cycle point': '1012', 'xtriggers': {'myxt': 'kustom_xt(feature=42)'}, 'graph': {'R1': '@myxt => foo'}, } }) Path(id_) with pytest.raises(XtriggerConfigError, match=r'This is only a test.'): validate(id_) @pytest.mark.parametrize('xtrig_call, expected_msg', [ pytest.param( 'xrandom()', r"missing a required argument: 'percent'", id="missing-arg" ), pytest.param( 'wall_clock(alan_grant=1)', r"unexpected keyword argument 'alan_grant'", id="unexpected-arg" ), ]) def test_xtrig_signature_validation( flow: 'Fixture', validate: 'Fixture', xtrig_call: str, expected_msg: str ): """Test automatic xtrigger function signature validation.""" id_ = flow({ 'scheduling': { 'initial cycle point': '2024', 'xtriggers': {'myxt': xtrig_call}, 'graph': {'R1': '@myxt => foo'}, } }) with pytest.raises(XtriggerConfigError, match=expected_msg): validate(id_) @pytest.mark.parametrize( 'left', ( param('@xrandom | @echo', id='xtrig-or-xtrig'), param('@xrandom | task', id='xtrig-or-task'), param('task | @echo', id='task-or-xtrig'), param('@xrandom | foo & bar', id='xtrig-or-complex'), param('@xrandom & bar | foo', id='complex-or-xtrig'), ) ) def test_xtrig_or_fails_validation( flow: "Fixture", validate: "Fixture", left: str ): """Xtriggers cannot be chained with the 'or' https://github.com/cylc/cylc-flow/issues/6771 https://github.com/cylc/cylc-flow/issues/2712 """ id_ = flow( { "scheduling": { "initial cycle point": "2024", "xtriggers": { "xrandom": "xrandom(100)", "echo": "echo(succeed=True)" }, "graph": {"R1": f"{left} => fin"}, } } ) expected_msg = ( "Xtriggers cannot be used in conditional graph expressions:\n") with pytest.raises(WorkflowConfigError, match=expected_msg): validate(id_) def test_special_task_non_word_names(flow: Fixture, validate: Fixture): """Test validation of special tasks names with non-word characters""" wid = flow({ 'scheduling': { 'initial cycle point': '2020', 'special tasks': { 'clock-trigger': 't-1, t+1, t%1, t@1', }, 'graph': { 'P1D': 't-1 & t+1 & t%1 & t@1', }, }, 'runtime': { 't-1, t+1, t%1, t@1': {'script': True}, }, }) validate(wid) async def test_glbl_cfg(monkeypatch, tmp_path, caplog): """Test accessing the global config via the glbl_cfg wrapper. Test the "cached" and "reload" kwargs to glbl_cfg. Also assert that accessing the global config during a reload operation does not cause issues. See https://github.com/cylc/cylc-flow/issues/6244 """ # wipe any previously cached config monkeypatch.setattr( 'cylc.flow.cfgspec.globalcfg.GlobalConfig._DEFAULT', None ) # load the global config from the test tmp directory monkeypatch.setenv('CYLC_CONF_PATH', str(tmp_path)) def write_global_config(cfg_str): """Write the global.cylc file.""" Path(tmp_path, 'global.cylc').write_text(cfg_str) def get_platforms(cfg_obj): """Return the platforms defined in the provided config instance.""" return set(cfg_obj.get(['platforms']).keys()) def expect_platforms_during_reload(platforms): """Test the platforms defined in glbl_cfg() during reload. Assert that the platforms defined in glbl_cfg() match the expected value, whilst the global config is in the process of being reloaded. In other words, this tests that the cached instance is not changed until after the reload has completed. See https://github.com/cylc/cylc-flow/issues/6244 """ caplog.set_level(logging.INFO) def _capture(fcn): def _inner(*args, **kwargs): cfg = glbl_cfg() assert get_platforms(cfg) == platforms logging.getLogger('test').info( 'ran expect_platforms_during_reload test' ) return fcn(*args, **kwargs) return _inner monkeypatch.setattr( 'cylc.flow.cfgspec.globalcfg.GlobalConfig._load', _capture(GlobalConfig._load) ) # write a global config write_global_config(''' [platforms] [[foo]] ''') # test the platforms defined in it assert get_platforms(glbl_cfg()) == {'localhost', 'foo'} # add a new platform the config write_global_config(''' [platforms] [[foo]] [[bar]] ''') # the new platform should not appear (due to the cached instance) assert get_platforms(glbl_cfg()) == {'localhost', 'foo'} # if we request an uncached instance, the new platform should appear assert get_platforms(glbl_cfg(cached=False)) == {'localhost', 'foo', 'bar'} # however, this should not affect the cached instance assert get_platforms(glbl_cfg()) == {'localhost', 'foo'} # * if we reload the cached instance, the new platform should appear # * but during the reload itself, the old config should persist # see https://github.com/cylc/cylc-flow/issues/6244 expect_platforms_during_reload({'localhost', 'foo'}) assert get_platforms(glbl_cfg(reload=True)) == {'localhost', 'foo', 'bar'} assert 'ran expect_platforms_during_reload test' in caplog.messages # the cache should have been updated by the reload assert get_platforms(glbl_cfg()) == {'localhost', 'foo', 'bar'} def test_nonlive_mode_validation(flow, validate, caplog, log_filter): """Nonlive tasks return a warning at validation. """ caplog.set_level(logging.INFO) msg1 = dedent( 'The following tasks are set to run in skip mode:\n * skip' ) wid = flow({ 'scheduling': { 'graph': { 'R1': 'live => skip => simulation => dummy => default' } }, 'runtime': { 'default': {}, 'live': {'run mode': 'live'}, 'skip': { 'run mode': 'skip', 'skip': {'outputs': 'started, submitted'} }, }, }) validate(wid) assert log_filter(contains=msg1) def test_skip_forbidden_as_output(flow, validate): """Run mode names are forbidden as task output names.""" wid = flow({ 'scheduling': {'graph': {'R1': 'task'}}, 'runtime': {'task': {'outputs': {'skip': 'message for skip'}}} }) with pytest.raises( WorkflowConfigError, match='Invalid task output .* cannot be: `skip`' ): validate(wid) def test_validate_workflow_run_mode( flow: Fixture, validate: Fixture, caplog: Fixture ): """Test that Cylc validate will only check simulation mode settings if validate --mode simulation or dummy. Discovered in: https://github.com/cylc/cylc-flow/pull/6213#issuecomment-2225365825 """ wid = flow( { 'scheduling': {'graph': {'R1': 'mytask'}}, 'runtime': { 'mytask': { 'simulation': {'fail cycle points': 'invalid'}, } }, } ) validate(wid) # It fails with run mode simulation: with pytest.raises(PointParsingError, match='Incompatible value'): validate(wid, run_mode='simulation') # It fails with run mode dummy: with pytest.raises(PointParsingError, match='Incompatible value'): validate(wid, run_mode='dummy') async def test_invalid_starttask(one_conf, flow, scheduler, start): """It should reject invalid starttask arguments.""" id_ = flow(one_conf) schd = scheduler(id_, starttask=['a///b']) with pytest.raises(InputError, match='a///b'): async with start(schd): pass async def test_CYLC_WORKFLOW_SRC_DIR_correctly_set(tmp_path, install, run_dir): """CYLC_WORKFLOW_SRC_DIR is set correctly: * In source dir * In installed dir (Not testing different permutations of installed dir as these are covered by testing of `get_workflow_source_dir`) * Created directly in the run dir. """ def process_file(target): """Run config through read_and_proc (~= cylc view --process) """ return read_and_proc( target, viewcfg={ 'mark': False, 'single': False, 'label': False, 'jinja2': True, 'contin': True, 'inline': True, }, ) # Setup a source directory: (tmp_path / 'flow.cylc').write_text( '#!jinja2\n{{ CYLC_WORKFLOW_SRC_DIR }}' ) # Check that the CYLC_SRC_DIRECTORY # points to the source directory (tmp_path): processed = process_file(tmp_path / 'flow.cylc') assert processed[0] == str(tmp_path) # After installation the CYLC_WORKFLOW_SRC_DIR # *still* points back to tmp_path: wid = await install(tmp_path) processed = process_file(run_dir / wid / 'flow.cylc') assert processed[0] == str(tmp_path) async def test_task_event_bad_custom_template( flow, validate, scheduler, start, log_filter ): """Validation fails if task event handler has a bad custom template. """ exception = ( r"bad task event handler template t1: echo %\(rubbish\)s:" r" KeyError\('rubbish'\)" ) events = { 'handlers': 'echo %(rubbish)s', 'handler events': 'succeeded' } wid = flow({ 'scheduling': {'graph': {'R1': 't1'}}, 'runtime': {'t1': {'events': events}}, }) with pytest.raises(WorkflowConfigError, match=exception): validate(wid) schd = scheduler(wid) with pytest.raises(WorkflowConfigError, match=exception): async with start(schd): pass async def test_icp_now_reload( flow, scheduler, start, monkeypatch: pytest.MonkeyPatch, log_filter ): """initial cycle point = 'now' should not change from original value on reload/restart, and sequences should remain intact. https://github.com/cylc/cylc-flow/issues/7047 """ def set_time(value): monkeypatch.setattr( 'cylc.flow.config.get_current_time_string', lambda *a, **k: f"2005-01-01T{value}Z", ) wid = flow({ 'scheduling': { 'initial cycle point': 'now', 'graph': { 'R1': 'cold => foo', 'PT15M': 'foo[-PT15M] => foo', }, }, }) schd: Scheduler = scheduler(wid) def main_check(icp): assert str(schd.config.initial_point) == icp assert schd.pool.get_task_ids() == { f'{icp}/cold', } assert {str(seq) for seq in schd.config.sequences} == { f'R1/{icp}/P0Y', f'R/{icp}/PT15M', } set_time('06:00') async with start(schd): expected_icp = '20050101T0600Z' main_check(expected_icp) set_time('06:03') await commands.run_cmd(commands.reload_workflow(schd)) main_check(expected_icp) await commands.run_cmd( commands.set_prereqs_and_outputs( schd, [f'{expected_icp}/cold'], [] ) ) # Downstream task should have spawned on sequence: assert schd.pool.get_task_ids() == { f'{expected_icp}/foo', } assert not log_filter(level=logging.WARNING) cylc-flow-8.7.0/tests/integration/test_task_job_mgr.py0000664000175000017500000002224515264420307023360 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from contextlib import suppress import json import logging from typing import Any as Fixture from unittest.mock import Mock from cylc.flow import CYLC_LOG from cylc.flow.job_runner_mgr import JOB_FILES_REMOVED_MESSAGE from cylc.flow.scheduler import Scheduler from cylc.flow.task_state import ( TASK_STATUS_FAILED, TASK_STATUS_RUNNING, ) async def test_run_job_cmd_no_hosts_error( flow, scheduler, start, mock_glbl_cfg, log_filter, ): """It should catch NoHostsError. NoHostsError's should be caught and handled rather than raised because they will cause problems (i.e. trigger shutdown) if they make it to the Scheduler. NoHostError's can occur in the poll & kill logic, this test ensures that these methods catch the NoHostsError and handle the event as a regular SSH failure by pushing the issue down the 255 callback. See https://github.com/cylc/cylc-flow/pull/5195 """ # define a platform mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[no-host-platform]] ''', ) # define a workflow with a task which runs on that platform id_ = flow({ 'scheduling': { 'graph': { 'R1': 'foo' } }, 'runtime': { 'foo': { 'platform': 'no-host-platform' } } }) # start the workflow schd: Scheduler = scheduler(id_) async with start(schd) as log: # set logging to debug level log.set_level(logging.DEBUG, CYLC_LOG) # tell Cylc the task is running on that platform schd.pool.get_tasks()[0].state_reset(TASK_STATUS_RUNNING) schd.pool.get_tasks()[0].platform = { 'name': 'no-host-platform', 'hosts': ['no-host-platform'], } # tell Cylc that that platform is not contactable # (i.e. all hosts are in bad_hosts) # (this casuses the NoHostsError to be raised) schd.task_job_mgr.bad_hosts.add('no-host-platform') # polling the task should not result in an error... schd.task_job_mgr.poll_task_jobs(schd.pool.get_tasks()) # ...but the failure should be logged assert log_filter( contains='No available hosts for no-host-platform', ) log.clear() # killing the task should not result in an error... schd.task_job_mgr.kill_task_jobs(schd.pool.get_tasks()) # ...but the failure should be logged assert log_filter( contains='No available hosts for no-host-platform', ) async def test__run_job_cmd_logs_platform_lookup_fail( one_conf: Fixture, flow: Fixture, scheduler: Fixture, run: Fixture, db_select: Fixture, caplog: Fixture ) -> None: """TaskJobMg._run_job_cmd handles failure to get platform.""" id_: str = flow(one_conf) schd: 'Scheduler' = scheduler(id_, paused_start=True) # Run async with run(schd): from types import SimpleNamespace schd.task_job_mgr._run_job_cmd( schd.task_job_mgr.JOBS_POLL, [SimpleNamespace(platform={'name': 'culdee fell summit'})], None, None ) warning = caplog.records[-1] assert warning.levelname == 'ERROR' assert 'Unable to run command jobs-poll' in warning.msg async def test__prep_submit_task_job_impl_handles_execution_time_limit( flow: Fixture, scheduler: Fixture, start: Fixture, ): """Ensure that emptying the execution time limit unsets it. Previously unsetting the etl by either broadcast or reload would not unset a previous etl. See https://github.com/cylc/cylc-flow/issues/5891 """ id_ = flow({ "scheduling": { "cycling mode": "integer", "graph": {"R1": "a"} }, "runtime": { "root": {}, "a": { "script": "sleep 10", "execution time limit": 'PT5S' } } }) # Run in live mode - function not called in sim mode. schd = scheduler(id_, run_mode='live') async with start(schd): task_a = schd.pool.get_tasks()[0] # We're not interested in the job file stuff, just # in the summary state. with suppress(FileExistsError): schd.task_job_mgr._prep_submit_task_job_impl( task_a, task_a.tdef.rtconfig ) assert task_a.summary['execution_time_limit'] == 5.0 # If we delete the etl it gets deleted in the summary: task_a.tdef.rtconfig['execution time limit'] = None with suppress(FileExistsError): schd.task_job_mgr._prep_submit_task_job_impl( task_a, task_a.tdef.rtconfig ) assert not task_a.summary.get('execution_time_limit', '') # put everything back and test broadcast too. task_a.tdef.rtconfig['execution time limit'] = 5.0 task_a.summary['execution_time_limit'] = 5.0 schd.broadcast_mgr.broadcasts = { '1': {'a': {'execution time limit': None}}} with suppress(FileExistsError): # We run a higher level function here to ensure # that the broadcast is applied. schd.task_job_mgr._prep_submit_task_job(task_a) assert not task_a.summary.get('execution_time_limit', '') async def test_broadcast_platform_change( mock_glbl_cfg, flow, scheduler, start, log_filter, ): """Broadcast can change task platform. Even after host selection failure. see https://github.com/cylc/cylc-flow/issues/6320 """ mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[foo]] hosts = food ''') id_ = flow({ "scheduling": {"graph": {"R1": "mytask"}}, # Platform = None doesn't cause this issue! "runtime": {"mytask": {"platform": "localhost"}}}) schd: Scheduler = scheduler(id_, run_mode='live') async with start(schd): # Change the task platform with broadcast: schd.broadcast_mgr.put_broadcast( ['1'], ['mytask'], [{'platform': 'foo'}]) # Simulate prior failure to contact hosts: schd.bad_hosts.add('food') # Attempt job submission: schd.submit_task_jobs(schd.pool.get_tasks()) # Check that task platform hasn't become "localhost": assert schd.pool.get_tasks()[0].platform['name'] == 'foo' # ... and that remote init failed because all hosts bad: assert log_filter(regex=r"platform: foo .*\(no hosts were reachable\)") async def test_poll_job_deleted_log_folder( one_conf, flow, scheduler, start, log_filter ): """Capture a task error caused by polling finding the job log dir deleted. https://github.com/cylc/cylc-flow/issues/6425 """ response = { 'run_signal': JOB_FILES_REMOVED_MESSAGE, 'run_status': 1, 'job_runner_exit_polled': 1, } schd: Scheduler = scheduler(flow(one_conf)) async with start(schd): itask = schd.pool.get_tasks()[0] itask.submit_num = 1 job_id = itask.job_tokens.relative_id schd.task_job_mgr._poll_task_job_callback( itask, cmd_ctx=Mock(), line=f'2025-02-13T12:08:30Z|{job_id}|{json.dumps(response)}', ) assert itask.state(TASK_STATUS_FAILED) assert log_filter( logging.ERROR, f"job log directory {job_id} no longer exists" ) async def test__prep_submit_task_job_impl_handles_all_old_platform_settings( flow: Fixture, scheduler: Fixture, start: Fixture, mock_glbl_cfg: Fixture, ): """Ensure that if the old host/batch system settings are set that task job manager will wait for the any hostname to be resolved. https://github.com/cylc/cylc-flow/pull/6990 """ mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[bakery]] hosts = localhost job runner = loaf ''', ) id_ = flow({ "scheduling": {"graph": {"R1": "a"}}, "runtime": {"a": {"job": {"batch system": "loaf"}}} }) schd = scheduler(id_, run_mode='live') async with start(schd): task_a = schd.pool.get_tasks()[0] with suppress(FileExistsError): schd.task_job_mgr._prep_submit_task_job(task_a) assert task_a.platform['name'] == 'bakery' cylc-flow-8.7.0/tests/integration/conftest.py0000664000175000017500000005365215264420307021513 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Default fixtures for functional tests.""" import asyncio from functools import partial from pathlib import Path import re from shutil import rmtree from time import time from typing import ( TYPE_CHECKING, List, Set, Tuple, Union, ) import pytest from cylc.flow.config import WorkflowConfig from cylc.flow.id import Tokens from cylc.flow.network.client import WorkflowRuntimeClient from cylc.flow.option_parsers import Options from cylc.flow.pathutil import get_cylc_run_dir from cylc.flow.run_modes import RunMode from cylc.flow.rundb import CylcWorkflowDAO from cylc.flow.scripts.install import ( get_option_parser as install_gop, install as cylc_install, ) from cylc.flow.scripts.show import ( ShowOptions, prereqs_and_outputs_query, ) from cylc.flow.scripts.validate import ValidateOptions from cylc.flow.task_state import ( TASK_STATUS_SUBMITTED, TASK_STATUS_SUCCEEDED, ) from cylc.flow.util import serialise_set from cylc.flow.wallclock import get_current_time_string from cylc.flow.workflow_files import infer_latest_run_from_id from cylc.flow.workflow_status import StopMode from .utils import _rm_if_empty from .utils.flow_tools import ( _make_flow, _make_scheduler, _make_src_flow, _run_flow, _start_flow, ) if TYPE_CHECKING: from cylc.flow.scheduler import Scheduler from cylc.flow.task_proxy import TaskProxy InstallOpts = Options(install_gop()) @pytest.hookimpl(tryfirst=True, hookwrapper=True) def pytest_runtest_makereport(item, call): """Expose the result of tests to their fixtures. This will add a variable to the "node" object which differs depending on the scope of the test. scope=function `_function_outcome` will be set to the result of the test function. scope=module `_module_outcome will be set to a list of all test results in the module. https://github.com/pytest-dev/pytest/issues/230#issuecomment-402580536 """ outcome = yield rep = outcome.get_result() # scope==function item._function_outcome = rep # scope==module _module_outcomes = getattr(item.module, '_module_outcomes', {}) _module_outcomes[(item.nodeid, rep.when)] = rep item.module._module_outcomes = _module_outcomes def _pytest_passed(request: pytest.FixtureRequest) -> bool: """Returns True if the test(s) a fixture was used in passed.""" if hasattr(request.node, '_function_outcome'): return request.node._function_outcome.outcome in {'passed', 'skipped'} return all(( report.outcome in {'passed', 'skipped'} for report in request.node.obj._module_outcomes.values() )) @pytest.fixture(scope='session') def run_dir(): """The cylc run directory for this host.""" path = Path(get_cylc_run_dir()) path.mkdir(exist_ok=True) yield path @pytest.fixture(scope='session') def ses_test_dir(request, run_dir): """The root run dir for test flows in this test session.""" timestamp = get_current_time_string(use_basic_format=True) uuid = f'cit-{timestamp}' path = Path(run_dir, uuid) path.mkdir(exist_ok=True) yield path _rm_if_empty(path) @pytest.fixture(scope='module') def mod_test_dir(request, ses_test_dir): """The root run dir for test flows in this test module.""" path = Path( ses_test_dir, # Shorten path by dropping `integration.` prefix: re.sub(r'^integration\.', '', request.module.__name__) ) path.mkdir(parents=True, exist_ok=True) yield path if _pytest_passed(request): # test passed -> remove all files rmtree(path, ignore_errors=False) else: # test failed -> remove the test dir if empty _rm_if_empty(path) @pytest.fixture def test_dir(request, mod_test_dir): """The root run dir for test flows in this test function.""" path = Path(mod_test_dir, request.function.__name__) path.mkdir(parents=True, exist_ok=True) yield path if _pytest_passed(request): # test passed -> remove all files rmtree(path, ignore_errors=False) else: # test failed -> remove the test dir if empty _rm_if_empty(path) @pytest.fixture(scope='module') def mod_flow(run_dir, mod_test_dir): """A function for creating module-level flows.""" yield partial(_make_flow, run_dir, mod_test_dir) @pytest.fixture def flow(run_dir, test_dir): """A function for creating function-level flows.""" yield partial(_make_flow, run_dir, test_dir) @pytest.fixture(scope='module') def mod_scheduler(): """Return a Scheduler object for a flow. Usage: see scheduler() below """ with _make_scheduler() as _scheduler: yield _scheduler @pytest.fixture def scheduler(): """Return a Scheduler object for a flow. Args: id_ (str): Workflow name. **opts (Any): Options to be passed to the Scheduler. """ with _make_scheduler() as _scheduler: yield _scheduler @pytest.fixture(scope='module') def mod_start(): """Start a scheduler but don't set it running (module scope).""" return partial(_start_flow, None) @pytest.fixture def start(caplog: pytest.LogCaptureFixture): """Start a scheduler but don't set it running.""" return partial(_start_flow, caplog) @pytest.fixture(scope='module') def mod_run(): """Start a scheduler and set it running (module scope).""" return partial(_run_flow, None) @pytest.fixture def run(caplog: pytest.LogCaptureFixture): """Start a scheduler and set it running.""" return partial(_run_flow, caplog) @pytest.fixture def one_conf(): return { 'scheduler': { 'allow implicit tasks': True }, 'scheduling': { 'graph': { 'R1': 'one' } } } @pytest.fixture(scope='module') def mod_one_conf(): return { 'scheduler': { 'allow implicit tasks': True }, 'scheduling': { 'graph': { 'R1': 'one' } } } @pytest.fixture def one(one_conf, flow, scheduler): """Return a Scheduler for the simple "R1 = one" graph.""" id_ = flow(one_conf) schd = scheduler(id_) return schd @pytest.fixture(scope='module') def mod_one(mod_one_conf, mod_flow, mod_scheduler): id_ = mod_flow(mod_one_conf) schd = mod_scheduler(id_) return schd @pytest.fixture(scope='module') def event_loop(): """This fixture defines the event loop used for each test. The default scoping for this fixture is "function" which means that all async fixtures must have "function" scoping. Defining `event_loop` as a module scoped fixture opens the door to module scoped fixtures but means all tests in a module will run in the same event loop. This is fine, it's actually an efficiency win but also something to be aware of. See: https://github.com/pytest-dev/pytest-asyncio/issues/171 """ loop = asyncio.get_event_loop_policy().new_event_loop() yield loop # gracefully exit async generators loop.run_until_complete(loop.shutdown_asyncgens()) # cancel any tasks still running in this event loop for task in asyncio.all_tasks(loop): task.cancel() loop.close() @pytest.fixture def db_select(): """Select columns from workflow database. Args: schd: The Scheduler object for the workflow. process_db_queue: Whether to process the scheduler's db queue before querying. table: The name of the database table to query. *columns (optional): The columns to select from the table. To select all columns, omit or use '*'. **where (optional): Kwargs specifying ='' for use in WHERE clauses. If more than one specified, they will be chained together using an AND operator. """ def _check_columns(table: str, *columns: str) -> None: all_columns = [x[0] for x in CylcWorkflowDAO.TABLES_ATTRS[table]] for col in columns: if col not in all_columns: raise ValueError(f"Column '{col}' not in table '{table}'") def _inner( schd: 'Scheduler', process_db_queue: bool, table: str, *columns: str, **where: str ) -> List[Tuple[str, ...]]: if process_db_queue: schd.process_workflow_db_queue() if table not in CylcWorkflowDAO.TABLES_ATTRS: raise ValueError(f"Table '{table}' not in database") if not columns: columns = ('*',) elif columns != ('*',): _check_columns(table, *columns) stmt = f'SELECT {",".join(columns)} FROM {table}' stmt_args = [] if where: _check_columns(table, *where.keys()) where_stmt = ' AND '.join([ f'{col}=?' for col in where.keys() ]) stmt += f' WHERE {where_stmt}' stmt_args = list(where.values()) with schd.workflow_db_mgr.get_pri_dao() as pri_dao: return list(pri_dao.connect().execute(stmt, stmt_args)) return _inner @pytest.fixture def gql_query(): """Execute a GraphQL query given a workflow runtime client.""" async def _gql_query( client: 'WorkflowRuntimeClient', query_str: str ) -> object: ret = await client.async_request( 'graphql', { 'request_string': 'query { ' + query_str + ' }' } ) return ret return _gql_query @pytest.fixture def validate(run_dir): """Provides a function for validating workflow configurations. Attempts to load the configuration, will raise exceptions if there are errors. Args: id_ - The flow to validate kwargs - Arguments to pass to ValidateOptions """ def _validate(id_: Union[str, Path], **kwargs) -> WorkflowConfig: id_ = str(id_) return WorkflowConfig( id_, str(Path(run_dir, id_, 'flow.cylc')), ValidateOptions(**kwargs) ) return _validate @pytest.fixture(scope='module') def mod_validate(run_dir): """Provides a function for validating workflow configurations. Attempts to load the configuration, will raise exceptions if there are errors. Args: id_ - The flow to validate kwargs - Arguments to pass to ValidateOptions """ def _validate(id_: Union[str, Path], **kwargs) -> WorkflowConfig: id_ = str(id_) return WorkflowConfig( id_, str(Path(run_dir, id_, 'flow.cylc')), ValidateOptions(**kwargs) ) return _validate @pytest.fixture def capture_submission(): """Suppress job submission and capture submitted tasks. Provides a function to run on a Scheduler *whilst started*, use like so: async with start(schd): submitted_tasks = capture_submission(schd) or: async with run(schd): submitted_tasks = capture_submission(schd) """ def _disable_submission(schd: 'Scheduler') -> 'Set[TaskProxy]': submitted_tasks: 'Set[TaskProxy]' = set() def _submit_task_jobs(itasks): for itask in itasks: itask.state_reset(TASK_STATUS_SUBMITTED) submitted_tasks.update(itasks) return itasks schd.submit_task_jobs = _submit_task_jobs # type: ignore return submitted_tasks return _disable_submission @pytest.fixture def capture_polling(): """Suppress job polling and capture polled tasks. Provides a function to run on a started Scheduler. async with start(schd): polled_tasks = capture_polling(schd) or: async with run(schd): polled_tasks = capture_polling(schd) """ def _disable_polling(schd: 'Scheduler') -> 'Set[TaskProxy]': polled_tasks: 'Set[TaskProxy]' = set() def run_job_cmd( _1, itasks, _3, _4=None ): polled_tasks.update(itasks) return itasks schd.task_job_mgr._run_job_cmd = run_job_cmd # type: ignore return polled_tasks return _disable_polling @pytest.fixture(scope='module') def mod_workflow_source(mod_flow, tmp_path_factory): """Create a workflow source directory. Args: cfg: Can be passed a config dictionary. Yields: Path to source directory. """ def _inner(cfg): src_dir = _make_src_flow(tmp_path_factory.getbasetemp(), cfg) return src_dir yield _inner @pytest.fixture def workflow_source(mod_flow, tmp_path): """Create a workflow source directory. Args: cfg: Can be passed a config dictionary. Yields: Path to source directory. """ def _inner(cfg): src_dir = _make_src_flow(tmp_path, cfg) return src_dir yield _inner @pytest.fixture def install(test_dir, run_dir): """Install a workflow from source Args: (Actually args for _inner, but what the fixture appears to take to the user) source: Directory containing the source. **kwargs: Options for cylc install. Returns: Workflow id, including run directory. """ async def _inner(source, **kwargs): opts = InstallOpts(**kwargs) # Note we append the source.name to the string rather than creating # a subfolder because the extra layer of directories would exceed # Cylc install's default limit. opts.workflow_name = ( f'{str(test_dir.relative_to(run_dir))}.{source.name}') workflow_id, _ = await cylc_install(opts, str(source)) workflow_id = infer_latest_run_from_id(workflow_id) return workflow_id yield _inner @pytest.fixture def reflog(): """Integration test version of the --reflog CLI option. This returns a set which captures task triggers. Note, you'll need to call this on the scheduler *after* you have started it. N.B. Trigger order is not stable; using a set ensures that tests check trigger logic rather than binding to specific trigger order which could change in the future, breaking the test. Args: schd: The scheduler to capture triggering information for. flow_nums: If True, the flow numbers of the task being triggered will be added to the end of each entry. Returns: tuple (task, triggers): If flow_nums == False (task, flow_nums, triggers): If flow_nums == True task: The [relative] task ID e.g. "1/a". flow_nums: The serialised flow nums e.g. ["1"]. triggers: Sorted tuple of the trigger IDs, e.g. ("1/a", "2/b"). """ def _reflog(schd: 'Scheduler', flow_nums: bool = False) -> Set[tuple]: submit_task_jobs = schd.submit_task_jobs triggers = set() def _submit_task_jobs(*args, **kwargs): itasks = submit_task_jobs(*args, **kwargs) for itask in itasks: deps = tuple(sorted(itask.state.get_resolved_dependencies())) if flow_nums: triggers.add( ( itask.identity, serialise_set(itask.flow_nums), deps or None, ) ) else: triggers.add((itask.identity, deps or None)) return itasks schd.submit_task_jobs = _submit_task_jobs return triggers return _reflog async def _complete( schd: 'Scheduler', *wait_tokens: Union[Tokens, str], stop_mode=StopMode.AUTO, timeout: int = 60, allow_paused: bool = False, ) -> None: """Wait for the workflow, or tasks within it to complete. Args: schd: The scheduler to await. wait_tokens: If specified, this will wait for the tasks represented by these tokens to be marked as completed by the task pool. Can use relative task ids as strings (e.g. '1/a') rather than tokens for convenience. stop_mode: If tokens_list is not provided, this will wait for the scheduler to be shutdown with the specified mode (default = AUTO, i.e. workflow completed normally). timeout: Max time to wait for the condition to be met. Note, if you need to increase this, you might want to rethink your test. Note, use this timeout rather than wrapping the complete call with async.timeout (handles shutdown logic more cleanly). allow_paused: This function will raise an Exception if the scheduler is paused (because this usually means the specified tasks cannot complete) unless allow_paused==True. Raises: AssertionError: In the event the scheduler shut down or the operation timed out. """ if schd.is_paused and not allow_paused: raise Exception( "You are waiting for a paused scheduler - if this is intended " "then use `complete(..., allow_paused=True)`" ) start_time = time() tokens_list: List[Tokens] = [] for tokens in wait_tokens: if isinstance(tokens, str): tokens = Tokens(tokens, relative=True) tokens_list.append(tokens.task) # capture task completion remove_if_complete = schd.pool.remove_if_complete def _remove_if_complete(itask, output=None): ret = remove_if_complete(itask) if ret and itask.tokens.task in tokens_list: tokens_list.remove(itask.tokens.task) return ret # capture workflow shutdown request set_stop = schd._set_stop stop_requested = False def _set_stop(mode=None): nonlocal stop_requested if mode == stop_mode: stop_requested = True return set_stop(mode) else: set_stop(mode) raise Exception(f'Workflow bailed with stop mode = {mode}') # determine the completion condition def done(): if wait_tokens: if not tokens_list: return True if not schd.contact_data: raise AssertionError( "Scheduler shut down before tasks completed: " + ", ".join(map(str, tokens_list)) ) return False # otherwise wait for the scheduler to shut down return stop_requested or not schd.contact_data with pytest.MonkeyPatch.context() as mp: mp.setattr(schd.pool, 'remove_if_complete', _remove_if_complete) mp.setattr(schd, '_set_stop', _set_stop) # wait for the condition to be met while not done(): # allow the main loop to advance await asyncio.sleep(0) if (time() - start_time) > timeout: msg = "Timeout waiting for " if wait_tokens: msg += ", ".join(map(str, tokens_list)) else: msg += "workflow to shut down" raise AssertionError(msg) @pytest.fixture def complete(): return _complete @pytest.fixture(scope='module') def mod_complete(): return _complete @pytest.fixture def reftest(run, reflog, complete): """Fixture that runs a simple reftest. Combines the `reflog` and `complete` fixtures. """ async def _reftest( schd: 'Scheduler', flow_nums: bool = False, ) -> Set[tuple]: async with run(schd): triggers = reflog(schd, flow_nums) await complete(schd) return triggers return _reftest @pytest.fixture def cylc_show(): """Fixture that runs `cylc show` on a scheduler, returning JSON object.""" async def _cylc_show(schd: 'Scheduler', *task_ids: str) -> dict: pclient = WorkflowRuntimeClient(schd.workflow) await schd.update_data_structure() json_filter: dict = {} await prereqs_and_outputs_query( schd.id, [Tokens(id_, relative=True) for id_ in task_ids], pclient, ShowOptions(json=True), json_filter, ) return json_filter return _cylc_show @pytest.fixture def capture_live_submissions(capcall, monkeypatch): """Capture live submission attempts. This prevents real jobs from being submitted to the system. If you call this fixture from a test, it will return a set of tasks that would have been submitted had this fixture not been used. """ def fake_submit(self, itasks, *_): self.submit_nonlive_task_jobs(itasks, RunMode.SIMULATION) for itask in itasks: for status in (TASK_STATUS_SUBMITTED, TASK_STATUS_SUCCEEDED): self.task_events_mgr.process_message( itask, 'INFO', status, '2000-01-01T00:00:00Z', '(received)', ) return itasks # suppress and capture live submissions submit_live_calls = capcall( 'cylc.flow.task_job_mgr.TaskJobManager.submit_livelike_task_jobs', fake_submit) def get_submissions(): return { itask.identity for ((_self, itasks, *_), _kwargs) in submit_live_calls for itask in itasks } return get_submissions cylc-flow-8.7.0/tests/integration/test_workflow_events.py0000664000175000017500000002534615264420307024162 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio from types import MethodType import pytest from cylc.flow.scheduler import SchedulerError from cylc.flow.workflow_events import WorkflowEventHandler EVENTS = ( 'startup', 'shutdown', 'abort', 'workflow timeout', 'stall', 'stall timeout', 'inactivity timeout', 'restart timeout', ) @pytest.fixture async def test_scheduler(flow, scheduler, capcall): events = capcall( 'cylc.flow.scheduler.Scheduler.run_event_handlers', ) def get_events(): return {e[0][1] for e in events} def _schd(event_config=None, config=None, **opts): assert not (event_config and config) if not config: config = { 'scheduler': { 'events': { 'mail events': ', '.join(EVENTS), **(event_config or {}), }, }, 'scheduling': { 'graph': { 'R1': 'a' } }, } id_ = flow(config) schd = scheduler(id_, **opts) schd.get_events = get_events return schd return _schd async def test_startup_and_shutdown(test_scheduler, run): """Test the startup and shutdown events. * "startup" should fire every time a scheduler is started. * "shutdown" should fire every time a scheduler does a controlled exit. (i.e. excluding aborts on unexpected internal errors). """ schd = test_scheduler() async with run(schd): # NOTE: the "startup" event is only yielded with "run" not "start" pass assert schd.get_events() == {'startup', 'shutdown'} async def test_workflow_timeout(test_scheduler, run): """Test the workflow timeout. This counts down from scheduler start. """ schd = test_scheduler({'workflow timeout': 'PT0S'}) async with asyncio.timeout(4): async with run(schd): await asyncio.sleep(0.1) assert schd.get_events() == {'startup', 'workflow timeout', 'shutdown'} async def test_inactivity_timeout(test_scheduler, start): """Test the inactivity timeout. This counts down from things like state changes. """ schd = test_scheduler({ 'inactivity timeout': 'PT0S', 'abort on inactivity timeout': 'True', }) async with asyncio.timeout(4): with pytest.raises(SchedulerError): async with start(schd): await asyncio.sleep(0) await schd._main_loop() assert schd.get_events() == {'inactivity timeout', 'shutdown'} async def test_abort(test_scheduler, run): """Test abort. This should fire when uncaught internal exceptions are raised. Note, this is orthogonal to shutdown (i.e. a scheduler either shuts down or aborts, not both). Note, this is orthogonal to the "abort on " configurations. """ schd = test_scheduler() # get the main-loop to raise an exception def killer(): raise Exception(':(') schd._main_loop = killer # start the scheduler and wait for it to hit the exception with pytest.raises(Exception): async with run(schd): for _ in range(10): # allow initialisation to complete await asyncio.sleep(0.1) # the abort event should be called # note, "abort" and "shutdown" are orthogonal assert schd.get_events() == {'startup', 'abort'} async def test_stall(test_scheduler, start): """Test the stall event. This should fire when the scheduler enters the stalled state. """ schd = test_scheduler() async with start(schd): # set the failed output schd.pool.spawn_on_output( schd.pool.get_tasks()[0], 'failed' ) # set the failed status schd.pool.get_tasks()[0].state_reset('failed') # check for workflow stall condition schd.is_paused = False schd.check_workflow_stalled() assert schd.get_events() == {'shutdown', 'stall'} async def test_restart_timeout_workflow_completion( test_scheduler, scheduler, run, complete, ): """Test restart timeout for completed workflows. This should fire when a completed workflow is restarted. """ schd = test_scheduler({'restart timeout': 'PT0S'}, paused_start=False) # run to completion async with run(schd): await complete(schd) assert schd.get_events() == {'startup', 'shutdown'} # restart schd2 = scheduler(schd.workflow) schd2.get_events = schd.get_events async with run(schd2): await asyncio.sleep(0.1) assert schd2.get_events() == {'startup', 'restart timeout', 'shutdown'} async def test_restart_timeout_workflow_stop_after_cycle_point( test_scheduler, scheduler, run, complete, ): """Test restart timeout with the "stop after cycle point" config. This should fire when a completed workflow is restarted. """ schd = test_scheduler( config={ 'scheduler': { 'cycle point format': 'CCYY', 'events': {'restart timeout': 'PT0S'}, }, 'scheduling': { 'initial cycle point': '2000', 'stop after cycle point': '2000', 'graph': { 'P1Y': 'foo[-P1Y] => foo', }, }, }, paused_start=False, ) # run to completion async with run(schd): await complete(schd) assert schd.get_events() == {'startup', 'shutdown'} # restart schd2 = scheduler(schd.workflow) schd2.get_events = schd.get_events async with run(schd2): await asyncio.sleep(0.1) assert schd2.get_events() == {'startup', 'restart timeout', 'shutdown'} async def test_shutdown_handler_timeout_kill( test_scheduler, run, monkeypatch, mock_glbl_cfg, caplog ): """Test shutdown handlers get killed on the process pool timeout. Has to be done differently as the process pool is closed during shutdown. See GitHub #6639 """ def mock_run_event_handlers(self, event, reason=""): """To replace scheduler.run_event_handlers(...). Run workflow event handlers even in simulation mode. """ self.workflow_event_handler.handle(self, event, str(reason)) # Configure a long-running shutdown handler. schd = test_scheduler({ 'shutdown handlers': 'sleep 10; echo', 'mail events': '', }) # Set a low process pool timeout value. mock_glbl_cfg( 'cylc.flow.subprocpool.glbl_cfg', ''' [scheduler] process pool timeout = PT1S ''' ) async with asyncio.timeout(30): async with run(schd): # Replace a scheduler method, to call handlers in simulation mode. monkeypatch.setattr( schd, 'run_event_handlers', MethodType(mock_run_event_handlers, schd), ) await asyncio.sleep(0.1) assert ( "[('workflow-event-handler-00', 'shutdown') err] killed on " "timeout (PT1S)" ) in caplog.text TEMPLATES = [ # perfectly valid pytest.param('%(workflow)s', id='good'), # no template variable of that name pytest.param('%(no_such_variable)s', id='bad'), # missing the 's' pytest.param('%(broken_syntax)', id='ugly'), ] @pytest.mark.parametrize('template', TEMPLATES) async def test_mail_footer_template( mod_one, # use the same scheduler for each test start, mock_glbl_cfg, log_filter, capcall, template, ): """It should handle templating issues with the mail footer.""" # prevent emails from being sent mail_calls = capcall( 'cylc.flow.workflow_events.WorkflowEventHandler._send_mail' ) # configure Cylc to send an email on startup with the configured footer mock_glbl_cfg( 'cylc.flow.workflow_events.glbl_cfg', f''' [scheduler] [[mail]] footer = 'footer={template}' [[events]] mail events = startup ''', ) # start the workflow and get it to send an email async with start(mod_one) as one_log: one_log.clear() # clear previous log messages mod_one.workflow_event_handler.handle( mod_one, WorkflowEventHandler.EVENT_STARTUP, 'event message' ) # warnings should appear only when the template is invalid should_log = 'workflow' not in template # check that template issues are handled correctly assert bool(log_filter( contains='Ignoring bad mail footer template', )) == should_log assert bool(log_filter( contains=template, )) == should_log # check that the mail is sent even if there are issues with the footer assert len(mail_calls) == 1 @pytest.mark.parametrize('template', TEMPLATES) async def test_custom_event_handler_template( mod_one, # use the same scheduler for each test start, mock_glbl_cfg, log_filter, template, ): """It should handle templating issues with custom event handlers.""" # configure Cylc to send an email on startup with the configured footer mock_glbl_cfg( 'cylc.flow.workflow_events.glbl_cfg', f''' [scheduler] [[events]] startup handlers = echo "{template}" ''' ) # start the workflow and get it to send an email async with start(mod_one) as one_log: one_log.clear() # clear previous log messages mod_one.workflow_event_handler.handle( mod_one, WorkflowEventHandler.EVENT_STARTUP, 'event message' ) # warnings should appear only when the template is invalid should_log = 'workflow' not in template # check that template issues are handled correctly assert bool(log_filter( contains='bad template', )) == should_log assert bool(log_filter( contains=template, )) == should_log cylc-flow-8.7.0/tests/integration/tui/0000775000175000017500000000000015264420307020102 5ustar alastairalastaircylc-flow-8.7.0/tests/integration/tui/test_mutations.py0000664000175000017500000002130415264420307023536 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio import pytest from cylc.flow.exceptions import ClientError async def process_command(schd, tries=10, interval=0.1): """Wait for command(s) to be queued and run. Waits for at least one command to be queued and for all queued commands to be run. """ # wait for the command to be queued for _try in range(tries): await asyncio.sleep(interval) if not schd.command_queue.empty(): break else: raise Exception(f'No command was queued after {tries * interval}s') # run the command await schd.process_command_queue() # push out updates await schd.update_data_structure() # make sure it ran assert schd.command_queue.empty(), 'command queue has not emptied' async def test_online_mutation( one_conf, flow, scheduler, start, rakiura, monkeypatch, log_filter, ): """Test a simple workflow with one task.""" id_ = flow(one_conf, name='one') schd = scheduler(id_) with rakiura(size='80,15') as rk: async with start(schd): await schd.update_data_structure() assert schd.command_queue.empty() # open the workflow rk.force_update() rk.user_input('down', 'right') rk.wait_until_loaded(schd.tokens.id) # focus on a task rk.user_input('down', 'right', 'down', 'right') rk.compare_screenshot( # take a screenshot to ensure we have focused on the task # successfully 'task-selected', 'the cursor should be on the task 1/foo', ) # focus on the hold mutation for a task rk.user_input('enter', 'down') rk.compare_screenshot( # take a screenshot to ensure we have focused on the mutation # successfully 'hold-mutation-selected', 'the cursor should be on the "hold" mutation', ) # run the hold mutation rk.user_input('enter') # the mutation should be in the scheduler's command_queue await asyncio.sleep(0) assert log_filter(contains="hold(tasks=['1/one'])") # close the dialogue and re-run the hold mutation rk.user_input('q', 'q', 'enter') rk.compare_screenshot( 'command-failed-workflow-stopped', 'an error should be visible explaining that the operation' ' cannot be performed on a stopped workflow', # NOTE: don't update so Tui still thinks the workflow is running force_update=False, ) # force mutations to raise ClientError def _get_client(*args, **kwargs): raise ClientError('mock error') monkeypatch.setattr( 'cylc.flow.tui.data.get_client', _get_client, ) # close the dialogue and re-run the hold mutation rk.user_input('q', 'q', 'enter') rk.compare_screenshot( 'command-failed-client-error', 'an error should be visible explaining that the operation' ' failed due to a client error', # NOTE: don't update so Tui still thinks the workflow is running force_update=False, ) @pytest.fixture def standardise_cli_cmds(monkeypatch): """This remove the variable bit of the workflow ID from CLI commands. The workflow ID changes from run to run. In order to make screenshots stable, this """ from cylc.flow.tui.data import extract_context def _extract_context(selection): context = extract_context(selection) if 'workflow' in context: context['workflow'] = [ workflow.rsplit('/', 1)[-1] for workflow in context.get('workflow', []) ] return context monkeypatch.setattr( 'cylc.flow.tui.data.extract_context', _extract_context, ) @pytest.fixture def capture_commands(monkeypatch): ret = [] returncode = [0] class _Popen: def __init__(self, *args, **kwargs): ret.append(args) def communicate(self): return 'mock-stdout', 'mock-stderr' @property def returncode(self): return returncode[0] monkeypatch.setattr( 'cylc.flow.tui.data.Popen', _Popen, ) return ret, returncode async def test_offline_mutation( one_conf, flow, rakiura, capture_commands, standardise_cli_cmds, ): flow(one_conf, name='one') commands, returncode = capture_commands with rakiura(size='80,15') as rk: # run the stop-all mutation rk.wait_until_loaded('root') rk.user_input('enter', 'down') rk.compare_screenshot( # take a screenshot to ensure we have focused on the task # successfully 'stop-all-mutation-selected', 'the stop-all mutation should be selected', ) rk.user_input('enter') # the command "cylc stop '*'" should have been run assert commands == [(['cylc', 'stop', '*'],)] commands.clear() # run the clean command on the workflow rk.user_input('down', 'enter', 'down') rk.compare_screenshot( # take a screenshot to ensure we have focused on the mutation # successfully 'clean-mutation-selected', 'the clean mutation should be selected', ) rk.user_input('enter') # the command "cylc clean " should have been run assert commands == [(['cylc', 'clean', '--yes', 'one'],)] commands.clear() # make commands fail returncode[:] = [1] rk.user_input('enter', 'down') rk.compare_screenshot( # take a screenshot to ensure we have focused on the mutation # successfully 'clean-mutation-selected', 'the clean mutation should be selected', ) rk.user_input('enter') assert commands == [(['cylc', 'clean', '--yes', 'one'],)] rk.compare_screenshot( # take a screenshot to ensure we have focused on the mutation # successfully 'clean-command-error', 'there should be a box displaying the error containing the stderr' ' returned by the command', ) async def test_set_mutation( flow, scheduler, start, rakiura, ): id_ = flow({ 'scheduling': { 'graph': { 'R1': 'a => z' }, }, }, name='one') schd = scheduler(id_) async with start(schd): await schd.update_data_structure() with rakiura(schd.tokens.id, size='80,15') as rk: # open the context menu on 1/a rk.user_input('down', 'down', 'down', 'enter') rk.force_update() # select the "set" mutation rk.user_input(*(('down',) * 7)) # 7th command down rk.compare_screenshot( # take a screenshot to ensure we have focused on the mutation # successfully 'set-command-selected', 'The command menu should be open for the task 1/a with the' ' set command selected.' ) # issue the "set" mutation rk.user_input('enter') # wait for the command to be received and run it await process_command(schd) await schd._main_loop() # close the error dialogue # NOTE: This hides an asyncio error that does not occur outside of # the tests rk.user_input('q', 'q', 'q') rk.compare_screenshot( # take a screenshot to ensure we have focused on the mutation # successfully 'task-state-updated', '1/a should now show as succeeded,' ' there should be no associated job.' ) cylc-flow-8.7.0/tests/integration/tui/__init__.py0000664000175000017500000000000015264420307022201 0ustar alastairalastaircylc-flow-8.7.0/tests/integration/tui/conftest.py0000664000175000017500000002311515264420307022303 0ustar alastairalastairfrom contextlib import contextmanager from difflib import unified_diff import os from pathlib import Path import re from time import sleep from secrets import token_hex import pytest from urwid.display import html_fragment from cylc.flow.id import Tokens from cylc.flow.tui.app import TuiApp from cylc.flow.tui.overlay import _get_display_id SCREENSHOT_DIR = Path(__file__).parent / 'screenshots' def configure_screenshot(v_term_size): """Configure Urwid HTML screenshots.""" screen = html_fragment.HtmlGenerator() screen.set_terminal_properties(256) screen.register_palette(TuiApp.palette) html_fragment.screenshot_init( [tuple(map(int, v_term_size.split(',')))], [] ) return screen, html_fragment def format_test_failure(expected, got, description): """Return HTML to represent a screenshot test failure. Args: expected: HTML fragment for the expected screenshot. got: HTML fragment for the test screenshot. description: Test description. """ diff = '\n'.join(unified_diff( expected.splitlines(), got.splitlines(), fromfile='expected', tofile='got', )) return f'''

{description}

Expected Got
{expected} {got}

Diff:

{diff}
''' class RakiuraSession: """Convenience class for accessing Rakiura functionality.""" def __init__(self, app, html_fragment, test_dir, test_name): self.app = app self.html_fragment = html_fragment self.test_dir = test_dir self.test_name = test_name def user_input(self, *keys): """Simulate a user pressing keys. Each "key" is a keyboard button e.g. "x" or "enter". If you provide more than one key, each one will be pressed, one after another. You can combine keys in a single string, e.g. "ctrl d". """ return self.app.loop.process_input(keys) def compare_screenshot( self, name, description, retries=3, delay=0.1, force_update=True, ): """Take a screenshot and compare it to one taken previously. To update the screenshot, set the environment variable "CYLC_UPDATE_SCREENSHOTS" to "true". Note, if the comparison fails, "force_update" is called and the test is retried. Arguments: name: The name to use for the screenshot, this is used in the filename for the generated HTML fragment. description: A description of the test to be used on test failure. retries: The maximum number of retries for this test before failing. delay: The delay between retries. This helps overcome timing issues with data provision. Raises: Exception: If the screenshot does not match the reference. """ filename = SCREENSHOT_DIR / f'{self.test_name}.{name}.html' exc = None for _try in range(retries): # load the expected result expected = '' if filename.exists(): with open(filename, 'r') as expected_file: expected = expected_file.read() # update to pick up latest data if force_update: self.force_update() # force urwid to draw the screen # (main loop isn't running so this doesn't happen automatically) self.app.loop.draw_screen() # take a screenshot screenshot = self.html_fragment.screenshot_collect()[-1] try: if expected != screenshot: # screenshot does not match # => write an html file with the visual diff out = self.test_dir / filename.name with open(out, 'w+') as out_file: out_file.write( format_test_failure( expected, screenshot, description, ) ) raise Exception( 'Screenshot differs:' '\n* Set "CYLC_UPDATE_SCREENSHOTS=true" to update' f'\n* To debug see: file:////{out}' ) break except Exception as exc_: exc = exc_ # wait a while to allow the updater to do its job sleep(delay) else: if os.environ.get('CYLC_UPDATE_SCREENSHOTS', '').lower() == 'true': with open(filename, 'w+') as expected_file: expected_file.write(screenshot) else: raise exc def force_update(self): """Run Tui's update method. This is done automatically by compare_screenshot but you may want to call it in a test, e.g. before pressing navigation keys. With Rakiura, the Tui event loop is not running so the data is never refreshed. You do NOT need to call this method for key presses, but you do need to call this if the data has changed (e.g. if you've changed a task state) OR if you've changed any filters (because filters are handled by the update code). """ # flush any prior updates self.app.get_update() # wait for the next update while not self.app.update(): pass def wait_until_loaded(self, *ids, retries=20): """Wait until the given ID appears in the Tui tree, then expand them. Useful for waiting whilst Tui loads a workflow. Note, this is a blocking wait with no timeout! """ exc = None try: ids = self.app.wait_until_loaded(*ids, retries=retries) except Exception as _exc: exc = _exc if ids: msg = ( 'Requested nodes did not appear in Tui after' f' {retries} retries: ' + ', '.join(ids) ) if exc: msg += f'\n{exc}' self.compare_screenshot(f'fail-{token_hex(4)}', msg, 1) @pytest.fixture def rakiura(test_dir, request, monkeypatch): """Visual regression test framework for Urwid apps. Like Cypress but for Tui so named after a NZ island with lots of Tuis. When called this yields a RakiuraSession object loaded with test utilities. All tests have default retries to avoid flaky tests. Similar to the "start" fixture, which starts a Scheduler without running the main loop, rakiura starts Tui without running the main loop. Arguments: workflow_id: The "WORKFLOW" argument of the "cylc tui" command line. size: The virtual terminal size for screenshots as a comma separated string e.g. "80,50" for 80 cols wide by 50 rows tall. Returns: A RakiuraSession context manager which provides useful utilities for testing. """ return _rakiura(test_dir, request, monkeypatch) @pytest.fixture def mod_rakiura(test_dir, request, monkeypatch): """Same as rakiura but configured to view module-scoped workflows. Note: This is *not* a module-scoped fixture (no need, creating Tui sessions is not especially slow), it is configured to display module-scoped "scheduler" fixtures (which may be more expensive to create/destroy). """ return _rakiura(test_dir.parent, request, monkeypatch) def _rakiura(test_dir, request, monkeypatch): # the user name and the prefix of workflow IDs are both variable # so we patch the render functions to make test output stable def get_display_id(id_): tokens = Tokens(id_) return _get_display_id( tokens.duplicate( user='cylc', workflow=tokens.get('workflow', '').rsplit('/', 1)[-1], ).id ) monkeypatch.setattr('cylc.flow.tui.util.ME', 'cylc') monkeypatch.setattr('cylc.flow.tui.app.CYLC_VERSION', '1.2.3') monkeypatch.setattr( 'cylc.flow.tui.util._display_workflow_id', lambda data: data['name'].rsplit('/', 1)[-1] ) monkeypatch.setattr( 'cylc.flow.tui.overlay._get_display_id', get_display_id, ) # standardise environment for tests monkeypatch.setenv('EDITOR', 'nvim') monkeypatch.setenv('GEDITOR', 'gvim -f') monkeypatch.setenv('PAGER', 'less') # filter Tui so that only workflows created within our test show up id_base = str(test_dir.relative_to(Path("~/cylc-run").expanduser())) workflow_filter = re.escape(id_base) + r'/.*' @contextmanager def _rakiura(workflow_id=None, size='80,50'): screen, html_fragment = configure_screenshot(size) app = TuiApp(screen=screen) with app.main( workflow_id, id_filter=workflow_filter, interactive=False, _is_tests=True, ): yield RakiuraSession( app, html_fragment, test_dir, request.function.__name__, ) return _rakiura cylc-flow-8.7.0/tests/integration/tui/screenshots/0000775000175000017500000000000015264420307022442 5ustar alastairalastaircylc-flow-8.7.0/tests/integration/tui/screenshots/test_states.workflow-context--paused.html0000664000175000017500000002430615264420307032566 0ustar alastairalastair
Cylc Tui 1.2.3 ┌────────────────────────────────────────────────┐               
               │  id: ~cylc/one                                 │               
- ~cylc        │  paused                                        │               
   - one - paus│                                                │               
      - ̎○ 1    │  Action                                        │               
           ̎● a │  < (cancel)                                 >  │               
           ̿○ b │                                                │               
           ̀○ c │  < clean                                    >  │               
               │  < log                                      >  │               
               │  < pause                                    >  │               
               │  < play                                     >  │               
               │  < reinstall-reload                         >  │               
               │                                                │               
quit: q  help: │ q to close                                     │↥ ↧ Home End   
filter tasks: T└────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_errors.open-error.html0000664000175000017500000002703715264420307030003 0ustar alastairalastair
┌──────────────────────────────────────────────────────────────────────────────┐
│  Error: Something went wrong :(                                              │
│                                                                              │
│  < Select File                                                            >  │
│  Open in:  < nvim >< gvim -f >< less >                                       │
│  (Configure apps to open logs via $EDITOR, $GEDITOR, $PAGER)                 │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│ q to close                                                                   │
└──────────────────────────────────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_offline_mutation.clean-command-error.html0000664000175000017500000002413715264420307033564 0ustar alastairalastair
Cylc Tui 1.2.3 ┌────┌────────────────────────────────────────────────┐          
               │  id│  Error                                         │          
- ~cylc        │  st│                                                │          
   + one - stop│    │  Error in command cylc clean --yes one         │          
               │  Ac│  mock-stderr                                   │          
               │  < │                                                │          
               │    │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │    │                                                │          
               │    │                                                │          
quit: q  help: │ q t│ q to close                                     │ome End   
filter tasks: T└────└────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_set_mutation.set-command-selected.html0000664000175000017500000002447015264420307033105 0ustar alastairalastair
Cylc Tui 1.2.3 ┌────────────────────────────────────────────────┐               
               │                                                │               
- ~cylc        │  Action                                        │               
   - one - paus│  < (cancel)                                 >  │               
      - ̿○ 1    │                                                │               
           ̿○ a │  < hold                                     >  │               
           ○ z │  < kill                                     >  │               
               │  < log                                      >  │               
               │  < poll                                     >  │               
               │  < release                                  >  │               
               │  < remove                                   >  │               
               │  < set                                      >  │               
               │                                                │               
quit: q  help: │ q to close                                     │↥ ↧ Home End   
filter tasks: T└────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-help.html0000664000175000017500000004520615264420307032063 0ustar alastairalastair
Cylc Tui 1┌──────────────────────────────────────────────────────────┐          
          │                                                          │          
- ~cylc   │                     _        _         _                 │          
          │                    | |      | |       (_)                │          
          │           ___ _   _| | ___  | |_ _   _ _                 │          
          │          / __| | | | |/ __| | __| | | | |                │          
          │         | (__| |_| | | (__  | |_| |_| | |                │          
          │          \___|\__, |_|\___|  \__|\__,_|_|                │          
          │                __/ |                                     │          
          │               |___/                                      │          
          │                                                          │          
          │            ( scroll using arrow keys )                   │          
          │                                                          │          
          │                                                          │          
          │                                                          │          
          │                             _,@@@@@@.                    │          
          │                           <=@@@, `@@@@@.                 │          
          │                              `-@@@@@@@@@@@'              │          
          │                                 :@@@@@@@@@@.             │          
          │                                (.@@@@@@@@@@@             │          
          │                               ( '@@@@@@@@@@@@.           │          
          │                              ;.@@@@@@@@@@@@@@@           │          
          │                            '@@@@@@@@@@@@@@@@@@,          │          
          │                          ,@@@@@@@@@@@@@@@@@@@@'          │          
          │                        :.@@@@@@@@@@@@@@@@@@@@@.          │          
          │                      .@@@@@@@@@@@@@@@@@@@@@@@@.          │          
          │                    '@@@@@@@@@@@@@@@@@@@@@@@@@.           │          
          │                  ;@@@@@@@@@@@@@@@@@@@@@@@@@@@            │          
          │                 .@@@@@@@@@@@@@@@@@@@@@@@@@@.             │          
          │                .@@@@@@@@@@@@@@@@@@@@@@@@@@,              │          
          │               .@@@@@@@@@@@@@@@@@@@@@@@@@'                │          
          │              .@@@@@@@@@@@@@@@@@@@@@@@@'     ,            │          
          │            :@@@@@@@@@@@@@@@@@@@@@..''';,,,;::-           │          
          │           '@@@@@@@@@@@@@@@@@@@.        `.   `            │          
          │          .@@@@@@.: ,.@@@@@@@.            `               │          
          │        :@@@@@@@,         ;.@,                            │          
          │       '@@@@@@.              `@'                          │          
          │                                                          │          
quit: q  h│ q to close                                               │ome End   
filter tas└──────────────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_task_states.filter-waiting-or-expired.html0000664000175000017500000002311115264420307033712 0ustar alastairalastair
Cylc Tui 1.2.3   tasks filtered (T - edit, R - reset)   workflows filtered (W - 
edit, E - reset)                                                                
                                                                                
- ~cylc                                                                         
   - test_task_states - paused 1■ 1■ 1■ 1■ 1■                                   
      - ̿⊗ 1                                                                     
         - ̿⊗ Y                                                                  
            - ◌ Y1                                                              
                 ◌ c                                                            
      - ̀○ 3                                                                     
         - ̀○ X                                                                  
              ̀○ a                                                               
         - ̀○ Y                                                                  
            - ̀○ Y1                                                              
                 ̀○ c                                                            
              ̀○ b                                                               
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html0000664000175000017500000002146415264420307034445 0ustar alastairalastair
Cylc Tui 1.2.3 ┌────────────────────────────────────────────────┐               
               │  id: ~cylc/one                                 │               
- ~cylc        │  stopped                                       │               
   + one - stop│                                                │               
               │  Action                                        │               
               │  < (cancel)                                 >  │               
               │                                                │               
               │  < clean                                    >  │               
               │  < log                                      >  │               
               │  < play                                     >  │               
               │  < reinstall-reload                         >  │               
               │                                                │               
               │                                                │               
quit: q  help: │ q to close                                     │↥ ↧ Home End   
filter tasks: T└────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html0000664000175000017500000003553715264420307033066 0ustar alastairalastair
┌──────────────────────────────────────────────────────────────────────────────┐
│  Host: myhost                                                                │
│  Path: mypath                                                                │
│  < Select File                                                            >  │
│  Open in:  < nvim >< gvim -f >< less >                                       │
│  (Configure apps to open logs via $EDITOR, $GEDITOR, $PAGER)                 │
│                                                                              │
│  this is the scheduler log file                                              │
│  line 2                                                                      │
│  line 3                                                                      │
│  line 4                                                                      │
│  line 5                                                                      │
│  line 6                                                                      │
│  line 7             ┌──────────────────────────────────────┐                 │
│  line 8             │  Select File                         │                 │
│  line 9             │                                      │                 │
│  line 10            │  < config/01-start-01.cylc        >  │                 │
│  line 11            │  < config/flow-processed.cylc     >  │                 │
│  line 12            │  < scheduler/01-start-01.log      >  │                 │
│  line 13            │                                      │                 │
│  line 14            │ q to close                           │                 │
│  line 15            └──────────────────────────────────────┘                 │
│  line 16                                                                     │
│  line 17                                                                     │
│  line 18                                                                     │
│  line 19                                                                     │
│  line 20                                                                     │
│                                                                              │
│ q to close                                                                   │
└──────────────────────────────────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html0000664000175000017500000001122415264420307031764 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   + one - stopping                                                             
   + tre - stopped                                                              
   + two - paused                                                               
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_job_logs.01-job.out.html0000664000175000017500000002703715264420307027773 0ustar alastairalastair
┌──────────────────────────────────────────────────────────────────────────────┐
│  Host: myhost                                                                │
│  Path: mypath                                                                │
│  < Select File                                                            >  │
│  Open in:  < nvim >< gvim -f >< less >                                       │
│  (Configure apps to open logs via $EDITOR, $GEDITOR, $PAGER)                 │
│                                                                              │
│  job: 1/a/01                                                                 │
│  this is a job log                                                           │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│ q to close                                                                   │
└──────────────────────────────────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_online_mutation.command-failed.html0000664000175000017500000002465715264420307032450 0ustar alastairalastair
Cylc Tui   work┌────┌────────────────────────────────────────────────┐          
               │  id│  Error                                         │          
- ~cylc        │    │                                                │          
   - one - paus│  Ac│  Cannot perform command hold on a stopped       │          
      - ̿○ 1    │  < │  workflow                                      │          
           ̿○ on│    │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │    │                                                │          
               │    │                                                │          
quit: q  help: │ q t│ q to close                                     │ome End   
filter tasks: T└────└────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_states.task-context--waiting+runahead.html0000664000175000017500000002430615264420307033622 0ustar alastairalastair
Cylc Tui 1.2.3 ┌────────────────────────────────────────────────┐               
               │  id: 1/c                                       │               
- ~cylc        │  waiting (runahead), flows=None                │               
   - one - paus│                                                │               
      - ̎○ 1    │  Action                                        │               
           ̎● a │  < (cancel)                                 >  │               
           ̿○ b │                                                │               
           ̀○ c │  < hold                                     >  │               
               │  < kill                                     >  │               
               │  < log                                      >  │               
               │  < poll                                     >  │               
               │  < release                                  >  │               
               │                                                │               
quit: q  help: │ q to close                                     │↥ ↧ Home End   
filter tasks: T└────────────────────────────────────────────────┘
././@LongLink0000644000000000000000000000015000000000000011577 Lustar rootrootcylc-flow-8.7.0/tests/integration/tui/screenshots/test_online_mutation.command-failed-client-error.htmlcylc-flow-8.7.0/tests/integration/tui/screenshots/test_online_mutation.command-failed-client-error.h0000664000175000017500000002520615264420307034325 0ustar alastairalastair
Cylc Tui 1.2.3 ┌────┌────────────────────────────────────────────────┐          
               │  id│  Error                                         │          
- ~cylc        │  wa│                                                │          
   - one - paus│    │  Error connecting to workflow: mock error      │          
      - ̿○ 1    │  Ac│                                                │          
           ̿○ on│  < │                                                │          
               │    │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │    │                                                │          
quit: q  help: │ q t│ q to close                                     │ome End   
filter tasks: T└────└────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_states.cycle-context--waiting.html0000664000175000017500000002430615264420307032174 0ustar alastairalastair
Cylc Tui 1.2.3 ┌────────────────────────────────────────────────┐               
               │  id: 1                                         │               
- ~cylc        │  waiting (held, runahead, queued)              │               
   - one - paus│                                                │               
      - ̎○ 1    │  Action                                        │               
           ̎● a │  < (cancel)                                 >  │               
           ̿○ b │                                                │               
           ̀○ c │  < hold                                     >  │               
               │  < kill                                     >  │               
               │  < poll                                     >  │               
               │  < release                                  >  │               
               │  < remove                                   >  │               
               │                                                │               
quit: q  help: │ q to close                                     │↥ ↧ Home End   
filter tasks: T└────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_tui_basics.test-rakiura.html0000664000175000017500000001570415264420307031135 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html0000664000175000017500000002703715264420307031220 0ustar alastairalastair
┌──────────────────────────────────────────────────────────────────────────────┐
│  Host: myhost                                                                │
│  Path: mypath                                                                │
│  < Select File                                                            >  │
│  Open in:  < nvim >< gvim -f >< less >                                       │
│  (Configure apps to open logs via $EDITOR, $GEDITOR, $PAGER)                 │
│                                                                              │
│  job: 1/a/02                                                                 │
│  this is a job error                                                         │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│ q to close                                                                   │
└──────────────────────────────────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_states.on-load.html0000664000175000017500000001241415264420307027224 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - one - paused 1■                                                            
      - ̎○ 1                                                                     
           ̎● a                                                                  
           ̿○ b                                                                  
           ̀○ c                                                                  
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html0000664000175000017500000002703715264420307033053 0ustar alastairalastair
┌──────────────────────────────────────────────────────────────────────────────┐
│  Host: myhost                                                                │
│  Path: mypath                                                                │
│  < Select File                                                            >  │
│  Open in:  < nvim >< gvim -f >< less >                                       │
│  (Configure apps to open logs via $EDITOR, $GEDITOR, $PAGER)                 │
│                                                                              │
│  this is the scheduler log file                                              │
│  line 2                                                                      │
│  line 3                                                                      │
│  line 4                                                                      │
│  line 5                                                                      │
│  line 6                                                                      │
│  line 7                                                                      │
│  line 8                                                                      │
│  line 9                                                                      │
│  line 10                                                                     │
│  line 11                                                                     │
│  line 12                                                                     │
│  line 13                                                                     │
│  line 14                                                                     │
│  line 15                                                                     │
│  line 16                                                                     │
│  line 17                                                                     │
│  line 18                                                                     │
│  line 19                                                                     │
│  line 20                                                                     │
│                                                                              │
│ q to close                                                                   │
└──────────────────────────────────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file-bottom.html0000664000175000017500000002454315264420307034354 0ustar alastairalastair
┌──────────────────────────────────────────────────────────────────────────────┐
│  line 974                                                                    │
│  line 975                                                                    │
│  line 976                                                                    │
│  line 977                                                                    │
│  line 978                                                                    │
│  line 979                                                                    │
│  line 980                                                                    │
│  line 981                                                                    │
│  line 982                                                                    │
│  line 983                                                                    │
│  line 984                                                                    │
│  line 985                                                                    │
│  line 986                                                                    │
│  line 987                                                                    │
│  line 988                                                                    │
│  line 989                                                                    │
│  line 990                                                                    │
│  line 991                                                                    │
│  line 992                                                                    │
│  line 993                                                                    │
│  line 994                                                                    │
│  line 995                                                                    │
│  line 996                                                                    │
│  line 997                                                                    │
│  line 998                                                                    │
│  line 999                                                                    │
│                                                                              │
│ q to close                                                                   │
└──────────────────────────────────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html0000664000175000017500000002047715264420307033637 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - one - paused                                                               
      - ̿○ 1                                                                     
         + ̿○ A                                                                  
         - ̿○ B                                                                  
            - ̿○ B1                                                              
                 ̿○ b11                                                          
                 ̿○ b12                                                          
            - ̿○ B2                                                              
                 ̿○ b21                                                          
                 ̿○ b22                                                          
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
././@LongLink0000644000000000000000000000015400000000000011603 Lustar rootrootcylc-flow-8.7.0/tests/integration/tui/screenshots/test_online_mutation.command-failed-workflow-stopped.htmlcylc-flow-8.7.0/tests/integration/tui/screenshots/test_online_mutation.command-failed-workflow-stopp0000664000175000017500000002520615264420307034507 0ustar alastairalastair
Cylc Tui 1.2.3 ┌────┌────────────────────────────────────────────────┐          
               │  id│  Error                                         │          
- ~cylc        │  wa│                                                │          
   - one - paus│    │  Cannot perform command hold on a stopped      │          
      - ̿○ 1    │  Ac│  workflow                                      │          
           ̿○ on│  < │                                                │          
               │    │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │    │                                                │          
quit: q  help: │ q t│ q to close                                     │ome End   
filter tasks: T└────└────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html0000664000175000017500000000775215264420307032577 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   + tre - stopped                                                              
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
././@LongLink0000644000000000000000000000014700000000000011605 Lustar rootrootcylc-flow-8.7.0/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.htmlcylc-flow-8.7.0/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.ht0000664000175000017500000002703715264420307034504 0ustar alastairalastair
┌──────────────────────────────────────────────────────────────────────────────┐
│  Host: myhost                                                                │
│  Path: mypath                                                                │
│  < Select File                                                            >  │
│  Open in:  < nvim >< gvim -f >< less >                                       │
│  (Configure apps to open logs via $EDITOR, $GEDITOR, $PAGER)                 │
│                                                                              │
│  [scheduling]                                                                │
│      [[graph]]                                                               │
│          R1 = a                                                              │
│  [runtime]                                                                   │
│      [[a]]                                                                   │
│      [[root]]                                                                │
│          [[[simulation]]]                                                    │
│              default run length = PT0S                                       │
│  [scheduler]                                                                 │
│      allow implicit tasks = True                                             │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│ q to close                                                                   │
└──────────────────────────────────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_task_states.filter-not-waiting.html0000664000175000017500000002466515264420307032453 0ustar alastairalastair
Cylc Tui 1.2.3   tasks filtered (T - edit, R - reset)   workflows filtered (W - 
edit, E - reset)                                                                
                                                                                
- ~cylc                                                                         
   - test_task_states - paused 1■ 1■ 1■ 1■ 1■                                   
      - ̿⊗ 1                                                                     
         - ̿● X                                                                  
              ̿● a                                                               
         - ̿⊗ Y                                                                  
            - ◌ Y1                                                              
                 ◌ c                                                            
              ̿⊗ b                                                               
      - ̎⊘ 2                                                                     
         - ̿⊙ X                                                                  
              ̿⊙ a                                                               
         - ̎⊘ Y                                                                  
            - ̎⊘ Y1                                                              
                 ̎⊘ c                                                            
              ̎⊙ b                                                               
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-enter.html0000664000175000017500000003314615264420307032250 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
               ┌────────────────────────────────────────────────┐               
               │  id: ~cylc/root                                │               
               │                                                │               
               │                                                │               
               │  Action                                        │               
               │  < (cancel)                                 >  │               
               │                                                │               
               │  < stop-all                                 >  │               
               │                                                │               
               │                                                │               
               │                                                │               
               │                                                │               
               │                                                │               
               │                                                │               
               │                                                │               
               │                                                │               
               │                                                │               
               │                                                │               
               │ q to close                                     │               
               └────────────────────────────────────────────────┘               
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_show.fail.html0000664000175000017500000005625515264420307026276 0ustar alastairalastair
Cylc Tui 1.2.3   wor┌────────────────────────────────────────────────┐          
                    │  Error                                         │          
- ~cylc             │                                                │          
   - one - paused   │  :(                                            │          
      - ̿○ 1         │                                                │          
           ̿○ foo    │                                                │          
                    │                                                │          
                    │                                                │          
                    │                                                │          
                    │                                                │          
               ┌────│                                                │          
               │  id│                                                │          
               │  wa│                                                │          
               │    │                                                │          
               │  Ac│                                                │          
               │  < │                                                │          
               │    │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │  < │                                                │          
               │    │                                                │          
               │    │                                                │          
               │ q t│                                                │          
               └────│                                                │          
                    │                                                │          
                    │                                                │          
                    │                                                │          
                    │                                                │          
                    │                                                │          
                    │                                                │          
                    │                                                │          
                    │                                                │          
quit: q  help: h  co│ q to close                                     │ome End   
filter tasks: T f s └────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_states.task-context--waiting+queued.html0000664000175000017500000002430615264420307033323 0ustar alastairalastair
Cylc Tui 1.2.3 ┌────────────────────────────────────────────────┐               
               │  id: 1/b                                       │               
- ~cylc        │  waiting (queued), flows=1,2                   │               
   - one - paus│                                                │               
      - ̎○ 1    │  Action                                        │               
           ̎● a │  < (cancel)                                 >  │               
           ̿○ b │                                                │               
           ̀○ c │  < hold                                     >  │               
               │  < kill                                     >  │               
               │  < log                                      >  │               
               │  < poll                                     >  │               
               │  < release                                  >  │               
               │                                                │               
quit: q  help: │ q to close                                     │↥ ↧ Home End   
filter tasks: T└────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html0000664000175000017500000001145715264420307033607 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - one - stopped                                                              
        Workflow is not running                                                 
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_errors.list-error.html0000664000175000017500000004763615264420307030024 0ustar alastairalastair
┌────────────────┌────────────────────────────────────────────────┐────────────┐
│  Error: Somethi│  Error                                         │            │
│                │                                                │            │
│  < Select File │  Something went wrong :(                       │         >  │
│  Open in:  < nv│                                                │            │
│  (Configure app│                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│                │                                                │            │
│ q to close     │ q to close                                     │            │
└────────────────└────────────────────────────────────────────────┘────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html0000664000175000017500000001047715264420307034472 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   + tre - stopped                                                              
   + two - paused                                                               
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_task_modifiers.task-context-menu.html0000664000175000017500000004070215264420307032762 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - test_task_modifiers - paused                                               
      - ̎○ 1                                                                     
           ̎○ al┌────────────────────────────────────────────────┐               
           ̎○ is│  id: 1/all                                     │               
           ̿○ is│  waiting (held, runahead, queued, retry        │               
           ⃗○ is│  scheduled, wallclock, xtriggered)             │               
           ̀○ is│                                                │               
           ፝○ is│  Action                                        │               
           ᷉○ is│  < (cancel)                                 >  │               
               │                                                │               
               │  < hold                                     >  │               
               │  < kill                                     >  │               
               │  < log                                      >  │               
               │  < poll                                     >  │               
               │  < release                                  >  │               
               │  < remove                                   >  │               
               │  < set                                      >  │               
               │  < show                                     >  │               
               │  < trigger                                  >  │               
               │                                                │               
               │ q to close                                     │               
               └────────────────────────────────────────────────┘               
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html0000664000175000017500000002300015264420307034137 0ustar alastairalastair
Cylc Tui 1.2.3 ┌────────────────────────────────────────────────┐               
               │  id: 1/one                                     │               
- ~cylc        │  waiting (queued)                              │               
   - one - paus│                                                │               
      - ̿○ 1    │  Action                                        │               
           ̿○ on│  < (cancel)                                 >  │               
               │                                                │               
               │  < hold                                     >  │               
               │  < kill                                     >  │               
               │  < log                                      >  │               
               │  < poll                                     >  │               
               │  < release                                  >  │               
               │                                                │               
quit: q  help: │ q to close                                     │↥ ↧ Home End   
filter tasks: T└────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_states.task-context--succeeded+held.html0000664000175000017500000002430615264420307033231 0ustar alastairalastair
Cylc Tui 1.2.3 ┌────────────────────────────────────────────────┐               
               │  id: 1/a                                       │               
- ~cylc        │  succeeded (held, queued)                      │               
   - one - paus│                                                │               
      - ̎○ 1    │  Action                                        │               
           ̎● a │  < (cancel)                                 >  │               
           ̿○ b │                                                │               
           ̀○ c │  < hold                                     >  │               
               │  < kill                                     >  │               
               │  < log                                      >  │               
               │  < poll                                     >  │               
               │  < release                                  >  │               
               │                                                │               
quit: q  help: │ q to close                                     │↥ ↧ Home End   
filter tasks: T└────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_show.success.html0000664000175000017500000003372315264420307027026 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - one - paused                                                               
      - ̿○ 1                                                                     
           ̿○ foo                                                                
                                                                                
                                                                                
                                                                                
                                                                                
               ┌────────────────────────────────────────────────┐               
               │  title: Foo                                    │               
               │  description: The first metasyntactic          │               
               │  variable.                                     │               
               │  URL: (not given)                              │               
               │  state: waiting (queued,run mode=Simulation)   │               
               │  prerequisites: (None)                         │               
               │  outputs: ('⨯': not completed)                 │               
               │    ⨯ 1/foo expired                             │               
               │    ⨯ 1/foo submitted                           │               
               │    ⨯ 1/foo submit-failed                       │               
               │    ⨯ 1/foo started                             │               
               │    ⨯ 1/foo succeeded                           │               
               │    ⨯ 1/foo failed                              │               
               │  output completion: incomplete                 │               
               │    ⨯ ┆  succeeded                              │               
               │                                                │               
               │                                                │               
               │ q to close                                     │               
               └────────────────────────────────────────────────┘               
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html0000664000175000017500000000775215264420307033456 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   + one - paused                                                               
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html0000664000175000017500000001226115264420307033602 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - one - paused                                                               
      - ̿○ 1                                                                     
           ̿○ one                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_navigation.on-load.html0000664000175000017500000001372315264420307030064 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   + one - paused                                                               
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_auto_expansion.on-load.html0000664000175000017500000001332315264420307030755 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - one - paused                                                               
      - ̿○ 1                                                                     
         - ̿○ A                                                                  
              ̿○ a                                                               
           ○ b                                                                  
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_task_states.filter-submitted.html0000664000175000017500000001705115264420307032202 0ustar alastairalastair
Cylc Tui 1.2.3   tasks filtered (T - edit, R - reset)   workflows filtered (W - 
edit, E - reset)                                                                
                                                                                
- ~cylc                                                                         
   - test_task_states - paused 1■ 1■ 1■ 1■ 1■                                   
      - ̎⊘ 2                                                                     
         - ̿⊙ X                                                                  
              ̿⊙ a                                                               
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html0000664000175000017500000001101615264420307033077 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - one - paused                                                               
      - ̿○ 1                                                                     
           ̿○ one                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_job_logs.02-job.out.html0000664000175000017500000002703715264420307027774 0ustar alastairalastair
┌──────────────────────────────────────────────────────────────────────────────┐
│  Host: myhost                                                                │
│  Path: mypath                                                                │
│  < Select File                                                            >  │
│  Open in:  < nvim >< gvim -f >< less >                                       │
│  (Configure apps to open logs via $EDITOR, $GEDITOR, $PAGER)                 │
│                                                                              │
│  job: 1/a/02                                                                 │
│  this is a job log                                                           │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│ q to close                                                                   │
└──────────────────────────────────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html0000664000175000017500000002171715264420307032175 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - one - paused                                                               
      - ̿○ 1                                                                     
         - ̿○ A                                                                  
              ̿○ a1                                                              
              ̿○ a2                                                              
         - ̿○ B                                                                  
            - ̿○ B1                                                              
                 ̿○ b11                                                          
                 ̿○ b12                                                          
            - ̿○ B2                                                              
                 ̿○ b21                                                          
                 ̿○ b22                                                          
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_auto_expansion.later-time.html0000664000175000017500000001435515264420307031475 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - one - paused                                                               
      - ● 1                                                                     
           ● b                                                                  
      - ○ 2                                                                     
         - ○ A                                                                  
              ○ a                                                               
           ○ b                                                                  
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_task_modifiers.task-modifiers.html0000664000175000017500000002065215264420307032317 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - test_task_modifiers - paused                                               
      - ̎○ 1                                                                     
           ̎○ all                                                                
           ̎○ is_held                                                            
           ̿○ is_queued                                                          
           ⃗○ is_retry                                                           
           ̀○ is_runahead                                                        
           ፝○ is_wallclock                                                       
           ᷉○ is_xtriggered                                                      
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_online_mutation.task-selected.html0000664000175000017500000001101615264420307032321 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - one - paused                                                               
      - ̿○ 1                                                                     
           ̿○ one                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
././@LongLink0000644000000000000000000000014600000000000011604 Lustar rootrootcylc-flow-8.7.0/tests/integration/tui/screenshots/test_task_states.filter-not-waiting-or-expired.htmlcylc-flow-8.7.0/tests/integration/tui/screenshots/test_task_states.filter-not-waiting-or-expired.htm0000664000175000017500000002362515264420307034346 0ustar alastairalastair
Cylc Tui 1.2.3   tasks filtered (T - edit, R - reset)   workflows filtered (W - 
edit, E - reset)                                                                
                                                                                
- ~cylc                                                                         
   - test_task_states - paused 1■ 1■ 1■ 1■ 1■                                   
      - ̿⊗ 1                                                                     
         - ̿● X                                                                  
              ̿● a                                                               
         - ̿⊗ Y                                                                  
              ̿⊗ b                                                               
      - ̎⊘ 2                                                                     
         - ̿⊙ X                                                                  
              ̿⊙ a                                                               
         - ̎⊘ Y                                                                  
            - ̎⊘ Y1                                                              
                 ̎⊘ c                                                            
              ̎⊙ b                                                               
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_task_states.unfiltered.html0000664000175000017500000002771515264420307031070 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - test_task_states - paused 1■ 1■ 1■ 1■ 1■                                   
      - ̿⊗ 1                                                                     
         - ̿● X                                                                  
              ̿● a                                                               
         - ̿⊗ Y                                                                  
            - ◌ Y1                                                              
                 ◌ c                                                            
              ̿⊗ b                                                               
      - ̎⊘ 2                                                                     
         - ̿⊙ X                                                                  
              ̿⊙ a                                                               
         - ̎⊘ Y                                                                  
            - ̎⊘ Y1                                                              
                 ̎⊘ c                                                            
              ̎⊙ b                                                               
      - ̀○ 3                                                                     
         - ̀○ X                                                                  
              ̀○ a                                                               
         - ̀○ Y                                                                  
            - ̀○ Y1                                                              
                 ̀○ c                                                            
              ̀○ b                                                               
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html0000664000175000017500000002703715264420307031237 0ustar alastairalastair
┌──────────────────────────────────────────────────────────────────────────────┐
│  Host: myhost                                                                │
│  Path: mypath                                                                │
│  < Select File                                                            >  │
│  Open in:  < nvim >< gvim -f >< less >                                       │
│  (Configure apps to open logs via $EDITOR, $GEDITOR, $PAGER)                 │
│                                                                              │
│  job: 1/a/02                                                                 │
│  this is a job log                                                           │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│                                                                              │
│ q to close                                                                   │
└──────────────────────────────────────────────────────────────────────────────┘
././@LongLink0000644000000000000000000000015000000000000011577 Lustar rootrootcylc-flow-8.7.0/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.htmlcylc-flow-8.7.0/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.h0000664000175000017500000002023215264420307034371 0ustar alastairalastair
Cylc Tui 1.2.3 ┌────────────────────────────────────────────────┐               
               │  id: ~cylc/root                                │               
- ~cylc        │                                                │               
   + one - stop│                                                │               
               │  Action                                        │               
               │  < (cancel)                                 >  │               
               │                                                │               
               │  < stop-all                                 >  │               
               │                                                │               
               │                                                │               
               │                                                │               
               │                                                │               
               │                                                │               
quit: q  help: │ q to close                                     │↥ ↧ Home End   
filter tasks: T└────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_workflow_states.filter-active.html0000664000175000017500000001047715264420307032372 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   + one - stopping                                                             
   + two - paused                                                               
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html0000664000175000017500000001047715264420307033651 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   + tre - stopped                                                              
   + two - paused                                                               
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_incompat_scheduler_version.on-load.html0000664000175000017500000001145715264420307033344 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - one - paused                                                               
        Error - Scheduler version 6.11.4 is not supported                       
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html0000664000175000017500000001226115264420307034121 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - one - paused                                                               
      - ̿○ 1                                                                     
           ̿○ one                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_set_mutation.task-state-updated.html0000664000175000017500000001152015264420307032604 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - one - paused                                                               
      - ○ 1                                                                     
           ● a                                                                  
           ○ z                                                                  
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_external_editor.before-opening-editor.html0000664000175000017500000002703715264420307033752 0ustar alastairalastair
┌──────────────────────────────────────────────────────────────────────────────┐
│  Host: myhost                                                                │
│  Path: mypath                                                                │
│  < Select File                                                            >  │
│  Open in:  < nvim >< gvim -f >< less >                                       │
│  (Configure apps to open logs via $EDITOR, $GEDITOR, $PAGER)                 │
│                                                                              │
│  this is the scheduler log file                                              │
│  line 2                                                                      │
│  line 3                                                                      │
│  line 4                                                                      │
│  line 5                                                                      │
│  line 6                                                                      │
│  line 7                                                                      │
│  line 8                                                                      │
│  line 9                                                                      │
│  line 10                                                                     │
│  line 11                                                                     │
│  line 12                                                                     │
│  line 13                                                                     │
│  line 14                                                                     │
│  line 15                                                                     │
│  line 16                                                                     │
│  line 17                                                                     │
│  line 18                                                                     │
│  line 19                                                                     │
│  line 20                                                                     │
│                                                                              │
│ q to close                                                                   │
└──────────────────────────────────────────────────────────────────────────────┘
cylc-flow-8.7.0/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html0000664000175000017500000002047715264420307032142 0ustar alastairalastair
Cylc Tui 1.2.3   workflows filtered (W - edit, E - reset)                       
                                                                                
- ~cylc                                                                         
   - one - paused                                                               
      - ̿○ 1                                                                     
         + ̿○ A                                                                  
         - ̿○ B                                                                  
            - ̿○ B1                                                              
                 ̿○ b11                                                          
                 ̿○ b12                                                          
            - ̿○ B2                                                              
                 ̿○ b21                                                          
                 ̿○ b22                                                          
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
                                                                                
quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
filter tasks: T f s r R  filter workflows: W E p
cylc-flow-8.7.0/tests/integration/tui/test_show.py0000664000175000017500000000446215264420307022501 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.exceptions import ClientError async def test_show(flow, scheduler, start, rakiura, monkeypatch): """Test "cylc show" support in Tui.""" id_ = flow({ 'scheduling': { 'graph': { 'R1': 'foo' }, }, 'runtime': { 'foo': { 'meta': { 'title': 'Foo', 'description': 'The first metasyntactic variable.' }, }, }, }, name='one') schd = scheduler(id_) async with start(schd): await schd.update_data_structure() with rakiura(size='80,40') as rk: rk.user_input('down', 'right') rk.wait_until_loaded(schd.tokens.id) # select a task rk.user_input('down', 'down', 'enter') # select the "show" context option rk.user_input(*(['down'] * 8), 'enter') rk.compare_screenshot( 'success', 'the show output should be displayed', ) # make it look like "cylc show" failed def cli_cmd_fail(*args, **kwargs): raise ClientError(':(') monkeypatch.setattr( 'cylc.flow.tui.data.cli_cmd', cli_cmd_fail, ) # select the "show" context option rk.user_input('q', 'enter', *(['down'] * 8), 'enter') rk.compare_screenshot( 'fail', 'the error should be displayed', ) cylc-flow-8.7.0/tests/integration/tui/test_updater.py0000664000175000017500000001731415264420307023165 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio from copy import deepcopy from pathlib import Path from queue import Queue import re from time import time import pytest from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.id import Tokens from cylc.flow.tui.updater import ( Updater, get_default_filters, ) from cylc.flow.workflow_status import WorkflowStatus @pytest.fixture def updater(test_dir): """Return an updater ready for testing.""" updater = Updater() # patch the update intervals so that everything runs for every update updater.BASE_UPDATE_INTERVAL = updater.BASE_SCAN_INTERVAL = 0 # swap multiprocessing.Queue for queue.Queue # (this means queued operations are instant making tests more stable) updater.update_queue = Queue() updater._command_queue = Queue() # set up the filters # (these filter for the workflows created in this test only) filters = get_default_filters() id_base = str(test_dir.relative_to(Path("~/cylc-run").expanduser())) filters['workflows']['id'] = f'^{re.escape(id_base)}/.*' updater._update_filters(filters) return updater def get_child_tokens(root_node, types, relative=False): """Return all ID of the specified types contained within the provided tree. Args: root_node: The Tui tree you want to look for IDs in. types: The Tui types (e.g. 'workflow' or 'task') you want to extract. relative: If True, the relative IDs will be returned. """ ret = set() stack = [root_node] while stack: node = stack.pop() stack.extend(node['children']) if node['type_'] in types: tokens = Tokens(node['id_']) if relative: ret.add(tokens.relative_id) else: ret.add(tokens.id) return ret async def test_subscribe(one_conf, flow, scheduler, run, updater): """It should subscribe and unsubscribe from workflows.""" id_ = flow(one_conf) schd = scheduler(id_) async with run(schd): # run the updater and the test async with asyncio.timeout(10): # wait for the first update root_node = await updater._update() # there should be a root root_node assert root_node['id_'] == 'root' # a single root_node representing the workflow assert root_node['children'][0]['id_'] == schd.tokens.id # and a "spring" root_node used to active the subscription # mechanism assert root_node['children'][0]['children'][0]['id_'] == '#spring' # subscribe to the workflow updater.subscribe(schd.tokens.id) root_node = await updater._update() # check the workflow contains one cycle with one task in it workflow_node = root_node['children'][0] assert len(workflow_node['children']) == 1 cycle_node = workflow_node['children'][0] assert Tokens(cycle_node['id_']).relative_id == '1' # cycle ID assert len(cycle_node['children']) == 1 task_node = cycle_node['children'][0] assert Tokens(task_node['id_']).relative_id == '1/one' # task ID # unsubscribe from the workflow updater.unsubscribe(schd.tokens.id) root_node = await updater._update() # the workflow should be replaced by a "spring" node again assert root_node['children'][0]['children'][0]['id_'] == '#spring' async def test_filters(one_conf, flow, scheduler, run, updater): """It should filter workflow and task states. Note: The workflow ID filter is not explicitly tested here, but it is indirectly tested, otherwise other workflows would show up in the updater results. """ one = scheduler(flow({ 'scheduler': { 'allow implicit tasks': 'True', }, 'scheduling': { 'graph': { 'R1': 'a & b & c', } }, 'runtime': { # TODO: remove this runtime section in # https://github.com/cylc/cylc-flow/pull/5721 'root': { 'simulation': { 'default run length': 'PT1M', }, }, }, }, name='one'), paused_start=True) two = scheduler(flow(one_conf, name='two')) tre = scheduler(flow(one_conf, name='tre')) # start workflow "one" async with run(one): # mark "1/a" as running and "1/b" as succeeded one_a = one.pool.get_task(IntegerPoint('1'), 'a') one_a.summary['started_time'] = time() one_a.state_reset('running') one.data_store_mgr.delta_task_state(one_a) one.pool.get_task(IntegerPoint('1'), 'b').state_reset('succeeded') # start workflow "two" async with run(two): # run the updater and the test filters = deepcopy(updater.filters) root_node = await updater._update() assert {child['id_'] for child in root_node['children']} == { one.tokens.id, two.tokens.id, tre.tokens.id, } # filter out paused workflows filters = deepcopy(filters) filters['workflows'][WorkflowStatus.STOPPED.value] = True filters['workflows'][WorkflowStatus.PAUSED.value] = False updater.update_filters(filters) # "one" and "two" should now be filtered out root_node = await updater._update() assert {child['id_'] for child in root_node['children']} == { tre.tokens.id, } # filter out stopped workflows filters = deepcopy(filters) filters['workflows'][WorkflowStatus.STOPPED.value] = False filters['workflows'][WorkflowStatus.PAUSED.value] = True updater.update_filters(filters) # "tre" should now be filtered out root_node = await updater._update() assert {child['id_'] for child in root_node['children']} == { one.tokens.id, two.tokens.id, } # subscribe to "one" updater._subscribe(one.tokens.id) root_node = await updater._update() assert get_child_tokens( root_node, types={'task'}, relative=True ) == { '1/a', '1/b', '1/c', } # filter out running tasks # TODO: see https://github.com/cylc/cylc-flow/issues/5716 # filters = deepcopy(filters) # filters['tasks'][TASK_STATUS_RUNNING] = False # updater.update_filters(filters) # root_node = await updater._update() # assert get_child_tokens( # root_node, # types={'task'}, # relative=True # ) == { # '1/b', # '1/c', # } cylc-flow-8.7.0/tests/integration/tui/test_app.py0000664000175000017500000005435115264420307022303 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from random import random import pytest import urwid from cylc.flow import __version__ from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.id import TaskTokens from cylc.flow.task_outputs import TASK_OUTPUT_SUCCEEDED from cylc.flow.task_state import ( TASK_STATUS_EXPIRED, TASK_STATUS_FAILED, TASK_STATUS_RUNNING, TASK_STATUS_SUBMIT_FAILED, TASK_STATUS_SUBMITTED, TASK_STATUS_SUCCEEDED, TASK_STATUS_WAITING, ) from cylc.flow.tui.util import MODIFIER_ATTR_MAPPING from cylc.flow.workflow_files import get_contact_file_path from cylc.flow.workflow_status import StopMode def _add_xtrig(schd, itask, sig, label, satisfied=False): """Register a new xtrigger. This is enough to get the data store to reflect the xtrig, but don't expect it to be functional! """ ( schd.data_store_mgr.xtrigger_tasks .setdefault(sig, set()) .add((itask.tokens.id, label)) ) schd.data_store_mgr.delta_xtrigger(sig, satisfied) def set_task_state(schd, task_states): """Force tasks into the desired states. Task states should be of the format (cycle, task, state, is_held). """ for cycle, task, state, modifiers in task_states: itask = schd.pool.get_task(cycle, task) if not itask: itask = schd.pool.spawn_task(task, cycle, {1}) if modifiers.pop('is_retry', None): _add_xtrig( schd, itask, 'wall_clock(offset=0)', f'_cylc_retry_{random()}' ) if modifiers.pop('is_wallclock', None): _add_xtrig(schd, itask, 'wall_clock(offset=1)', 'my_clock') if modifiers.pop('is_xtriggered', None): _add_xtrig(schd, itask, 'my_custom()', 'my_custom') itask.state_reset(state, **modifiers) schd.data_store_mgr.delta_task_state(itask) schd.data_store_mgr.increment_graph_window( itask.tokens, cycle, ) async def test_tui_basics(rakiura): """Test basic Tui interaction with no workflows.""" with rakiura(size='80,40') as rk: # the app should open rk.compare_screenshot('test-rakiura', 'the app should have loaded') # "h" should bring up the onscreen help rk.user_input('h') rk.compare_screenshot( 'test-rakiura-help', 'the help screen should be visible' ) # "q" should close the popup rk.user_input('q') rk.compare_screenshot( 'test-rakiura', 'the help screen should have closed', ) # "enter" should bring up the context menu rk.user_input('enter') rk.compare_screenshot( 'test-rakiura-enter', 'the context menu should have opened', ) # "enter" again should close it via the "cancel" button rk.user_input('enter') rk.compare_screenshot( 'test-rakiura', 'the context menu should have closed', ) # "ctrl d" should exit Tui with pytest.raises(urwid.ExitMainLoop): rk.user_input('ctrl d') # "q" should exit Tui with pytest.raises(urwid.ExitMainLoop): rk.user_input('q') async def test_subscribe_unsubscribe( one_conf, flow, scheduler, start, rakiura ): """Test a simple workflow with one task.""" id_ = flow(one_conf, name='one') schd = scheduler(id_) async with start(schd): await schd.update_data_structure() with rakiura(size='80,15') as rk: rk.compare_screenshot( 'unsubscribed', 'the workflow should be collapsed' ' (no subscription no state totals)', ) # expand the workflow to subscribe to it rk.user_input('down', 'right') rk.wait_until_loaded() rk.compare_screenshot( 'subscribed', 'the workflow should be expanded', ) # collapse the workflow to unsubscribe from it rk.user_input('left', 'up') rk.force_update() rk.compare_screenshot( 'unsubscribed', 'the workflow should be collapsed' ' (no subscription no state totals)', ) async def test_workflow_states(one_conf, flow, scheduler, start, rakiura): """Test viewing multiple workflows in different states.""" # one => stopping id_1 = flow(one_conf, name='one') schd_1 = scheduler(id_1) # two => paused id_2 = flow(one_conf, name='two') schd_2 = scheduler(id_2) # tre => stopped flow(one_conf, name='tre') async with start(schd_1): schd_1.stop_mode = StopMode.AUTO # make it look like we're stopping await schd_1.update_data_structure() async with start(schd_2): await schd_2.update_data_structure() with rakiura(size='80,15') as rk: rk.compare_screenshot( 'unfiltered', 'All workflows should be visible (one, two, tree)', ) # filter for active workflows (i.e. paused, running, stopping) rk.user_input('p') rk.compare_screenshot( 'filter-active', 'Only active workflows should be visible (one, two)' ) # invert the filter so we are filtering for stopped workflows rk.user_input('W', 'enter', 'q') rk.compare_screenshot( 'filter-stopped', 'Only stopped workflow should be visible (tre)' ) # filter in paused workflows rk.user_input('W', 'down', 'enter', 'q') rk.force_update() rk.compare_screenshot( 'filter-stopped-or-paused', 'Only stopped or paused workflows should be visible' ' (two, tre)', ) # reset the state filters rk.user_input('W', 'down', 'down', 'enter', 'down', 'enter') # scroll to the id filter text box rk.user_input('down', 'down', 'down', 'down') # scroll to the end of the ID rk.user_input(*['right'] * ( len(schd_1.tokens['workflow'].rsplit('/', 1)[0]) + 1) ) # type the letter "t" # (this should filter for workflows starting with "t") rk.user_input('t') rk.force_update() # this is required for the tests rk.user_input('page up', 'q') # close the dialogue rk.compare_screenshot( 'filter-starts-with-t', 'Only workflows starting with the letter "t" should be' ' visible (two, tre)', ) async def test_task_states(flow, scheduler, start, rakiura): id_ = flow({ 'scheduler': { 'allow implicit tasks': 'true', }, 'scheduling': { 'initial cycle point': '1', 'cycling mode': 'integer', 'runahead limit': 'P1', 'graph': { 'P1': ''' a & b & c ''' }, }, 'runtime': { 'X': {}, 'Y': {}, 'Y1': {'inherit': 'Y'}, 'a': {'inherit': 'X'}, 'b': {'inherit': 'Y'}, 'c': {'inherit': 'Y1'}, }, }, name='test_task_states') schd = scheduler(id_) async with start(schd): set_task_state( schd, [ ( IntegerPoint('1'), 'a', TASK_STATUS_SUCCEEDED, {'is_held': False} ), ( IntegerPoint('1'), 'b', TASK_STATUS_FAILED, {'is_held': False} ), ( IntegerPoint('1'), 'c', TASK_STATUS_EXPIRED, {'is_held': False} ), ( IntegerPoint('2'), 'a', TASK_STATUS_SUBMITTED, {'is_held': False} ), ( IntegerPoint('2'), 'b', TASK_STATUS_RUNNING, {'is_held': True} ), ( IntegerPoint('2'), 'c', TASK_STATUS_SUBMIT_FAILED, {'is_held': True} ), ] ) await schd.update_data_structure() with rakiura(schd.tokens.id, size='80,30') as rk: rk.compare_screenshot( 'unfiltered', 'all tasks should be displayed' ' (i.e. 1/*, 2/* and 3/* should be displayed)', ) # filter OUT waiting tasks rk.user_input('T', 'down', 'enter', 'q') # select waiting rk.compare_screenshot( 'filter-not-waiting', 'waiting tasks should be filtered out' ' (i.e. 1/* and 2/* should be displayed)', ) # filter OUT waiting & expired tasks rk.user_input('T', 'down', 'down', 'enter', 'q') # select expired rk.compare_screenshot( 'filter-not-waiting-or-expired', 'waiting & expired tasks should be filtered out' ' (i.e. only 1/a, 1/b and 2/* should be displayed)', ) # filter FOR waiting & expired tasks rk.user_input('T', 'enter', 'q') # select invert rk.compare_screenshot( 'filter-waiting-or-expired', 'only waiting and expired tasks should be displayed' ' (i.e. only 1/c and 3/* should be displayed)', ) # filter FOR submitted tasks (using shortcuts) rk.user_input('R', 's') # reset filters and apply submitted filter rk.compare_screenshot( 'filter-submitted', 'only submitted tasks should be displayed' ' (i.e. only 2/a should be displayed)', ) async def test_task_modifiers(flow, scheduler, start, rakiura): """It should display task modifiers and text summaries of them.""" id_ = flow({ 'scheduling': { 'graph': { 'R1': '\n'.join( modifier for modifier, _ in MODIFIER_ATTR_MAPPING.values() ) + '\nall' }, }, }, name='test_task_modifiers') schd = scheduler(id_) async with start(schd): set_task_state( schd, [ *[ ( IntegerPoint('1'), modifier, TASK_STATUS_WAITING, # NOTE: set is_queued=False as the default because # parentless tasks are autoqueued on startup {**{'is_queued': False}, modifier: True}, ) for modifier, _ in MODIFIER_ATTR_MAPPING.values() ], ( IntegerPoint('1'), 'all', TASK_STATUS_WAITING, { modifier: True for modifier, _ in MODIFIER_ATTR_MAPPING.values() }, ) ] ) await schd.update_data_structure() with rakiura(schd.tokens.id, size='80,30') as rk: # test modifier icon rendering rk.compare_screenshot( 'task-modifiers', 'all tasks should be displayed along with their modifiers' ) # test modifier text summary rk.user_input('down', 'down', 'down', 'enter') # select task "all" rk.compare_screenshot( 'task-context-menu', 'all modifiers should be listed along with the task state' ' (i.e. the text "held, runahead, queued, retry scheduled,' ' wallclock, xtriggered") should be present' ) async def test_navigation(flow, scheduler, start, rakiura): """Test navigating with the arrow keys.""" id_ = flow({ 'scheduling': { 'graph': { 'R1': 'A & B1 & B2', } }, 'runtime': { 'A': {}, 'B': {}, 'B1': {'inherit': 'B'}, 'B2': {'inherit': 'B'}, 'a1': {'inherit': 'A'}, 'a2': {'inherit': 'A'}, 'b11': {'inherit': 'B1'}, 'b12': {'inherit': 'B1'}, 'b21': {'inherit': 'B2'}, 'b22': {'inherit': 'B2'}, } }, name='one') schd = scheduler(id_) async with start(schd): await schd.update_data_structure() with rakiura(size='80,30') as rk: # wait for the workflow to appear (collapsed) rk.wait_until_loaded('#spring') rk.compare_screenshot( 'on-load', 'the workflow should be collapsed when Tui is loaded', ) # pressing "right" should connect to the workflow # and expand it once the data arrives rk.user_input('down', 'right') rk.wait_until_loaded(schd.tokens.id) rk.compare_screenshot( 'workflow-expanded', 'the workflow should be expanded', ) # pressing "left" should collapse the node rk.user_input('down', 'down', 'left') rk.compare_screenshot( 'family-A-collapsed', 'the family "1/A" should be collapsed', ) # the "page up" and "page down" buttons should navigate to the top # and bottom of the screen rk.user_input('page down') rk.compare_screenshot( 'cursor-at-bottom-of-screen', 'the cursor should be at the bottom of the screen', ) async def test_auto_expansion(flow, scheduler, start, rakiura): """It should automatically expand cycles and top-level families. When a workflow is expanded, Tui should auto expand cycles and top-level families. Any new cycles and top-level families should be auto-expanded when added. """ id_ = flow({ 'scheduling': { 'runahead limit': 'P1', 'initial cycle point': '1', 'cycling mode': 'integer', 'graph': { 'P1': 'b[-P1] => a => b' }, }, 'runtime': { 'A': {}, 'a': {'inherit': 'A'}, 'b': {}, }, }, name='one') schd = scheduler(id_) with rakiura(size='80,20') as rk: async with start(schd): await schd.update_data_structure() # wait for the workflow to appear (collapsed) rk.wait_until_loaded('#spring') # open the workflow rk.force_update() rk.user_input('down', 'right') rk.wait_until_loaded(schd.tokens.id) rk.compare_screenshot( 'on-load', 'cycle "1" and top-level family "1/A" should be expanded', ) for task in ('a', 'b'): schd.pool.set_prereqs_and_outputs( items={TaskTokens('1', task)}, outputs=[TASK_OUTPUT_SUCCEEDED], prereqs=[], flow=[] ) await schd._main_loop() await schd.update_data_structure() schd.update_data_store() rk.compare_screenshot( 'later-time', 'cycle "2" and top-level family "2/A" should be expanded', ) async def test_restart_reconnect(one_conf, flow, scheduler, start, rakiura): """It should handle workflow shutdown and restart. The Cylc client can raise exceptions e.g. WorkflowStopped. Any text written to stdout/err will mess with Tui. The purpose of this test is to ensure Tui can handle shutdown / restart without any errors occurring and any spurious text appearing on the screen. """ with rakiura(size='80,20') as rk: schd = scheduler(flow(one_conf, name='one')) # 1- start the workflow async with start(schd): await schd.update_data_structure() # wait for the workflow to appear (collapsed) rk.wait_until_loaded('#spring') # expand the workflow (subscribes to updates from it) rk.force_update() rk.user_input('down', 'right') # wait for workflow to appear (expanded) rk.wait_until_loaded(schd.tokens.id) rk.compare_screenshot( '1-workflow-running', 'the workflow should appear in tui and be expanded', ) # 2 - stop the worlflow rk.compare_screenshot( '2-workflow-stopped', 'the stopped workflow should be collapsed with a message saying' ' workflow stopped', ) # 3- restart the workflow schd = scheduler(flow(one_conf, name='one')) async with start(schd): await schd._main_loop() await schd.update_data_structure() rk.wait_until_loaded(schd.tokens.id) rk.compare_screenshot( '3-workflow-restarted', 'the restarted workflow should be expanded', ) async def test_states(flow, scheduler, start, rakiura): """It should dim no-flow tasks and display state summary in context menus. """ id_ = flow( { 'scheduling': { 'graph': { 'R1': 'a & b & c', }, }, }, name='one', ) from cylc.flow.scheduler import Scheduler schd: Scheduler = scheduler(id_) async with start(schd): a = schd.pool.get_task(IntegerPoint('1'), 'a') b = schd.pool.get_task(IntegerPoint('1'), 'b') c = schd.pool.get_task(IntegerPoint('1'), 'c') assert a and b and c # set task flow numbers assert a.flow_nums == {1} b.flow_nums = {1, 2} c.flow_nums = {} # set task state a.state_reset(TASK_STATUS_SUCCEEDED, is_held=True) b.state_reset(TASK_STATUS_WAITING, is_queued=True) c.state_reset(TASK_STATUS_WAITING, is_queued=False, is_runahead=True) # update data store for task in (a, b, c): schd.data_store_mgr.delta_task_state(task) schd.data_store_mgr.delta_task_flow_nums(task) await schd.update_data_structure() with rakiura(schd.tokens.id, size='80,15') as rk: rk.compare_screenshot( 'on-load', 'the workflow should be expanded,' ' no-flow task 1/c should be dimmed' ) # workflow node rk.user_input('down', 'enter') rk.compare_screenshot( 'workflow-context--paused', 'the workflow should show as paused in the context menu', ) # cycle: 1 rk.user_input('q', 'down', 'enter') rk.compare_screenshot( 'cycle-context--waiting', 'the cycle should show as waiting in the context menu' ) # task:a rk.user_input('q', 'down', 'enter') rk.compare_screenshot( 'task-context--succeeded+held', 'the task should show as succeeded+held in the context menu,' ' no flow numbers should be displayed', ) # task:b rk.user_input('q', 'down', 'enter') rk.compare_screenshot( 'task-context--waiting+queued', 'the task should show as waiting+queued in the context menu,' ' the flow numbers 1,2 should be displayed', ) # task:c rk.user_input('q', 'down', 'enter') rk.compare_screenshot( 'task-context--waiting+runahead', 'the task should show as waiting+runahead in the context menu,' ' the task should be marked as flows=None' ) async def test_incompat_scheduler_version( one_conf, flow, scheduler, start, rakiura ): """It should handle workflows it cannot subscribe to.""" schd = scheduler(flow(one_conf, name='one')) async with start(schd): # make it look like the scheduler is reallllyyyy old contact = get_contact_file_path(schd.workflow) with open(contact, 'r') as contact_file: contact_lines = contact_file.read().replace(__version__, '6.11.4') with open(contact, 'w') as contact_file: contact_file.write(''.join(contact_lines)) with rakiura(size='80,20') as rk: await schd.update_data_structure() # wait for the workflow to appear (collapsed) rk.wait_until_loaded('#spring') # expand the workflow (subscribes to updates from it) rk.force_update() rk.user_input('down', 'right') # it should be marked as incompatible rk.compare_screenshot( 'on-load', 'the workflow should be marked as incompatible', ) cylc-flow-8.7.0/tests/integration/tui/test_logs.py0000664000175000017500000003456415264420307022473 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio from pathlib import Path from typing import TYPE_CHECKING from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.exceptions import ClientError from cylc.flow.scheduler import Scheduler from cylc.flow.task_job_logs import get_task_job_log from cylc.flow.task_state import ( TASK_STATUS_FAILED, TASK_STATUS_SUCCEEDED, ) from cylc.flow.tui.data import _get_log import pytest if TYPE_CHECKING: from cylc.flow.id import Tokens def get_job_log(tokens: 'Tokens', suffix: str) -> Path: """Return the path to a job log file. Args: tokens: Job tokens. suffix: Filename. """ return Path(get_task_job_log( tokens['workflow'], tokens['cycle'], tokens['task'], tokens['job'], suffix=suffix, )) @pytest.fixture(scope='module') def standarise_host_and_path(mod_monkeypatch): """Replace variable content in the log view. The log view displays the "Host" and "Path" of the log file. These will differer from user to user, so we mock away the difference to produce stable results. """ def _parse_log_header(contents): _header, text = contents.split('\n', 1) return 'myhost', 'mypath', text mod_monkeypatch.setattr( 'cylc.flow.tui.data._parse_log_header', _parse_log_header, ) @pytest.fixture def wait_log_loaded(monkeypatch): """Wait for Tui to successfully open a log file.""" # previous log open count before = 0 # live log open count count = 0 # wrap the Tui "_get_log" method to count the number of times it has # returned def __get_log(*args, **kwargs): nonlocal count try: ret = _get_log(*args, **kwargs) except ClientError as exc: count += 1 raise exc count += 1 return ret monkeypatch.setattr( 'cylc.flow.tui.data._get_log', __get_log, ) async def _wait_log_loaded(tries: int = 25, delay: float = 0.1): """Wait for the log file to be loaded. Args: tries: The number of (re)tries to attempt before failing. delay: The delay between retries. """ nonlocal before for _try in range(tries): if count > before: await asyncio.sleep(0) before += 1 return await asyncio.sleep(delay) raise Exception(f'Log file was not loaded within {delay * tries}s') return _wait_log_loaded @pytest.fixture(scope='module') async def workflow( mod_flow, mod_scheduler, mod_start, standarise_host_and_path ): """Test fixture providing a workflow with some log files to poke at.""" id_ = mod_flow({ 'scheduling': { 'graph': { 'R1': 'a', } }, 'runtime': { 'a': {}, } }, name='one') schd: Scheduler = mod_scheduler(id_) async with mod_start(schd): # create some log files for tests to inspect # create a scheduler log # (note the scheduler log doesn't get created in integration tests) scheduler_log = Path(schd.workflow_log_dir, '01-start-01.log') with open(scheduler_log, 'w+') as logfile: logfile.write( 'this is the scheduler log file' + '\n' + '\n'.join(f'line {x}' for x in range(2, 1000)) ) # task 1/a itask = schd.pool.get_task(IntegerPoint('1'), 'a') # mark 1/a/01 as failed job_1 = schd.tokens.duplicate(cycle='1', task='a', job='01') schd.data_store_mgr.insert_job( itask, TASK_STATUS_SUCCEEDED, {'submit_num': 1, 'platform': {'name': 'x'}} ) schd.data_store_mgr.delta_job_state(itask, TASK_STATUS_FAILED) # mark 1/a/02 as succeeded itask.submit_num = 2 job_2 = schd.tokens.duplicate(cycle='1', task='a', job='02') schd.data_store_mgr.insert_job( itask, TASK_STATUS_SUCCEEDED, {'submit_num': 2, 'platform': {'name': 'x'}} ) schd.data_store_mgr.delta_job_state(itask, TASK_STATUS_SUCCEEDED) schd.data_store_mgr.delta_task_state(itask) # mark 1/a as succeeded itask.state_reset(TASK_STATUS_SUCCEEDED) schd.data_store_mgr.delta_task_state(itask) # 1/a/01 - job.out job_1_out = get_job_log(job_1, 'job.out') job_1_out.parent.mkdir(parents=True) with open(job_1_out, 'w+') as log: log.write(f'job: {job_1.relative_id}\nthis is a job log\n') # 1/a/02 - job.out job_2_out = get_job_log(job_2, 'job.out') job_2_out.parent.mkdir(parents=True) with open(job_2_out, 'w+') as log: log.write(f'job: {job_2.relative_id}\nthis is a job log\n') # 1/a/02 - job.err job_2_err = get_job_log(job_2, 'job.err') with open(job_2_err, 'w+') as log: log.write(f'job: {job_2.relative_id}\nthis is a job error\n') # 1/a/NN -> 1/a/02 (job_2_out.parent.parent / 'NN').symlink_to( (job_2_out.parent.parent / '02'), target_is_directory=True, ) # populate the data store await schd.update_data_structure() yield schd async def test_scheduler_logs( workflow, mod_rakiura, wait_log_loaded, ): """Test viewing the scheduler log files.""" with mod_rakiura(size='80,30') as rk: # wait for the workflow to appear (collapsed) rk.wait_until_loaded('#spring') # open the workflow in Tui rk.user_input('down', 'right') rk.wait_until_loaded(workflow.tokens.id) # open the log view for the workflow rk.user_input('enter') rk.user_input('down', 'down', 'enter') # wait for the default log file to load await wait_log_loaded() rk.compare_screenshot( 'scheduler-log-file', 'the scheduler log file should be open', ) # jump to the bottom of the file rk.user_input('end') rk.compare_screenshot( 'scheduler-log-file-bottom', 'we should be looking at the bottom of the file' ) # jump back to the top of the file rk.user_input('home') rk.compare_screenshot( 'scheduler-log-file', 'we should be looking at the bottom of the file' ) # open the list of log files rk.user_input('enter') rk.compare_screenshot( 'log-file-selection', 'the list of available log files should be displayed' ) # select the processed workflow configuration file rk.user_input('down', 'enter') # wait for the file to load await wait_log_loaded() rk.compare_screenshot( 'workflow-configuration-file', 'the workflow configuration file should be open' ) async def test_task_logs( workflow, mod_rakiura, wait_log_loaded, ): """Test viewing task log files. I.E. Test viewing job log files by opening the log view on a task. """ with mod_rakiura(size='80,30') as rk: # wait for the workflow to appear (collapsed) rk.wait_until_loaded('#spring') # open the workflow in Tui rk.user_input('down', 'right') rk.wait_until_loaded(workflow.tokens.id) # open the context menu for the task 1/a rk.user_input('down', 'down', 'enter') # open the log view for the task 1/a rk.user_input('down', 'down', 'down', 'enter') # wait for the default log file to load await wait_log_loaded() rk.compare_screenshot( 'latest-job.out', 'the job.out file for the second job should be open', ) rk.user_input('enter') rk.user_input('enter') # wait for the job.err file to load await wait_log_loaded() rk.compare_screenshot( 'latest-job.err', 'the job.out file for the second job should be open', ) async def test_job_logs( workflow, mod_rakiura, wait_log_loaded, ): """Test viewing the job log files. I.E. Test viewing job log files by opening the log view on a job. """ with mod_rakiura(size='80,30') as rk: # wait for the workflow to appear (collapsed) rk.wait_until_loaded('#spring') # open the workflow in Tui rk.user_input('down', 'right') rk.wait_until_loaded(workflow.tokens.id) # open the context menu for the job 1/a/02 rk.user_input('down', 'down', 'right', 'down', 'enter') # open the log view for the job 1/a/02 rk.user_input('down', 'down', 'down', 'enter') # wait for the default log file to load await wait_log_loaded() rk.compare_screenshot( '02-job.out', 'the job.out file for the *second* job should be open', ) # close log view rk.user_input('q') # open the log view for the job 1/a/01 rk.user_input('down', 'enter') rk.user_input('down', 'down', 'down', 'enter') # wait for the default log file to load await wait_log_loaded() rk.compare_screenshot( '01-job.out', 'the job.out file for the *first* job should be open', ) async def test_errors( workflow, mod_rakiura, wait_log_loaded, monkeypatch, ): """Test error handing of cat-log commands.""" # make it look like cat-log commands are failing def cli_cmd_fail(*args, **kwargs): raise ClientError('Something went wrong :(') monkeypatch.setattr( 'cylc.flow.tui.data.cli_cmd', cli_cmd_fail, ) with mod_rakiura(size='80,30') as rk: # wait for the workflow to appear (collapsed) rk.wait_until_loaded('#spring') # open the log view on scheduler rk.user_input('down', 'enter', 'down', 'down', 'enter') # it will fail to open await wait_log_loaded() rk.compare_screenshot( 'open-error', 'the error message should be displayed in the log view header', ) # open the file selector rk.user_input('enter') # it will fail to list available log files rk.compare_screenshot( 'list-error', 'the error message should be displayed in a pop up', ) async def test_external_editor( workflow, mod_rakiura, wait_log_loaded, monkeypatch, capsys, ): """Test the "open in external editor" functionality. This test covers the relevant code about as well as we can in an integration test. * The integration tests write HTML fragments to a file rather ANSI to a terminal. * Suspending / restoring the Tui session involves shell interaction that we cannot simulate here. * We're also not testing subprocesses in this integration test. But this test passing tells us that the relevant code does indeed run without falling over in a heap, so it will detect interface breakages and the like which is useful. """ fake_popen_instances = [] class FakePopen: def __init__(self, cmd, *args, raises=None, **kwargs): fake_popen_instances.append(self) self.cmd = cmd self.args = args self.kwargs = kwargs self.raises = raises def wait(self): if self.raises: raise self.raises() return 0 # mock out subprocess.Popen monkeypatch.setattr( 'cylc.flow.tui.overlay.Popen', FakePopen, ) # mock out time.sleep monkeypatch.setattr( 'cylc.flow.tui.overlay.sleep', lambda x: None, ) with mod_rakiura(size='80,30') as rk: # wait for the workflow to appear (collapsed) rk.wait_until_loaded('#spring') # open the log view on scheduler rk.user_input('down', 'enter', 'down', 'down', 'enter') # it will fail to open await wait_log_loaded() assert len(fake_popen_instances) == 0 assert capsys.readouterr()[1] == '' # select the open in "$EDITOR" option rk.user_input('down', 'left', 'left', 'left') # make a note of what the screen looks like rk.compare_screenshot( 'before-opening-editor', 'The open in $EDITOR option should be selected', ) # launch the external tool rk.user_input('enter') # the subprocess should be started and a message logged to stderr assert len(fake_popen_instances) == 1 assert 'launching external tool' in capsys.readouterr()[1].lower() # once the subprocess exist, the Tui session should be restored # exactly as it was before rk.compare_screenshot( 'before-opening-editor', 'The Tui session should restore exactly as it was before', ) # get the subprocess to fail in a nasty way from functools import partial monkeypatch.setattr( 'cylc.flow.tui.overlay.Popen', partial(FakePopen, raises=OSError), ) # launch the external tool rk.user_input('enter') # the subprocess should be started, the error should be logged # to stderr assert len(fake_popen_instances) == 2 assert 'error running' in capsys.readouterr()[1].lower() # once the subprocess exist, the Tui session should be restored # exactly as it was before rk.compare_screenshot( 'before-opening-editor', 'The Tui session should restore exactly as it was before', ) cylc-flow-8.7.0/tests/integration/test_id_cli.py0000664000175000017500000000675715264420307022154 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Some integration tests running on live workflows to test filtering properly. """ import pytest from cylc.flow.pathutil import get_cylc_run_dir from cylc.flow.id_cli import parse_ids_async @pytest.fixture(scope='module') async def harness( mod_run, mod_scheduler, mod_flow, mod_one_conf, mod_test_dir, ): """Create three workflows, two running, one stopped.""" reg_prefix = mod_test_dir.relative_to(get_cylc_run_dir()) # abc:running reg1 = mod_flow(mod_one_conf, name='abc') schd1 = mod_scheduler(reg1) # def:running reg2 = mod_flow(mod_one_conf, name='def') schd2 = mod_scheduler(reg2) # ghi:stopped reg3 = mod_flow(mod_one_conf, name='ghi') async with mod_run(schd1): async with mod_run(schd2): yield reg_prefix, reg1, reg2, reg3 async def test_glob_wildcard(harness): """It should search for workflows using globs.""" reg_prefix, reg1, reg2, reg3 = harness # '*' should return all workflows workflows, _ = await parse_ids_async( f'{reg_prefix / "*"}', constraint='workflows', match_workflows=True, ) assert sorted(workflows) == sorted([reg1, reg2]) workflows, _ = await parse_ids_async( f'{reg_prefix / "z*"}', constraint='workflows', match_workflows=True, ) assert sorted(workflows) == sorted([]) async def test_glob_pattern(harness): """It should support fnmatch syntax including square brackets.""" # [a]* should match workflows starting with "a" reg_prefix, reg1, reg2, reg3 = harness workflows, _ = await parse_ids_async( f'{reg_prefix / "[a]*"}', constraint='workflows', match_workflows=True, ) assert sorted(workflows) == sorted([reg1]) workflows, _ = await parse_ids_async( f'{reg_prefix / "[z]*"}', constraint='workflows', match_workflows=True, ) assert sorted(workflows) == sorted([]) async def test_state_filter(harness): """It should filter by workflow state.""" reg_prefix, reg1, reg2, reg3 = harness # '*' should return all workflows workflows, _ = await parse_ids_async( f'{reg_prefix / "*"}', constraint='workflows', match_workflows=True, match_active=None, ) assert sorted(workflows) == sorted([reg1, reg2, reg3]) workflows, _ = await parse_ids_async( f'{reg_prefix / "*"}', constraint='workflows', match_workflows=True, match_active=True, ) assert sorted(workflows) == sorted([reg1, reg2]) workflows, _ = await parse_ids_async( f'{reg_prefix / "*"}', constraint='workflows', match_workflows=True, match_active=False, ) assert sorted(workflows) == sorted([reg3]) cylc-flow-8.7.0/tests/integration/plugins/0000775000175000017500000000000015264420307020762 5ustar alastairalastaircylc-flow-8.7.0/tests/integration/plugins/test_global_template_variables.py0000664000175000017500000000441115264420307027556 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) NIWA & British Crown (Met Office) & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.util import sstrip async def test_global_template_variables( flow, scheduler, start, mock_glbl_cfg, one_conf ): """It should ingest template variables defined in the global config. This test makes sure that global tempalte variables are provided, but also that they are provided as the appropriate type by attempting to mutate them (e.g. `STRING + ' '`). """ mock_glbl_cfg( 'cylc.flow.plugins.global_template_variables.glbl_cfg', ''' [template variables] # NOTE: both sets of quotes should be preserved STRING = '"answer"' INT = 40 BOOL = True LIST = [5, 6] TUPLE = (1, "7") DICT = {'a': 0, 1: 'possible'} ''', ) id_ = flow( { '#!Jinja2': '', 'meta': { 'description': ''' The {{ STRING + ' ' }}is {{ INT + 1 + BOOL }}. The question, what do you get if you multiply {{ LIST[0] + 1 }} by {{ TUPLE[1] + '?' }} Or is it {{ DICT[TUPLE[DICT['a']]] + ' to know both' }}? ''' }, **one_conf, } ) schd = scheduler(id_) async with start(schd): assert schd.config.cfg['meta']['description'] == sstrip(''' The "answer" is 42. The question, what do you get if you multiply 6 by 7? Or is it possible to know both? ''') cylc-flow-8.7.0/tests/integration/test_workflow_files.py0000664000175000017500000002221615264420307023751 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from itertools import product import logging from os import unlink from pathlib import Path from textwrap import dedent from uuid import uuid4 import pytest from cylc.flow import CYLC_LOG from cylc.flow.exceptions import ( SchedulerAlive, CylcError, ) from cylc.flow.workflow_files import ( ContactFileFields as CFF, WorkflowFiles, _is_process_running, detect_old_contact_file, dump_contact_file, load_contact_file, load_contact_file_async, ) @pytest.fixture(scope='module') async def myflow(mod_flow, mod_scheduler, mod_run, mod_one_conf): id_ = mod_flow(mod_one_conf) schd = mod_scheduler(id_) async with mod_run(schd): yield schd def test_load_contact_file(myflow): cont = load_contact_file(myflow.workflow) assert cont[CFF.HOST] == myflow.host async def test_load_contact_file_async(myflow): cont = await load_contact_file_async(myflow.workflow) assert cont[CFF.HOST] == myflow.host # compare the async interface to the sync interface cont2 = load_contact_file(myflow.workflow) assert cont == cont2 @pytest.fixture async def workflow(flow, scheduler, one_conf, run_dir): id_ = flow(one_conf) schd = scheduler(id_) await schd.install() from collections import namedtuple Server = namedtuple('Server', ['port', 'pub_port']) schd.server = Server(1234, pub_port=2345) schd.uuid_str = str(uuid4()) contact_data = schd.get_contact_data() contact_file = Path( run_dir, id_, WorkflowFiles.Service.DIRNAME, WorkflowFiles.Service.CONTACT ) def dump_contact(**kwargs): dump_contact_file( id_, { **contact_data, **kwargs } ) assert contact_file.exists() dump_contact() Fixture = namedtuple( 'TextFixture', [ 'id_', 'contact_file', 'contact_data', 'dump_contact', ] ) return Fixture(id_, contact_file, contact_data, dump_contact) def test_detect_old_contact_file_running(workflow): """It should raise an error if the workflow is running.""" # the workflow is running so we should get a ServiceFileError with pytest.raises(SchedulerAlive): detect_old_contact_file(workflow.id_) # the contact file is valid so should be left alone assert workflow.contact_file.exists() def test_detect_old_contact_file_network_issue(workflow): """It should raise an error if there are network issues.""" # modify the contact file to make it look like the PID has changed workflow.dump_contact( **{ # set the HOST to a non existent host CFF.HOST: 'not-a-host.no-such.domain' } ) # detect_old_contact_file should report that it can't tell if the workflow # is running or not with pytest.raises(CylcError) as exc_ctx: detect_old_contact_file(workflow.id_) assert ( 'Cannot determine whether workflow is running' in str(exc_ctx.value) ) # the contact file should be left alone assert workflow.contact_file.exists() def test_detect_old_contact_file_old_run(workflow, caplog, log_filter): """It should remove the contact file from an old run.""" # modify the contact file to make it look like the COMMAND has changed workflow.dump_contact( **{ CFF.COMMAND: 'foo bar baz' } ) caplog.set_level(logging.INFO, logger=CYLC_LOG) # the workflow should not appear to be running (according to the contact # data) so detect_old_contact_file should not raise any errors detect_old_contact_file(workflow.id_) # as a side effect the contact file should have been removed assert not workflow.contact_file.exists() assert log_filter(contains='Removed contact file') def test_detect_old_contact_file_none(workflow): """It should do nothing if there is no contact file.""" # remove the contact file workflow.contact_file.unlink() assert not workflow.contact_file.exists() # detect_old_contact_file should return detect_old_contact_file(workflow.id_) # it should not recreate the contact file assert not workflow.contact_file.exists() @pytest.mark.parametrize( 'process_running,contact_present_after,raises_error', filter( lambda x: x != (False, False, True), # logically impossible product([True, False], repeat=3), ) ) def test_detect_old_contact_file_removal_errors( workflow, monkeypatch, caplog, log_filter, process_running, contact_present_after, raises_error, ): """Test issues with removing the contact file are handled correctly. Args: process_running: If True we will make it look like the workflow process is still running (i.e. the workflow is still running). In this case detect_old_contact_file should *not* attempt to remove the contact file. contact_present_after: If False we will make the contact file disappear midway through the operation. This can happen because: * detect_old_contact_file in another client. * cylc clean. * Aliens. This is fine, nothing should be logged. raises_error: If True we will make it look like removing the contact file resulted in an OS error (not a FileNotFoundError). This error should be logged. """ # patch the is_process_running method def mocked_is_process_running(*args): if not contact_present_after: # remove the contact file midway through detect_old_contact_file unlink(workflow.contact_file) return process_running monkeypatch.setattr( 'cylc.flow.workflow_files._is_process_running', mocked_is_process_running, ) # patch the contact file removal def _unlink(*args): raise OSError('mocked-os-error') if raises_error: # force os.unlink to raise an arbitrary error monkeypatch.setattr( 'cylc.flow.workflow_files.os.unlink', _unlink, ) caplog.set_level(logging.INFO, logger=CYLC_LOG) # try to remove the contact file if process_running: # this should error if the process is running with pytest.raises(SchedulerAlive): detect_old_contact_file(workflow.id_) else: detect_old_contact_file(workflow.id_) # decide which log messages we should expect to see if process_running: remove_succeeded = False remove_failed = False else: if contact_present_after: if raises_error: remove_succeeded = False remove_failed = True else: remove_succeeded = True remove_failed = False else: remove_succeeded = False remove_failed = False # check the appropriate messages were logged assert bool(log_filter( contains='Removed contact file', )) is remove_succeeded assert bool(log_filter( contains=( f'Failed to remove contact file for {workflow.id_}:' '\nmocked-os-error' ), )) is remove_failed def test_is_process_running_dirty_output(monkeypatch, caplog): """Ensure _is_process_running can handle polluted output. E.G. this can happen if there is an echo statement in the `.bashrc`. """ stdout = None class Popen(): def __init__(self, *args, **kwargs): self.returncode = 0 def communicate(self, *args, **kwargs): return (stdout, '') monkeypatch.setattr( 'cylc.flow.workflow_files.Popen', Popen, ) # respond with something Cylc should be able to make sense of stdout = dedent(''' % simulated stdout pollution % [["expected", "command"]] ''') caplog.set_level(logging.WARN, logger=CYLC_LOG) assert _is_process_running('localhost', 1, 'expected command') assert not caplog.record_tuples assert not _is_process_running('localhost', 1, 'slartibartfast') assert not caplog.record_tuples # respond with something totally non-sensical stdout = 'sir henry' with pytest.raises(CylcError): _is_process_running('localhost', 1, 'expected command') # the command output should be in the debug message assert 'sir henry' in caplog.records[0].message cylc-flow-8.7.0/tests/integration/test_sequential_xtriggers.py0000664000175000017500000002641115264420307025166 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # mypy: disable-error-code=union-attr """Test interactions with sequential xtriggers.""" from unittest.mock import patch import pytest from cylc.flow.commands import ( run_cmd, force_trigger_tasks, remove_tasks, set_prereqs_and_outputs ) from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.cycling.iso8601 import ISO8601Point from cylc.flow.exceptions import XtriggerConfigError from cylc.flow.id import TaskTokens from cylc.flow.scheduler import Scheduler def list_cycles(schd: Scheduler): """List the task instance cycle points present in the pool.""" return sorted(itask.tokens['cycle'] for itask in schd.pool.get_tasks()) @pytest.fixture() def sequential(flow, scheduler): id_ = flow({ 'scheduler': { 'cycle point format': 'CCYY', }, 'scheduling': { 'runahead limit': 'P2', 'initial cycle point': '2000', 'graph': { 'P1Y': '@wall_clock => foo', } } }) return scheduler(id_) async def test_remove(sequential: Scheduler, start): """It should not spawn the next instance when a task is removed. Ensure that manually removing a task with a sequential xtrigger does not spawn the next, while the default removal does. """ async with start(sequential): # starts with the first task in the first cycle assert list_cycles(sequential) == ['2000'] # removing it should spawn the next sequential instance # internal remove should spawn the next cycle sequential.pool.remove( sequential.pool.get_task(ISO8601Point('2000'), 'foo'), reason='because' ) assert list_cycles(sequential) == ['2001'] # this sequentially spawns out to the runahead limit for year in range(2001, 2010): foo = sequential.pool.get_task(ISO8601Point(f'{year}'), 'foo') if foo.state(is_runahead=True): break sequential.xtrigger_mgr.call_xtriggers_async(foo) for _ in range(4): await sequential._main_loop() assert list_cycles(sequential) == [ '2001', '2002', '2003', '2004', ] # default remove of RH PSX should spawn the next cycle await run_cmd(remove_tasks(sequential, ['2004'], ["1"])) assert '2005' in list_cycles(sequential) # remove no-spawn should not spawn next RH task await run_cmd(remove_tasks(sequential, ['2005'], ["1"], True)) assert list_cycles(sequential) == [ '2001', '2002', '2003', ] # Default remove of already xtrigger spawned task should # spawn next again (because it's not tracked). await run_cmd(remove_tasks(sequential, ['2003'], ["1"])) assert list_cycles(sequential) == [ '2001', '2002', '2004', ] # Now, let's just command/manual remove all tasks in the pool await run_cmd(remove_tasks(sequential, ['*'], ["1"], True)) # the workflow should be empty assert not list_cycles(sequential) async def test_trigger(sequential, start): """It should spawn its next instance if triggered ahead of time. If you manually trigger a sequentially spawned task before its xtriggers have become satisfied, then the sequential spawning chain is broken. The task pool should defend against this to ensure that triggering a task doesn't cancel it's future instances. """ async with start(sequential): assert list_cycles(sequential) == ['2000'] foo1 = sequential.pool.get_task(ISO8601Point('2000'), 'foo') await run_cmd(force_trigger_tasks(sequential, [foo1.identity], ["1"])) await sequential._main_loop() assert list_cycles(sequential) == ['2001'] foo2 = sequential.pool.get_task(ISO8601Point('2001'), 'foo') foo2.state_reset(is_runahead=False) await run_cmd( set_prereqs_and_outputs(sequential, [foo2.identity], ["1"])) assert list_cycles(sequential) == ['2002'] async def test_set_outputs(sequential, start): """It should spawn its next instance if outputs are set ahead of time. If you set outputs of a sequentially spawned task before its xtriggers have become satisfied, then the sequential spawning chain is broken. The task pool should defend against this to ensure that setting outputs doesn't cancel it's future instances and their downstream tasks. """ async with start(sequential): assert list_cycles(sequential) == ['2000'] sequential.pool.get_task(ISO8601Point('2000'), 'foo') # set foo:succeeded it should spawn next instance sequential.pool.set_prereqs_and_outputs( {TaskTokens('2000', 'foo')}, ["succeeded"], [], []) assert list_cycles(sequential) == ['2001'] async def test_set_prereqs(sequential, start): """It should spawn next after manual xtrigger prereq satisfaction.""" async with start(sequential): assert list_cycles(sequential) == ['2000'] sequential.pool.get_task(ISO8601Point('2000'), 'foo') # satisfy foo's xtriggers - it should spawn next instance sequential.pool.set_prereqs_and_outputs( {TaskTokens('2000', 'foo')}, [], ['xtrigger/all:succeeded'], []) await sequential._main_loop() assert list_cycles(sequential) == ['2000', '2001'] async def test_reload(sequential, start): """It should set the is_xtrigger_sequential flag on reload. TODO: test that changes to the sequential status in the config get picked up on reload """ async with start(sequential): # the task should be marked as sequential pre_reload = sequential.pool.get_task(ISO8601Point('2000'), 'foo') assert pre_reload.is_xtrigger_sequential is True # reload the workflow sequential.pool.reload(sequential.config) # the original task proxy should have been replaced post_reload = sequential.pool.get_task(ISO8601Point('2000'), 'foo') assert id(pre_reload) != id(post_reload) # the new task should be marked as sequential assert post_reload.is_xtrigger_sequential is True @pytest.mark.parametrize('is_sequential', [True, False]) @pytest.mark.parametrize('xtrig_def', [ 'wall_clock(sequential={})', 'wall_clock(PT1H, sequential={})', 'xrandom(1, 1, sequential={})', ]) async def test_sequential_arg_ok( flow, scheduler, start, xtrig_def: str, is_sequential: bool ): """Test passing the sequential argument to xtriggers.""" wid = flow({ 'scheduler': { 'cycle point format': 'CCYY', }, 'scheduling': { 'initial cycle point': '2000', 'runahead limit': 'P1', 'xtriggers': { 'myxt': xtrig_def.format(is_sequential), }, 'graph': { 'P1Y': '@myxt => foo', } } }) schd: Scheduler = scheduler(wid) expected_num_cycles = 1 if is_sequential else 3 async with start(schd): itask = schd.pool.get_task(ISO8601Point('2000'), 'foo') assert itask.is_xtrigger_sequential is is_sequential assert len(list_cycles(schd)) == expected_num_cycles def test_sequential_arg_bad(flow, validate): """Test validation of 'sequential' arg for custom xtrigger function def""" wid = flow({ 'scheduling': { 'xtriggers': { 'myxt': 'custom_xt(42)' }, 'graph': { 'R1': '@myxt => foo' } } }) def xtrig1(x, sequential): """This uses 'sequential' without a default value""" return True def xtrig2(x, sequential='True'): """This uses 'sequential' with a default of wrong type""" return True for xtrig in (xtrig1, xtrig2): with patch( 'cylc.flow.xtrigger_mgr.get_xtrig_func', return_value=xtrig ): with pytest.raises(XtriggerConfigError) as excinfo: validate(wid) assert ( "reserved argument 'sequential' with no boolean default" ) in str(excinfo.value) def test_sequential_arg_bad2(flow, validate): """Test validation of 'sequential' arg for xtrigger calls""" wid = flow({ 'scheduling': { 'initial cycle point': '2000', 'xtriggers': { 'clock': 'wall_clock(sequential=3)', }, 'graph': { 'R1': '@clock => foo', }, }, }) with pytest.raises(XtriggerConfigError) as excinfo: validate(wid) assert ( "invalid argument 'sequential=3' - must be boolean" ) in str(excinfo.value) @pytest.mark.parametrize('is_sequential', [True, False]) async def test_any_sequential(flow, scheduler, start, is_sequential: bool): """Test that a task is marked as sequential if any of its xtriggers are.""" wid = flow({ 'scheduling': { 'xtriggers': { 'xt1': 'custom_xt()', 'xt2': f'custom_xt(sequential={is_sequential})', 'xt3': 'custom_xt(sequential=False)', }, 'graph': { 'R1': '@xt1 & @xt2 & @xt3 => foo', } } }) with patch( 'cylc.flow.xtrigger_mgr.get_xtrig_func', return_value=lambda *a, **k: True ): schd: Scheduler = scheduler(wid) async with start(schd): itask = schd.pool.get_task(IntegerPoint('1'), 'foo') assert itask.is_xtrigger_sequential is is_sequential async def test_override(flow, scheduler, start): """Test that the 'sequential=False' arg can override a default of True.""" wid = flow({ 'scheduling': { 'sequential xtriggers': True, 'xtriggers': { 'xt1': 'custom_xt()', 'xt2': 'custom_xt(sequential=False)', }, 'graph': { 'R1': ''' @xt1 => foo @xt2 => bar ''', } } }) with patch( 'cylc.flow.xtrigger_mgr.get_xtrig_func', return_value=lambda *a, **k: True ): schd: Scheduler = scheduler(wid) async with start(schd): foo = schd.pool.get_task(IntegerPoint('1'), 'foo') assert foo.is_xtrigger_sequential is True bar = schd.pool.get_task(IntegerPoint('1'), 'bar') assert bar.is_xtrigger_sequential is False cylc-flow-8.7.0/tests/integration/test_broadcast_mgr.py0000664000175000017500000002247315264420307023531 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests for Broadcast Manager.""" import asyncio import pytest from cylc.flow import commands from cylc.flow.cycling.integer import IntegerInterval, IntegerPoint from cylc.flow.cycling.iso8601 import ISO8601Interval, ISO8601Point from cylc.flow.task_state import TASK_STATUS_FAILED async def test_reject_valid_broadcast_is_remote_clash_with_config( one_conf, flow, start, scheduler, log_filter ): """`put_broadcast` gracefully rejects invalid broadcast: Existing config = [task][remote]host = foo Broadcast = [task]platform = bar https://github.com/cylc/cylc-flow/issues/6693 """ one_conf.update({'runtime': {'root': {'platform': 'foo'}}}) wid = flow(one_conf) schd = scheduler(wid) async with start(schd): bc_mgr = schd.broadcast_mgr good, bad = bc_mgr.put_broadcast( point_strings=['1'], namespaces=['one'], settings=[{'remote': {'host': 'bar'}}] ) # the error should be reported in the workflow log assert log_filter(contains='Cannot apply broadcast') assert bc_mgr.broadcasts == {'1': {}} # the bad setting should be reported assert good == [] assert bad == { 'settings': [ {'remote': {'host': 'bar'}}, ] } async def test_reject_valid_broadcast_is_remote_clash_with_broadcast( one_conf, flow, start, scheduler, log_filter ): """`put_broadcast` gracefully rejects invalid broadcast: Existing Broadcast = [task][remote]host = foo New Broadcast = [task]platform = bar https://github.com/cylc/cylc-flow/pull/6711/files#r2033457964 """ schd = scheduler(flow(one_conf)) async with start(schd): bc_mgr = schd.broadcast_mgr _, bad = bc_mgr.put_broadcast( point_strings=['1'], namespaces=['one'], settings=[{'remote': {'host': 'bar'}}] ) assert bad == {} # broadcast should be successful # this should not be allowed, if it is the scheduler will crash # when unpaused: good, bad = bc_mgr.put_broadcast( point_strings=['1'], namespaces=['one'], settings=[{'platform': 'foo'}] ) # the error should be reported in the workflow log assert log_filter(contains='Cannot apply broadcast') assert bc_mgr.broadcasts == {'1': {'one': {'remote': {'host': 'bar'}}}} # the bad setting should be reported assert good == [] assert bad == { 'settings': [ {'platform': 'foo'}, ] } @pytest.mark.parametrize('cycling_mode', ('integer', 'gregorian', '360_day')) async def test_broadcast_expire_limit( cycling_mode, flow, scheduler, run, complete, capcall, ): """Test automatic broadcast expiry. To prevent broadcasts from piling up and causing a memory leak, we expire (aka clear) them. The broadcast expiry limit is the oldest active cycle MINUS the longest cycling sequence. See https://github.com/cylc/cylc-flow/pull/6964 """ # capture broadcast expiry calls _expires = capcall('cylc.flow.broadcast_mgr.BroadcastMgr.expire_broadcast') def expires(): """Return a list of the cycle limit expired since the last call.""" ret = [x[0][1] for x in _expires] _expires.clear() return ret def cycle(number): """Return a cycle point object in the relevant format.""" if cycling_mode == 'integer': return IntegerPoint(str(number)) else: return ISO8601Point(f'000{number}') def interval(number): """Return an integer object in the relevant format.""" if cycling_mode == 'integer': return IntegerInterval(sequence(number)) else: return ISO8601Interval(sequence(number)) def sequence(number): """Return a sequence string in the relevant format.""" if cycling_mode == 'integer': return f'P{number}' else: return f'P{number}Y' # a workflow with a sequential task id_ = flow({ 'scheduler': { 'cycle point format': 'CCYY' } if cycling_mode != 'integer' else {}, 'scheduling': { 'cycling mode': cycling_mode, 'initial cycle point': cycle(1), 'graph': { # the sequence with the sequential task sequence(1): f'a[-{sequence(1)}] => a', # a longer sequence to make the offset more interesting sequence(3): 'a', } } }) schd = scheduler(id_, paused_start=False) async with run(schd): # the longest cycling sequence has a step of "3" assert schd.config.interval_of_longest_sequence == interval(3) # no broadcast expires should happen on startup assert expires() == [] # when a cycle closes, auto broadcast expiry should happen # NOTE: datetimes cannot be negative, so this expiry will be skipped # for datetimetime cycling workflows await complete(schd, f'{cycle(1)}/a') assert expires() in ([], [cycle(-1)]) await complete(schd, f'{cycle(2)}/a') assert expires() == [cycle(0)] await complete(schd, f'{cycle(3)}/a') assert expires() == [cycle(1)] async def test_broadcast_expiry_async( one_conf, flow, scheduler, run, complete, capcall ): """Test auto broadcast expiry with async workflows. Auto broadcast expiry should not happen in async workflows as there is only one cycle so it doesn't make sense. See https://github.com/cylc/cylc-flow/pull/6964 """ # capture broadcast expiry calls expires = capcall('cylc.flow.broadcast_mgr.BroadcastMgr.expire_broadcast') id_ = flow(one_conf) schd = scheduler(id_, paused_start=False) async with run(schd): # this is an async workflow so the longest cycling interval is a # null interval assert ( schd.config.interval_of_longest_sequence == IntegerInterval.get_null() ) await complete(schd) # no auto-expiry should take place assert expires == [] async def test_broadcast_old_cycle(flow, scheduler, run, complete): """It should not expire broadcasts whilst the scheduler is paused. This tests the use case of broadcasting to a historical cycle (whilst the workflow is paused) before triggering it to run to ensure that the broadcast is not expired before the operator is able to run the trigger command. For context, see https://github.com/cylc/cylc-flow/pull/6499 and https://github.com/cylc/cylc-flow/pull/6192#issuecomment-2486785465 """ id_ = flow({ 'scheduling': { 'initial cycle point': '1', 'cycling mode': 'integer', 'graph': { 'P1': 'a[-P1] => a', }, }, }) schd = scheduler(id_, paused_start=False) async with run(schd): # issue a broadcast into the first cycle schd.broadcast_mgr.put_broadcast( point_strings=['1'], namespaces=['a'], settings=[{'environment': {'ANSWER': '42'}}] ) assert list(schd.broadcast_mgr.broadcasts) == ['1'] # the broadcast should expire after the workflow passes cycle "3" await complete(schd, '3/a') assert list(schd.broadcast_mgr.broadcasts) == [] # pause the workflow await commands.run_cmd(commands.pause(schd)) # issue a broadcast into the first cycle (now behind the broadcast # expire point) schd.broadcast_mgr.put_broadcast( point_strings=['1'], namespaces=['a'], settings=[{'simulation': {'fail cycle points': '1'}}] ) # this should not be expired whilst the scheduler is paused await schd._main_loop() # run one iteration of the main loop assert list(schd.broadcast_mgr.broadcasts) == ['1'] # trigger the first cycle and resume the workflow await commands.run_cmd(commands.force_trigger_tasks(schd, ['1'], [])) await commands.run_cmd(commands.resume(schd)) # the broadcast should still be there await schd._main_loop() # run one iteration of the main loop assert list(schd.broadcast_mgr.broadcasts) == ['1'] # and should take effect a_1 = schd.pool._get_task_by_id('1/a') async with asyncio.timeout(5): while True: await asyncio.sleep(0.1) if a_1.state(TASK_STATUS_FAILED): break cylc-flow-8.7.0/tests/integration/test_id_match.py0000664000175000017500000001425015264420307022464 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from typing import ( TYPE_CHECKING, Set, Tuple, cast, ) from cylc.flow import commands from cylc.flow.id import Tokens from cylc.flow.scheduler import Scheduler if TYPE_CHECKING: from cylc.flow.id import TaskTokens async def test_id_match(flow, scheduler, start, caplog): id_ = flow({ 'scheduling': { 'cycling mode': 'integer', 'initial cycle point': '1', 'final cycle point': '3', 'graph': { 'P2': ''' a1 => b1 => c1 a2 => b2 => c2 b1[-P1] => b1 b2[-P1] => b2 ''', }, }, 'runtime': { 'a1, a2': {'inherit': 'A'}, 'A': {}, 'b1, b2': {'inherit': 'B'}, 'B': {}, 'c1, c2': {}, }, }) schd: Scheduler = scheduler(id_) def match(*ids: str) -> Tuple[Set[str], Set[str]]: matched, unmatched = schd.pool.id_match( {cast('TaskTokens', Tokens(id_, relative=True)) for id_ in ids}, ) return {id_.relative_id for id_ in matched}, { id_.relative_id_with_selectors for id_ in unmatched } async with start(schd): await commands.run_cmd( commands.set_prereqs_and_outputs( schd, ['1/a2'], ['1'], ['succeeded'], None ) ) await commands.run_cmd( commands.set_prereqs_and_outputs( schd, ['1/b2'], ['1'], ['failed'], None ) ) # task pool state: # * cycle 1 # * n=0 a1 waiting # * n=1 b1 waiting # * n=2 c1 waiting # * n=1 a2 succeeded # * n=0 b2 failed # * n=1 c2 waiting # * cycle 2 # * n=0 a1 waiting # * n=1 b1 waiting # * n=2 c1 waiting # * n=0 a2 waiting # * n=1 b2 waiting # * n=2 c2 waiting # check the n=0 window matches expectations before proceeding assert { itask.tokens.relative_id for itask in schd.pool.get_tasks() } == {'1/a1', '1/b2', '3/a1', '3/a2'} # test active task selection: waiting selector assert ( match('*:waiting') == match('*/root:waiting') == match('*/*:waiting') == match('*/A:waiting') == match('*/a*:waiting') == match('1/a1:waiting', '3/a1:waiting', '3/a2:waiting') == match('^/a1:waiting', '3/a1:waiting', '$/a2:waiting') == ({'1/a1', '3/a1', '3/a2'}, set()) ) # test active task selection: failed selector assert ( match('*:failed') == match('*/root:failed') == match('*/*:failed') == match('*/B:failed') == match('*/b*:failed') == match('1/b2:failed') == match('^/b2:failed') == ({'1/b2'}, set()) ) # test active task selection: failed selector assert ( match('1/b1:failed', '1/b2:failed') == match('1/B:failed', '1/b1:failed') == match('*:failed', '1/B:failed', '1/b1:failed') == ({'1/b2'}, {'1/b1:failed'}) ) # test active task selection: submit-failed selector assert match('*:submit-failed') == (set(), {'*:submit-failed'}) # test globs, cycle and family matching assert ( match('*') == match('*/*') == match('*/root') == match('*/A', '*/B', '*/c1', '*/c2') == match('*/a*', '*/b*', '*/c*') == ( { '1/a1', '1/a2', '1/b1', '1/b2', '1/c1', '1/c2', '3/a1', '3/a2', '3/b1', '3/b2', '3/c1', '3/c2', }, set(), ) ) # test globs, cycle and family matching assert ( match('1/A') == match('1/a*') == match('^/a*') == match('1/a1', '1/a2') == ({'1/a1', '1/a2'}, set()) ) assert match('1/X') == (set(), {'1/X'}) assert match('5:waiting') == (set(), {'5:waiting'}) # test invalid IDs assert match('not-a-cycle/*', '*/not_a_task', '*:not-a-state') == ( set(), {'not-a-cycle/*', '*/not_a_task', '*:not-a-state'}, ) # test invalid cycle point in combination with a selector assert match('x/y:succeeded') == (set(), {'x/y:succeeded'}) # ensure that off-sequence IDs are filtered out # NOTE: cycle 2 is not on sequence for task a1 assert match('1/a1', '2/a1', '3/a1') == ({'1/a1', '3/a1'}, {'2/a1'}) # ensure warnings are raised for off-sequence tasks if the user # explicitly specified them # NOTE: 2/a1 should result in a warning (because the user asked for # this exact combination), however, "2/a*" should not (because there # may be a combination of tasks which are or are not valid at the given # cycle(s) which match the task name pattern) caplog.clear() assert match('2/a1', '2/a*') == (set(), {'2/a1', '2/a*'}) assert caplog.messages == ['Invalid cycle point for task: a1, 2'] cylc-flow-8.7.0/tests/integration/validate/0000775000175000017500000000000015264420307021072 5ustar alastairalastaircylc-flow-8.7.0/tests/integration/validate/test_filter.py0000664000175000017500000000272615264420307023777 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.parsec.exceptions import IllegalItemError def test_filtered_keys_error(flow, validate): """ Test evaluates an invalid key: 'retry delays', and checks that this returns an IllegalItemError. REGEX for the event possible keys change. """ id_ = flow({ 'scheduling': { 'graph': { 'R1': 'foo', }, }, 'runtime': { 'foo': { 'retry delays': 'PT1S' }, }, }) with pytest.raises(IllegalItemError, match=( r'\[runtime\]\[foo\]retry delays.* did you ' r'mean execution retry delays.*submission retry delays.*' )): validate(id_) cylc-flow-8.7.0/tests/integration/validate/test_validation.py0000664000175000017500000004237115264420307024644 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.parsec.exceptions import ( IllegalValueError, IllegalItemError, UpgradeError, ) from cylc.flow.exceptions import ( GraphParseError, WorkflowConfigError, PointParsingError, SequenceDegenerateError, ) from metomi.isodatetime.exceptions import ( TimePointDumperBoundsError, ) def test_multi_inheritance(flow, validate): """Test validating simple multi-inheritance workflows.""" id_ = flow({ 'scheduling': { 'graph': { 'R1': '"""foo"""', }, }, 'runtime': { 'FOO': { }, 'BAR': { }, 'foo': { 'inherit': 'FOO, BAR', }, }, }) validate(id_) def test_periodical(flow, validate): """Test validating Daily, Monthly and Yearly type tasks.""" id_ = flow({ 'scheduling': { 'initial cycle point': '20100101T00', 'final cycle point': '20100102T00', 'graph': { 'P1D': '"daily"', 'P1M': '"monthly"', 'P1Y': '"yearly"', }, }, 'runtime': { 'root': { 'script': '"true"', }, }, }) validate(id_) def test_scripting_quotes(flow, validate): """Test that validating: script = "foo"bar"baz" fails""" id_ = flow({ 'scheduling': { 'graph': { 'R1': '"foo"', }, }, 'runtime': { 'foo': { 'script': '"foo"bar"baz"', }, }, }) with pytest.raises(IllegalValueError, match=( r'\(type=string\) \[runtime\]\[foo\].*' )): validate(id_) def test_bad_recurrence(flow, validate): """Test validation for a bad recurrences""" id_ = flow({ 'scheduling': { 'initial cycle point': '20140101T00', 'final cycle point': '20140201T00', 'graph': { 'R/T00/PT5D': '"foo"', # PT5D is invalid - should be P5D }, }, 'runtime': { 'foo': { 'script': 'true', }, }, }) with pytest.raises(WorkflowConfigError, match=( r'Cannot process recurrence.*' )): validate(id_) def test_fail_cylc6_inter_cycle_syntax(flow, validate): """Test validation with a new-style cycle point and a prev-style offset.""" id_ = flow({ 'scheduling': { 'initial cycle point': '20100101T00', 'graph': { 'T00': '"foo[T-24] => foo"', }, }, }) with pytest.raises(GraphParseError, match=( r'Illegal graph node.*' )): validate(id_) def test_fail_cylc6_timeout_syntax(flow, validate): """Test validation with a new-style cycle point and a prev-style limit.""" id_ = flow({ 'scheduling': { 'initial cycle point': '20100101T00', 'graph': { 'T00,T06,T12': '"foo"', }, }, 'runtime': { 'root': { 'events': { 'execution timeout': '3', }, }, }, }) with pytest.raises(IllegalValueError, match=( r'\(type=ISO 8601 interval\) \[runtime\]\[root\]\[events\].*' )): validate(id_) def test_fail_cylc6_cycle_point(flow, validate): """Test validation with a prev-style cycle point and a new-style cycling section""" id_ = flow({ 'scheduling': { 'initial cycle point': '2010010100', 'graph': { 'T12': '"foo"', }, }, }) with pytest.raises(PointParsingError, match=( r'Incompatible value for.* Invalid ISO.*' )): validate(id_) def test_fail_cylc6_recurrence(flow, validate): """Test validation with a new-style cycle point and an async graph.""" id_ = flow({ 'scheduling': { 'initial cycle point': '20100101T00', 'graph': { 'R1': '"cold_foo"', '12': '"cold_foo => foo"', }, }, }) with pytest.raises(WorkflowConfigError, match=( r'Cannot process recurrence.*' )): validate(id_) def test_fail_no_scheduling(flow, validate): """Test validation for a workflow with no "[scheduling]" section.""" id_ = flow({ }) with pytest.raises(WorkflowConfigError, match=( r'missing \[scheduling\] section.*' )): validate(id_) def test_fail_empty_graph(flow, validate): """Test validation of an empty graph.""" id_ = flow({ 'scheduling': { 'graph': { }, }, }) with pytest.raises(WorkflowConfigError, match=( r'No workflow dependency graph defined.*' )): validate(id_) def test_fail_empty_graph_2(flow, validate): """Test validation of an empty graph.""" id_ = flow({ 'scheduling': { 'graph': { 'R1': '""', }, }, }) with pytest.raises(WorkflowConfigError, match=( r'No workflow dependency graph defined.' )): validate(id_) def test_num_expanded_year_digits(flow, validate): """Test validation for "cycle point num expanded year digits".""" # it should fail a cycle point that uses expanded digits if Cylc has not # been configured to use them conf = { 'scheduling': { 'initial cycle point': '+10000-01-01T00', 'graph': { 'T00': 'foo', }, }, } id_ = flow(conf) with pytest.raises(PointParsingError, match=( r'Incompatible value for.*: \+10000-01-01T00:.* ' r'\(incompatible with \[cylc\]cycle point ' r'num expanded year digits = 0 \?\)' )): validate(id_) # it should fail validation if the number of expanded digits is configured # but does not match for number in (0, 2, 10): conf['scheduler']['cycle point num expanded year digits'] = str(number) id_ = flow(conf) with pytest.raises(PointParsingError, match=( r'Incompatible value for.*: \+10000-01-01T00:.* ' r'\(incompatible with \[cylc\]cycle point ' rf'num expanded year digits = {number}' )): validate(id_) # it should pass otherwise conf['scheduler']['cycle point num expanded year digits'] = '1' id_ = flow(conf) validate(id_) def test_fail_initial_greater_final(flow, validate): """Test validation fails for initial cycle point greater than the final.""" id_ = flow({ 'scheduling': { 'initial cycle point': '20141208T0000Z', 'final cycle point': '20141207T0000Z', 'graph': { 'T00': 'A => B', }, }, 'runtime': { 'A': { }, 'B': { }, }, }) with pytest.raises(WorkflowConfigError, match=( r'The initial cycle point.* is after the final cycle point.*' )): validate(id_) def test_9999_rollover(flow, validate): """Test intercycle dependencies.""" id_ = flow({ 'scheduling': { 'initial cycle point': '99991231T2200', 'graph': { 'R3//PT1H': '"foo"', }, }, }) with pytest.raises(TimePointDumperBoundsError, match=( r'Cannot dump TimePoint year:.*' )): validate(id_) def test_constrained_initial(flow, validate): """Test validating simple multi-inheritance workflows.""" conf = { 'scheduling': { 'initial cycle point': '20100101T06', 'initial cycle point constraints': 'T00, T06, T12, T18', 'graph': { 'T00, T06, T12, T18': 'foo', }, }, 'runtime': { 'FOO': { }, 'BAR': { }, 'foo': { 'inherit': 'FOO, BAR', }, }, } id_ = flow(conf) validate(id_) conf['scheduling']['initial cycle point'] = '20100101T03' id_ = flow(conf) with pytest.raises(WorkflowConfigError, match=( r'Initial cycle point .* does not meet the constraints.*' )): validate(id_) def test_constrained_final(flow, validate): """Test validating simple multi-inheritance workflows.""" conf = { 'scheduling': { 'initial cycle point': '20100101T06', 'final cycle point': '20100101T18', 'final cycle point constraints': 'T00, T06, T12, T18', 'graph': { 'T00, T06, T12, T18': 'foo', }, }, 'runtime': { 'FOO': { }, 'BAR': { }, 'foo': { 'inherit': 'FOO, BAR', }, }, } id_ = flow(conf) validate(id_) conf['scheduling']['initial cycle point'] = '20100101T03' conf['scheduling']['final cycle point'] = '20100102T17' id_ = flow(conf) with pytest.raises(WorkflowConfigError, match=( r'Final cycle point .* does not meet the constraints.*' )): validate(id_) def test_fail_not_integer(flow, validate): """Test clash of integer and ISO cycling.""" id_ = flow({ 'scheduling': { 'initial cycle point': '2015-01-01', 'final cycle point': '2015-01-01', 'graph': { 'P1': 'foo', }, }, }) with pytest.raises(WorkflowConfigError, match=( r'Cannot process recurrence.*' )): validate(id_) def test_no_clock_int_cycle(flow, validate): """Test validation fails when the cycle point format is less precise than a graph recurrence interval""" id_ = flow({ 'scheduler': { 'cycle point format': '%Y-%m', }, 'scheduling': { 'initial cycle point': '2015-08', 'graph': { 'P1D': 'foo', }, }, }) with pytest.raises(SequenceDegenerateError, match=( r'R/2015-08/P1D, point format.*' )): validate(id_) def test_hyphen_fam_1(flow, validate): """Test validation of task name with a XXX-FAM pattern.""" id_ = flow({ 'scheduling': { 'graph': { 'R1': '"baz-foo => bar"', }, }, 'runtime': { 'foo': { }, 'bar, baz-foo': { 'inherit': 'foo', }, }, }) validate(id_) def test_hyphen_fam_2(flow, validate): """Test validation of task name with a XXX-FAM pattern.""" id_ = flow({ 'scheduling': { 'graph': { 'R1': '"foo-baz => bar"', }, }, 'runtime': { 'foo': { }, 'bar, foo-baz': { 'inherit': 'foo', }, }, }) validate(id_) def test_null_timeout(flow, validate): """Test explicit unset timeout intervals validate (GitHub #1865).""" id_ = flow({ 'scheduling': { 'graph': { 'R1': 'foo', }, }, 'runtime': { 'foo': { 'events': { 'execution timeout': '', }, }, }, }) validate(id_) def test_hyphen_finish(flow, validate): """Test hyphen in task name + ":finish". See cylc/cylc-flow#1949.""" id_ = flow({ 'scheduling': { 'graph': { 'R1': 'foo-bar:finish => baz', }, }, 'runtime': { 'foo-bar,baz': { 'script': 'true', }, }, }) validate(id_) def test_succeed_sub(flow, validate): """In graph lines where we have multiple triggers of the same task, ensure:succeed does not get substituted to symbols that already have a trigger.""" id_ = flow({ 'scheduling': { 'graph': { 'R1': 'foo:fail? | (foo? & bar:fail) => something', }, }, }) validate(id_) def test_offset_no_offset(flow, validate): """GitHub PR #2002 - validation of "foo | foo[-P1D] => bar" was failing becausethe explicit ':succeed' trigger was being substituted before the offset instead of after, creating an invalid trigger expression.""" id_ = flow({ 'scheduling': { 'initial cycle point': '2010', 'graph': { 'P1D': 'foo | foo[-P1D] => bar', }, }, }) validate(id_) def test_icp_quoted_now(flow, validate): """Quoted "now" for initial cycle point was failing.""" id_ = flow({ 'scheduler': { 'cycle point format': '%Y%m%d', }, 'scheduling': { 'initial cycle point': '"now"', 'graph': { 'P1D': 't1', }, }, }) validate(id_) def test_bad_task_event_handler_tmpl(flow, validate): """Test validation fails on bad task event handler templates.""" id_ = flow({ 'scheduling': { 'graph': { 'R1': 't1', }, }, 'runtime': { 't1': { 'script': 'true', 'events': { 'failed handlers': 'echo %(id)s, echo %(rubbish)s', }, }, }, }) with pytest.raises(WorkflowConfigError, match=( r'bad task event handler template.*' )): validate(id_) def test_no_clock_int_cycle_bad_value(flow, validate): """Test validation fails on bad task event handler templates.""" id_ = flow({ 'scheduling': { 'graph': { 'R1': 't1', }, }, 'runtime': { 't1': { 'events': { 'failed handlers': 'echo %(ids', }, }, }, }) with pytest.raises(WorkflowConfigError, match=( r'bad task event handler template.*' )): validate(id_) def test_no_clock_int_cycle_xtrigger(flow, validate): """Test that clock xtriggers are not allowed with integer cycling.""" id_ = flow({ 'scheduling': { 'cycling mode': 'integer', 'initial cycle point': '1', 'final cycle point': '2', 'xtriggers': { 'c1': 'wall_clock(offset=P0Y)', }, 'graph': { 'R/^/P1': '"@c1 & foo[-P1] => foo"', }, }, }) with pytest.raises(WorkflowConfigError, match=( r'Clock xtriggers require datetime cycling.*' )): validate(id_) def test_Valid_xtrigger_name(flow, validate): """Test validating xtrigger names in workflow.""" id_ = flow({ 'scheduling': { 'initial cycle point': '2000', 'xtriggers': { 'foo': 'wall_clock():PT1S', }, 'graph': { 'R1': '@foo => bar', }, }, }) validate(id_) def test_Invalid_xtrigger_name(flow, validate): """Test validating xtrigger names in workflow.""" id_ = flow({ 'scheduling': { 'initial cycle point': '2000', 'xtriggers': { 'foo-1': 'wall_clock():PT1S', }, 'graph': { 'R1': '@foo-1 => bar', }, }, }) with pytest.raises(WorkflowConfigError, match=( r'Invalid xtrigger name.*' )): validate(id_) def test_section_as_setting_normal(flow, validate): """Test handling of mixed up sections vs settings 1. section as setting (normal)""" id_ = flow({ 'runtime': { 'foo': { 'environment': '42', }, }, }) with pytest.raises(IllegalItemError, match=( r'\[runtime\]\[foo\]environment.*' )): validate(id_) def test_section_as_setting_upgrader(flow, validate): """Test handling of mixed up sections vs settings 2. section as setting (via upgrader)""" id_ = flow({ 'scheduling': '22', }) with pytest.raises(UpgradeError, match=( r'\[scheduling\].*' )): validate(id_) def test_setting_as_section(flow, validate): """Test handling of mixed up sections vs settings 3. setting as section""" id_ = flow({ 'scheduling': { 'initial cycle point': { }, }, }) with pytest.raises(IllegalItemError, match=( r'\[scheduling\]initial cycle point.*' )): validate(id_) cylc-flow-8.7.0/tests/integration/validate/test_jinja2.py0000664000175000017500000000460115264420307023661 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import re from textwrap import dedent from cylc.flow.exceptions import InputError from cylc.flow.parsec.exceptions import Jinja2Error import pytest @pytest.fixture def flow_cylc(tmp_path): """Write a flow.cylc file containing the provided text.""" def _inner(text): (tmp_path / 'flow.cylc').write_text(dedent(text).strip()) return tmp_path return _inner @pytest.mark.parametrize( 'line', [ pytest.param("raise('some error message')", id='raise'), pytest.param("assert(False, 'some error message')", id='assert'), ], ) def test_raise_helper(flow_cylc, validate, line, monkeypatch): """It should raise an error from within Jinja2.""" # it should raise a minimal InputError # (because assert/raise are used to validate inputs) # - intended for users of the workflow src_dir = flow_cylc(f''' #!Jinja2 {{{{ {line} }}}} ''') with pytest.raises( InputError, match=( r'^some error message' r'\n\(add --verbose for more context\)$' ), ): validate(src_dir) # in verbose mode, it should raise the full error # (this includes the Jinja2 context including file/line info) # - intended for developers of the workflow monkeypatch.setattr('cylc.flow.flags.verbosity', 1) with pytest.raises( Jinja2Error, match=( r'^some error message' r'\nFile /.*/pytest-.*/test_.*/flow.cylc' r'\n #!Jinja2' r'\n \{\{ ' + re.escape(line) + r' \}\}' r'\t<-- Jinja2AssertionError$' ) ): validate(src_dir) cylc-flow-8.7.0/tests/integration/validate/test_outputs.py0000664000175000017500000002677315264420307024245 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test validation of the [runtime][][outputs] section.""" from random import random import pytest from cylc.flow.exceptions import ( GraphParseError, WorkflowConfigError, WorkflowFilesError, ) from cylc.flow.scripts.validate import ( ValidateOptions, run as validate, ) from cylc.flow.unicode_rules import ( TaskMessageValidator, TaskOutputValidator, ) from cylc.flow.workflow_files import WorkflowFiles @pytest.mark.parametrize( 'outputs, valid', [ pytest.param( [ 'foo', 'foo-bar', 'foo_bar', '0foo0', '123', ], True, id='valid', ), pytest.param( [ # special prefix '_cylc', # nasty chars 'foo bar', 'foo,bar', 'foo/bar', 'foo+bar', # keywords 'required', 'optional', 'all', # built-in qualifiers 'succeeded', 'succeed-all', # alternative qualifiers 'succeed', ], False, id='invalid', ), ], ) def test_outputs(outputs, valid, flow, validate): """It should validate task outputs. Outputs i.e. the keys of the [outputs] section. We don't want users adding outputs that override built-in outputs (e.g. succeeded, failed) or qualifiers (e.g. succeed-all). We don't want users adding outputs that conflict with keywords e.g. "required" or "all". """ # test that each output validates correctly for output in outputs: assert TaskOutputValidator.validate(output)[0] is valid # test that output validation is actually being performed id_ = flow({ 'scheduling': { 'graph': {'R1': 'foo'} }, 'runtime': { 'foo': { 'outputs': { output: str(random()) for output in outputs } } }, }) val = lambda: validate(id_) if valid: val() else: with pytest.raises(WorkflowConfigError): val() @pytest.mark.parametrize( 'messages, valid', [ pytest.param( [ 'foo bar baz', 'WARN:foo bar baz' ], True, id='valid', ), pytest.param( [ # special prefix '_cylc', # invalid colon usage 'foo bar: baz' # built-in qualifiers 'succeeded', 'succeed-all', # alternative qualifiers 'succeed', ], False, id='invalid', ), ], ) def test_messages(messages, valid, flow, validate): """It should validate task messages. Messages i.e. the values of the [outputs] section. We don't want users adding messages that override built-in outputs (e.g. succeeded, failed). To avoid confusion it's best to prohibit outputs which override built-in qualifiers (e.g. succeed-all) too. There's a special use of the colon character which users need to conform with too. """ # test that each message validates correctly for message in messages: assert TaskMessageValidator.validate(message)[0] is valid # test that output validation is actually being performed id_ = flow({ 'scheduling': { 'graph': {'R1': 'foo'} }, 'runtime': { 'foo': { 'outputs': { str(random())[2:]: message for message in messages } } }, }) val = lambda: validate(id_) if valid: val() else: with pytest.raises(WorkflowConfigError): val() @pytest.mark.parametrize( 'graph, expression, message', [ pytest.param( 'foo:x', 'succeeded and (x or y)', r'foo:x is required in the graph.*' r' but optional in the completion expression', id='required-in-graph-optional-in-completion', ), pytest.param( 'foo:x?', 'succeeded and x', r'foo:x is optional in the graph.*' r' but required in the completion expression', id='optional-in-graph-required-in-completion', ), pytest.param( 'foo:x', 'succeeded', 'foo:x is required in the graph.*' 'but not referenced in the completion expression', id='required-in-graph-not-referenced-in-completion', ), pytest.param( # tests proposal point 4: # https://cylc.github.io/cylc-admin/proposal-optional-output-extension.html#proposal 'foo:expired', 'succeeded', 'foo:expired must be optional', id='expire-required-in-graph', ), pytest.param( 'foo:expired?', 'succeeded', 'foo:expired is permitted in the graph.*' '\nTry: completion = "succeeded or expired"', id='expire-optional-in-graph-but-not-used-in-completion' ), pytest.param( # tests part of proposal point 5: # https://cylc.github.io/cylc-admin/proposal-optional-output-extension.html#proposal 'foo', 'finished and x', '"finished" output cannot be used in completion expressions', id='finished-output-used-in-completion-expression', ), pytest.param( # https://github.com/cylc/cylc-flow/pull/6046#issuecomment-2059266086 'foo?', 'x and failed', 'foo:failed is optional in the graph.*' 'but required in the completion expression', id='failed-implicitly-optional-in-graph-required-in-completion', ), pytest.param( 'foo', '(succeed and x) or failed', 'Use "succeeded" not "succeed" in completion expressions', id='alt-compvar1', ), pytest.param( 'foo? & foo:submitted?', 'submit_fail or succeeded', 'Use "submit_failed" not "submit_fail" in completion expressions', id='alt-compvar2', ), pytest.param( 'foo? & foo:submitted?', 'submit-failed or succeeded', 'Use "submit_failed" rather than "submit-failed"' ' in completion expressions.', id='submit-failed used in completion expression', ), pytest.param( 'foo:file-1', 'succeeded or file-1', 'Replace hyphens with underscores in task outputs when' ' used in completion expressions.', id='Hyphen used in completion expression', ), pytest.param( 'foo:x', 'not succeeded or x', 'Error in .*' '\nInvalid expression', id='Non-whitelisted syntax used in completion expression', ), ] ) def test_completion_expression_invalid( flow, validate, graph, expression, message, ): """It should ensure the completion is logically consistent with the graph. Tests proposal point 5: https://cylc.github.io/cylc-admin/proposal-optional-output-extension.html#proposal """ id_ = flow({ 'scheduling': { 'graph': {'R1': graph}, }, 'runtime': { 'foo': { 'completion': expression, 'outputs': { 'x': 'xxx', 'y': 'yyy', 'file-1': 'asdf' }, }, }, }) with pytest.raises(WorkflowConfigError, match=message): validate(id_) @pytest.mark.parametrize( 'graph, expression', [ ('foo', 'succeeded and (x or y or z)'), ('foo?', 'succeeded and (x or y or z) or failed or expired'), ('foo', '(succeeded and x) or (expired and y)'), ] ) def test_completion_expression_valid( flow, validate, graph, expression, ): id_ = flow({ 'scheduling': { 'graph': {'R1': graph}, }, 'runtime': { 'foo': { 'completion': expression, 'outputs': { 'x': 'xxx', 'y': 'yyy', 'z': 'zzz', }, }, }, }) validate(id_) def test_unique_messages( flow, validate ): """Task messages must be unique in the [outputs] section. See: https://github.com/cylc/cylc-flow/issues/6056 """ id_ = flow({ 'scheduling': { 'graph': {'R1': 'foo'} }, 'runtime': { 'foo': { 'outputs': { 'a': 'foo', 'b': 'bar', 'c': 'baz', 'd': 'foo', } }, } }) with pytest.raises( WorkflowConfigError, match=( r'"\[runtime\]\[foo\]\[outputs\]d = foo"' ' - messages must be unique' ), ): validate(id_) async def test_suite_rc(tmp_path): """It should reject workflows with suite.rc files.""" # suite.rc present (tmp_path / WorkflowFiles.SUITE_RC).touch() with pytest.raises( WorkflowFilesError, match=f'suite.rc found in.*{tmp_path}' ): await validate(ValidateOptions(), str(tmp_path)) # suite.rc and flow.cylc present (tmp_path / WorkflowFiles.FLOW_FILE).touch() with pytest.raises( WorkflowFilesError, match=f'Both flow.cylc and suite.rc.*{tmp_path}' ): await validate(ValidateOptions(), str(tmp_path)) # flow.cylc present (tmp_path / WorkflowFiles.SUITE_RC).unlink() with pytest.raises( # blank flow.cylc -> WorkflowConfigError (not WorkflowFilesError) WorkflowConfigError, match=r'missing \[scheduling\] section' ): await validate(ValidateOptions(), str(tmp_path)) async def test_opposite_outputs(flow, validate): """Test logically opposite outputs are reported if misused.""" id_ = flow({ 'scheduling': { 'graph': { 'R1': ''' foo => bar foo:fail => baz foo => !baz ''' }, }, }) with pytest.raises( GraphParseError, match=( 'Opposite outputs foo:failed and foo:succeeded must both be' ' optional if both are used' ), ): validate(id_) cylc-flow-8.7.0/tests/integration/test_xtrigger_mgr.py0000664000175000017500000006442515264420307023425 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests for the behaviour of xtrigger manager.""" import asyncio from pathlib import Path from textwrap import dedent from typing import cast, Iterable from cylc.flow import commands from cylc.flow.data_messages_pb2 import PbTaskProxy from cylc.flow.data_store_mgr import FAMILY_PROXIES, TASK_PROXIES from cylc.flow.id import TaskTokens from cylc.flow.pathutil import get_workflow_run_dir from cylc.flow.scheduler import Scheduler from cylc.flow.subprocctx import SubFuncContext from cylc.flow.task_outputs import TASK_OUTPUT_SUCCEEDED async def test_2_xtriggers(flow, start, scheduler, monkeypatch): """Test that if an itask has 4 wall_clock triggers with different offsets that xtrigger manager gets all of them. https://github.com/cylc/cylc-flow/issues/5783 n.b. Clock 3 exists to check the memoization path is followed, and causing this test to give greater coverage. Clock 4 & 5 test higher precision offsets than the CPF. """ task_point = 1588636800 # 2020-05-05 ten_years_ahead = 1904169600 # 2030-05-05 PT2H35M31S_ahead = 1588646131 # 2020-05-05 02:35:31 PT2H35M31S_behind = 1588627469 # 2020-05-04 21:24:29 monkeypatch.setattr( 'cylc.flow.xtriggers.wall_clock.time', lambda: ten_years_ahead - 1 ) id_ = flow({ 'scheduler': { 'cycle point format': 'CCYY-MM-DD', }, 'scheduling': { 'initial cycle point': '2020-05-05', 'xtriggers': { 'clock_1': 'wall_clock()', 'clock_2': 'wall_clock(offset=P10Y)', 'clock_3': 'wall_clock(offset=P10Y)', 'clock_4': 'wall_clock(offset=PT2H35M31S)', 'clock_5': 'wall_clock(offset=-PT2H35M31S)', }, 'graph': { 'R1': ( '@clock_1 & @clock_2 & @clock_3 & @clock_4 & @clock_5' ' => foo' ) } } }) schd = scheduler(id_) async with start(schd): foo_proxy = schd.pool.get_tasks()[0] clock_1_ctx = schd.xtrigger_mgr.get_xtrig_ctx(foo_proxy, 'clock_1') clock_2_ctx = schd.xtrigger_mgr.get_xtrig_ctx(foo_proxy, 'clock_2') clock_3_ctx = schd.xtrigger_mgr.get_xtrig_ctx(foo_proxy, 'clock_2') clock_4_ctx = schd.xtrigger_mgr.get_xtrig_ctx(foo_proxy, 'clock_4') clock_5_ctx = schd.xtrigger_mgr.get_xtrig_ctx(foo_proxy, 'clock_5') assert clock_1_ctx.func_kwargs['trigger_time'] == task_point assert clock_2_ctx.func_kwargs['trigger_time'] == ten_years_ahead assert clock_3_ctx.func_kwargs['trigger_time'] == ten_years_ahead assert clock_4_ctx.func_kwargs['trigger_time'] == PT2H35M31S_ahead assert clock_5_ctx.func_kwargs['trigger_time'] == PT2H35M31S_behind schd.xtrigger_mgr.call_xtriggers_async(foo_proxy) assert foo_proxy.state.xtriggers == { 'clock_1': True, 'clock_2': False, 'clock_3': False, 'clock_4': True, 'clock_5': True, } async def test_1_xtrigger_2_tasks(flow, start, scheduler, mocker): """ If two tasks depend on the same satisfied xtrigger, put_xtriggers should only be called once - when the xtrigger gets satisfied. - https://github.com/cylc/cylc-flow/pull/5908 So long as both tasks are active at once: - https://github.com/cylc/cylc-flow/issues/7027 """ id_ = flow({ 'scheduling': { 'initial cycle point': '2020', 'graph': { 'R1': '@wall_clock => foo & bar' } } }) schd = scheduler(id_) spy = mocker.spy(schd.workflow_db_mgr, 'put_xtriggers') async with start(schd): # Call the clock trigger via its dependent tasks, to get it satisfied. for task in schd.pool.get_tasks(): # (For clock triggers this is synchronous) schd.xtrigger_mgr.call_xtriggers_async(task) # It should now be satisfied. assert task.state.xtriggers == {'wall_clock': True} # Check one put_xtriggers call only, not two. assert spy.call_count == 1 # Note on master prior to GH #5908 the call is made from the # scheduler main loop when the two tasks become satisfied, # resulting in two calls to put_xtriggers. This test fails # on master, but with call count 0 (not 2) because the main # loop doesn't run in this test. async def test_1_xtrigger_2_tasks_async( flow, start, scheduler, mocker, caplog ): """ Like test_1_xtrigger_2_tasks but for async (not clock) xtriggers. If two tasks depend on the same satisfied xtrigger, put_xtriggers should only be called once - when the xtrigger gets satisfied (so long as both are active at once - https://github.com/cylc/cylc-flow/issues/7027 """ id_ = flow({ 'scheduling': { 'cycling mode': 'integer', 'xtriggers': { 'echo': 'echo("whatever", succeed=False)', }, 'graph': { 'R1': ''' @echo => foo & bar ''' }, }, }) schd = scheduler(id_) spy = mocker.spy(schd.workflow_db_mgr, 'put_xtriggers') async with start(schd): foo = schd.pool._get_task_by_id('1/foo') bar = schd.pool._get_task_by_id('1/bar') schd.xtrigger_mgr.call_xtriggers_async(foo) assert "Commencing xtrigger" in caplog.text caplog.clear() schd.xtrigger_mgr.call_xtriggers_async(bar) assert "Commencing xtrigger" not in caplog.text caplog.clear() satisfy_xtrigger_functions(schd) # mock results assert "xtrigger succeeded" in caplog.text caplog.clear() schd.xtrigger_mgr.call_xtriggers_async(foo) # process callbacks assert "satisfying xtrigger" in caplog.text caplog.clear() schd.xtrigger_mgr.call_xtriggers_async(bar) # process callbacks assert "satisfying xtrigger" in caplog.text caplog.clear() # It should now be satisfied. assert foo.state.xtriggers == {'echo': True} assert bar.state.xtriggers == {'echo': True} # Check put_xtriggers called once, not twice. assert spy.call_count == 1 async def test_1_xtrigger_2_tasks_later( flow, start, scheduler, mocker, caplog ): """ If two tasks depend on the same satisfied xtrigger, but are not both active at the same time, the xtrigger will need to be satisfied twice - https://github.com/cylc/cylc-flow/issues/7027 """ id_ = flow({ 'scheduling': { 'cycling mode': 'integer', 'xtriggers': { 'echo': 'echo("whatever", succeed=False)', }, 'graph': { 'R1': ''' @echo => foo & bar # spawn bar after @echo is housekept post satisfying foo foo => bar ''' }, }, }) schd = scheduler(id_) spy = mocker.spy(schd.workflow_db_mgr, 'put_xtriggers') async with start(schd): # Get foo at startup foo = schd.pool._get_task_by_id('1/foo') # call the xtrigger schd.xtrigger_mgr.call_xtriggers_async(foo) assert "Commencing xtrigger" in caplog.text caplog.clear() # satisfy it satisfy_xtrigger_functions(schd) # mock results assert "xtrigger succeeded" in caplog.text caplog.clear() # process callback schd.xtrigger_mgr.call_xtriggers_async(foo) assert "satisfying xtrigger" in caplog.text caplog.clear() # foo should now be satisfied. assert foo.state.xtriggers == {'echo': True} # this will delete the xtrigger - nothing else depends on it schd.xtrigger_mgr.housekeep([foo]) # Spawn bar and remove foo schd.pool.spawn_on_output(foo, TASK_OUTPUT_SUCCEEDED) bar = schd.pool._get_task_by_id('1/bar') # the xtrigger should be called again for bar schd.xtrigger_mgr.call_xtriggers_async(bar) assert "Commencing xtrigger" in caplog.text caplog.clear() # satisfy it satisfy_xtrigger_functions(schd) # mock results assert "xtrigger succeeded" in caplog.text caplog.clear() # process callback schd.xtrigger_mgr.call_xtriggers_async(bar) assert "satisfying xtrigger" in caplog.text caplog.clear() # bar should now be satisfied. assert bar.state.xtriggers == {'echo': True} # Check put_xtriggers called twice, not once. assert spy.call_count == 2 async def test_xtriggers_restart(flow, start, scheduler, db_select): """It should write satisfied xtriggers to the DB and load on restart. This checks persistence of xtrigger function results across restarts. See also test_set_xtriggers_restart for task dependence on xtriggers. """ id_ = flow({ 'scheduling': { 'xtriggers': { 'x100': 'xrandom(100)', # always succeeds 'x0': 'xrandom(0)' # never succeeds }, 'graph': { 'R1': ''' @x100 => foo @x0 = > bar ''' }, } }) # start the workflow & run the xtrigger schd = scheduler(id_) async with start(schd): # run all xtriggers for task in schd.pool.get_tasks(): schd.xtrigger_mgr.call_xtriggers_async(task) # two xtriggers should have been scheduled to run # and x100 should succeed assert len(schd.proc_pool.queuings) + len(schd.proc_pool.runnings) == 2 # wait for it to return for _ in range(50): await asyncio.sleep(0.1) schd.proc_pool.process() if len(schd.proc_pool.runnings) == 0: break else: raise Exception('Process pool did not clear') # the satisfied x100 should be written to the DB db_xtriggers = db_select(schd, True, 'xtriggers') assert len(db_xtriggers) == 1 assert db_xtriggers[0][0] == 'xrandom(100)' assert db_xtriggers[0][1].startswith('{"COLOR":') # (xrandom result dict) # restart the workflow, the xtrigger should *not* run again schd = scheduler(id_) async with start(schd): # run all xtriggers for task in schd.pool.get_tasks(): schd.xtrigger_mgr.call_xtriggers_async(task) # satisfied x100 should have been loaded from the DB # so only one xtrigger should be scheduled to run now assert len(schd.proc_pool.queuings) + len(schd.proc_pool.runnings) == 1 # x0 should not be satisfied bar = schd.pool._get_task_by_id('1/bar') assert not bar.state.xtriggers["x0"] # x100 should now be satisfied in the task pool and the datastore foo = schd.pool._get_task_by_id('1/foo') assert foo.state.xtriggers["x100"] await schd.update_data_structure() [xtrig] = [ p for t in cast( 'Iterable[PbTaskProxy]', schd.data_store_mgr.data[ schd.data_store_mgr.workflow_id ][ TASK_PROXIES ].values() ) for p in t.xtriggers.values() if p.label == "x100" ] assert xtrig.id == "xrandom(100)" assert xtrig.satisfied # check the DB to ensure no additional entries have been created assert db_select(schd, True, 'xtriggers') == db_xtriggers async def test_set_xtrig_prereq_restart(flow, start, scheduler, db_select): """Satisfied xtrigger prerequisites should persist across restart. (Task prerequisites can be artificially satisfied by "cylc set"). See also test_xtriggers_restart, for persistence of xtrigger results. """ id_ = flow({ 'scheduling': { 'xtriggers': { 'x0': 'xrandom(0)' # never succeeds naturally }, 'graph': { 'R1': ''' @x0 = > foo & bar ''' }, } }) schd = scheduler(id_) async with start(schd): # artificially set dependence of foo on x0 schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'foo')}, [], ['xtrigger/x0:succeeded'], [] ) # the satisfied x0 prerequisite should be written to the DB [db_pre] = db_select(schd, True, 'task_prerequisites') assert db_pre == ('1', 'foo', '[1]', 'x0', 'xtrigger', 'succeeded', '1') # restart the workflow schd = scheduler(id_) async with start(schd): # run all xtriggers for task in schd.pool.get_tasks(): schd.xtrigger_mgr.call_xtriggers_async(task) # foo's dependence on x0 should be satisfied from the DB # but bar still depends on it so the scheduler should still call it. assert len(schd.proc_pool.queuings) + len(schd.proc_pool.runnings) == 1 # "@x0 => bar" should not be satisfied bar = schd.pool._get_task_by_id('1/bar') assert not bar.state.xtriggers["x0"] # but "x0 => foo" should be, in the task pool and the datastore foo = schd.pool._get_task_by_id('1/foo') assert foo.state.xtriggers["x0"] await schd.update_data_structure() xtrigs = [ (t.id, p) for t in cast( 'Iterable[PbTaskProxy]', schd.data_store_mgr.data[ schd.data_store_mgr.workflow_id ][ TASK_PROXIES ].values() ) for p in t.xtriggers.values() ] assert len(xtrigs) == 2 for (id, xtrig) in xtrigs: if id.endswith('foo'): assert xtrig.id == "xrandom(0)" assert xtrig.satisfied elif id.endswith('bar'): assert xtrig.id == "xrandom(0)" assert not xtrig.satisfied else: assert False async def test_error_in_xtrigger(flow, start, scheduler): """Failure in an xtrigger is handled nicely. """ id_ = flow({ 'scheduling': { 'xtriggers': { 'mytrig': 'mytrig()' }, 'graph': { 'R1': '@mytrig => foo' }, } }) # add a custom xtrigger to the workflow run_dir = Path(get_workflow_run_dir(id_)) xtrig_dir = run_dir / 'lib/python' xtrig_dir.mkdir(parents=True) (xtrig_dir / 'mytrig.py').write_text(dedent(''' def mytrig(*args, **kwargs): raise Exception('This Xtrigger is broken') ''')) schd = scheduler(id_) async with start(schd) as log: foo = schd.pool.get_tasks()[0] schd.xtrigger_mgr.call_xtriggers_async(foo) for _ in range(50): await asyncio.sleep(0.1) schd.proc_pool.process() if len(schd.proc_pool.runnings) == 0: break else: raise Exception('Process pool did not clear') error = log.messages[-1].split('\n') assert error[-2] == 'Exception: This Xtrigger is broken' assert error[0] == 'ERROR in xtrigger mytrig()' async def test_1_seq_clock_trigger_2_tasks(flow, start, scheduler): """Test that all tasks dependent on a sequential clock trigger continue to spawn after the first cycle. See https://github.com/cylc/cylc-flow/issues/6204 """ id_ = flow({ 'scheduler': { 'cycle point format': 'CCYY', }, 'scheduling': { 'initial cycle point': '1990', 'graph': { 'P1Y': '@wall_clock => foo & bar', }, }, }) schd: Scheduler = scheduler(id_) async with start(schd): start_task_pool = schd.pool.get_task_ids() assert start_task_pool == {'1990/foo', '1990/bar'} for _ in range(3): await schd._main_loop() assert schd.pool.get_task_ids() == start_task_pool.union( f'{year}/{name}' for year in range(1991, 1994) for name in ('foo', 'bar') ) async def test_set_xtrig_prereq_reload(flow, start, scheduler, db_select): """Satisfied xtrigger prerequisites should persist across reload. (Task prerequisites can be artificially satisfied by "cylc set"). """ id_ = flow({ 'scheduling': { 'xtriggers': { 'x0': 'xrandom(0)' # never succeeds naturally }, 'graph': { 'R1': ''' @x0 = > foo & bar ''' }, } }) schd = scheduler(id_) async with start(schd): # artificially set dependence of foo on x0 schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'foo')}, [], ['xtrigger/x0:succeeded'], [] ) # reload the workflow await commands.run_cmd(commands.reload_workflow(schd)) # run all xtriggers for task in schd.pool.get_tasks(): schd.xtrigger_mgr.call_xtriggers_async(task) # foo's dependence on x0 should remain satisfied, but bar still depends # on it so the function should still be called by the scheduler. assert len(schd.proc_pool.queuings) + len(schd.proc_pool.runnings) == 1 # "@x0 => bar" should not be satisfied bar = schd.pool._get_task_by_id('1/bar') assert not bar.state.xtriggers["x0"] # but "x0 => foo" should be, in the task pool and the datastore foo = schd.pool._get_task_by_id('1/foo') assert foo.state.xtriggers["x0"] await schd.update_data_structure() xtrigs = [ (t.id, p) for t in cast( 'Iterable[PbTaskProxy]', schd.data_store_mgr.data[ schd.data_store_mgr.workflow_id ][ TASK_PROXIES ].values() ) for p in t.xtriggers.values() ] assert len(xtrigs) == 2 for (id, xtrig) in xtrigs: if id.endswith('foo'): assert xtrig.id == "xrandom(0)" assert xtrig.satisfied elif id.endswith('bar'): assert xtrig.id == "xrandom(0)" assert not xtrig.satisfied else: assert False async def test_force_satisfy(flow, start, scheduler, log_filter): """It should satisfy valid xtriggers and ignore invalid ones.""" id_ = flow({ 'scheduling': { 'xtriggers': { 'x': 'xrandom(0)' }, 'graph': { 'R1': '@x => foo' }, } }) schd = scheduler(id_) async with start(schd): foo = schd.pool.get_tasks()[0] # check x not satisfied yet assert not foo.state.xtriggers['x'] # not satisfied # force satisfy it xtrigs = { "x": True, # it should satisfy this one "y": True # it should just ignore this one } schd.xtrigger_mgr.force_satisfy(foo, xtrigs) assert foo.state.xtriggers['x'] # satisfied assert log_filter( contains=('prerequisite force-satisfied: x = xrandom(0)')) # force satisfy it again schd.xtrigger_mgr.force_satisfy(foo, xtrigs) assert foo.state.xtriggers['x'] # satisfied assert log_filter( contains=('prerequisite already satisfied: x = xrandom(0)')) # force unsatisfy it schd.xtrigger_mgr.force_satisfy(foo, {"x": False}) assert not foo.state.xtriggers['x'] # not satisfied assert log_filter( contains=('prerequisite force-unsatisfied: x = xrandom(0)')) # force unsatisfy it again schd.xtrigger_mgr.force_satisfy(foo, {"x": False}) assert not foo.state.xtriggers['x'] # not satisfied assert log_filter( contains=('prerequisite already unsatisfied: x = xrandom(0)') ) async def test_data_store(flow, start, scheduler): """It should update the data store with xtrigger state.""" id_ = flow({ 'scheduling': { 'initial cycle point': 'previous(T00)', 'xtriggers': { 'clock1': 'wall_clock()', 'clock2': 'wall_clock(offset="P1Y")', }, 'graph': { 'R1': ''' @clock1 => foo @clock2 => foo ''' } } }) schd: Scheduler = scheduler(id_) async with start(schd): await schd.update_data_structure() itask = schd.pool.get_tasks()[0] # extract xtrigger entry from the data store xtriggers = schd.data_store_mgr.data[ schd.tokens.id ][TASK_PROXIES][itask.tokens.id].xtriggers clock1, clock2 = sorted(x for x in xtriggers if 'wall_clock' in x) # it should not be satisfied (yet) assert xtriggers[clock1].label == 'clock1' assert xtriggers[clock1].satisfied is False # execute the xtrigger schd.xtrigger_mgr.call_xtriggers_async(itask) # an update delta should be produced # NOTE: both xtriggers should be present in the update # (see https://github.com/cylc/cylc-flow/issues/6307) task_delta = schd.data_store_mgr.updated[TASK_PROXIES][itask.tokens.id] assert task_delta.xtriggers[clock1].satisfied is True assert task_delta.xtriggers[clock2].satisfied is False # the xtrigger should be satisfied in the data store await schd.update_data_structure() assert xtriggers[clock1].label == 'clock1' assert xtriggers[clock1].satisfied is True def satisfy_xtrigger_functions(schd, stdout='[true, {}]', ret_code=0): """Satisfy and dequeue any xtrigger subprocesses.""" for item in list(schd.proc_pool.queuings): ctx, _, callback, *_ = item if isinstance(schd.proc_pool.queuings[0][0], SubFuncContext): # dequeue from the proc pool schd.proc_pool.queuings.remove(item) # mock the xtrigger output ctx.ret_code = ret_code ctx.out = stdout # run the callback callback(ctx) async def test_xtrigger_modifiers(flow, scheduler, start): """It should update xtrigger derived task modifiers.""" id_ = flow({ 'scheduling': { 'initial cycle point': 'previous(T00)', 'xtriggers': { 'echo': 'echo("whatever", succeed=True)', }, 'graph': { 'R1': ''' @wall_clock => foo @echo => foo ''' }, }, 'runtime': { 'foo': { 'execution retry delays': 'PT0S', }, }, }) schd = scheduler(id_) async with start(schd): # configure the task to retry itask = schd.pool.get_tasks()[0] schd.task_events_mgr._retry_task(itask, 0) await schd.update_data_structure() # the task proxy object in the data store ds_tproxy = schd.data_store_mgr.data[schd.tokens.id][TASK_PROXIES][ itask.tokens.id ] # the "root" family proxy object in the data store ds_fproxy = schd.data_store_mgr.data[schd.tokens.id][FAMILY_PROXIES][ itask.tokens.duplicate(task='root').id ] # the task modifiers should be initialised on task creation assert len(itask.state.xtriggers) == 3 assert ds_tproxy.is_retry is True assert ds_tproxy.is_wallclock is True assert ds_tproxy.is_xtriggered is True # the modifiers should bubble up the family tree assert ds_fproxy.is_retry is True assert ds_fproxy.is_wallclock is True assert ds_fproxy.is_xtriggered is True # run the xtriggers schd.xtrigger_mgr.call_xtriggers_async(itask) # run xtrigs satisfy_xtrigger_functions(schd) # mock results schd.xtrigger_mgr.call_xtriggers_async(itask) # process callbacks await schd.update_data_structure() # the task modifiers should have been updated assert ds_tproxy.is_retry is False assert ds_tproxy.is_wallclock is False assert ds_tproxy.is_xtriggered is False # the modifiers should bubble up the family tree assert ds_fproxy.is_retry is False assert ds_fproxy.is_wallclock is False assert ds_fproxy.is_xtriggered is False async def test_orphaned_task_with_xtrigger_on_reload( flow, start, scheduler, log_filter ): """Removing a task with an xtrigger dependency and reloading should not crash. """ conf = { 'scheduling': { 'xtriggers': { 'my_xtrig': 'xrandom(0)', }, 'graph': { 'R1': ''' @my_xtrig => foo bar ''' }, }, } id_ = flow(conf) schd: Scheduler = scheduler(id_) async with start(schd): # confirm the task with the xtrigger is in the pool assert schd.pool._get_task_by_id('1/foo') is not None # remove task with xtrigger dependency (no foo in R1) conf['scheduling']['graph']['R1'] = 'bar' flow(conf, workflow_id=id_) # reload should not crash await commands.run_cmd(commands.reload_workflow(schd)) # the orphaned task should have been removed assert schd.pool._get_task_by_id('1/foo') is None # the reload should have completed successfully assert log_filter(contains='Reload completed') cylc-flow-8.7.0/tests/integration/remote/0000775000175000017500000000000015264420307020574 5ustar alastairalastaircylc-flow-8.7.0/tests/integration/remote/test_watch_and_kill.py0000664000175000017500000000513015264420307025147 0ustar alastairalastairimport asyncio import signal from subprocess import ( PIPE, Popen, ) import sys from time import sleep from cylc.flow.remote import watch_and_kill from cylc.flow.scripts.cylc import pycoverage if __name__ == '__main__': # Ignore SIGHUP (death of controlling process) for all processes, as # this might give a false positive test result: signal.signal(signal.SIGHUP, lambda *_: None) def log_start(proc_type: str): print(f'start {proc_type}') def log_term(proc_type: str): print(f'term {proc_type}') sys.exit(1) def log_exit(proc_type: str): print(f'exit {proc_type}') def parent() -> None: log_start('parent') signal.signal(signal.SIGTERM, lambda *_: log_term('parent')) child = Popen([sys.executable, __file__, 'child']) asyncio.run(watch_and_kill(child, interval=0.1)) child.wait() log_exit('parent') def child() -> None: log_start('child') signal.signal(signal.SIGTERM, lambda *_: log_term('child')) sleep(60) log_exit('child') def launcher() -> None: log_start('launcher') signal.signal(signal.SIGTERM, lambda *_: log_term('launcher')) Popen([sys.executable, __file__, 'parent']).wait() log_exit('launcher') with pycoverage(sys.argv): if 'parent' in sys.argv: parent() elif 'child' in sys.argv: child() elif 'launcher' in sys.argv: launcher() else: raise ValueError('Unknown or missing type argument') sys.exit(0) def test_watch_and_kill(): """It should detect changes in the process tree. When this test runs, we get the following process tree: pytest `-- launcher `-- parent `-- child The test then kills the launcher process, changing the process tree. The parent running watch_and_kill() should detect this when monitoring the child's process tree and terminate the child. """ launcher = Popen( [sys.executable, __file__, 'launcher'], stdout=PIPE, text=True ) sleep(5) launcher.terminate() out, err = launcher.communicate(timeout=10) print(out) # for debugging print(err, file=sys.stderr) assert out.splitlines() == [ # all three processes start 'start launcher', 'start parent', 'start child', # the launcher is terminated 'term launcher', # the parent detects its pid changing and terminates the child 'term child', # the parent exits cleanly 'exit parent', ] cylc-flow-8.7.0/tests/integration/test_task_events_mgr.py0000664000175000017500000003706415264420307024117 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging from pathlib import Path from types import SimpleNamespace from typing import Any as Fixture import pytest from cylc.flow import CYLC_LOG from cylc.flow.data_store_mgr import ( JOBS, TASK_STATUS_WAITING, ) from cylc.flow.id import Tokens from cylc.flow.network.resolvers import TaskMsg from cylc.flow.run_modes import RunMode from cylc.flow.scheduler import Scheduler from cylc.flow.task_events_mgr import ( EventKey, TaskEventsManager, TaskJobLogsRetrieveContext, ) from cylc.flow.task_job_logs import get_task_job_log from cylc.flow.task_state import ( TASK_STATUS_PREPARING, TASK_STATUS_SUBMIT_FAILED, ) from .test_workflow_events import TEMPLATES # NOTE: we do not test custom event handlers here because these are tested # as a part of workflow validation (now also performed by cylc play) async def test_process_job_logs_retrieval_warns_no_platform( one_conf: Fixture, flow: Fixture, scheduler: Fixture, run: Fixture, db_select: Fixture, caplog: Fixture ): """Job log retrieval handles `NoHostsError`""" ctx = TaskJobLogsRetrieveContext( platform_name='skarloey', max_size=256, key='skarloey' ) id_: str = flow(one_conf) schd: 'Scheduler' = scheduler(id_, paused_start=True) # Run async with run(schd): schd.task_events_mgr._process_job_logs_retrieval( schd, ctx, 'foo' ) warning = caplog.records[-1] assert warning.levelname == 'WARNING' assert 'Unable to retrieve' in warning.msg async def test__reset_job_timers( one_conf: Fixture, flow: Fixture, scheduler: Fixture, start: Fixture, caplog: Fixture, mock_glbl_cfg: Fixture, ): """Integration test of pathway leading to process_execution_polling_intervals. """ schd = scheduler(flow(one_conf)) async with start(schd, level=logging.DEBUG): itask = schd.pool.get_tasks()[0] itask.state.status = 'running' itask.platform['execution polling intervals'] = [25] itask.platform['execution time limit polling intervals'] = [10] itask.summary['execution_time_limit'] = 30 caplog.records.clear() schd.task_events_mgr._reset_job_timers(itask) assert ( 'polling intervals=PT25S,PT15S,PT10S,...' in caplog.records[0].msg ) async def test__insert_task_job(flow, one_conf, scheduler, start, validate): """Simulation mode tasks are inserted into the Data Store, with correct submit number. """ conf = { 'scheduling': {'graph': {'R1': 'rhenas'}}, 'runtime': { 'rhenas': { 'simulation': { 'fail cycle points': '1', 'fail try 1 only': False, } } }, } id_ = flow(conf) schd = scheduler(id_) async with start(schd): # Set task to running: itask = schd.pool.get_tasks()[0] itask.state.status = 'running' itask.submit_num += 1 itask.run_mode = RunMode.SIMULATION # Not run _insert_task_job yet: assert not schd.data_store_mgr.added['jobs'].keys() # Insert task (twice): schd.task_events_mgr._insert_task_job(itask, 'now', 1) itask.submit_num += 1 schd.task_events_mgr._insert_task_job(itask, 'now', 1) # Check that there are two entries with correct submit # numbers waiting for data-store insertion: assert [ i.submit_num for i in schd.data_store_mgr.added['jobs'].values() ] == [1, 2] async def test__always_insert_task_job( flow, scheduler, mock_glbl_cfg, start ): """Insert Task Job _Always_ inserts a task into the data store. Bug https://github.com/cylc/cylc-flow/issues/6172 was caused by passing task state to data_store_mgr.insert_job: Where a submission retry was in progress the task state would be "waiting" which caused the data_store_mgr.insert_job to return without adding the task to the data store. This is testing two different cases: * Could not select host from platform * Could not select host from platform group """ global_config = """ [platforms] [[broken1]] hosts = no-such-host-1 job runner = abc [[broken2]] hosts = no-such-host-2 job runner = def [platform groups] [[broken_group]] platforms = broken1 """ mock_glbl_cfg('cylc.flow.platforms.glbl_cfg', global_config) id_ = flow({ 'scheduling': {'graph': {'R1': 'foo & bar'}}, 'runtime': { 'root': {'submission retry delays': 'PT10M'}, 'foo': {'platform': 'broken_group'}, 'bar': {'platform': 'broken2'} } }) schd: Scheduler = scheduler(id_, run_mode='live') schd.bad_hosts.update({'no-such-host-1', 'no-such-host-2'}) async with start(schd): schd.submit_task_jobs(schd.pool.get_tasks()) await schd.update_data_structure() # Both tasks are in a waiting state: assert all( i.state.status == TASK_STATUS_WAITING for i in schd.pool.get_tasks() ) # Both jobs are in the data store with submit-failed state: ds_jobs = schd.data_store_mgr.data[schd.id][JOBS] updates = { id_.split('//')[-1]: (job.state, job.platform, job.job_runner_name) for id_, job in ds_jobs.items() } assert updates == { '1/foo/01': ('submit-failed', 'broken_group', ''), '1/bar/01': ('submit-failed', 'broken2', 'def'), } for job in ds_jobs.values(): assert job.submitted_time async def test__submit_failed_job_id(flow, scheduler, start, db_select): """If a job is killed in the submitted state, the job ID should still be in the DB/data store. See https://github.com/cylc/cylc-flow/pull/6926 """ async def get_ds_job_id(schd: Scheduler): await schd.update_data_structure() return list(schd.data_store_mgr.data[schd.id][JOBS].values())[0].job_id id_ = flow('foo') schd: Scheduler = scheduler(id_) job_id = '1234' async with start(schd): itask = schd.pool.get_tasks()[0] itask.state_reset(TASK_STATUS_PREPARING) itask.submit_num = 1 itask.summary['submit_method_id'] = job_id schd.workflow_db_mgr.put_insert_task_jobs(itask, {}) schd.task_events_mgr.process_message( itask, 'INFO', schd.task_events_mgr.EVENT_SUBMITTED ) assert await get_ds_job_id(schd) == job_id schd.task_events_mgr.process_message( itask, 'CRITICAL', schd.task_events_mgr.EVENT_SUBMIT_FAILED ) assert itask.state(TASK_STATUS_SUBMIT_FAILED) assert await get_ds_job_id(schd) == job_id assert db_select(schd, False, 'task_jobs', 'job_id', 'submit_status') == [ (job_id, 1) ] # Restart and check data store again: schd = scheduler(id_) async with start(schd): assert await get_ds_job_id(schd) == job_id async def test__process_message_failed_with_retry( one: Scheduler, start, log_filter ): """Log job failure, even if a retry is scheduled. See: https://github.com/cylc/cylc-flow/pull/6169 """ async with start(one) as LOG: fail_once = one.pool.get_tasks()[0] # Add retry timers: one.task_job_mgr._set_retry_timers( fail_once, { 'execution retry delays': [1], 'submission retry delays': [1] }) # Process submit failed message with and without retries: one.task_events_mgr._process_message_submit_failed(fail_once, None) record = log_filter(contains='1/one:waiting(queued)] retrying in') assert record[0][0] == logging.WARNING one.task_events_mgr._process_message_submit_failed(fail_once, None) failed_record = log_filter(level=logging.ERROR)[-1] assert 'submission failed' in failed_record[1] # Process failed message with and without retries: one.task_events_mgr._process_message_failed( fail_once, None, 'failed', False, 'failed/OOK') last_record = LOG.records[-1] assert last_record.levelno == logging.WARNING assert 'retrying in' in last_record.message one.task_events_mgr._process_message_failed( fail_once, None, 'failed', False, 'failed/OOK') failed_record = log_filter(level=logging.ERROR)[-1] assert 'failed/OOK' in failed_record[1] @pytest.mark.parametrize('id_', ['1/no_such_task/01', '1/no_job']) async def test__unhandled_message(id_, one: Scheduler, start, log_filter): """It should log unhandled messages.""" async with start(one): one.message_queue.put( TaskMsg( Tokens(id_, relative=True), "time", 'INFO', "the quick brown" ) ) one.process_queued_task_messages() _, warning_msg = log_filter(level=logging.WARNING)[-1] assert ( 'Undeliverable task messages received and ignored:' in warning_msg ) assert f'{id_}: INFO - "the quick brown"' in warning_msg @pytest.mark.parametrize('template', TEMPLATES) async def test_mail_footer_template( mod_one, # use the same scheduler for each test start, mock_glbl_cfg, log_filter, capcall, template, ): """It should handle templating issues with the mail footer.""" # prevent emails from being sent mail_calls = capcall( 'cylc.flow.task_events_mgr.TaskEventsManager._send_mail' ) # configure mail footer mock_glbl_cfg( 'cylc.flow.workflow_events.glbl_cfg', f''' [scheduler] [[mail]] footer = 'footer={template}' ''', ) # start the workflow and get it to send an email ctx = SimpleNamespace(mail_to=None, mail_from=None) id_keys = [EventKey('none', 'failed', 'failed', Tokens('//1/a'))] async with start(mod_one): mod_one.task_events_mgr._process_event_email(mod_one, ctx, id_keys) # warnings should appear only when the template is invalid should_log = 'workflow' not in template # check that template issues are handled correctly assert bool(log_filter( contains='Ignoring bad mail footer template', )) == should_log assert bool(log_filter( contains=template, )) == should_log # check that the mail is sent even if there are issues with the footer assert len(mail_calls) == 1 async def test_event_email_body( mod_one, start, capcall, ): """It should send an email with the event context.""" mail_calls = capcall( 'cylc.flow.task_events_mgr.TaskEventsManager._send_mail' ) # start the workflow and get it to send an email ctx = SimpleNamespace(mail_to=None, mail_from=None) async with start(mod_one): # send a custom task message with the warning severity level id_keys = [ EventKey('none', 'warning', 'warning message', Tokens('//1/a/01')) ] mod_one.task_events_mgr._process_event_email(mod_one, ctx, id_keys) # test the email which would have been sent for this message email_body = mail_calls[0][0][3] assert 'event: warning' assert 'job: 1/a/01' in email_body assert 'message: warning message' in email_body assert f'workflow: {mod_one.tokens["workflow"]}' in email_body assert f'host: {mod_one.host}' in email_body assert f'port: {mod_one.server.port}' in email_body assert f'owner: {mod_one.owner}' in email_body async def test_job_log_retrieval_success_condition( one: 'Scheduler', start, caplog, mock_glbl_cfg, log_filter ): """Test the success condition for job log retrieval. Job log retrieval may be retried automatically if configured. It will stop when the configured list of files has been retrieved. """ mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[localhost]] retrieve job log expected files = job.forty-two ''' ) # capture event timer calls _remove_event_timer_calls = [] _unset_waiting_event_timer_calls = [] # called if retrieval complete def _remove_event_timer(id_key): _remove_event_timer_calls.append(id_key) # called if retrieval incomplete def _unset_waiting_event_timer(id_key): _unset_waiting_event_timer_calls.append(id_key) def job_logs_retrieve(*retrieved_files): """Run simulated job log retrieval. Any files specified will be created on the filesystem (simulating their retrieval). """ # request job log retrieval ctx = TaskJobLogsRetrieveContext( TaskEventsManager.HANDLER_JOB_LOGS_RETRIEVE, 'localhost', None ) id_key = EventKey( handler='job-logs-retrieve', event='succeeded', message='succeeded', tokens=one.pool.get_tasks()[0].tokens, ) one.task_events_mgr._process_job_logs_retrieval( one, ctx, [id_key], ) # simulate job log retrieval ctx, _, callback, callback_args, *__ = one.proc_pool.queuings.popleft() for fname in ('job-activity.log', *retrieved_files): # create the job log dir and any requested files job_log_file = Path(get_task_job_log( one.workflow, id_key.tokens['cycle'], id_key.tokens['task'], id_key.tokens['job'], fname, )) job_log_file.parent.mkdir(parents=True, exist_ok=True) job_log_file.touch() ctx.ret_code = 0 callback(ctx, *callback_args) async with start(one): one.task_events_mgr.remove_event_timer = _remove_event_timer one.task_events_mgr.unset_waiting_event_timer = ( _unset_waiting_event_timer ) caplog.set_level(logging.DEBUG, CYLC_LOG) # run retrieval -> no files are retrieval caplog.clear() job_logs_retrieve() assert log_filter( contains='File(s) not retrieved: job.forty-two job.out' ) assert len(_unset_waiting_event_timer_calls) == 1 assert len(_remove_event_timer_calls) == 0 # run retrieval -> "job.forty-two" is retrieved caplog.clear() job_logs_retrieve('job.forty-two') assert log_filter(contains='File(s) not retrieved: job.out') assert len(_unset_waiting_event_timer_calls) == 2 assert len(_remove_event_timer_calls) == 0 # run retrieval -> "job.out" is retrieved caplog.clear() job_logs_retrieve('job.out') assert not log_filter(contains='File(s) not retrieved') assert len(_unset_waiting_event_timer_calls) == 2 assert len(_remove_event_timer_calls) == 1 cylc-flow-8.7.0/tests/integration/test_mode_on_restart.py0000664000175000017500000000407215264420307024101 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """What happens to the mode on restart? """ import pytest from cylc.flow.exceptions import InputError from cylc.flow.scheduler import Scheduler MODES = [('live'), ('simulation'), ('dummy')] @pytest.mark.parametrize('mode_after', MODES) @pytest.mark.parametrize('mode_before', MODES + [None]) async def test_restart_mode( flow, run, scheduler, start, one_conf, mode_before, mode_after ): """Restarting a workflow in live mode leads to workflow in live mode. N.B - we need use run because the check in question only happens on start. """ schd: Scheduler id_ = flow(one_conf) schd = scheduler(id_, run_mode=mode_before) async with start(schd): if not mode_before: mode_before = 'live' assert schd.get_run_mode().value == mode_before schd = scheduler(id_, run_mode=mode_after) if ( mode_before == mode_after or not mode_before and mode_after != 'live' ): # Restarting in the same mode is fine. async with run(schd): assert schd.get_run_mode().value == mode_before else: # Restarting in a new mode is not: errormsg = f'^This.*{mode_before} mode: You.*{mode_after} mode.$' with pytest.raises(InputError, match=errormsg): async with run(schd): pass cylc-flow-8.7.0/tests/integration/test_task_remote_mgr.py0000664000175000017500000001437015264420307024101 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import cylc from cylc.flow.task_remote_mgr import ( REMOTE_FILE_INSTALL_DONE, REMOTE_FILE_INSTALL_FAILED, ) async def test_remote_tidy( flow, scheduler, start, mock_glbl_cfg, one_conf, monkeypatch ): """Remote tidy gets platforms for install targets. In particular, referencing https://github.com/cylc/cylc-flow/issues/5429, ensure that install targets defined implicitly by platform name are found. Mock remote init map: - Include an install target (quiz) with message != REMOTE_FILE_INSTALL_DONE to ensure that this is picked out. - Install targets where we can get a platform - foo - Install target is implicitly the platform name. - bar9 - The install target is implicitly the platform name, and the platform name matches a platform regex. - baz - Install target is set explicitly. - An install target (qux) where we cannot get a platform: Ensure that we get the desired error. Test that platforms with no good hosts (no host not in bad hosts). """ # Monkeypatch away subprocess.Popen calls - prevent any interaction with # remotes actually happening: class MockProc: def __init__(self, *args, **kwargs): self.poll = lambda: True if ( 'baum' in args[0] or 'bay' in args[0] ): self.returncode = 255 else: self.returncode = 0 self.communicate = lambda: ('out', 'err') monkeypatch.setattr( cylc.flow.task_remote_mgr, 'Popen', lambda *args, **kwargs: MockProc(*args, **kwargs) ) # Monkeypath function to add a sort order which we don't need in the # real code but rely to prevent the test becoming flaky: def mock_get_install_target_platforms_map(*args, **kwargs): """Add sort to original function to ensure test consistency""" from cylc.flow.platforms import get_install_target_to_platforms_map result = get_install_target_to_platforms_map(*args, **kwargs) sorted_result = {} for key in sorted(result): sorted_result[key] = sorted( result[key], key=lambda x: x['name'], reverse=True) return sorted_result monkeypatch.setattr( cylc.flow.task_remote_mgr, 'get_install_target_to_platforms_map', mock_get_install_target_platforms_map ) # Set up global config mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[foo]] # install target = foo (implicit) # hosts = foo (implicit) [[bar.]] # install target = bar1 to bar9 (implicit) # hosts = bar1 to bar9 (implicit) [[baz]] install target = baz # baum and bay should be uncontactable: hosts = baum, bay, baz [[[selection]]] method = definition order [[notthisone]] install target = baz hosts = baum, bay [[bay]] ''', ) # Get a scheduler: id_ = flow(one_conf) schd = scheduler(id_) async with start(schd) as log: # Write database with 6 tasks using 3 platforms: platforms = ['baz', 'bar9', 'foo', 'notthisone', 'bay'] line = r"('', '', {}, 0, 1, '', '', 0,'', '', '', 0, '', '{}', 4, '')" stmt = r"INSERT INTO task_jobs VALUES" + r','.join([ line.format(i, platform) for i, platform in enumerate(platforms) ]) schd.workflow_db_mgr.pri_dao.connect().execute(stmt) schd.workflow_db_mgr.pri_dao.connect().commit() # Mock a remote init map. schd.task_job_mgr.task_remote_mgr.remote_init_map = { 'baz': REMOTE_FILE_INSTALL_DONE, # Should match platform baz 'bar9': REMOTE_FILE_INSTALL_DONE, # Should match platform bar. 'foo': REMOTE_FILE_INSTALL_DONE, # Should match platform foo 'qux': REMOTE_FILE_INSTALL_DONE, # Should not match a platform 'quiz': REMOTE_FILE_INSTALL_FAILED, # Should not be considered 'bay': REMOTE_FILE_INSTALL_DONE, # Should return NoPlatforms } # Clear the log, run the test: log.clear() schd.task_job_mgr.task_remote_mgr.bad_hosts.update(['baum', 'bay']) schd.task_job_mgr.task_remote_mgr.remote_tidy() pass records = [str(r.msg) for r in log.records] # We can't get qux, no defined platform has a matching install target: qux_msg = 'No platforms available to remote tidy install targets:\n * qux' assert qux_msg in records # We can get foo bar baz, and we try to remote tidy them. # (This will ultimately fail, but past the point we are testing). for target in ['foo', 'bar9', 'baz']: msg = f'platform: {target} - remote tidy (on {target})' assert msg in records # We haven't done anything with Quiz because we're only looking # at cases where platform == REMOTE_FILE_INSTALL_DONE assert not [r for r in records if 'quiz' in r] assert ( 'platform: notthisone - clean up did not complete' '\nUnable to find contactable host for platform: notthisone' ) in records assert ( 'Unable to find a contactable platform for install target' ' bay during remote tidy' ) in records cylc-flow-8.7.0/tests/integration/test_job_runner_mgr.py0000664000175000017500000001107015264420307023721 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import errno import logging from pathlib import Path import re from textwrap import dedent from cylc.flow.job_runner_mgr import JobRunnerManager from cylc.flow.pathutil import get_workflow_run_job_dir from cylc.flow.task_state import TASK_STATUS_RUNNING from cylc.flow.subprocctx import SubProcContext from cylc.flow.task_job_logs import JOB_LOG_OUT, JOB_LOG_ERR async def test_kill_error(one, start, test_dir, capsys, log_filter): """It should report the failure to kill a job.""" async with start(one): # make it look like the task is running itask = one.pool.get_tasks()[0] itask.submit_num += 1 itask.state_reset(TASK_STATUS_RUNNING) # fake job details workflow_job_log_dir = Path(get_workflow_run_job_dir(one.workflow)) job_id = itask.job_tokens.relative_id job_log_dir = Path(workflow_job_log_dir, job_id) # create job status file (give it a fake pid) job_log_dir.mkdir(parents=True) (job_log_dir / 'job.status').write_text(dedent(''' CYLC_JOB_RUNNER_NAME=background CYLC_JOB_ID=99999999 CYLC_JOB_PID=99999999 ''')) # attempt to kill the job using the jobs-kill script # (note this is normally run via a subprocess) capsys.readouterr() JobRunnerManager().jobs_kill(str(workflow_job_log_dir), [job_id]) # the kill should fail, the failure should be written to stdout # (the jobs-kill callback will read this in and handle it) out, err = capsys.readouterr() assert re.search( # # NOTE: ESRCH = no such process rf'TASK JOB ERROR.*{job_id}.*Errno {errno.ESRCH}', out, ) # feed this jobs-kill output into the scheduler # (as if we had run the jobs-kill script as a subprocess) one.task_job_mgr._kill_task_jobs_callback( # mock the subprocess SubProcContext( one.task_job_mgr.JOBS_KILL, ['mock-cmd'], # provide it with the out/err the script produced out=out, err=err, ), [itask], ) # a warning should be logged assert log_filter( regex=r'1/one/01:running.*job kill failed', level=logging.WARNING, ) assert itask.state(TASK_STATUS_RUNNING) async def test_create_nn_new(one, start): """Test _create_nn. It should create the NN symlink. """ async with start(one): itask = one.pool.get_tasks()[0] workflow_job_log_dir = Path(get_workflow_run_job_dir(one.workflow)) job_id = itask.tokens.duplicate(job='01').relative_id job_log_dir = Path(workflow_job_log_dir, job_id) job_log_dir.mkdir(parents=True) # call _create_nn JobRunnerManager()._create_nn(job_log_dir / 'job') # check the symlink exists assert (job_log_dir.parent / "NN").is_symlink() async def test_create_nn_old(one, start): """Test _create_nn. It should remove existing job logs, if the dir already exists. """ async with start(one): itask = one.pool.get_tasks()[0] # fake some old job logs workflow_job_log_dir = Path(get_workflow_run_job_dir(one.workflow)) job_id = itask.tokens.duplicate(job='01').relative_id job_log_dir = Path(workflow_job_log_dir, job_id) job_log_dir.mkdir(parents=True) job_logs = [] for name in JOB_LOG_OUT, JOB_LOG_ERR: job_logs.append(job_log_dir / name) # create the logs for job_log in job_logs: job_log.touch() # call _create_nn JobRunnerManager()._create_nn(job_log_dir / 'job') # check they were removed for job_log in job_logs: assert not job_log.is_file() cylc-flow-8.7.0/tests/integration/reftests/0000775000175000017500000000000015264420307021140 5ustar alastairalastaircylc-flow-8.7.0/tests/integration/reftests/test_pre_initial.py0000664000175000017500000001233115264420307025050 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . async def test_basic(flow, scheduler, reftest): """Test simplification of basic conditionals in pre-initial cycling""" wid = flow({ 'scheduling': { 'initial cycle point': '2010-01-01', 'final cycle point': '2010-01-02', 'graph': { 'T00': 'a[-P1D] & b => a', }, }, }) schd = scheduler(wid, paused_start=False) assert await reftest(schd) == { ('20100101T0000Z/b', None), ('20100102T0000Z/b', None), ('20100101T0000Z/a', ('20091231T0000Z/a', '20100101T0000Z/b')), ('20100102T0000Z/a', ('20100101T0000Z/a', '20100102T0000Z/b')), } async def test_advanced(flow, scheduler, reftest): """Test nested conditional simplification for pre-initial cycling.""" wid = flow({ 'scheduling': { 'initial cycle point': '2010-01-01', 'final cycle point': '2010-01-02', 'graph': { 'PT6H': '(a[-PT6H] & b) & c[-PT6H] => a & c', }, }, }) schd = scheduler(wid, paused_start=False) assert await reftest(schd) == { ('20100101T0000Z/b', None), ('20100101T0600Z/b', None), ( '20100101T0000Z/a', ('20091231T1800Z/a', '20091231T1800Z/c', '20100101T0000Z/b'), ), ( '20100101T0000Z/c', ('20091231T1800Z/a', '20091231T1800Z/c', '20100101T0000Z/b'), ), ('20100101T1200Z/b', None), ( '20100101T0600Z/a', ('20100101T0000Z/a', '20100101T0000Z/c', '20100101T0600Z/b'), ), ( '20100101T0600Z/c', ('20100101T0000Z/a', '20100101T0000Z/c', '20100101T0600Z/b'), ), ('20100101T1800Z/b', None), ( '20100101T1200Z/a', ('20100101T0600Z/a', '20100101T0600Z/c', '20100101T1200Z/b'), ), ( '20100101T1200Z/c', ('20100101T0600Z/a', '20100101T0600Z/c', '20100101T1200Z/b'), ), ('20100102T0000Z/b', None), ( '20100101T1800Z/c', ('20100101T1200Z/a', '20100101T1200Z/c', '20100101T1800Z/b'), ), ( '20100101T1800Z/a', ('20100101T1200Z/a', '20100101T1200Z/c', '20100101T1800Z/b'), ), ( '20100102T0000Z/a', ('20100101T1800Z/a', '20100101T1800Z/c', '20100102T0000Z/b'), ), ( '20100102T0000Z/c', ('20100101T1800Z/a', '20100101T1800Z/c', '20100102T0000Z/b'), ), } async def test_drop(flow, scheduler, reftest): """Test the case of dropping a conditional based on pre-initial cycling""" wid = flow({ 'scheduling': { 'initial cycle point': '2010-01-01', 'final cycle point': '2010-01-02', 'graph': { 'PT6H': 'a[-PT6H] & b[-PT6H] => a => b', }, }, }) schd = scheduler(wid, paused_start=False) assert await reftest(schd) == { ('20100101T0000Z/a', ('20091231T1800Z/a', '20091231T1800Z/b')), ('20100101T0000Z/b', ('20100101T0000Z/a',)), ('20100101T0600Z/a', ('20100101T0000Z/a', '20100101T0000Z/b')), ('20100101T0600Z/b', ('20100101T0600Z/a',)), ('20100101T1200Z/a', ('20100101T0600Z/a', '20100101T0600Z/b')), ('20100101T1200Z/b', ('20100101T1200Z/a',)), ('20100101T1800Z/a', ('20100101T1200Z/a', '20100101T1200Z/b')), ('20100101T1800Z/b', ('20100101T1800Z/a',)), ('20100102T0000Z/a', ('20100101T1800Z/a', '20100101T1800Z/b')), ('20100102T0000Z/b', ('20100102T0000Z/a',)), } async def test_over_bracketed(flow, scheduler, reftest, validate): """Test nested conditional simplification for pre-initial cycling.""" wid = flow({ 'scheduling': { 'initial cycle point': '2013-12-25T12:00Z', 'final cycle point': '2013-12-25T12:00Z', 'graph': { 'T12': ''' (a[-P1D]:fail? | b[-P1D]:fail? | c[-P1D]:fail?) => d a? & b? & c? # Implied by implicit cycling now... ''', }, }, }) validate(wid) schd = scheduler(wid, paused_start=False) assert await reftest(schd) == { ('20131225T1200Z/c', None), ( '20131225T1200Z/d', ('20131224T1200Z/a', '20131224T1200Z/b', '20131224T1200Z/c'), ), ('20131225T1200Z/a', None), ('20131225T1200Z/b', None), } cylc-flow-8.7.0/tests/integration/reftests/test_triggering.py0000664000175000017500000000236215264420307024715 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . async def test_fail(flow, scheduler, reftest): """Test triggering on :fail""" id_ = flow({ 'scheduling': { 'graph': { 'R1': 'foo:failed => bar' } }, 'runtime': { 'foo': { 'simulation': {'fail cycle points': 'all'} } } }) schd = scheduler(id_, paused_start=False) assert await reftest(schd) == { ('1/foo', None), ('1/bar', ('1/foo',)), } cylc-flow-8.7.0/tests/integration/reftests/test_cyclers.py0000664000175000017500000001041615264420307024217 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . async def test_360_calendar(flow, scheduler, reftest): """Test 360 day calendar.""" wid = flow({ 'scheduling': { 'initial cycle point': '2013-02-28', 'final cycle point': '2013-03-01', 'cycling mode': '360day', 'graph': { 'P1D': 'foo[-P1D] => foo' }, }, }) schd = scheduler(wid, paused_start=False) assert await reftest(schd) == { ('20130228T0000Z/foo', ('20130227T0000Z/foo',)), ('20130229T0000Z/foo', ('20130228T0000Z/foo',)), ('20130230T0000Z/foo', ('20130229T0000Z/foo',)), ('20130301T0000Z/foo', ('20130230T0000Z/foo',)), } async def test_365_calendar(flow, scheduler, reftest): """Test 365 day calendar.""" wid = flow({ 'scheduling': { 'initial cycle point': '2012-02-28', 'final cycle point': '2012-03-01', 'cycling mode': '365day', 'graph': { 'P1D': 'foo[-P1D] => foo' }, }, }) schd = scheduler(wid, paused_start=False) assert await reftest(schd) == { ('20120228T0000Z/foo', ('20120227T0000Z/foo',)), ('20120301T0000Z/foo', ('20120228T0000Z/foo',)), } async def test_366_calendar(flow, scheduler, reftest): """Test 366 day calendar.""" wid = flow({ 'scheduling': { 'initial cycle point': '2013-02-28', 'final cycle point': '2013-03-01', 'cycling mode': '366day', 'graph': { 'P1D': 'foo[-P1D] => foo' }, }, }) schd = scheduler(wid, paused_start=False) assert await reftest(schd) == { ('20130228T0000Z/foo', ('20130227T0000Z/foo',)), ('20130229T0000Z/foo', ('20130228T0000Z/foo',)), ('20130301T0000Z/foo', ('20130229T0000Z/foo',)), } async def test_icp_fcp_notation(flow, scheduler, reftest): """Test initial and final cycle point special notation (^, $)""" wid = flow({ 'scheduling': { 'initial cycle point': '2016-01-01', 'final cycle point': '2016-01-02', 'graph': { 'R1': 'foo', 'R1/^': 'bar', 'R1/^+PT1H': 'baz', 'R1/$-PT1H': 'boo', 'R1/$': 'foo[^] & bar[^] & baz[^+PT1H] & boo[^+PT23H] => bot' }, }, }) schd = scheduler(wid, paused_start=False) assert await reftest(schd) == { ('20160101T0000Z/foo', None), ('20160101T0000Z/bar', None), ('20160101T0100Z/baz', None), ('20160101T2300Z/boo', None), ( '20160102T0000Z/bot', ( '20160101T0000Z/bar', '20160101T0000Z/foo', '20160101T0100Z/baz', '20160101T2300Z/boo', ), ), } async def test_recurrence_format_1(flow, scheduler, reftest): """Test ISO 8601 recurrence format no. 1 with unbounded repetitions.""" wid = flow({ 'scheduler': { 'cycle point format': 'CCYY-MM-DD', }, 'scheduling': { 'initial cycle point': '2010-01-01', 'final cycle point': '2010-01-10', 'graph': { 'R/2010-01-01/2010-01-04': 'worf', # 3-day interval }, }, }) schd = scheduler(wid, paused_start=False) assert await reftest(schd) == { ('2010-01-01/worf', None), ('2010-01-04/worf', None), ('2010-01-07/worf', None), ('2010-01-10/worf', None), } cylc-flow-8.7.0/tests/integration/README.md0000664000175000017500000000536315264420307020567 0ustar alastairalastair# Integration Tests This directory contains Cylc integration tests. ## How To Run These Tests ```console $ pytest tests/i $ pytest tests/i -n 5 # run up to 5 tests in parallel $ pytest tests/i --dist=no -n0 # turn off xdist (allows --pdb etc) ``` ## What Are Integration Tests Integration tests aren't end-to-end tests. They focus on targeted interactions of multiple modules and may do a bit of monkeypatching to achieve that result. With Cylc this typically involves running workflows. The general approach is: 1) Start a workflow. 2) Put it in a funny state. 3) Test how components interact to handle this state. I.e., the integration test framework runs the scheduler. The only thing it's really cutting out is the CLI. You can do everything, up to and including reference tests with it if so inclined, although that would really be a functional test implemented in Python: async with run(schd) as log: # run the workflow with a timeout of 60 seconds await asyncio.sleep(60) assert reftest(log) == ''' 1/b triggered off [1/a] 1/c triggered off [1/b] ''' For a more integration'y approach to reftests we can do something like this which is essentially just another way of getting the "triggered off" information without having to run the main loop and bring race conditions into play: async with start(schd): assert set(schd.pool.get_tasks()) == {'1/a'} # setting a:succeeded should spawn b schd.command_reset('1/a', 'succeeded') assert set(schd.pool.get_tasks()) == {'1/b'} # setting b:x should spawn c schd.command_reset('1/b', 'x') assert set(schd.pool.get_tasks()) == {'1/b', '1/c'} ## Guidelines Don't write functional tests here: * No sleep statements! * Avoid interaction with the command line. * Avoid testing interaction with other systems. * Don't get workflows to call out to executables. * Put workflows into funny states via artificial means rather than by getting the workflow to actually run to the desired state. * Avoid testing specific log messages or output formats where more general testing is possible. Don't write unit tests here: * No testing of odd methods and functions. * If it runs *really* quickly, it's likely a unit test. ## How To Write Integration Tests Common test patterns are documented in `test_examples.py`. Workflows can be run in two ways: ``` with start(schd): # starts the Scheduler but does not start the main loop # (always the better option if its possible) ... with run(schd): # starts the Scheduler and sets the main loop running await asyncio.sleep(0) # yield control to the main loop ... ``` These methods both shut down the workflow / clean up after themselves. It is necessary to shut down workflows correctly to clean up resources and running tasks. cylc-flow-8.7.0/tests/integration/test_optional_outputs.py0000664000175000017500000006051115264420307024345 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests optional output and task completion logic. This functionality is defined by the "optional-output-extension" proposal: https://cylc.github.io/cylc-admin/proposal-optional-output-extension.html#proposal """ from itertools import combinations from typing import TYPE_CHECKING import logging import pytest import itertools from cylc.flow.commands import ( run_cmd, force_trigger_tasks ) from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.cycling.iso8601 import ISO8601Point from cylc.flow.exceptions import WorkflowConfigError from cylc.flow.id import TaskTokens, Tokens from cylc.flow.network.resolvers import TaskMsg from cylc.flow.task_events_mgr import ( TaskEventsManager, ) from cylc.flow.task_outputs import ( TASK_OUTPUTS, TASK_OUTPUT_EXPIRED, TASK_OUTPUT_FINISHED, TASK_OUTPUT_SUCCEEDED, get_completion_expression, ) from cylc.flow.task_state import ( TASK_STATUS_EXPIRED, TASK_STATUS_PREPARING, TASK_STATUS_WAITING, ) if TYPE_CHECKING: from cylc.flow.task_proxy import TaskProxy from cylc.flow.scheduler import Scheduler OPT_BOTH_ERR = "Output {} can't be both required and optional" def reset_outputs(itask: 'TaskProxy'): """Undo the consequences of setting task outputs. This assumes you haven't completed the task. """ itask.state.outputs._completed = { message: False for message in itask.state.outputs._completed } itask.state_reset( TASK_STATUS_WAITING, is_queued=False, is_held=False, is_runahead=False, ) @pytest.mark.parametrize( 'graph, completion_outputs', [ pytest.param( 'a:x', [{TASK_OUTPUT_SUCCEEDED, 'x'}], id='1', ), pytest.param( 'a\na:x\na:expired?', [{TASK_OUTPUT_SUCCEEDED, 'x'}, {TASK_OUTPUT_EXPIRED}], id='2', ), ], ) async def test_task_completion( flow, scheduler, start, graph, completion_outputs, capcall, ): """Ensure that task completion is watertight. Run through every possible permutation of outputs MINUS the ones that would actually complete a task to ensure that task completion is correctly handled. Note, the building and evaluation of completion expressions is also tested, this is more of an end-to-end test to ensure everything is connected properly. """ # prevent tasks from being removed from the pool when complete capcall( 'cylc.flow.task_pool.TaskPool.remove_if_complete' ) id_ = flow({ 'scheduling': { 'graph': {'R1': graph}, }, 'runtime': { 'a': { 'outputs': { 'x': 'xxx', }, }, }, }) schd = scheduler(id_) all_outputs = { # all built-in outputs *TASK_OUTPUTS, # all registered custom outputs 'x' # but not the finished pseudo output } - {TASK_OUTPUT_FINISHED} async with start(schd): a1 = schd.pool.get_task(IntegerPoint('1'), 'a') # try every set of outputs that *shouldn't* complete the task for combination in { comb # every possible combination of outputs for _length in range(1, len(all_outputs)) for comb in combinations(all_outputs, _length) # that doesn't contain the outputs that would satisfy the task if not any( set(comb) & output_set == output_set for output_set in completion_outputs ) }: # set the combination of outputs schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'a')}, combination, [], ['1'], ) # ensure these outputs do *not* complete the task assert not a1.state.outputs.is_complete() # reset any changes reset_outputs(a1) # now try the outputs that *should* satisfy the task for combination in completion_outputs: # set the combination of outputs schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'a')}, combination, [], ['1'], ) # ensure the task *is* completed assert a1.state.outputs.is_complete() # reset any changes reset_outputs(a1) async def test_expire_orthogonality(flow, scheduler, start): """Ensure "expired?" does not infer "succeeded?". Asserts proposal point 2: https://cylc.github.io/cylc-admin/proposal-optional-output-extension.html#proposal """ id_ = flow({ 'scheduling': { 'graph': { 'R1': 'a:expire? => e' }, }, }) schd: 'Scheduler' = scheduler(id_, paused_start=False) async with start(schd): a_1 = schd.pool.get_task(IntegerPoint('1'), 'a') # wait for the task to submit while not a_1.state(TASK_STATUS_WAITING, TASK_STATUS_PREPARING): schd.release_tasks_to_run() # NOTE: The submit number isn't presently incremented via this code # pathway so we have to hack it here. If the task messages in this test # get ignored because of some future change, then you can safely remove # this line (it's not what this test is testing). a_1.submit_num += 1 # tell the scheduler that the task *submit-failed* schd.message_queue.put( TaskMsg( Tokens('//1/a/01'), '2000-01-01T00:00:00+00', 'INFO', TaskEventsManager.EVENT_SUBMIT_FAILED ), ) schd.process_queued_task_messages() # ensure that the scheduler is stalled assert not a_1.state.outputs.is_complete() assert schd.pool.is_stalled() # tell the scheduler that the task *failed* schd.message_queue.put( TaskMsg( Tokens('//1/a/01'), '2000-01-01T00:00:00+00', 'INFO', TaskEventsManager.EVENT_FAILED, ), ) schd.process_queued_task_messages() # ensure that the scheduler is stalled assert not a_1.state.outputs.is_complete() assert schd.pool.is_stalled() # tell the scheduler that the task *expired* schd.message_queue.put( TaskMsg( Tokens('//1/a/01'), '2000-01-01T00:00:00+00', 'INFO', TaskEventsManager.EVENT_EXPIRED, ), ) schd.process_queued_task_messages() # ensure that the scheduler is *not* stalled assert a_1.state.outputs.is_complete() assert not schd.pool.is_stalled() @pytest.fixture(scope='module') def implicit_completion_config(mod_flow, mod_validate): id_ = mod_flow({ 'scheduling': { 'graph': { 'R1': ''' a b? c:x d:x? d:y? d:z? e:x e:y e:z f? f:x g:expired? h:succeeded? h:expired? i:expired? i:submitted j:expired? j:submitted? k:submit-failed? k:succeeded? l:expired? l:submit-failed? l:succeeded? ''' } }, 'runtime': { 'root': { 'outputs': {x: f'{x * 3}' for x in 'abcdefghijklxyz'} } } }) return mod_validate(id_) @pytest.mark.parametrize( 'task, condition', [ pytest.param('a', 'succeeded', id='a'), pytest.param('b', 'succeeded or failed', id='b'), pytest.param('c', '(succeeded and x)', id='c'), pytest.param('d', 'succeeded', id='d'), pytest.param('e', '(succeeded and x and y and z)', id='e'), pytest.param('f', '(x and succeeded) or failed', id='f'), pytest.param('g', 'succeeded or expired', id='h'), pytest.param('h', 'succeeded or failed or expired', id='h'), pytest.param('i', '(submitted and succeeded) or expired', id='i'), pytest.param('j', 'succeeded or submit_failed or expired', id='j'), pytest.param('k', 'succeeded or failed or submit_failed', id='k'), pytest.param( 'l', 'succeeded or failed or submit_failed or expired', id='l' ), ], ) async def test_implicit_completion_expression( implicit_completion_config, task, condition, ): """It should generate a completion expression from the graph. If no completion expression is provided in the runtime section, then it should auto generate one inferring whether outputs are required or not from the graph. """ completion_expression = get_completion_expression( implicit_completion_config.taskdefs[task] ) assert completion_expression == condition async def test_clock_expire_partially_satisfied_task( flow, scheduler, start, ): """Clock expire should take effect on a partially satisfied task. Tests proposal point 8: https://cylc.github.io/cylc-admin/proposal-optional-output-extension.html#proposal """ id_ = flow({ 'scheduling': { 'initial cycle point': '2000', 'runahead limit': 'P0', 'special tasks': { 'clock-expire': 'e', }, 'graph': { 'P1D': ''' # this prerequisite we will satisfy a => e # this prerequisite we will leave unsatisfied creating a # partially-satisfied task b => e ''' }, }, }) schd = scheduler(id_) async with start(schd): # satisfy one of the prerequisites a = schd.pool.get_task(ISO8601Point('20000101T0000Z'), 'a') assert a schd.pool.spawn_on_output(a, TASK_OUTPUT_SUCCEEDED) # the task "e" should now be spawned e = schd.pool.get_task(ISO8601Point('20000101T0000Z'), 'e') assert e # check for clock-expired tasks schd.pool.clock_expire_tasks() # the task should now be in the expired state assert e.state(TASK_STATUS_EXPIRED) async def test_clock_expiry( flow, scheduler, start, ): """Waiting tasks should be considered for clock-expiry. Tests two things: * Manually triggered tasks should not be considered for clock-expiry. Tests proposal point 10: https://cylc.github.io/cylc-admin/proposal-optional-output-extension.html#proposal * Active tasks should not be considered for clock-expiry. Closes https://github.com/cylc/cylc-flow/issues/6025 """ id_ = flow({ 'scheduling': { 'initial cycle point': '2000', 'runahead limit': 'P1', 'special tasks': { 'clock-expire': 'x' }, 'graph': { 'P1Y': 'x' }, }, }) schd = scheduler(id_) async with start(schd): # the first task (waiting) one = schd.pool.get_task(ISO8601Point('20000101T0000Z'), 'x') assert one # the second task (preparing) two = schd.pool.get_task(ISO8601Point('20010101T0000Z'), 'x') assert two two.state_reset(TASK_STATUS_PREPARING) # the third task (force-triggered) await run_cmd(force_trigger_tasks(schd, ['20100101T0000Z/x'], ['1'])) three = schd.pool.get_task(ISO8601Point('20100101T0000Z'), 'x') assert three # check for expiry schd.pool.clock_expire_tasks() # the first task should be expired (it was waiting) assert one.state(TASK_STATUS_EXPIRED) assert one.state.outputs.is_message_complete(TASK_OUTPUT_EXPIRED) # the second task should *not* be expired (it was active) assert not two.state(TASK_STATUS_EXPIRED) assert not two.state.outputs.is_message_complete(TASK_OUTPUT_EXPIRED) # the third task should *not* be expired (it was a manual submit) assert not three.state(TASK_STATUS_EXPIRED) assert not three.state.outputs.is_message_complete(TASK_OUTPUT_EXPIRED) @pytest.mark.parametrize( 'graph, err', [ pytest.param( """ a a? """, OPT_BOTH_ERR.format("a:succeeded"), ), pytest.param( """ a => b a? """, OPT_BOTH_ERR.format("a:succeeded"), ), pytest.param( """ a? => b a """, OPT_BOTH_ERR.format("a:succeeded"), ), pytest.param( """ a => b? b """, OPT_BOTH_ERR.format("b:succeeded"), ), pytest.param( """ a => b:succeeded b? """, OPT_BOTH_ERR.format("b:succeeded"), ), pytest.param( """ a => b:succeeded c => b? """, OPT_BOTH_ERR.format("b:succeeded"), ), pytest.param( """ c:x => d a => c:x? """, OPT_BOTH_ERR.format("c:x"), ), pytest.param( """ c:x? => d a => c:x """, OPT_BOTH_ERR.format("c:x"), ), pytest.param( """ FAM:finish-all? """, "Family pseudo-output FAM:finish-all can't be optional", ), pytest.param( """ a => b => c b? """, OPT_BOTH_ERR.format("b:succeeded"), ), pytest.param( """ a => FAM => c """, "Family trigger required: FAM => c", ), ], ids=itertools.count() ) async def test_optional_outputs_consistency(flow, validate, graph, err): """Check that inconsistent output optionality fails validation.""" id_ = flow( { 'scheduling': { 'graph': { 'R1': graph }, }, 'runtime': { 'FAM': {}, 'm1, m2': { 'inherit': 'FAM', }, 'c': { 'outputs': { 'x': 'x', }, }, }, }, ) with pytest.raises(WorkflowConfigError) as exc_ctx: validate(id_) assert err in str(exc_ctx.value) @pytest.mark.parametrize( 'graph, expected', [ pytest.param( "a => b", { ("a", "succeeded"): True, # inferred ("b", "succeeded"): True, # default ("a", "failed"): None, # (not set) ("b", "failed"): None, # (not set) }, ), pytest.param( """ a => b b? """, { ("a", "succeeded"): True, # inferred ("b", "succeeded"): False, # inferred ("b", "failed"): None, # (not set) }, ), pytest.param( """ a:failed => b """, { ("a", "failed"): True, ("b", "succeeded"): True, ("a", "succeeded"): None, ("b", "failed"): None, }, ), pytest.param( """ a => b b """, { ("a", "succeeded"): True, ("b", "succeeded"): True, }, ), pytest.param( """ a => b b? """, { ("a", "succeeded"): True, ("b", "succeeded"): False, }, ), pytest.param( """ a? => b """, { ("a", "succeeded"): False, ("b", "succeeded"): True, }, ), pytest.param( """ a? => b b? """, { ("a", "succeeded"): False, ("b", "succeeded"): False, }, ), pytest.param( """ a? => b? """, { ("a", "succeeded"): False, ("b", "succeeded"): False, }, ), pytest.param( """ FAM """, { ("m1", "succeeded"): True, # family default ("m2", "succeeded"): True, # family default }, ), pytest.param( """ FAM:succeed-all? """, { ("m1", "succeeded"): False, # family default ("m2", "succeeded"): False, # family default }, ), pytest.param( """ FAM m1? """, { ("m1", "succeeded"): False, # inferred ("m2", "succeeded"): True, # family default }, ), pytest.param( """ a => FAM """, { ("a", "succeeded"): True, # inferred ("m1", "succeeded"): True, # default ("m2", "succeeded"): True, # default }, ), pytest.param( """ a => FAM m2? """, { ("a", "succeeded"): True, # inferred ("m1", "succeeded"): True, # default ("m2", "succeeded"): False, # explicit "?" }, ), pytest.param( """ a => FAM:finish-all """, { ("a", "succeeded"): True, # inferred ("m1", "succeeded"): False, # family default ("m2", "succeeded"): False, # family default }, ), pytest.param( """ FAM:succeed-any => a """, { ("a", "succeeded"): True, # inferred ("m1", "succeeded"): True, # family default ("m2", "succeeded"): True, # family default }, ), pytest.param( """ FAM:succeed-any? => a """, { ("a", "succeeded"): True, # inferred ("m1", "succeeded"): False, # family default ("m2", "succeeded"): False, # family default }, ), pytest.param( """ FAM:succeed-any => a m1? """, { ("a", "succeeded"): True, ("m1", "succeeded"): False, ("m2", "succeeded"): True, }, ), pytest.param( """ a & b? => c """, { ("a", "succeeded"): True, # inferred ("b", "succeeded"): False, # explicit "?" ("c", "succeeded"): True, # default }, ), pytest.param( """ a => c:x """, { ("a", "succeeded"): True, # inferred ("c", "succeeded"): True, # default ("c", "x"): True, # explicit ":x" }, ), pytest.param( """ a => c:x? """, { ("a", "succeeded"): True, # inferred ("c", "succeeded"): True, # default ("c", "x"): False, # explicit ":x?" }, ), pytest.param( """ a => b => c # infer :succeeded for b inside chain """, { ("a", "succeeded"): True, # inferred ("b", "succeeded"): True, # inferred ("c", "succeeded"): True, # default }, ), pytest.param( # Check we don't infer c:succeeded at end-of-chain # when there's an & at the end. """ a => b & c c? """, { ("a", "succeeded"): True, # inferred ("b", "succeeded"): True, # default ("c", "succeeded"): False, # inferred }, ), ], ids=itertools.count() ) async def test_optional_outputs_inference( flow, validate, graph, expected ): """Check task output optionality after graph parsing. This checks taskdef.outputs, which holds inferred and default values. """ id = flow( { 'scheduling': { 'graph': { 'R1': graph }, }, 'runtime': { 'FAM': {}, 'm1, m2': { 'inherit': 'FAM', }, 'c': { 'outputs': { 'x': 'x', }, }, }, } ) config = validate(id) for (task, output), exp in expected.items(): tdef = config.get_taskdef(task) (_, required) = tdef.outputs[output] assert required == exp async def test_log_outputs(flow, validate, caplog, monkeypatch): """Test logging of optional and required outputs inferred from the graph. This probes output optionality inferred by the graph parser, so it does not include RHS-only tasks that just default to :succeeded required. """ id = flow( { 'scheduling': { 'graph': { 'R1': """ # (b:succeeded required by default, not by inference) a? => FAM:succeed-all? => b m1 a? => c:x? a? => c:y """, }, }, 'runtime': { 'FAM': {}, 'm1, m2': { 'inherit': 'FAM', }, 'c': { "outputs": { "x": "x", "y": "y" } } } } ) monkeypatch.setattr('cylc.flow.flags.verbosity', 2) caplog.set_level(logging.DEBUG) validate(id) found_opt = False found_req = False for record in caplog.records: msg = record.message if "Optional outputs inferred from the graph:" in msg: found_opt = True for output in ["a:succeeded", "m2:succeeded", "c:x"]: assert output in msg for output in [ "b:succeeded", "m1:succeeded", "c:y", "c:succeeded" ]: assert output not in msg elif "Required outputs inferred from the graph:" in msg: found_req = True for output in ["m1:succeeded", "c:y"]: assert output in msg for output in [ "m2:succeeded", "b:succeeded", "a:succeeded", "c:x", "c:succeeded" ]: assert output not in msg assert found_opt assert found_req cylc-flow-8.7.0/tests/integration/test_scan_api.py0000664000175000017500000002510615264420307022473 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test the cylc scan Python API (which is equivalent to the CLI).""" import json from pathlib import Path from shutil import ( copytree, rmtree ) import pytest from cylc.flow.scripts.scan import ( main, ScanOptions ) from cylc.flow.workflow_files import ( ContactFileFields, WorkflowFiles, dump_contact_file, load_contact_file ) @pytest.fixture(scope='module') async def flows(mod_flow, mod_scheduler, mod_run, mod_one_conf): """Three workflows in different states. One stopped, one paused and one that thinks its running. TODO: Start one of the workflows with tasks in funny states in order to test the state totals functionality properly. Requires: https://github.com/cylc/cylc-flow/pull/3668 """ # a simple workflow we will leave stopped mod_flow(mod_one_conf, name='-stopped-') # a simply hierarchically registered workflow we will leave stopped mod_flow(mod_one_conf, name='a/b/c') # a simple workflow we will leave paused reg1 = mod_flow(mod_one_conf, name='-paused-') schd1 = mod_scheduler(reg1, paused_start=True) # a workflow with some metadata we will make look like it's running reg2 = mod_flow( { 'meta': { 'title': 'Foo', 'description': ''' Here we find a multi line description ''' }, 'scheduler': { 'allow implicit tasks': True }, 'scheduling': { 'graph': { 'R1': 'foo' } }, 'runtime': { 'foo': { 'simulation': {'default run length': 'PT10S'} } } }, name='-running-' ) schd2 = mod_scheduler(reg2, paused_start=False) # run cylc run async with mod_run(schd1): async with mod_run(schd2): yield async def test_state_filter(flows, mod_test_dir): """It should filter flows by state.""" # one stopped flow opts = ScanOptions(states='stopped', sort=True) lines = [] await main(opts, write=lines.append, scan_dir=mod_test_dir) assert len(lines) == 2 assert '-stopped-' in lines[0] assert 'a/b/c' in lines[1] # one paused flow opts = ScanOptions(states='paused') lines = [] await main(opts, write=lines.append, scan_dir=mod_test_dir) assert len(lines) == 1 assert '-paused-' in lines[0] # one running flow opts = ScanOptions(states='running') lines = [] await main(opts, write=lines.append, scan_dir=mod_test_dir) assert len(lines) == 1 assert '-running-' in lines[0] # two active flows opts = ScanOptions(states='paused,running') lines = [] await main(opts, write=lines.append, scan_dir=mod_test_dir) assert len(lines) == 2 # three registered flows opts = ScanOptions(states='paused,running,stopped') lines = [] await main(opts, write=lines.append, scan_dir=mod_test_dir) assert len(lines) == 4 async def test_name_filter(flows, mod_test_dir): """It should filter flows by name regex.""" # one stopped flow opts = ScanOptions(states='all', name=['.*paused.*']) lines = [] await main(opts, write=lines.append, scan_dir=mod_test_dir) assert len(lines) == 1 assert '-paused-' in lines[0] async def test_name_sort(flows, mod_test_dir): """It should sort flows by name.""" # one stopped flow opts = ScanOptions(states='all', sort=True) lines = [] await main(opts, write=lines.append, scan_dir=mod_test_dir) assert len(lines) == 4 assert '-paused-' in lines[0] assert '-running-' in lines[1] assert '-stopped-' in lines[2] assert 'a/b/c' in lines[3] async def test_format_json(flows, mod_test_dir): """It should dump results in json format.""" # one stopped flow opts = ScanOptions(states='all', format='json') lines = [] await main(opts, write=lines.append, scan_dir=mod_test_dir) data = json.loads(lines[0]) assert len(data) == 4 assert data[0]['name'] async def test_format_tree(flows, run_dir, ses_test_dir, mod_test_dir): """It should dump results in an ascii tree format.""" # one stopped flow opts = ScanOptions(states='running', format='tree') workflows = [] await main(opts, write=workflows.append, scan_dir=mod_test_dir) assert len(workflows) == 1 lines = workflows[0].splitlines() # this flow is hierarchically registered in the run dir already # it should be registered as // assert ses_test_dir.name in lines[0] assert mod_test_dir.name in lines[1] assert '-running-' in lines[2] async def test_format_rich(flows, mod_test_dir): """It should print results in a long human-friendly format.""" # one stopped flow (--colour-blind) opts = ScanOptions(states='running', format='rich', colour_blind=True) workflows = [] await main(opts, write=workflows.append, scan_dir=mod_test_dir) assert len(workflows) == 1 lines = workflows[0].splitlines() # test that the multi-line description was output correctly # with trailing lines indented correctly desc_lines = [ 'Here we find a', 'multi', 'line', 'description' ] prev_ind = -1 prev_offset = -1 for expected in desc_lines: for ind, line in enumerate(lines): if expected in line: offset = line.index(expected) if prev_ind < 1: prev_ind = ind prev_offset = offset else: if ind != prev_ind + 1: raise Exception( f'Lines found in wrong order: {line}') if offset != prev_offset: raise Exception('Line incorrectly indented: {line}') break else: raise Exception(f'Missing line: {line}') # test that the state totals show one task running (colour_blind mode) for line in lines: if 'running:1' in line: break else: raise Exception('missing state totals line (colour_blind)') # one stopped flow (--colour=always) opts = ScanOptions(states='running', format='rich') workflows = [] await main( opts, write=workflows.append, scan_dir=mod_test_dir, color='always' ) assert len(workflows) == 1 lines = workflows[0].splitlines() # test that the state totals show one task running (colour mode) for line in lines: if '1 ■' in line: break else: raise Exception('missing state totals line (colourful)') async def test_scan_cleans_stuck_contact_files( start, scheduler, flow, one_conf, run_dir, test_dir, ): """Ensure scan tidies up contact files from crashed flows.""" # create a flow id_ = flow(one_conf, name='-crashed-') schd = scheduler(id_) srv_dir = Path(run_dir, id_, WorkflowFiles.Service.DIRNAME) tmp_dir = test_dir / 'srv' cont = srv_dir / WorkflowFiles.Service.CONTACT # run the flow, copy the contact, stop the flow, copy back the contact async with start(schd): copytree(srv_dir, tmp_dir) rmtree(srv_dir) copytree(tmp_dir, srv_dir) rmtree(tmp_dir) # the old contact file check uses the CLI command that the flow was run # with to check whether the flow is running. Because this is an # integration test the process is the pytest process and it is still # running so we need to change the command so that Cylc sees the flow as # having crashed contact_info = load_contact_file(id_) contact_info[ContactFileFields.COMMAND] += 'xyz' dump_contact_file(id_, contact_info) # make sure this flow shows for a regular filesystem-only scan opts = ScanOptions(states='running,paused', format='name') flows = [] await main(opts, write=flows.append, scan_dir=test_dir) assert len(flows) == 1 assert '-crashed-' in flows[0] # the contact file should still be there assert cont.exists() # make sure this flow shows for a regular filesystem-only scan opts = ScanOptions(states='running,paused', format='name', ping=True) flows = [] await main(opts, write=flows.append, scan_dir=test_dir) assert len(flows) == 0 # the contact file should have been removed by the scan assert not cont.exists() async def test_scan_fail_well_when_client_unreachable( start, scheduler, flow, one_conf, run_dir, test_dir, caplog, ): """It handles WorkflowRuntimeClient.async_request raising a WorkflowStopped elegantly. """ # create a flow id_ = flow(one_conf, name='-crashed-') schd = scheduler(id_) srv_dir = Path(run_dir, id_, WorkflowFiles.Service.DIRNAME) tmp_dir = test_dir / 'srv' # run the flow, copy the contact, stop the flow, copy back the contact async with start(schd): copytree(srv_dir, tmp_dir) rmtree(srv_dir) copytree(tmp_dir, srv_dir) rmtree(tmp_dir) # the old contact file check uses the CLI command that the flow was run # with to check whether the flow is running. Because this is an # integration test the process is the pytest process and it is still # running so we need to change the command so that Cylc sees the flow as # having crashed contact_info = load_contact_file(id_) contact_info[ContactFileFields.COMMAND] += 'xyz' dump_contact_file(id_, contact_info) # Run Cylc Scan opts = ScanOptions(states='all', format='rich', ping=True) flows = [] await main(opts, write=flows.append, scan_dir=test_dir) # Check that the records contain a message but not an error rec = caplog.records[-1] assert not rec.exc_text assert 'Workflow not running' in rec.msg cylc-flow-8.7.0/tests/integration/network/0000775000175000017500000000000015264420307020772 5ustar alastairalastaircylc-flow-8.7.0/tests/integration/network/key_setup.py0000664000175000017500000000366515264420307023366 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Creates authentication keys for use in testing""" from cylc.flow.workflow_files import ( create_server_keys, get_workflow_srv_dir, KeyInfo, KeyOwner, KeyType, remove_keys_on_server) from cylc.flow.task_remote_cmd import ( remove_keys_on_client, create_client_keys) def setup_keys(workflow_name): workflow_srv_dir = get_workflow_srv_dir(workflow_name) server_keys = { "client_public_key": KeyInfo( KeyType.PUBLIC, KeyOwner.CLIENT, workflow_srv_dir=workflow_srv_dir), "client_private_key": KeyInfo( KeyType.PRIVATE, KeyOwner.CLIENT, workflow_srv_dir=workflow_srv_dir), "server_public_key": KeyInfo( KeyType.PUBLIC, KeyOwner.SERVER, workflow_srv_dir=workflow_srv_dir), "server_private_key": KeyInfo( KeyType.PRIVATE, KeyOwner.SERVER, workflow_srv_dir=workflow_srv_dir) } remove_keys_on_server(server_keys) remove_keys_on_client(workflow_srv_dir, None, full_clean=True) create_server_keys(server_keys, workflow_srv_dir) create_client_keys(workflow_srv_dir, None) cylc-flow-8.7.0/tests/integration/network/test_client.py0000664000175000017500000001004215264420307023656 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test cylc.flow.client.WorkflowRuntimeClient.""" import json from unittest.mock import Mock import pytest from cylc.flow.exceptions import ClientError from cylc.flow.network.client import WorkflowRuntimeClient from cylc.flow.network.server import PB_METHOD_MAP @pytest.fixture(scope='module') async def harness(mod_flow, mod_scheduler, mod_run, mod_one_conf): id_ = mod_flow(mod_one_conf) schd = mod_scheduler(id_) async with mod_run(schd): client = WorkflowRuntimeClient(id_) yield schd, client async def test_graphql(harness): """It should return True if running.""" schd, client = harness ret = await client.async_request( 'graphql', {'request_string': 'query { workflows { id } }'} ) workflows = ret['workflows'] assert len(workflows) == 1 workflow = workflows[0] assert schd.workflow in workflow['id'] async def test_protobuf(harness): """It should return True if running.""" schd, client = harness ret = await client.async_request('pb_entire_workflow') pb_data = PB_METHOD_MAP['pb_entire_workflow']() pb_data.ParseFromString(ret) assert schd.workflow in pb_data.workflow.id async def test_command_validation_failure(harness): """It should send the correct response if a command fails validation. Command arguments are validated before the command is queued. Any issues at this stage will be communicated back via the mutation "result". See https://github.com/cylc/cylc-flow/pull/6112 """ schd, client = harness # run a mutation that will fail validation response = await client.async_request( 'graphql', { 'request_string': ''' mutation { set( workflows: ["*"], tasks: ["*"], # this list of prerequisites fails validation: prerequisites: ["1/a", "all"] ) { result } } ''' }, ) # the validation error should be returned to the client assert response['set']['result'] == [ { 'id': schd.id, 'response': [False, '--pre=all must be used alone'], } ] @pytest.mark.parametrize( 'sock_response, expected', [ pytest.param({'error': 'message'}, r"^message$", id="basic"), pytest.param( {'foo': 1}, r"^Received invalid response for" r" Cylc 8\.[\w.]+: \{'foo': 1[^}]*\}$", id="no-err-field", ), pytest.param( {'cylc_version': '8.x.y'}, r"^Received invalid.+\n\(Workflow is running in Cylc 8.x.y\)$", id="no-err-field-with-version", ), ], ) async def test_async_request_err( one, start, monkeypatch: pytest.MonkeyPatch, sock_response, expected ): async def mock_recv(): return json.dumps(sock_response).encode() async with start(one): client = WorkflowRuntimeClient(one.workflow) with monkeypatch.context() as mp: mp.setattr(client, 'socket', Mock(recv=mock_recv)) mp.setattr(client, 'poller', Mock()) with pytest.raises(ClientError, match=expected): await client.async_request('graphql') cylc-flow-8.7.0/tests/integration/network/test_publisher.py0000664000175000017500000000337715264420307024412 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio from cylc.flow.network.subscriber import ( WorkflowSubscriber, process_delta_msg, ) async def test_publisher(flow, scheduler, run, one_conf, port_range): """It should publish deltas when the flow starts.""" id_ = flow(one_conf) schd = scheduler(id_, paused_start=False) async with run(schd): # create a subscriber subscriber = WorkflowSubscriber( schd.workflow, host=schd.host, port=schd.server.pub_port, topics=[b'workflow'] ) async with asyncio.timeout(2): # wait for the first delta from the workflow btopic, msg = await subscriber.socket.recv_multipart() _, delta = process_delta_msg(btopic, msg, None) for key in ('added', 'updated'): if getattr(getattr(delta, key), 'id', None): assert schd.id == getattr(delta, key).id break else: raise Exception("Delta wasn't added or updated") cylc-flow-8.7.0/tests/integration/network/__init__.py0000664000175000017500000000000015264420307023071 0ustar alastairalastaircylc-flow-8.7.0/tests/integration/network/test_graphql.py0000664000175000017500000003222315264420307024043 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test the top-level (root) GraphQL queries.""" from contextlib import suppress import pytest from typing import TYPE_CHECKING from graphql import parse, MiddlewareManager from cylc.flow.data_store_mgr import create_delta_store from cylc.flow.id import TaskTokens, Tokens from cylc.flow.network.client import WorkflowRuntimeClient from cylc.flow.network.schema import schema, SUB_RESOLVER_MAPPING from cylc.flow.network.graphql import ( CylcExecutionContext, IgnoreFieldMiddleware, instantiate_middleware, ) from cylc.flow.network.graphql_subscribe import subscribe from cylc.flow.workflow_status import get_workflow_status if TYPE_CHECKING: from cylc.flow.scheduler import Scheduler # NOTE: These tests mutate the data store, so running them in isolation may # see failures when they actually pass if you run the whole file def job_config(schd): return { 'owner': schd.owner, 'submit_num': 1, 'task_id': '1/foo', 'job_runner_name': 'background', 'env-script': None, 'err-script': None, 'exit-script': None, 'execution_time_limit': None, 'init-script': None, 'post-script': None, 'pre-script': None, 'script': 'sleep 5; echo "I come in peace"', 'work_d': None, 'directives': {}, 'environment': {}, 'param_var': {}, 'platform': {'name': 'platform'}, } def gather_subscription_args(schd, request_string): kwargs = { "variable_values": {}, "operation_name": None, "context_value": { 'op_id': 1, 'resolvers': schd.server.resolvers, 'meta': {}, }, "subscribe_resolver_map": SUB_RESOLVER_MAPPING, "middleware": MiddlewareManager( *list( instantiate_middleware( [IgnoreFieldMiddleware] ) ) ), "execution_context_class": CylcExecutionContext, } document = parse(request_string) return (document, kwargs) @pytest.fixture def job_db_row(): return [ '1', 'foo', 'running', 4, '2020-04-03T13:40:18+13:00', '2020-04-03T13:40:20+13:00', '2020-04-03T13:40:30+13:00', 'background', '20542', 'localhost', ] @pytest.fixture(scope='module') async def harness(mod_flow, mod_scheduler, mod_run): flow_def = { 'scheduler': { 'allow implicit tasks': True }, 'scheduling': { 'graph': { 'R1': 'a => b & c => d' } }, 'runtime': { 'A': { }, 'B': { 'inherit': 'A', }, 'b': { 'inherit': 'B', }, }, } id_: str = mod_flow(flow_def) schd: 'Scheduler' = mod_scheduler(id_) async with mod_run(schd): client = WorkflowRuntimeClient(id_) schd.pool.hold_tasks({TaskTokens('*', 'root')}) schd.resume_workflow() # Think this is needed to save the data state at first start (?) # Fails without it.. and a test needs to overwrite schd data with this. # data = schd.data_store_mgr.data[schd.data_store_mgr.workflow_id] workflow_tokens = Tokens( user=schd.owner, workflow=schd.workflow, ) yield schd, client, workflow_tokens async def test_workflows(harness): schd, client, w_tokens = harness ret = await client.async_request( 'graphql', {'request_string': 'query { workflows { id } }'} ) assert ret == { 'workflows': [ { 'id': f'{w_tokens}' } ] } async def test_tasks(harness): schd, client, w_tokens = harness # query "tasks" ret = await client.async_request( 'graphql', {'request_string': 'query { tasks { id } }'} ) ids = [ w_tokens.duplicate(cycle=f'$namespace|{namespace}').id for namespace in ('a', 'b', 'c', 'd') ] ret['tasks'].sort(key=lambda x: x['id']) assert ret == { 'tasks': [ {'id': id_} for id_ in ids ] } # query "task" for id_ in ids: ret = await client.async_request( 'graphql', {'request_string': 'query { task(id: "%s") { id } }' % id_} ) assert ret == { 'task': {'id': id_} } async def test_families(harness): schd, client, w_tokens = harness # query "tasks" ret = await client.async_request( 'graphql', {'request_string': 'query { families { id } }'} ) ids = [ w_tokens.duplicate( cycle=f'$namespace|{namespace}' ).id for namespace in ('A', 'B', 'root') ] ret['families'].sort(key=lambda x: x['id']) assert ret == { 'families': [ {'id': id_} for id_ in ids ] } # query "task" for id_ in ids: ret = await client.async_request( 'graphql', {'request_string': 'query { family(id: "%s") { id } }' % id_} ) assert ret == { 'family': {'id': id_} } async def test_task_proxies(harness): schd, client, w_tokens = harness # query "tasks" ret = await client.async_request( 'graphql', {'request_string': 'query { taskProxies { id } }'} ) ids = [ w_tokens.duplicate( cycle='1', task=namespace, ) # NOTE: task "d" is not in the n=1 window yet for namespace in ('a', 'b', 'c') ] ret['taskProxies'].sort(key=lambda x: x['id']) assert ret == { 'taskProxies': [ {'id': id_.id} for id_ in ids ] } # query "task" ret = await client.async_request( 'graphql', {'request_string': 'query { taskProxy(id: "%s") { id } }' % ids[0].id} ) assert ret == { 'taskProxy': {'id': ids[0].id} } # query "taskProxies" fragment with null stripping ret = await client.async_request( 'graphql', { 'request_string': ''' fragment wf on Workflow { taskProxies (ids: ["%s"], stripNull: true) { id } } query { workflows (ids: ["%s"]) { ...wf } } ''' % (ids[0].relative_id, ids[0].workflow_id) } ) assert ret == {'workflows': [{'taskProxies': [{'id': ids[0].id}]}]} async def test_family_proxies(harness): schd, client, w_tokens = harness # query "familys" ret = await client.async_request( 'graphql', {'request_string': 'query { familyProxies { id } }'} ) ids = [ w_tokens.duplicate( cycle='1', task=namespace, ).id # NOTE: family "d" is not in the n=1 window yet for namespace in ('A', 'B', 'root') ] ret['familyProxies'].sort(key=lambda x: x['id']) assert ret == { 'familyProxies': [ {'id': id_} for id_ in ids ] } # query "family" for id_ in ids: ret = await client.async_request( 'graphql', {'request_string': 'query { familyProxy(id: "%s") { id } }' % id_} ) assert ret == { 'familyProxy': {'id': id_} } async def test_edges(harness): schd, client, w_tokens = harness t_tokens = [ w_tokens.duplicate( cycle='1', task=namespace, ) # NOTE: task "d" is not in the n=1 window yet for namespace in ('a', 'b', 'c') ] edges = [ (t_tokens[0], t_tokens[1]), (t_tokens[0], t_tokens[2]), ] e_ids = sorted([ w_tokens.duplicate( cycle=( '$edge' f'|{left.relative_id}' f'|{right.relative_id}' ) ).id for left, right in edges ]) # query "edges" ret = await client.async_request( 'graphql', {'request_string': 'query { edges { id } }'} ) ret['edges'].sort(key=lambda x: x['id']) assert ret == { 'edges': [ {'id': id_} for id_ in e_ids ] } # query "nodesEdges" ret = await client.async_request( 'graphql', {'request_string': 'query { nodesEdges { nodes {id}\nedges {id} } }'} ) ret['nodesEdges']['nodes'].sort(key=lambda x: x['id']) ret['nodesEdges']['edges'].sort(key=lambda x: x['id']) assert ret == { 'nodesEdges': { 'nodes': [ {'id': tokens.id} for tokens in t_tokens ], 'edges': [ {'id': id_} for id_ in e_ids ], }, } async def test_jobs(harness): schd: Scheduler schd, client, w_tokens = harness # add a job itask = schd.pool._get_task_by_id('1/a') schd.data_store_mgr.insert_job(itask, 'submitted', job_config(schd)) schd.data_store_mgr.update_data_structure() j_tokens = w_tokens.duplicate( cycle='1', task='a', job='01', ) j_id = j_tokens.id # query "jobs" ret = await client.async_request( 'graphql', {'request_string': 'query { jobs { id } }'} ) assert ret == { 'jobs': [ {'id': f'{j_id}'} ] } # query "job" ret = await client.async_request( 'graphql', {'request_string': 'query { job(id: "%s") { id } }' % j_id} ) assert ret == { 'job': {'id': f'{j_id}'} } # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Test the GraphQL subscription infrastructure. # (currently only used at UIS) @pytest.mark.asyncio(loop_scope="module") async def test_subscription_basic(harness): """Test a basic subscription that uses the resolver's sub_resolver code.""" schd, _, w_tokens = harness document, kwargs = gather_subscription_args( schd, 'subscription { workflows { id } }', ) subscription = await subscribe( schema.graphql_schema, document, **kwargs ) has_item = False with suppress(GeneratorExit): async for response in subscription: has_item = True assert response.data['workflows'][0]['id'] == w_tokens.id await subscription.aclose() assert has_item @pytest.mark.asyncio(loop_scope="module") async def test_subscription_deltas(one, start): """Test the full subscription with null-stripping and delta handling.""" async with start(one): document, kwargs = gather_subscription_args( one, ''' subscription { deltas (stripNull: true) { id added { workflow { id host status } } updated { workflow { id host status } } } } ''', ) subscription = await subscribe( schema.graphql_schema, document, **kwargs ) aitem = await subscription.__anext__() assert aitem.data['deltas']['added']['workflow'] == { 'id': one.id, 'host': one.host, 'status': 'paused', } # Workflow one is paused on start await one.update_data_structure() assert ( one.data_store_mgr.data[one.id]['workflow'].status == get_workflow_status(one).value ) # Get the all delta, process, then add it to the subscription queue. btopic, delta, _ = one.data_store_mgr.publish_deltas[-1] _, sub_queue = next( iter(one.data_store_mgr.delta_queues[one.id].items()) ) sub_queue.put( ( one.id, btopic.decode('utf-8'), create_delta_store(delta, one.id) ) ) aitem = await subscription.__anext__() assert aitem.data['deltas']['updated']['workflow'] == { 'id': one.id, } with suppress(GeneratorExit): await subscription.aclose() cylc-flow-8.7.0/tests/integration/network/test_server.py0000664000175000017500000001404715264420307023717 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio import logging from threading import BrokenBarrierError from typing import Callable import pytest from cylc.flow import __version__ as CYLC_VERSION from cylc.flow.network.server import PB_METHOD_MAP, WorkflowRuntimeServer from cylc.flow.scheduler import Scheduler @pytest.fixture(scope='module') async def myflow(mod_flow, mod_scheduler, mod_run, mod_one_conf): id_ = mod_flow(mod_one_conf) schd = mod_scheduler(id_) async with mod_run(schd): yield schd def test_graphql(myflow): """Test GraphQL endpoint method.""" request_string = f''' query {{ workflows(ids: ["{myflow.id}"]) {{ id }} }} ''' data = myflow.server.graphql(request_string) assert myflow.id == data['workflows'][0]['id'] def test_graphql_error(myflow): """Test GraphQL endpoint method.""" request_string = f''' query {{ workflows(ids: ["{myflow.id}"]) {{ id notafield alsonotafield }} }} ''' with pytest.raises(Exception) as excinfo: myflow.server.graphql(request_string) assert "Cannot query field 'notafield'" in excinfo assert "Cannot query field 'alsonotafield'" in excinfo def test_pb_data_elements(myflow): """Test Protobuf elements endpoint method.""" element_type = 'workflow' data = PB_METHOD_MAP['pb_data_elements'][element_type]() data.ParseFromString( myflow.server.pb_data_elements(element_type) ) assert data.added.id == myflow.id def test_pb_entire_workflow(myflow): """Test Protobuf entire workflow endpoint method.""" data = PB_METHOD_MAP['pb_entire_workflow']() data.ParseFromString( myflow.server.pb_entire_workflow() ) assert data.workflow.id == myflow.id async def test_stop(one: Scheduler, start): """Test stop.""" async with start(one): assert one.server.thread.is_alive() assert one.server.publisher assert one.server.curve_auth.is_alive() async with asyncio.timeout(2): # Wait for the server to consume the STOP signal. # If it doesn't, the test will fail with a asyncio.timeout error. await one.server.stop('TESTING') assert not one.server.thread.is_alive() assert not one.server.publisher assert not one.server.curve_auth.is_alive() assert not one.server.loop.is_running() @pytest.mark.filterwarnings( r"ignore:.*:pytest.PytestUnhandledThreadExceptionWarning" ) async def test_broken_barrier_shutdown( one: Scheduler, start, monkeypatch: pytest.MonkeyPatch ): """Test that the server shuts down if the barrier is broken on startup. Broken barrier has been observed to happen if the scheduler host is sickly https://github.com/cylc/cylc-flow/issues/7425 """ class MockServer(WorkflowRuntimeServer): def start(self, barrier): barrier.abort() super().start(barrier) monkeypatch.setattr( 'cylc.flow.scheduler.WorkflowRuntimeServer', MockServer ) with pytest.raises(BrokenBarrierError): async with start(one): pass assert not one.server.thread.is_alive() async def test_receiver_basic(one: Scheduler, start, log_filter): """Test the receiver with different message objects.""" async with asyncio.timeout(5): async with start(one): # start with a message that works msg = {'command': 'api', 'user': 'bono', 'args': {}} res = one.server.receiver(msg) assert not res.get('error') assert res['data'] assert res['cylc_version'] == CYLC_VERSION # simulate a command failure with the original message # (the one which worked earlier) - should error def _api(*args, **kwargs): raise Exception('oopsie') one.server.api = _api res = one.server.receiver(msg) assert res == { 'error': {'message': 'oopsie'}, 'cylc_version': CYLC_VERSION, } assert log_filter(logging.ERROR, 'oopsie') @pytest.mark.parametrize( 'msg, expected', [ pytest.param( {'user': 'bono', 'args': {}}, "Request missing field 'command' required for" f" Cylc {CYLC_VERSION}", id='missing-command', ), pytest.param( {'command': 'foobar', 'user': 'bono', 'args': {}}, f"No method by the name 'foobar' at Cylc {CYLC_VERSION}", id='bad-command', ), ], ) async def test_receiver_bad_requests(one: Scheduler, start, msg, expected): """Test the receiver with different bad requests.""" async with asyncio.timeout(5): async with start(one): res = one.server.receiver(msg) assert res == { 'error': {'message': expected}, 'cylc_version': CYLC_VERSION, } async def test_publish_before_shutdown( one: Scheduler, start: Callable ): """Test that the server publishes final deltas before shutting down.""" async with start(one): one.server.publish_queue.put([(b'fake', b'blah')]) await one.server.stop('i said stop!') assert not one.server.publish_queue.qsize() cylc-flow-8.7.0/tests/integration/network/test_replier.py0000664000175000017500000000367315264420307024056 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio import pytest from cylc.flow import __version__ as CYLC_VERSION from cylc.flow.network import deserialize from cylc.flow.network.client import WorkflowRuntimeClient from cylc.flow.scheduler import Scheduler async def test_listener(one: Scheduler, start): """Test listener.""" async with start(one): # Test listener handles an invalid message from client # (without directly calling listener): client = WorkflowRuntimeClient(one.workflow) client.socket.send_string(r'Not JSON') res = deserialize( (await client.socket.recv()).decode() ) assert res['error'] assert 'data' not in res # Check other fields are present: assert res['cylc_version'] == CYLC_VERSION one.server.replier.queue.put('STOP') async with asyncio.timeout(2): # wait for the server to consume the STOP item from the queue while not one.server.replier.queue.empty(): await asyncio.sleep(0.01) # ensure the server is "closed" one.server.replier.queue.put('foobar') with pytest.raises(ValueError): one.server.replier.listener() cylc-flow-8.7.0/tests/integration/network/test_zmq.py0000664000175000017500000000445015264420307023215 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest import zmq from cylc.flow.exceptions import CylcError from cylc.flow.network import ZMQSocketBase from .key_setup import setup_keys @pytest.fixture(scope='module') def myflow(mod_flow, mod_one_conf): return mod_flow(mod_one_conf) def test_single_port(myflow, port_range): """Test server on a single port and port in use exception.""" context = zmq.Context() setup_keys(myflow) # auth keys are required for comms serv1 = ZMQSocketBase( zmq.REP, context=context, workflow=myflow, bind=True) serv2 = ZMQSocketBase( zmq.REP, context=context, workflow=myflow, bind=True) serv1._socket_bind(*port_range) port = serv1.port with pytest.raises(CylcError, match=r"Address already in use"): serv2._socket_bind(port, port) serv2.stop() serv1.stop() context.destroy() def test_start(myflow, port_range): """Test socket start.""" setup_keys(myflow) # auth keys are required for comms publisher = ZMQSocketBase(zmq.PUB, workflow=myflow, bind=True) assert publisher.loop is None assert publisher.port is None publisher.start(*port_range) assert publisher.loop is not None assert publisher.port is not None publisher.stop() def test_stop(myflow, port_range): """Test socket/thread stop.""" setup_keys(myflow) # auth keys are required for comms publisher = ZMQSocketBase(zmq.PUB, workflow=myflow, bind=True) publisher.start(*port_range) assert not publisher.socket.closed publisher.stop() assert publisher.socket.closed cylc-flow-8.7.0/tests/integration/network/test_resolvers.py0000664000175000017500000002247215264420307024436 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging from typing import AsyncGenerator, Callable from unittest.mock import Mock import pytest from cylc.flow.data_store_mgr import EDGES, TASK_PROXIES from cylc.flow.id import Tokens from cylc.flow import CYLC_LOG from cylc.flow.network.resolvers import Resolvers from cylc.flow.scheduler import Scheduler from cylc.flow.workflow_status import StopMode @pytest.fixture def flow_args(): return { 'workflows': [], 'exworkflows': [], } @pytest.fixture def node_args(): return { 'workflows': [], 'exworkflows': [], 'ids': [], 'exids': [], 'states': [], 'exstates': [], 'mindepth': -1, 'maxdepth': -1, } @pytest.fixture(scope='module') async def mock_flow( mod_flow: Callable[..., str], mod_scheduler: Callable[..., Scheduler], mod_start, ) -> AsyncGenerator[Scheduler, None]: ret = Mock() ret.id_ = mod_flow({ 'scheduler': { 'allow implicit tasks': True }, 'scheduling': { 'initial cycle point': '2000', 'dependencies': { 'R1': 'prep => foo', 'PT12H': 'foo[-PT12H] => foo => bar' } } }) ret.schd = mod_scheduler(ret.id_, paused_start=True) async with mod_start(ret.schd): ret.schd.pool.release_runahead_tasks() ret.schd.data_store_mgr.initiate_data_model() ret.owner = ret.schd.owner ret.name = ret.schd.workflow ret.id = list(ret.schd.data_store_mgr.data.keys())[0] ret.resolvers = Resolvers( ret.schd.data_store_mgr, schd=ret.schd ) ret.data = ret.schd.data_store_mgr.data[ret.id] ret.node_ids = [ node.id for node in ret.data[TASK_PROXIES].values() ] ret.edge_ids = [ edge.id for edge in ret.data[EDGES].values() ] yield ret async def test_get_workflows(mock_flow, flow_args): """Test method returning workflow messages satisfying filter args.""" flow_args['workflows'].append({ 'user': mock_flow.owner, 'workflow': mock_flow.name, 'workflow_sel': None, }) flow_msgs = await mock_flow.resolvers.get_workflows(flow_args) assert len(flow_msgs) == 1 async def test_get_nodes_all(mock_flow, node_args): """Test method returning workflow(s) node message satisfying filter args. """ node_args['workflows'].append({ 'user': mock_flow.owner, 'workflow': mock_flow.name, 'workflow_sel': None, }) node_args['states'].append('failed') nodes = await mock_flow.resolvers.get_nodes_all(TASK_PROXIES, node_args) assert len(nodes) == 0 node_args['states'] = [] node_args['ids'].append(Tokens(mock_flow.node_ids[0])) nodes = [ n for n in await mock_flow.resolvers.get_nodes_all( TASK_PROXIES, node_args) if n in mock_flow.data[TASK_PROXIES].values() ] assert len(nodes) == 1 async def test_get_nodes_by_ids(mock_flow, node_args): """Test method returning workflow(s) node messages who's ID is a match to any given.""" node_args['workflows'].append({ 'user': mock_flow.owner, 'workflow': mock_flow.name, 'workflow_sel': None }) nodes = await mock_flow.resolvers.get_nodes_by_ids(TASK_PROXIES, node_args) assert len(nodes) == 0 node_args['native_ids'] = mock_flow.node_ids nodes = [ n for n in await mock_flow.resolvers.get_nodes_by_ids( TASK_PROXIES, node_args ) if n in mock_flow.data[TASK_PROXIES].values() ] assert len(nodes) > 0 async def test_get_node_by_id(mock_flow, node_args): """Test method returning a workflow node message who's ID is a match to that given.""" node_args['id'] = Tokens( user='me', workflow='mine', cycle='20500808T00', task='jin', ).id node_args['workflows'].append({ 'user': mock_flow.owner, 'workflow': mock_flow.name, 'workflow_sel': None }) node = await mock_flow.resolvers.get_node_by_id(TASK_PROXIES, node_args) assert node is None node_args['id'] = mock_flow.node_ids[0] node = await mock_flow.resolvers.get_node_by_id(TASK_PROXIES, node_args) assert node in mock_flow.data[TASK_PROXIES].values() async def test_get_edges_all(mock_flow, flow_args): """Test method returning all workflow(s) edges.""" edges = [ e for e in await mock_flow.resolvers.get_edges_all(flow_args) if e in mock_flow.data[EDGES].values() ] assert len(edges) > 0 async def test_get_edges_by_ids(mock_flow, node_args): """Test method returning workflow(s) edge messages who's ID is a match to any given edge IDs.""" edges = await mock_flow.resolvers.get_edges_by_ids(node_args) assert len(edges) == 0 node_args['native_ids'] = mock_flow.edge_ids edges = [ e for e in await mock_flow.resolvers.get_edges_by_ids(node_args) if e in mock_flow.data[EDGES].values() ] assert len(edges) > 0 async def test_mutator(mock_flow, flow_args): """Test the mutation method.""" flow_args['workflows'].append({ 'user': mock_flow.owner, 'workflow': mock_flow.name, 'workflow_sel': None }) args = {} meta = {} response = await mock_flow.resolvers.mutator( None, 'pause', flow_args, args, meta ) assert response[0]['id'] == mock_flow.id async def test_mutation_mapper(mock_flow): """Test the mapping of mutations to internal command methods.""" meta = {} response = await mock_flow.resolvers._mutation_mapper('pause', {}, meta) assert response[0] is True # (True, command-uuid-str) with pytest.raises(ValueError): await mock_flow.resolvers._mutation_mapper('non_exist', {}, meta) async def test_command_logging(mock_flow, caplog, log_filter): """The command log message should include non-owner name.""" meta = {} caplog.set_level(logging.INFO, CYLC_LOG) await mock_flow.resolvers._mutation_mapper( "stop", {'mode': StopMode.REQUEST_CLEAN.value}, meta, ) assert log_filter(contains='Command "stop" received') # put_messages: only log for owner kwargs = { "task_job": "1/foo/01", "event_time": "bedtime", "messages": [[logging.CRITICAL, "it's late"]] } meta["auth_user"] = mock_flow.owner await mock_flow.resolvers._mutation_mapper("put_messages", kwargs, meta) assert not log_filter(contains='Command "put_messages" received:') meta["auth_user"] = "Dr Spock" await mock_flow.resolvers._mutation_mapper("put_messages", kwargs, meta) assert log_filter(contains='Command "put_messages" received from Dr Spock') async def test_command_validation_failure( mock_flow, caplog, flow_args, monkeypatch, ): """It should log command validation failures server side.""" caplog.set_level(logging.DEBUG, None) flow_args['workflows'].append( { 'user': mock_flow.owner, 'workflow': mock_flow.name, 'workflow_sel': None, } ) # submit a command with invalid arguments: async def submit_invalid_command(verbosity=0): monkeypatch.setattr('cylc.flow.flags.verbosity', verbosity) caplog.clear() return await mock_flow.resolvers.mutator( None, 'stop', flow_args, {'task': 'cycle/task/job', 'mode': 'not-a-mode'}, {}, ) # submitting the invalid command should result in this error msg = 'This command does not take job IDs: cycle/task/job' # test submitting the command at *default* verbosity response = await submit_invalid_command() # the error should be sent back to the client: assert response[0]['response'][1] == msg # it should also be logged by the server: assert caplog.records[-1].levelno == logging.WARNING assert msg in caplog.records[-1].message # test submitting the command at *debug* verbosity response = await submit_invalid_command(verbosity=2) # the error should be sent back to the client: assert response[0]['response'][1] == msg # it should be logged at the server assert caplog.records[-2].levelno == logging.WARNING assert msg in caplog.records[-2].message # the traceback should also be logged # (note traceback gets logged at the ERROR level and shows up funny in # caplog) assert caplog.records[-1].levelno == logging.ERROR assert msg in caplog.records[-1].message cylc-flow-8.7.0/tests/integration/network/test_scan.py0000664000175000017500000003273515264420307023341 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test file-system interaction aspects of scan functionality.""" from contextlib import suppress from pathlib import Path import re from shutil import rmtree from tempfile import TemporaryDirectory from typing import List import pytest from cylc.flow.network.scan import ( filter_name, graphql_query, is_active, scan, scan_multi, workflow_params, ) from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager from cylc.flow.workflow_files import WorkflowFiles SRV_DIR = Path(WorkflowFiles.Service.DIRNAME) CONTACT = Path(WorkflowFiles.Service.CONTACT) RUN_N = Path(WorkflowFiles.RUN_N) INSTALL = Path(WorkflowFiles.Install.DIRNAME) def init_flows(tmp_run_path=None, running=None, registered=None, un_registered=None, tmp_src_path=None, src=None): """Create some dummy workflows for scan to discover. Assume "run1, run2, ..., runN" structure if flow name contains "run". Optionally create workflow source dirs in a give location too. """ def make_registered(name, running=False): run_d = Path(tmp_run_path, name) run_d.mkdir(parents=True, exist_ok=True) (run_d / "flow.cylc").touch() if "run" in name: root = Path(tmp_run_path, name).parent with suppress(FileExistsError): (root / "runN").symlink_to(run_d, target_is_directory=True) else: root = run_d (root / INSTALL).mkdir(parents=True, exist_ok=True) srv_d = (run_d / SRV_DIR) srv_d.mkdir(parents=True, exist_ok=True) if running: (srv_d / CONTACT).touch() def make_src(name): src_d = Path(tmp_src_path, name) src_d.mkdir(parents=True, exist_ok=True) (src_d / "flow.cylc").touch() for name in (running or []): make_registered(name, running=True) for name in (registered or []): make_registered(name) for name in (un_registered or []): Path(tmp_run_path, name).mkdir(parents=True, exist_ok=True) for name in (src or []): make_src(name) @pytest.fixture(scope='session') def sample_run_dir(): tmp_path = Path(TemporaryDirectory().name) tmp_path.mkdir() init_flows( tmp_path, running=('foo', 'bar/pub', 'cheese/run2'), registered=('baz', 'cheese/run1'), un_registered=('qux',) ) yield tmp_path rmtree(tmp_path) @pytest.fixture def badly_messed_up_cylc_run_dir( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> Path: monkeypatch.setattr('cylc.flow.pathutil._CYLC_RUN_DIR', tmp_path) # one regular workflow init_flows( tmp_path, running=('foo',) ) # and an erroneous service dir at the top level for no reason Path(tmp_path, SRV_DIR).mkdir() return tmp_path @pytest.fixture(scope='session') def run_dir_with_symlinks(): tmp_path = Path(TemporaryDirectory().name) tmp_path.mkdir() # one regular workflow init_flows( tmp_path, running=('foo',) ) # one symlinked workflow tmp_path2 = Path(TemporaryDirectory().name) tmp_path2.mkdir() init_flows( tmp_path2, # make it nested to prove that the link is followed running=('bar/baz',) ) Path(tmp_path, 'bar').symlink_to(Path(tmp_path2, 'bar')) yield tmp_path rmtree(tmp_path) @pytest.fixture(scope='session') def run_dir_with_nasty_symlinks(): tmp_path = Path(TemporaryDirectory().name) tmp_path.mkdir() # one regular workflow init_flows( tmp_path, running=('foo',) ) # and a symlink pointing back at it in the same dir Path(tmp_path, 'bar').symlink_to(Path(tmp_path, 'foo')) yield tmp_path rmtree(tmp_path) @pytest.fixture(scope='session') def nested_dir(): tmp_path = Path(TemporaryDirectory().name) tmp_path.mkdir() init_flows( tmp_path, running=('a', 'b/c', 'd/e/f', 'g/h/i/j'), ) yield tmp_path rmtree(tmp_path) @pytest.fixture def source_dirs(mock_glbl_cfg): src = Path(TemporaryDirectory().name) src.mkdir() src1 = src / '1' src1.mkdir() init_flows( tmp_src_path=src1, src=('a', 'b/c') ) src2 = src / '2' src2.mkdir() init_flows( tmp_src_path=src2, src=('d', 'e/f') ) mock_glbl_cfg( 'cylc.flow.scripts.scan.glbl_cfg', f''' [install] source dirs = {src1}, {src2} ''' ) yield [src1, src2] rmtree(src) async def listify(async_gen, field='name'): """Convert an async generator into a list.""" ret = [] async for item in async_gen: ret.append(item[field]) ret.sort() return ret async def test_scan(sample_run_dir): """It should list all flows.""" assert await listify( scan(sample_run_dir) ) == [ 'bar/pub', 'baz', 'cheese/run1', 'cheese/run2', 'foo' ] async def test_scan_with_files(sample_run_dir): """It shouldn't be perturbed by arbitrary files.""" Path(sample_run_dir, 'abc').touch() Path(sample_run_dir, 'def').touch() assert await listify( scan(sample_run_dir) ) == [ 'bar/pub', 'baz', 'cheese/run1', 'cheese/run2', 'foo', ] async def test_scan_horrible_mess(badly_messed_up_cylc_run_dir): """It shouldn't be affected by erroneous cylc files/dirs. How could you end up with a .service dir in ~/cylc-run? Well misuse of Cylc7 can result in this situation so this test ensures Cylc7 workflows can't mess up a Cylc8 scan. """ assert await listify( scan(badly_messed_up_cylc_run_dir) ) == [ 'foo' ] async def test_scan_symlinks(run_dir_with_symlinks): """It should follow symlinks to flows in other dirs.""" assert await listify( scan(run_dir_with_symlinks) ) == [ 'bar/baz', 'foo' ] async def test_scan_nasty_symlinks(run_dir_with_nasty_symlinks): """It should handle strange symlinks because users can be nasty.""" assert await listify( scan(run_dir_with_nasty_symlinks) ) == [ 'bar', # well you got what you asked for 'foo' ] async def test_scan_non_exist(tmp_path: Path): """Calling scan() on a scan_dir that doesn't exist should not raise.""" assert await listify( scan(scan_dir=(tmp_path / 'HORSE')) ) == [] async def test_is_active(sample_run_dir): """It should filter flows by presence of a contact file.""" # running flows assert await is_active.func( {'path': sample_run_dir / 'foo'}, True ) assert await is_active.func( {'path': sample_run_dir / 'bar/pub'}, True ) # registered flows assert not await is_active.func( {'path': sample_run_dir / 'baz'}, True ) # unregistered flows assert not await is_active.func( {'path': sample_run_dir / 'qux'}, True ) # non-existent flows assert not await is_active.func( {'path': sample_run_dir / 'elephant'}, True ) @pytest.mark.parametrize( 'depth, expected', [ (1, ['a']), (3, ['a', 'b/c', 'd/e/f']) ] ) async def test_max_depth(nested_dir, depth: int, expected: List[str]): """It should descend only as far as permitted.""" assert await listify( scan(nested_dir, max_depth=depth) ) == expected async def test_max_depth_configurable(nested_dir, mock_glbl_cfg): """Default scan depth should be configurable in global.cylc.""" mock_glbl_cfg( 'cylc.flow.network.scan.glbl_cfg', ''' [install] max depth = 2 ''' ) assert await listify( scan(nested_dir) ) == [ 'a', 'b/c', ] async def test_scan_one(one, start, test_dir): """Ensure that a running workflow appears in the scan results.""" async with start(one): pipe = ( # scan just this workflow scan(scan_dir=test_dir) | filter_name(rf'^{re.escape(one.workflow)}$') | is_active(True) | workflow_params ) async for flow in pipe: assert flow['name'] == one.workflow break else: raise Exception('Expected one scan result') async def test_workflow_params( one, start, one_conf, run_dir, mod_test_dir ): """It should extract workflow params from the workflow database. Note: For this test we ensure that the workflow UUID is present in the params table. """ async with start(one): pipe = ( # scan just this workflow scan(scan_dir=mod_test_dir) | filter_name(rf'^{re.escape(one.workflow)}$') | is_active(True) | workflow_params ) async for flow in pipe: # check the workflow_params field has been provided assert 'workflow_params' in flow # check the workflow uuid key has been read from the DB uuid_key = WorkflowDatabaseManager.KEY_UUID_STR assert uuid_key in flow['workflow_params'] # check the workflow uuid key matches the scheduler value assert flow['workflow_params'][uuid_key] == one.uuid_str break else: raise Exception('Expected one scan result') async def test_source_dirs(source_dirs): """It should list uninstalled workflows from configured source dirs.""" src1, src2 = source_dirs assert await listify( scan_multi(source_dirs, max_depth=3) ) == [ # NOTE: flow names from scan_multi are full paths (src1 / 'a'), (src1 / 'b/c'), (src2 / 'd'), (src2 / 'e/f'), ] async def test_scan_sigstop( flow, scheduler, start, one_conf, test_dir, caplog, ): """It should log warnings if workflows are un-contactable. Note: This replaces tests/functional/cylc-scan/02-sigstop.t last found in Cylc Flow 8.0a2 which used sigstop to make the flow unresponsive. """ # run a workflow id_ = flow(one_conf) schd = scheduler(id_) async with start(schd): # stop the server to make the flow un-responsive await schd.server.stop('make-unresponsive') # try scanning the workflow pipe = scan(test_dir) | graphql_query(['status']) caplog.clear() async for flow in pipe: raise Exception("There shouldn't be any scan results") # there should, however, be a warning name = Path(id_).name assert ( (30, f'Workflow not running: {name}') in [(level, msg) for _, level, msg in caplog.record_tuples] ) @pytest.fixture def cylc7_run_dir(tmp_path): """A run directory containing three Cylc 7 workflows.""" # a workflow that has not yet been run # (could be run by Cylc <8.7 so should appear in scan results) either = tmp_path / 'either' either.mkdir() (either / WorkflowFiles.SUITE_RC).touch() # a Cylc 7 workflow that has been / is being run by Cylc 7 # (should not appear in scan results) cylc7 = tmp_path / 'cylc7' cylc7.mkdir() (cylc7 / WorkflowFiles.SUITE_RC).touch() Path(cylc7, WorkflowFiles.LogDir.DIRNAME, 'suite').mkdir(parents=True) Path(cylc7, WorkflowFiles.LogDir.DIRNAME, 'suite', 'log').touch() # a Cylc 7 workflow running under Cylc 8 in compatibility mode # (should appear in scan results) cylc8 = tmp_path / 'cylc8' cylc8.mkdir() (cylc8 / WorkflowFiles.SUITE_RC).touch() Path(cylc8, WorkflowFiles.LogDir.DIRNAME, 'scheduler').mkdir(parents=True) Path(cylc8, WorkflowFiles.LogDir.DIRNAME, 'scheduler', 'log').touch() # a Cylc 7 workflow installed by Cylc 8 but not run yet. # (should appear in scan results) cylc8a = tmp_path / 'cylc8a' cylc8a.mkdir() (cylc8a / WorkflowFiles.SUITE_RC).touch() Path(cylc8a, WorkflowFiles.LogDir.DIRNAME, 'install').mkdir(parents=True) # crazy niche case of a Cylc 7 workflow that has had its DB removed # and re-run under Cylc 8 # (should appear in scan results) cylc8 = tmp_path / 'cylc78' cylc8.mkdir() (cylc8 / WorkflowFiles.SUITE_RC).touch() Path(cylc8, WorkflowFiles.LogDir.DIRNAME, 'suite').mkdir(parents=True) Path(cylc8, WorkflowFiles.LogDir.DIRNAME, 'suite', 'log').touch() Path(cylc8, WorkflowFiles.LogDir.DIRNAME, 'scheduler').mkdir(parents=True) Path(cylc8, WorkflowFiles.LogDir.DIRNAME, 'scheduler', 'log').touch() return tmp_path async def test_scan_cylc7(cylc7_run_dir): """It should exclude Cylc 7 workflows from scan results. Unless they are running under Cylc 8 in Cylc 7 compatibility mode. """ assert await listify( scan(cylc7_run_dir) ) == [ 'cylc78', 'cylc8', 'cylc8a', 'either' ] cylc-flow-8.7.0/tests/integration/test_scheduler.py0000664000175000017500000005176415264420307022705 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio import logging from pathlib import Path import re from signal import ( SIGHUP, SIGINT, SIGTERM, ) from typing import ( Any, Callable, ) from unittest.mock import Mock import pytest from cylc.flow import commands from cylc.flow.exceptions import CylcError, WorkflowFilesError from cylc.flow.id import Tokens from cylc.flow.network.resolvers import TaskMsg from cylc.flow.parsec.exceptions import ParsecError from cylc.flow.scheduler import ( Scheduler, SchedulerStop, ) from cylc.flow.scheduler_cli import RunOptions from cylc.flow.task_outputs import TASK_OUTPUT_STARTED from cylc.flow.task_remote_mgr import ( REMOTE_FILE_INSTALL_255, REMOTE_FILE_INSTALL_DONE, REMOTE_FILE_INSTALL_FAILED, REMOTE_INIT_255, REMOTE_INIT_DONE, REMOTE_INIT_FAILED, ) from cylc.flow.task_state import ( TASK_STATUS_FAILED, TASK_STATUS_RUNNING, TASK_STATUS_SUBMIT_FAILED, TASK_STATUS_SUBMITTED, TASK_STATUS_SUCCEEDED, TASK_STATUS_WAITING, ) from cylc.flow.workflow_files import WorkflowFiles from cylc.flow.workflow_status import ( AutoRestartMode, StopMode, ) Fixture = Any TRACEBACK_MSG = "Traceback (most recent call last):" async def test_is_paused_after_stop( one_conf: Fixture, flow: Fixture, scheduler: Fixture, run: Fixture, db_select: Fixture): """Test the paused status is unset on normal shutdown.""" id_: str = flow(one_conf) schd: 'Scheduler' = scheduler(id_, paused_start=True) # Run async with run(schd): assert not schd.is_restart assert schd.is_paused # Stopped assert ('is_paused', '1') not in db_select(schd, False, 'workflow_params') # Restart schd = scheduler(id_, paused_start=None) async with run(schd): assert schd.is_restart assert not schd.is_paused async def test_is_paused_after_crash( one_conf: Fixture, flow: Fixture, scheduler: Fixture, run: Fixture, db_select: Fixture): """Test the paused status is not unset for an interrupted workflow.""" id_: str = flow(one_conf) schd: 'Scheduler' = scheduler(id_, paused_start=True) def ctrl_c(): raise asyncio.CancelledError("Mock keyboard interrupt") # Patch this part of the main loop _schd_workflow_shutdown = schd.workflow_shutdown setattr(schd, 'workflow_shutdown', ctrl_c) # Run with pytest.raises(asyncio.CancelledError): async with run(schd): assert not schd.is_restart assert schd.is_paused # Stopped assert ('is_paused', '1') in db_select(schd, False, 'workflow_params') # Reset patched method setattr(schd, 'workflow_shutdown', _schd_workflow_shutdown) # Restart schd = scheduler(id_, paused_start=None) async with run(schd): assert schd.is_restart assert schd.is_paused async def test_shutdown_CylcError_log(one: Scheduler, run: Callable): """Test that if a CylcError occurs during shutdown, it is logged in one line.""" schd = one # patch the shutdown to raise an error async def mock_shutdown(*a, **k): raise CylcError("Error on shutdown") setattr(schd, '_shutdown', mock_shutdown) # run the workflow log: pytest.LogCaptureFixture with pytest.raises(CylcError) as exc: async with run(schd) as log: pass # check the log records after attempted shutdown assert str(exc.value) == "Error on shutdown" last_record = log.records[-1] assert last_record.message == "CylcError: Error on shutdown" assert last_record.levelno == logging.ERROR # shut down the scheduler properly await Scheduler._shutdown(schd, SchedulerStop('because I said so')) async def test_shutdown_general_exception_log(one: Scheduler, run: Callable): """Test that if a non-CylcError occurs during shutdown, it is logged with traceback (but not excessive).""" schd = one # patch the shutdown to raise an error async def mock_shutdown(*a, **k): raise ValueError("Error on shutdown") setattr(schd, '_shutdown', mock_shutdown) # run the workflow log: pytest.LogCaptureFixture with pytest.raises(ValueError) as exc: async with run(schd) as log: pass # check the log records after attempted shutdown assert str(exc.value) == "Error on shutdown" last_record = log.records[-1] assert last_record.message == "Error on shutdown" assert last_record.levelno == logging.ERROR assert last_record.exc_text is not None assert last_record.exc_text.startswith(TRACEBACK_MSG) assert ("During handling of the above exception, " "another exception occurred") not in last_record.exc_text # shut down the scheduler properly await Scheduler._shutdown(schd, SchedulerStop('because I said so')) async def test_holding_tasks_whilst_scheduler_paused( capture_submission, flow, one_conf, start, scheduler, ): """It should hold tasks irrespective of workflow state. See https://github.com/cylc/cylc-flow/issues/4278 """ id_ = flow(one_conf) one = scheduler(id_, paused_start=True) # run the workflow async with start(one): # capture any job submissions submitted_tasks = capture_submission(one) assert submitted_tasks == set() # release runahead/queued tasks # (nothing should happen because the scheduler is paused) one.pool.release_runahead_tasks() one.release_tasks_to_run() assert submitted_tasks == set() # hold all tasks & resume the workflow await commands.run_cmd(commands.hold(one, ['*/*'])) one.resume_workflow() # release queued tasks # (there should be no change because the task is still held) one.release_tasks_to_run() assert submitted_tasks == set() # release all tasks await commands.run_cmd(commands.release(one, ['*/*'])) # release queued tasks # (the task should be submitted) one.release_tasks_to_run() assert len(submitted_tasks) == 1 async def test_no_poll_waiting_tasks( capture_polling, flow, one_conf, start, scheduler, ): """Waiting tasks shouldn't be polled. If a waiting task previously it will have the submit number of its previous job, and polling would erroneously return the state of that job. See https://github.com/cylc/cylc-flow/issues/4658 """ id_: str = flow(one_conf) # start the scheduler in live mode in order to activate regular polling # logic one: Scheduler = scheduler(id_, run_mode='live') log: pytest.LogCaptureFixture async with start(one) as log: # Test assumes start up with a waiting task. task = (one.pool.get_tasks())[0] assert task.state.status == TASK_STATUS_WAITING polled_tasks = capture_polling(one) # Waiting tasks should not be polled. await commands.run_cmd(commands.poll_tasks(one, ['*/*'])) assert polled_tasks == set() # Even if they have a submit number. task.submit_num = 1 await commands.run_cmd(commands.poll_tasks(one, ['*/*'])) assert len(polled_tasks) == 0 # But these states should be: for state in [ TASK_STATUS_SUBMIT_FAILED, TASK_STATUS_FAILED, TASK_STATUS_SUBMITTED, TASK_STATUS_RUNNING ]: task.state.status = state await commands.run_cmd(commands.poll_tasks(one, ['*/*'])) assert len(polled_tasks) == 1 polled_tasks.clear() # Shut down with a running task. task.state.status = TASK_STATUS_RUNNING # For good measure, check the faked running task is reported at shutdown. assert "Orphaned tasks:\n* 1/one (running)" in log.messages async def test_unexpected_ParsecError( one: Scheduler, start: Callable, log_filter: Callable, monkeypatch: pytest.MonkeyPatch ): """Test that ParsecErrors - that occur at any time other than config load when running a workflow - are displayed with traceback, because they are not expected.""" log: pytest.LogCaptureFixture def raise_ParsecError(*a, **k): raise ParsecError("Mock error") monkeypatch.setattr(one, '_configure_contact', raise_ParsecError) with pytest.raises(ParsecError): async with start(one) as log: pass assert log_filter( logging.CRITICAL, exact_match="Workflow shutting down - Mock error" ) assert TRACEBACK_MSG in log.text async def test_error_during_auto_restart( one: Scheduler, run: Callable, log_filter: Callable, monkeypatch: pytest.MonkeyPatch, ): """Test that an error during auto-restart does not get swallowed""" log: pytest.LogCaptureFixture err_msg = "Mock error: sugar in water" def mock_auto_restart(*a, **k): raise RuntimeError(err_msg) monkeypatch.setattr(one, 'workflow_auto_restart', mock_auto_restart) monkeypatch.setattr( one, 'auto_restart_mode', AutoRestartMode.RESTART_NORMAL ) with pytest.raises(RuntimeError, match=err_msg): async with run(one) as log: pass assert log_filter(logging.ERROR, err_msg) assert TRACEBACK_MSG in log.text async def test_uuid_unchanged_on_restart( one: Scheduler, scheduler: Callable, start: Callable, ): """Restart gets UUID from Database: See https://github.com/cylc/cylc-flow/issues/5615 Process: * Create a scheduler then shut it down. * Create a new scheduler for the same workflow and check that it has retrieved the UUID from the Daatabase. """ uuid_re = re.compile('CYLC_WORKFLOW_UUID=(.*)') contact_file = Path(one.workflow_run_dir) / '.service/contact' async with start(one): pass schd = scheduler(one.workflow_name, paused_start=True) async with start(schd): # UUID in contact file should be the same as that set in the database # and the scheduler. cf_uuid = uuid_re.findall(contact_file.read_text()) assert cf_uuid == [schd.uuid_str] async def test_restart_timeout( flow, one_conf, scheduler, run, log_filter, complete, capture_submission, ): """It should wait for user input if there are no tasks in the pool. When restarting a completed workflow there are no tasks in the pool so the scheduler is inclined to shutdown before the user has had the chance to trigger tasks in order to allow the workflow to continue. In order to make this easier, the scheduler should enter the paused state and wait around for a configured period before shutting itself down. See: https://github.com/cylc/cylc-flow/issues/5078 """ id_ = flow(one_conf) # run the workflow to completion schd: Scheduler = scheduler(id_, paused_start=False) async with run(schd): await complete(schd) # restart the completed workflow schd = scheduler(id_, paused_start=False) async with run(schd): # it should detect that the workflow has completed and alert the user assert log_filter( logging.WARNING, contains='This workflow already ran to completion.' ) # it should activate a timeout assert log_filter(logging.WARNING, contains='restart timer starts NOW') capture_submission(schd) # when we trigger tasks the timeout should be cleared await commands.run_cmd( commands.force_trigger_tasks(schd, ['1/one'], [])) await asyncio.sleep(0) # yield control to the main loop assert log_filter(logging.INFO, contains='restart timer stopped') @pytest.mark.parametrize("signal", ((SIGHUP), (SIGINT), (SIGTERM))) async def test_signal_escallation(one, start, signal, log_filter): """Double signal should escalate shutdown. If a term-like signal is received whilst the workflow is already stopping in NOW mode, then the shutdown should be escalated to NOW_NOW mode. See https://github.com/cylc/cylc-flow/pull/6444 """ async with start(one): # put the workflow in the stopping state one._set_stop(StopMode.REQUEST_CLEAN) assert one.stop_mode.name == 'REQUEST_CLEAN' # one signal should escalate this from CLEAN to NOW one._handle_signal(signal, None) assert log_filter(contains=signal.name) assert one.stop_mode.name == 'REQUEST_NOW' # two signals should escalate this from NOW to NOW_NOW one._handle_signal(signal, None) assert one.stop_mode.name == 'REQUEST_NOW_NOW' async def test_set_stall_interaction(flow, scheduler, start): """Test the interaction between manual set and workflow stall. A set operation can put a scheduler into the stalled state and can also get a scheduler out of the stalled state. """ id_ = flow({ 'scheduling': { 'xtriggers': {'never': 'xrandom(0)'}, 'graph': { 'R1': ''' # add a pending xtrigger so that when we un-stall the # workflow (via "cylc set") there are no downstream impacts # See https://github.com/cylc/cylc-flow/issues/7157 @never => b a => b ''' }, }, }) schd: 'Scheduler' = scheduler(id_, paused_start=False) async with start(schd): # set 1/a to failed await commands.run_cmd( commands.set_prereqs_and_outputs( schd, ['^/a'], [], [TASK_STATUS_FAILED] ) ) # the scheduler should be stalled await schd._main_loop() await schd._main_loop() assert schd.is_stalled is True assert ( schd.data_store_mgr.data[schd.tokens.id]['workflow'].status_msg == 'stalled' ) # set 1/a to succeeded await commands.run_cmd( commands.set_prereqs_and_outputs( schd, ['^/a'], [], [TASK_STATUS_SUCCEEDED] ) ) # the scheduler should NOT be stalled await schd._main_loop() await schd._main_loop() assert schd.is_stalled is False assert ( schd.data_store_mgr.data[schd.tokens.id]['workflow'].status_msg != 'stalled' ) async def test_cylc_message(one, start, log_filter, caplog): message = ( "20000101T0000Z", # event_time "INFO", # severity 'some-message', # message ) async with start(one): # send a message for an invalid task (task names cannot contain spaces) one.message_queue.put( TaskMsg( Tokens("1/Invalid task", relative=True), *message, ) ) one.process_queued_task_messages() assert log_filter(contains="Illegal task name: Invalid task") # send a message for a valid task caplog.clear() one.message_queue.put( TaskMsg( Tokens("1/one", relative=True), *message, ) ) one.process_queued_task_messages() assert not log_filter(contains="Illegal task name") assert log_filter(contains='(received)some-message') async def test_manage_remote_init_retry_on_255( flow, scheduler, start, monkeypatch: pytest.MonkeyPatch ): """Test that manage_remote_init retries remote init/file install on 255. See https://github.com/cylc/cylc-flow/issues/7286 On restart, if remote-init fails with exit code 255 (SSH connection failure), manage_remote_init should retry rather than silently dropping the install target. """ missing_target = 'Alderaan' fake_platform = {'install target': missing_target, 'name': 'test_plat'} schd: Scheduler = scheduler(flow('one'), run_mode='live') async with start(schd): remote_mgr = schd.task_job_mgr.task_remote_mgr mock_remote_init = Mock() monkeypatch.setattr(remote_mgr, 'remote_init', mock_remote_init) mock_file_install = Mock() monkeypatch.setattr(remote_mgr, 'file_install', mock_file_install) # Simulate restart remote-init has been kicked off and is being # tracked via incomplete_ri_map: schd.incomplete_ri_map[missing_target] = fake_platform # REMOTE_INIT_255: retry remote_init (not file_install) remote_mgr.remote_init_map[missing_target] = REMOTE_INIT_255 schd.manage_remote_init() mock_remote_init.assert_called_once_with(fake_platform) mock_file_install.assert_not_called() assert missing_target in schd.incomplete_ri_map, ( "Install target should remain in incomplete_ri_map for tracking" ) mock_remote_init.reset_mock() # REMOTE_INIT_DONE: should trigger file_install (not remote_init) remote_mgr.remote_init_map[missing_target] = REMOTE_INIT_DONE schd.manage_remote_init() mock_file_install.assert_called_once_with(fake_platform) mock_remote_init.assert_not_called() assert missing_target in schd.incomplete_ri_map mock_file_install.reset_mock() # REMOTE_FILE_INSTALL_255: should retry file_install (not remote_init) remote_mgr.remote_init_map[missing_target] = REMOTE_FILE_INSTALL_255 schd.manage_remote_init() mock_file_install.assert_called_once_with(fake_platform) mock_remote_init.assert_not_called() assert missing_target in schd.incomplete_ri_map mock_file_install.reset_mock() # REMOTE_FILE_INSTALL_DONE: should remove from ri map remote_mgr.remote_init_map[missing_target] = REMOTE_FILE_INSTALL_DONE schd.manage_remote_init() mock_remote_init.assert_not_called() mock_file_install.assert_not_called() assert missing_target not in schd.incomplete_ri_map # REMOTE_INIT_FAILED: should remove from ri map schd.incomplete_ri_map[missing_target] = fake_platform remote_mgr.remote_init_map[missing_target] = REMOTE_INIT_FAILED schd.manage_remote_init() mock_remote_init.assert_not_called() mock_file_install.assert_not_called() assert missing_target not in schd.incomplete_ri_map # REMOTE_FILE_INSTALL_FAILED: should remove from ri map schd.incomplete_ri_map[missing_target] = fake_platform remote_mgr.remote_init_map[missing_target] = REMOTE_FILE_INSTALL_FAILED schd.manage_remote_init() mock_remote_init.assert_not_called() mock_file_install.assert_not_called() assert missing_target not in schd.incomplete_ri_map async def test_suite_rc(test_dir, run_dir, start): """It should reject workflows with suite.rc files.""" (test_dir / WorkflowFiles.SUITE_RC).touch() workflow_id = str(test_dir.relative_to(run_dir)) schd = Scheduler(workflow_id, RunOptions()) # suite.rc present with pytest.raises( WorkflowFilesError, match=f'suite.rc found in .*{test_dir.name}' ): async with start(schd): pass # suite.rc and flow.cylc present (test_dir / WorkflowFiles.FLOW_FILE).touch() with pytest.raises( WorkflowFilesError, match=f'Both flow.cylc and suite.rc.*{test_dir.name}', ): async with start(schd): pass async def test_custom_output_runahead_release(flow, scheduler, run, complete): """Test that a custom output can trigger runahead release of the dependent task. https://github.com/cylc/cylc-flow/pull/7516 """ wid = flow({ 'scheduling': { 'graph': {'R1': 'a:x => b'}, }, 'runtime': { 'a': { 'outputs': {'x': 'xray'}, 'simulation': {'default run length': 'PT30S'}, }, }, }) schd: Scheduler = scheduler(wid, paused_start=False) async with run(schd): task_a = schd.pool._get_task_by_id('1/a') task_a.submit_num = 1 schd.task_events_mgr.process_message( task_a, logging.INFO, TASK_OUTPUT_STARTED ) # Ensure started (& other, implied standard) outputs are air-gapped # from the custom output by a main loop iteration, otherwise the bug # may not be triggered: await schd._main_loop() schd.task_events_mgr.process_message(task_a, logging.INFO, 'xray') # b should release and run: await complete(schd, '1/b', timeout=5) cylc-flow-8.7.0/tests/integration/run_modes/0000775000175000017500000000000015264420307021274 5ustar alastairalastaircylc-flow-8.7.0/tests/integration/run_modes/test_nonlive.py0000664000175000017500000001322715264420307024364 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from typing import Any, Dict from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.cycling.iso8601 import ISO8601Point from cylc.flow.id import TaskTokens from cylc.flow.scheduler import Scheduler # Define here to ensure test doesn't just mirror code: KGO = { 'live': { 'flow_nums': '[1]', 'is_manual_submit': 0, 'try_num': 1, 'submit_status': 0, 'run_signal': None, 'run_status': 0, # capture_live_submissions fixture submits jobs in sim mode 'platform_name': 'simulation', 'job_runner_name': 'simulation', 'job_id': None, }, 'skip': { 'flow_nums': '[1]', 'is_manual_submit': 0, 'try_num': 1, 'submit_status': 0, 'run_signal': None, 'run_status': 0, 'platform_name': 'skip', 'job_runner_name': 'skip', 'job_id': None, }, } def not_time(data: Dict[str, Any]): """Filter out fields containing times to reduce risk of flakiness""" return {k: v for k, v in data.items() if 'time' not in k} @pytest.fixture def submit_and_check_db(): """Wraps up testing that we want to do repeatedly in test_db_task_jobs. """ def _inner(schd): # Submit task jobs: schd.submit_task_jobs(schd.pool.get_tasks()) # Make sure that db changes are enacted: schd.workflow_db_mgr.process_queued_ops() for mode, kgo in KGO.items(): task_jobs = schd.workflow_db_mgr.pub_dao.select_task_job(1, mode) # Check all non-datetime items against KGO: assert not_time(task_jobs) == kgo, ( f'Mode {mode}: incorrect db entries.') # Check that timestamps have been created: for timestamp in [ 'time_submit', 'time_submit_exit', 'time_run', 'time_run_exit' ]: assert task_jobs[timestamp] is not None return _inner async def test_db_task_jobs( flow, scheduler, start, capture_live_submissions, submit_and_check_db ): """Ensure that task job data is added to the database correctly for each run mode. """ schd: Scheduler = scheduler( flow({ 'scheduling': { 'graph': { 'R1': ' & '.join(KGO) } }, 'runtime': { mode: {'run mode': mode} for mode in KGO }, }), run_mode='live' ) async with start(schd): # Reference all task proxies so we can examine them # at the end of the test: itask_skip = schd.pool.get_task(IntegerPoint('1'), 'skip') itask_live = schd.pool.get_task(IntegerPoint('1'), 'live') submit_and_check_db(schd) # Set outputs to failed: schd.pool.set_prereqs_and_outputs( {TaskTokens('*', 'root')}, ['failed'], [], [] ) submit_and_check_db(schd) # capture_live_submissions fixture submits jobs in sim mode assert itask_live.run_mode.value == 'simulation' assert itask_skip.run_mode.value == 'skip' async def test_db_task_states( one_conf, flow, scheduler, start ): """Test that tasks will have the same information entered into the task state database whichever mode is used. """ conf = one_conf conf['runtime'] = {'one': {'run mode': 'skip'}} schd = scheduler(flow(conf)) async with start(schd): schd.submit_task_jobs(schd.pool.get_tasks()) schd.workflow_db_mgr.process_queued_ops() result = schd.workflow_db_mgr.pri_dao.connect().execute( 'SELECT * FROM task_states').fetchone() # Submit number has been added to the table: assert result[5] == 1 # time_created added to the table assert result[3] async def test_mean_task_time( flow, scheduler, start, complete, capture_live_submissions ): """Non-live tasks are not added to the list of task times, so skipping tasks will not affect how long Cylc expects tasks to run. """ schd = scheduler(flow({ 'scheduling': { 'initial cycle point': '1000', 'final cycle point': '1002', 'graph': {'P1Y': 'foo'}} }), run_mode='live') async with start(schd): itask = schd.pool.get_task(ISO8601Point('10000101T0000Z'), 'foo') assert list(itask.tdef.elapsed_times) == [] # Make the task run in skip mode at one cycle: schd.broadcast_mgr.put_broadcast( ['1000'], ['foo'], [{'run mode': 'skip'}]) # Fake adding some other examples of the task: itask.tdef.elapsed_times.extend([133.0, 132.4]) # Submit two tasks: schd.submit_task_jobs([itask]) # Ensure that the skipped task has succeeded, and that the # number of items in the elapsed_times has not changed. assert itask.state.status == 'succeeded' assert len(itask.tdef.elapsed_times) == 2 cylc-flow-8.7.0/tests/integration/run_modes/test_mode_overrides.py0000664000175000017500000001251015264420307025712 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test that using [runtime][TASK]run mode works in each mode. Point 3 of the Skip Mode proposal https://github.com/cylc/cylc-admin/blob/master/docs/proposal-skip-mode.md | The run mode should be controlled by a new task configuration | [runtime][]run mode with the default being live. | As a runtime configuration, this can be defined in the workflow | for development / testing purposes or set by cylc broadcast. n.b: This is pretty much a functional test and probably ought to be labelled as such, but uses the integration test framework. """ import pytest from cylc.flow.cycling.iso8601 import ISO8601Point from cylc.flow.id import TaskTokens from cylc.flow.run_modes import WORKFLOW_RUN_MODES, RunMode from cylc.flow.scheduler import Scheduler, SchedulerStop from cylc.flow.task_state import TASK_STATUS_WAITING from cylc.flow.commands import ( run_cmd, force_trigger_tasks ) @pytest.mark.parametrize('workflow_run_mode', sorted(WORKFLOW_RUN_MODES)) async def test_run_mode_override_from_config( capture_live_submissions, flow, scheduler, run, complete, workflow_run_mode, validate ): """Test that `[runtime][]run mode` overrides workflow modes.""" id_ = flow({ 'scheduling': { 'graph': { 'R1': 'live & skip', }, }, 'runtime': { 'live': {'run mode': 'live'}, 'skip': {'run mode': 'skip'}, } }) run_mode = RunMode(workflow_run_mode) validate(id_) schd = scheduler(id_, run_mode=run_mode, paused_start=False) async with run(schd): await complete(schd) if workflow_run_mode == 'live': assert capture_live_submissions() == {'1/live'} elif workflow_run_mode == 'dummy': # Skip mode doesn't override dummy mode: assert capture_live_submissions() == {'1/live', '1/skip'} else: assert capture_live_submissions() == set() async def test_force_trigger_does_not_override_run_mode( flow, scheduler, start, ): """Force-triggering a task will not override the run mode. Taken from spec at https://github.com/cylc/cylc-admin/blob/master/docs/proposal-skip-mode.md#proposal """ wid = flow({ 'scheduling': {'graph': {'R1': 'foo'}}, 'runtime': {'foo': {'run mode': 'skip'}} }) schd = scheduler(wid, run_mode="live") async with start(schd): foo = schd.pool.get_tasks()[0] # Force trigger task: await run_cmd(force_trigger_tasks(schd, ['1/foo'], ['1'])) # ... but job submission will always change this to the correct mode: schd.submit_task_jobs([foo]) assert foo.run_mode.value == 'skip' async def test_run_mode_skip_abides_by_held(flow, scheduler, run): """Tasks with run mode = skip will continue to abide by the is_held flag as normal. Taken from spec at https://github.com/cylc/cylc-admin/blob/master/docs/proposal-skip-mode.md#proposal """ wid = flow({ 'scheduling': {'graph': {'R1': 'foo'}}, 'runtime': {'foo': {'run mode': 'skip'}} }) schd: Scheduler = scheduler(wid, run_mode="live", paused_start=False) async with run(schd): foo = schd.pool.get_tasks()[0] assert not foo.state.is_held # Hold task, check that it's held: schd.pool.hold_tasks({TaskTokens('1', 'foo')}) await schd._main_loop() assert foo.state(TASK_STATUS_WAITING, is_held=True) schd.pool.release_held_tasks({TaskTokens('1', 'foo')}) assert not foo.state.is_held with pytest.raises(SchedulerStop): # Will shut down as foo has run await schd._main_loop() async def test_run_mode_override_from_broadcast( flow, scheduler, start, complete, log_filter, capture_live_submissions ): """Test that run_mode modifications only apply to one task. """ cfg = { "scheduler": {"cycle point format": "%Y"}, "scheduling": { "initial cycle point": "1000", "final cycle point": "1001", "graph": {"P1Y": "foo"}}, "runtime": { } } id_ = flow(cfg) schd = scheduler(id_, run_mode='live', paused_start=False) async with start(schd): schd.broadcast_mgr.put_broadcast( ['1000'], ['foo'], [{'run mode': 'skip'}]) foo_1000 = schd.pool.get_task(ISO8601Point('1000'), 'foo') foo_1001 = schd.pool.get_task(ISO8601Point('1001'), 'foo') schd.submit_task_jobs([foo_1000, foo_1001]) assert foo_1000.run_mode.value == 'skip' assert capture_live_submissions() == {'1001/foo'} cylc-flow-8.7.0/tests/integration/run_modes/test_simulation.py0000664000175000017500000004102015264420307025066 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test the workings of simulation mode""" import logging from pathlib import Path import pytest from pytest import param from cylc.flow import commands from cylc.flow.cycling.iso8601 import ISO8601Point from cylc.flow.run_modes import RunMode from cylc.flow.run_modes.simulation import sim_time_check async def test_started_trigger(flow, reftest, scheduler): """Does the started task output trigger downstream tasks in sim mode? Long standing Bug discovered in Skip Mode work. https://github.com/cylc/cylc-flow/pull/6039#issuecomment-2321147445 """ schd = scheduler( flow( { 'scheduler': { 'events': { 'stall timeout': 'PT0S', 'abort on stall timeout': True, } }, 'scheduling': {'graph': {'R1': 'a:started => b'}}, } ), paused_start=False, ) assert await reftest(schd) == { ('1/a', None), ('1/b', ('1/a',)) } @pytest.fixture def monkeytime(monkeypatch): """Convenience function monkeypatching time.""" def _inner(time_: int): monkeypatch.setattr('cylc.flow.task_job_mgr.time', lambda: time_) monkeypatch.setattr( 'cylc.flow.run_modes.simulation.time', lambda: time_) return _inner @pytest.fixture def run_simjob(monkeytime): """Run a simulated job to completion. Returns the output status. """ def _run_simjob(schd, point, task): itask = schd.pool.get_task(point, task) itask.state.is_queued = False monkeytime(0) schd.task_job_mgr.submit_nonlive_task_jobs([itask], RunMode.SIMULATION) monkeytime(itask.mode_settings.timeout + 1) # Run Time Check assert sim_time_check( schd.task_events_mgr, [itask], schd.workflow_db_mgr ) is True # Capture result process queue. return itask return _run_simjob @pytest.fixture(scope='module') async def sim_time_check_setup( mod_flow, mod_scheduler, mod_start, mod_one_conf, ): schd = mod_scheduler(mod_flow({ 'scheduler': {'cycle point format': '%Y'}, 'scheduling': { 'initial cycle point': '1066', 'graph': { 'R1': 'one & fail_all & fast_forward', 'P1Y': 'fail_once & fail_all_submits' } }, 'runtime': { 'one': {}, 'fail_all': { 'simulation': { 'fail cycle points': 'all', 'fail try 1 only': False }, 'outputs': {'foo': 'bar'} }, # This task ought not be finished quickly, but for the speed up 'fast_forward': { 'execution time limit': 'PT1M', 'simulation': {'speedup factor': 2} }, 'fail_once': { 'simulation': { 'fail cycle points': '1066, 1068', } }, 'fail_all_submits': { 'simulation': { 'fail cycle points': '1066', 'fail try 1 only': False, } } } })) async with mod_start(schd): itasks = schd.pool.get_tasks() [schd.task_job_mgr._set_retry_timers(i) for i in itasks] yield schd, itasks def test_false_if_not_running( sim_time_check_setup, monkeypatch ): schd, itasks = sim_time_check_setup itasks = [i for i in itasks if i.state.status != 'running'] # False if task status not running: assert sim_time_check(schd.task_events_mgr, itasks, '') is False @pytest.mark.parametrize( 'itask, point, results', ( # Task fails this CP, first submit. param( 'fail_once', '1066', (True, False, False), id='only-fail-on-submit-1'), # Task succeeds this CP, all submits. param( 'fail_once', '1067', (False, False, False), id='do-not-fail-this-cp'), # Task fails this CP, first submit. param( 'fail_once', '1068', (True, False, False), id='and-another-cp'), # Task fails this CP, all submits. param( 'fail_all_submits', '1066', (True, True, True), id='fail-all-submits'), # Task succeeds this CP, all submits. param( 'fail_all_submits', '1067', (False, False, False), id='fail-no-submits'), ) ) def test_fail_once(sim_time_check_setup, itask, point, results, monkeypatch): """A task with a fail cycle point only fails at that cycle point, and then only on the first submission. """ schd, _ = sim_time_check_setup itask = schd.pool.get_task( ISO8601Point(point), itask) for i, result in enumerate(results): itask.try_timers['execution-retry'].num = i schd.task_job_mgr.submit_nonlive_task_jobs([itask], RunMode.SIMULATION) assert itask.mode_settings.sim_task_fails is result def test_task_finishes(sim_time_check_setup, monkeytime, caplog): """...and an appropriate message sent. Checks that failed and bar are output if a task is set to fail. Does NOT check every possible cause of an outcome - this is done in unit tests. """ schd, _ = sim_time_check_setup monkeytime(0) # Setup a task to fail, submit it. fail_all_1066 = schd.pool.get_task(ISO8601Point('1066'), 'fail_all') fail_all_1066.state.status = 'running' fail_all_1066.state.is_queued = False schd.task_job_mgr.submit_nonlive_task_jobs( [fail_all_1066], RunMode.SIMULATION ) # For the purpose of the test delete the started time set by # submit_nonlive_task_jobs. fail_all_1066.summary['started_time'] = 0 # Before simulation time is up: assert sim_time_check(schd.task_events_mgr, [fail_all_1066], '') is False # Time's up... monkeytime(12) # After simulation time is up it Fails and records custom outputs: assert sim_time_check(schd.task_events_mgr, [fail_all_1066], '') is True outputs = fail_all_1066.state.outputs assert outputs.is_message_complete('succeeded') is False assert outputs.is_message_complete('bar') is True assert outputs.is_message_complete('failed') is True def test_task_sped_up(sim_time_check_setup, monkeytime): """Task will speed up by a factor set in config.""" schd, _ = sim_time_check_setup fast_forward_1066 = schd.pool.get_task( ISO8601Point('1066'), 'fast_forward') # Run the job submission method: monkeytime(0) schd.task_job_mgr.submit_nonlive_task_jobs( [fast_forward_1066], RunMode.SIMULATION ) fast_forward_1066.state.is_queued = False result = sim_time_check(schd.task_events_mgr, [fast_forward_1066], '') assert result is False monkeytime(29) result = sim_time_check(schd.task_events_mgr, [fast_forward_1066], '') assert result is False monkeytime(31) result = sim_time_check(schd.task_events_mgr, [fast_forward_1066], '') assert result is True async def test_settings_restart(monkeytime, flow, scheduler, start): """Check that simulation mode settings are correctly restored upon restart. In the case of start time this is collected from the database from task_jobs.start_time. tasks: one: Runs straighforwardly. two: Test case where database is missing started_time because it was upgraded from an earlier version of Cylc. """ id_ = flow({ 'scheduler': {'cycle point format': '%Y'}, 'scheduling': { 'initial cycle point': '1066', 'graph': { 'R1': 'one & two' } }, 'runtime': { 'root': { 'execution time limit': 'PT1M', 'execution retry delays': 'P0Y', 'simulation': { 'speedup factor': 1, 'fail cycle points': 'all', 'fail try 1 only': True, } }, } }) schd = scheduler(id_) # Start the workflow: async with start(schd): og_timeouts = {} for itask in schd.pool.get_tasks(): schd.task_job_mgr.submit_nonlive_task_jobs( [itask], RunMode.SIMULATION ) og_timeouts[itask.identity] = itask.mode_settings.timeout # Mock wallclock < sim end timeout monkeytime(itask.mode_settings.timeout - 1) assert sim_time_check( schd.task_events_mgr, [itask], schd.workflow_db_mgr ) is False # Stop and restart the scheduler: schd = scheduler(id_) async with start(schd): for itask in schd.pool.get_tasks(): # Check that we haven't got mode settings back: assert itask.mode_settings is None if itask.identity == '1066/two': # Delete the database entry for `two`: Ensure that # we don't break sim mode on upgrade to this version of Cylc. schd.workflow_db_mgr.pri_dao.connect().execute( 'UPDATE task_jobs' '\n SET time_submit = NULL' '\n WHERE (name == \'two\')' ) schd.workflow_db_mgr.process_queued_ops() monkeytime(42) else: monkeytime(og_timeouts[itask.identity] - 1) assert sim_time_check( schd.task_events_mgr, [itask], schd.workflow_db_mgr ) is False # Check that the itask.mode_settings is now re-created assert itask.mode_settings.simulated_run_length == 60.0 assert itask.mode_settings.sim_task_fails is True async def test_settings_reload( flow, scheduler, start, run_simjob ): """Check that simulation mode settings are changed for future pseudo jobs on reload. """ id_ = flow({ 'scheduler': {'cycle point format': '%Y'}, 'scheduling': { 'initial cycle point': '1066', 'graph': {'R1': 'one'} }, 'runtime': { 'one': { 'execution time limit': 'PT1M', 'execution retry delays': 'P0Y', 'simulation': { 'speedup factor': 1, 'fail cycle points': 'all', 'fail try 1 only': False, } }, } }) schd = scheduler(id_) async with start(schd): # Submit first psuedo-job and "run" to failure: one_1066 = schd.pool.get_task(ISO8601Point('1066'), 'one') itask = run_simjob(schd, one_1066.point, 'one') assert itask.state.outputs.is_message_complete('failed') is False # Modify config as if reinstall had taken place: conf_file = Path(schd.workflow_run_dir) / 'flow.cylc' conf_file.write_text( conf_file.read_text().replace('False', 'True')) # Reload Workflow: await commands.run_cmd(commands.reload_workflow(schd)) # Submit second psuedo-job and "run" to success: itask = run_simjob(schd, one_1066.point, 'one') assert itask.state.outputs.is_message_complete('succeeded') is True async def test_settings_broadcast( flow, scheduler, start, monkeytime, log_filter ): """Assert that broadcasting a change in the settings for a task affects subsequent psuedo-submissions. """ id_ = flow({ 'scheduler': {'cycle point format': '%Y'}, 'scheduling': { 'initial cycle point': '1066', 'graph': {'R1': 'one'} }, 'runtime': { 'one': { 'execution time limit': 'PT1S', 'execution retry delays': '2*PT5S', 'simulation': { 'speedup factor': 1, 'fail cycle points': '1066', 'fail try 1 only': False } }, } }, defaults=False) schd = scheduler(id_, paused_start=False, run_mode='simulation') async with start(schd) as log: itask = schd.pool.get_task(ISO8601Point('1066'), 'one') itask.state.is_queued = False # Submit the first - the sim task will fail: schd.task_job_mgr.submit_nonlive_task_jobs([itask], RunMode.SIMULATION) assert itask.mode_settings.sim_task_fails is True # Let task finish. monkeytime(itask.mode_settings.timeout + 1) assert sim_time_check( schd.task_events_mgr, [itask], schd.workflow_db_mgr ) is True # The mode_settings object has been cleared: assert itask.mode_settings is None # Change a setting using broadcast: schd.broadcast_mgr.put_broadcast( ['1066'], ['one'], [{ 'simulation': {'fail cycle points': ''}, }]) # Submit again - result is different: schd.task_job_mgr.submit_nonlive_task_jobs([itask], RunMode.SIMULATION) assert itask.mode_settings.sim_task_fails is False # Assert that setting run mode on a simulation mode task fails with # warning: good, bad = schd.broadcast_mgr.put_broadcast( ['1066'], ['one'], [{ 'run mode': 'live', }]) assert good == [] assert bad == {'settings': [{'run mode': 'live'}]} record = log_filter(contains='will not be actioned')[0] assert record[0] == logging.WARNING assert 'run mode' not in schd.broadcast_mgr.broadcasts # Assert Clearing the broadcast works schd.broadcast_mgr.clear_broadcast() schd.task_job_mgr.submit_nonlive_task_jobs([itask], RunMode.SIMULATION) assert itask.mode_settings.sim_task_fails is True # Assert that list of broadcasts doesn't change if we submit # Invalid fail cycle points to broadcast. itask.mode_settings = None schd.broadcast_mgr.put_broadcast( ['1066'], ['one'], [{ 'simulation': {'fail cycle points': 'higadfuhasgiurguj'} }]) schd.task_job_mgr.submit_nonlive_task_jobs([itask], RunMode.SIMULATION) assert ( 'Invalid ISO 8601 date representation: higadfuhasgiurguj' in log.messages[-1]) # Check that the invalid broadcast hasn't # changed the itask sim mode settings: assert itask.mode_settings.sim_task_fails is True schd.broadcast_mgr.put_broadcast( ['1066'], ['one'], [{ 'simulation': {'fail cycle points': '1'} }]) schd.task_job_mgr.submit_nonlive_task_jobs([itask], RunMode.SIMULATION) assert ( 'Invalid ISO 8601 date representation: 1' in log.messages[-1]) # Broadcast tasks will reparse correctly: schd.broadcast_mgr.put_broadcast( ['1066'], ['one'], [{ 'simulation': {'fail cycle points': '1945, 1977, 1066'}, 'execution retry delays': '3*PT2S' }]) schd.task_job_mgr.submit_nonlive_task_jobs([itask], RunMode.SIMULATION) assert itask.mode_settings.sim_task_fails is True assert itask.try_timers['execution-retry'].delays == [2.0, 2.0, 2.0] # n.b. rtconfig should remain unchanged, lest we cancel broadcasts: assert itask.tdef.rtconfig['execution retry delays'] == [5.0, 5.0] async def test_db_submit_num( flow, one_conf, scheduler, run, complete, db_select ): """Test simulation mode correctly increments the submit_num in the DB.""" one_conf['runtime'] = { 'one': {'simulation': {'default run length': 'PT0S'}} } schd = scheduler(flow(one_conf), paused_start=False) async with run(schd): await complete(schd, '1/one', timeout=10) assert db_select(schd, False, 'task_states', 'submit_num', 'status') == [ (1, 'succeeded'), ] cylc-flow-8.7.0/tests/integration/run_modes/test_skip.py0000664000175000017500000002230615264420307023656 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test for skip mode integration. """ from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.id import TaskTokens from cylc.flow.scheduler import Scheduler from cylc.flow.task_outputs import TASK_OUTPUT_FAILED from cylc.flow.task_state import ( TASK_STATUS_FAILED, TASK_STATUS_SUCCEEDED, ) async def test_settings_override_from_broadcast( flow, scheduler, start, complete, log_filter ): """Test that skip mode runs differently if settings are modified. """ cfg = { "scheduling": {"graph": {"R1": "foo:failed => bar"}}, "runtime": { "foo": { "events": { 'handler events': 'failed', "handlers": 'echo "HELLO"' } } } } id_ = flow(cfg) schd = scheduler(id_, run_mode='live') async with start(schd): schd.broadcast_mgr.put_broadcast( ['1'], ['foo'], [ {'run mode': 'skip'}, {'skip': {'outputs': 'failed'}}, {'skip': {'disable task event handlers': "False"}} ] ) foo, = schd.pool.get_tasks() schd.submit_task_jobs(schd.pool.get_tasks()) # Run mode has changed: assert foo.platform['name'] == 'skip' # Output failed emitted: assert foo.state.status == 'failed' # After processing events there is a handler in the subprocpool: schd.task_events_mgr.process_events(schd) assert 'echo "HELLO"' in schd.proc_pool.is_not_done()[0][0].cmd async def test_broadcast_changes_set_skip_outputs( flow, scheduler, start ): """When cylc set --out skip is used, task outputs are updated with broadcasts. Skip mode proposal point 4 https://github.com/cylc/cylc-admin/blob/master/docs/proposal-skip-mode.md | The cylc set --out option should accept the skip value which should | set the outputs defined in [runtime][][skip]outputs. | The skip keyword should not be allowed in custom outputs. """ wid = flow({ 'scheduling': {'graph': {'R1': 'foo:x?\nfoo:y?'}}, 'runtime': {'foo': {'outputs': { 'x': 'some message', 'y': 'another message'}}} }) schd = scheduler(wid, run_mode='live') async with start(schd): schd.broadcast_mgr.put_broadcast( ['1'], ['foo'], [{'skip': {'outputs': 'x'}}], ) foo, = schd.pool.get_tasks() schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'foo')}, ['skip'], [], []) foo_outputs = foo.state.outputs.get_completed_outputs() assert foo_outputs == { 'submitted': '(manually completed)', 'started': '(manually completed)', 'succeeded': '(manually completed)', 'x': '(manually completed)'} async def test_skip_mode_outputs( flow, scheduler, reftest, ): """Skip mode can be configured by the `[runtime][][skip]` section. Skip mode proposal point 2 https://github.com/cylc/cylc-admin/blob/master/docs/proposal-skip-mode.md """ graph = r""" # By default, all required outputs will be generated # plus succeeded if success is optional: foo? & foo:required_out => success_if_optional & required_outs # The outputs submitted and started are always produced # and do not need to be defined in [runtime][X][skip]outputs: foo:submitted => submitted_always foo:started => started_always # If outputs is specified and does not include either # succeeded or failed then succeeded will be produced. opt:optional_out? => optional_outs_produced should_fail:fail => did_fail """ wid = flow({ 'scheduling': {'graph': {'R1': graph}}, 'runtime': { 'root': { 'run mode': 'skip', 'outputs': { 'required_out': 'the plans have been on display...', 'optional_out': 'its only four light years away...' } }, 'opt': { 'skip': { 'outputs': 'optional_out' } }, 'should_fail': { 'skip': { 'outputs': 'failed' } } } }) schd = scheduler(wid, run_mode='live', paused_start=False) assert await reftest(schd) == { ('1/did_fail', ('1/should_fail',),), ('1/foo', None,), ('1/opt', None,), ('1/optional_outs_produced', ('1/opt',),), ('1/required_outs', ('1/foo', '1/foo',),), ('1/should_fail', None,), ('1/started_always', ('1/foo',),), ('1/submitted_always', ('1/foo',),), ('1/success_if_optional', ('1/foo', '1/foo',),), } async def test_doesnt_release_held_tasks( one_conf, flow, scheduler, start, log_filter, capture_live_submissions ): """Point 5 of the proposal https://github.com/cylc/cylc-admin/blob/master/docs/proposal-skip-mode.md | Tasks with run mode = skip will continue to abide by the is_held | flag as normal. """ one_conf['runtime'] = {'one': {'run mode': 'skip'}} schd = scheduler(flow(one_conf), run_mode='live', paused_start=False) async with start(schd): msg = 'held tasks shoudn\'t {}' # Set task to held and check submission in skip mode doesn't happen: schd.pool.hold_tasks({TaskTokens('1', 'one')}) schd.release_tasks_to_run() assert not log_filter(contains='=> running'), msg.format('run') assert not log_filter(contains='=> succeeded'), msg.format('succeed') # Release held task and assert that it now skips successfully: schd.pool.release_held_tasks({TaskTokens('1', 'one')}) schd.release_tasks_to_run() assert log_filter(contains='=> running'), msg.format('run') assert log_filter(contains='=> succeeded'), msg.format('succeed') async def test_prereqs_marked_satisfied_by_skip_mode( flow, scheduler, start, log_filter, complete ): """Point 8 from the skip mode proposal https://github.com/cylc/cylc-admin/blob/master/docs/proposal-skip-mode.md | When tasks are run in skip mode, the prerequisites which correspond | to the outputs they generate should be marked as "satisfied by skip mode" | rather than "satisfied naturally" for provenance reasons. """ schd = scheduler(flow({ 'scheduling': {'graph': {'R1': 'foo => bar'}}, 'runtime': {'foo': {'run mode': 'skip'}} }), run_mode='live') async with start(schd): foo = schd.pool.get_task(IntegerPoint(1), 'foo') schd.submit_task_jobs([foo]) bar = schd.pool.get_task(IntegerPoint(1), 'bar') satisfied_message, = bar.state.prerequisites[0]._satisfied.values() assert satisfied_message == 'satisfied by skip mode' async def test_outputs_can_be_changed( one_conf, flow, start, scheduler, validate ): schd = scheduler(flow(one_conf), run_mode='live') async with start(schd): # Broadcast the task into skip mode, output failed and submit it: schd.broadcast_mgr.put_broadcast( ["1"], ["one"], [ {"run mode": "skip"}, {"skip": {"outputs": "failed"}}, ], ) schd.submit_task_jobs(schd.pool.get_tasks()) # Broadcast the task into skip mode, output succeeded and submit it: schd.broadcast_mgr.put_broadcast( ['1'], ['one'], [{'skip': {'outputs': 'succeeded'}}] ) schd.submit_task_jobs(schd.pool.get_tasks()) async def test_rerun_after_skip_mode_broadcast( flow, one_conf, scheduler, start ): """Test re-running a task after it has been set to skip. See https://github.com/cylc/cylc-flow/pull/6940 """ id_ = flow({ **one_conf, "runtime": { "one": { "execution time limit": "PT1M", }, }, }) schd: Scheduler = scheduler(id_, run_mode='live') async with start(schd): itask = schd.pool.get_tasks()[0] schd.submit_task_jobs([itask]) schd.task_events_mgr.process_message( itask, 'CRITICAL', TASK_OUTPUT_FAILED ) assert itask.state(TASK_STATUS_FAILED) schd.broadcast_mgr.put_broadcast( ['1'], ['root'], [{'run mode': 'skip'}] ) schd.submit_task_jobs([itask]) assert itask.state(TASK_STATUS_SUCCEEDED) cylc-flow-8.7.0/tests/integration/test_template_variables_cli.py0000664000175000017500000000276615264420307025417 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.scripts.view import get_option_parser, _main as view from cylc.flow.option_parsers import Options async def test_list_tvars(tmp_path, capsys): """View shows that lists of comma separated args are converted into strings: """ (tmp_path / 'flow.cylc').write_text( '#!jinja2\n' '{% for i in FOO %}\n' '# {{i}} is string: {{i is string}}\n' '{% endfor %}\n' ) options = Options(get_option_parser())() options.jinja2 = True options.templatevars_lists = ['FOO="w,x",y,z'] await view(options, str(tmp_path)) result = capsys.readouterr().out.split('\n') for string in ['w,x', 'y', 'z']: assert f'# {string} is string: True' in result cylc-flow-8.7.0/tests/integration/test_taskdef.py0000664000175000017500000000527715264420307022346 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.config import WorkflowConfig from cylc.flow.pathutil import get_workflow_run_dir from cylc.flow.scheduler_cli import RunOptions from cylc.flow.task_outputs import TASK_OUTPUT_SUCCEEDED from cylc.flow.workflow_files import WorkflowFiles async def test_almost_self_suicide(flow, scheduler, start): """Suicide triggers should not count as upstream tasks when looking to spawn parentless tasks. https://github.com/cylc/cylc-flow/issues/6594 For the example under test, pre-requisites for ``!a`` should not be considered the same as pre-requisites for ``a``. If the are then then is parentless return false for all cases of ``a`` not in the initial cycle and subsequent cycles never run. """ wid = flow({ 'scheduler': {'cycle point format': '%Y'}, 'scheduling': { 'initial cycle point': 1990, 'final cycle point': 1992, 'graph': { 'R1': 'install_cold', 'P1Y': 'install_cold[^] => a? => b?\nb:fail? => !a?' } } }) schd = scheduler(wid) async with start(schd): tasks = [str(t) for t in schd.pool.get_tasks()] for task in ['1990/a:waiting', '1991/a:waiting', '1992/a:waiting']: assert task in tasks def test_graph_children(flow): """TaskDef.graph_children should not include duplicates. https://github.com/cylc/cylc-flow/issues/6619#issuecomment-2668932069 """ wid = flow({ 'scheduling': { 'graph': { 'R1': 'foo | bar => fin', }, }, 'task parameters': { 'n': '1..3', }, }) config = WorkflowConfig( wid, get_workflow_run_dir(wid, WorkflowFiles.FLOW_FILE), RunOptions() ) foo = config.taskdefs['foo'] graph_children = list(foo.graph_children.values())[0] assert [name for name, _ in graph_children[TASK_OUTPUT_SUCCEEDED]] == [ 'fin' ] cylc-flow-8.7.0/tests/integration/test_task_pool.py0000664000175000017500000022057715264420307022722 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from json import loads import logging import re from typing import ( TYPE_CHECKING, AsyncGenerator, Callable, Iterable, List, Tuple, Union, cast, ) import pytest from pytest import param from cylc.flow import ( CYLC_LOG, commands, ) from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.cycling.iso8601 import ISO8601Point from cylc.flow.data_messages_pb2 import PbPrerequisite from cylc.flow.data_store_mgr import TASK_PROXIES from cylc.flow.exceptions import WorkflowConfigError from cylc.flow.flow_mgr import FLOW_NONE from cylc.flow.id import ( TaskTokens, Tokens, ) from cylc.flow.run_modes import RunMode from cylc.flow.task_events_mgr import TaskEventsManager from cylc.flow.task_outputs import ( TASK_OUTPUT_FAILED, TASK_OUTPUT_SUCCEEDED, ) from cylc.flow.task_pool import TaskPool from cylc.flow.task_state import ( TASK_STATUS_EXPIRED, TASK_STATUS_FAILED, TASK_STATUS_PREPARING, TASK_STATUS_RUNNING, TASK_STATUS_SUBMIT_FAILED, TASK_STATUS_SUBMITTED, TASK_STATUS_SUCCEEDED, TASK_STATUS_WAITING, ) if TYPE_CHECKING: from cylc.flow.cycling import PointBase from cylc.flow.scheduler import Scheduler # NOTE: foo and bar have no parents so at start-up (even with the workflow # paused) they get spawned out to the runahead limit. 2/pub spawns # immediately too, because we spawn autospawn absolute-triggered tasks as # well as parentless tasks. 3/asd does not spawn at start, however. EXAMPLE_FLOW_CFG = { 'scheduling': { 'cycling mode': 'integer', 'initial cycle point': 1, 'final cycle point': 10, 'runahead limit': 'P3', 'graph': { 'P1': 'foo & bar', 'R1/2': 'foo[1] => pub', 'R1/3': 'foo[-P1] => asd' } }, 'runtime': { 'FAM': {}, 'bar': {'inherit': 'FAM'} } } EXAMPLE_FLOW_2_CFG = { 'scheduler': { 'UTC mode': True }, 'scheduling': { 'initial cycle point': '2001', 'runahead limit': 'P3Y', 'graph': { 'P1Y': 'foo', 'R/2025/P1Y': 'foo => bar', } }, } def get_task_ids( name_point_list: Iterable[Tuple[str, Union['PointBase', str, int]]] ) -> List[str]: """Helper function to return sorted task identities from a list of (name, point) tuples.""" return sorted(f'{point}/{name}' for name, point in name_point_list) def assert_expected_log( caplog_instance: pytest.LogCaptureFixture, expected_log_substrings: List[str] ) -> List[str]: """Helper function to check that expected (substrings of) log messages are actually in the log. Returns the list of actual logged messages. Args: caplog_instance: The instance of the caplog fixture for the particular test. expected_log_substrings: The expected, possibly partial, log messages. """ logged_messages = [i[2] for i in caplog_instance.record_tuples] assert len(logged_messages) == len(expected_log_substrings) for actual, expected in zip( sorted(logged_messages), sorted(expected_log_substrings)): assert expected in actual return logged_messages @pytest.fixture(scope='module') async def mod_example_flow( mod_flow: Callable, mod_scheduler: Callable, mod_run: Callable ) -> AsyncGenerator['Scheduler', None]: """Return a scheduler for interrogating its task pool. This is module-scoped so faster than example_flow, but should only be used where the test does not mutate the state of the scheduler or task pool. """ id_ = mod_flow(EXAMPLE_FLOW_CFG) schd: 'Scheduler' = mod_scheduler(id_, paused_start=True) async with mod_run(schd, level=logging.DEBUG): yield schd @pytest.fixture async def example_flow( flow: Callable, scheduler: Callable, start, caplog: pytest.LogCaptureFixture, ) -> AsyncGenerator['Scheduler', None]: """Return a scheduler for interrogating its task pool. This is function-scoped so slower than mod_example_flow; only use this when the test mutates the scheduler or task pool. """ # The run(schd) fixture doesn't work for modifying the DB, so have to # set up caplog and do schd.install()/.initialise()/.configure() instead caplog.set_level(logging.INFO, CYLC_LOG) id_ = flow(EXAMPLE_FLOW_CFG) schd: 'Scheduler' = scheduler(id_) async with start(schd, level=logging.DEBUG): yield schd @pytest.fixture(scope='module') async def mod_example_flow_2( mod_flow: Callable, mod_scheduler: Callable, mod_run: Callable ) -> AsyncGenerator['Scheduler', None]: """Return a scheduler for interrogating its task pool. This is module-scoped so faster than example_flow, but should only be used where the test does not mutate the state of the scheduler or task pool. """ id_ = mod_flow(EXAMPLE_FLOW_2_CFG) schd: 'Scheduler' = mod_scheduler(id_, paused_start=True) async with mod_run(schd): yield schd @pytest.mark.parametrize( 'ids, expected_tasks_to_hold_ids', [ param( ['1/foo', '3/asd'], ['1/foo', '3/asd'], id="Active & inactive tasks", ), param( ['1/FAM', '2/FAM', '6/FAM'], ['1/bar', '2/bar', '6/bar'], id="Family names hold active and future tasks", ), param( ['1/grogu', 'H/foo', '20/foo', '1/pub'], [], id="Non-existent task name or invalid cycle point", ), param( ['1/foo:waiting', '1/foo:failed', '6/bar:waiting'], ['1/foo'], id=( "Specifying task state works for active tasks," " not inactive tasks" ), ), ], ) async def test_hold_tasks( ids: List[str], expected_tasks_to_hold_ids: List[str], example_flow: 'Scheduler', caplog: pytest.LogCaptureFixture, db_select: Callable ) -> None: """Test TaskPool.hold_tasks(). Also tests TaskPool_explicit_match_tasks_to_hold() in the process; kills 2 birds with 1 stone. Params: items: Arg passed to hold_tasks(). expected_tasks_to_hold_ids: Expected IDs of the tasks that get put in the TaskPool.tasks_to_hold set, of the form "{point}/{name}"/ expected_warnings: Expected to be logged. """ expected_tasks_to_hold_ids = sorted(expected_tasks_to_hold_ids) caplog.set_level(logging.WARNING, CYLC_LOG) task_pool = example_flow.pool task_pool.hold_tasks( {cast('TaskTokens', Tokens(id_, relative=True)) for id_ in ids} ) for itask in task_pool.get_tasks(): hold_expected = itask.identity in expected_tasks_to_hold_ids assert itask.state.is_held is hold_expected assert get_task_ids(task_pool.tasks_to_hold) == expected_tasks_to_hold_ids db_held_tasks = db_select(example_flow, True, 'tasks_to_hold') assert get_task_ids(db_held_tasks) == expected_tasks_to_hold_ids async def test_release_held_tasks( example_flow: 'Scheduler', db_select: Callable ) -> None: """Test TaskPool.release_held_tasks(). For a workflow with held active tasks 1/foo & 1/bar, and held inactive task 3/asd. We skip testing the matching logic here because it would be slow using the function-scoped example_flow fixture, and it would repeat what is covered in test_hold_tasks(). """ # Setup task_pool = example_flow.pool expected_tasks_to_hold_ids = sorted(['1/foo', '1/bar', '3/asd']) task_pool.hold_tasks( { TaskTokens('1', 'foo'), TaskTokens('1', 'bar'), TaskTokens('3', 'asd'), } ) for itask in task_pool.get_tasks(): hold_expected = itask.identity in expected_tasks_to_hold_ids assert itask.state.is_held is hold_expected assert get_task_ids(task_pool.tasks_to_hold) == expected_tasks_to_hold_ids db_tasks_to_hold = db_select(example_flow, True, 'tasks_to_hold') assert get_task_ids(db_tasks_to_hold) == expected_tasks_to_hold_ids # Test task_pool.release_held_tasks( {TaskTokens('1', 'foo'), TaskTokens('3', 'asd')} ) for itask in task_pool.get_tasks(): assert itask.state.is_held is (itask.identity == '1/bar') expected_tasks_to_hold_ids = sorted(['1/bar']) assert get_task_ids(task_pool.tasks_to_hold) == expected_tasks_to_hold_ids db_tasks_to_hold = db_select(example_flow, True, 'tasks_to_hold') assert get_task_ids(db_tasks_to_hold) == expected_tasks_to_hold_ids @pytest.mark.parametrize( 'hold_after_point, expected_held_task_ids', [ ( '0', [ '1/foo', '1/bar', '2/foo', '2/bar', '2/pub', '3/foo', '3/bar', '4/foo', '4/bar', '5/foo', '5/bar', ], ), ( '1', [ '2/foo', '2/bar', '2/pub', '3/foo', '3/bar', '4/foo', '4/bar', '5/foo', '5/bar', ], ), ], ) async def test_hold_point( hold_after_point: str, expected_held_task_ids: List[str], example_flow: 'Scheduler', db_select: Callable ) -> None: """Test TaskPool.set_hold_point() and .release_hold_point()""" expected_held_task_ids = sorted(expected_held_task_ids) task_pool = example_flow.pool # Test hold task_pool.set_hold_point(IntegerPoint(hold_after_point)) assert ('holdcp', str(hold_after_point)) in db_select( example_flow, True, 'workflow_params') for itask in task_pool.get_tasks(): hold_expected = itask.identity in expected_held_task_ids assert itask.state.is_held is hold_expected assert get_task_ids(task_pool.tasks_to_hold) == expected_held_task_ids db_tasks_to_hold = db_select(example_flow, True, 'tasks_to_hold') assert get_task_ids(db_tasks_to_hold) == expected_held_task_ids # Test release task_pool.release_hold_point() assert db_select(example_flow, True, 'workflow_params', key='holdcp') == [ ('holdcp', None) ] for itask in task_pool.get_tasks(): assert itask.state.is_held is False assert task_pool.tasks_to_hold == set() assert db_select(example_flow, True, 'tasks_to_hold') == [] @pytest.mark.parametrize( 'status,should_trigger', [ (TASK_STATUS_WAITING, True), (TASK_STATUS_PREPARING, False), (TASK_STATUS_SUBMITTED, False), (TASK_STATUS_RUNNING, False), (TASK_STATUS_SUCCEEDED, True), ] ) async def test_trigger_states( status: str, should_trigger: bool, one: 'Scheduler', start: Callable ): """It should only trigger tasks in compatible states.""" async with start(one): itask = one.pool.get_task(IntegerPoint('1'), 'one') # reset task a to the provided state itask.state.reset(status) # try triggering the task await commands.run_cmd( commands.force_trigger_tasks(one, ['1/one'], [])) # retrieve the task again - the original may have been removed itask = one.pool.get_task(IntegerPoint('1'), 'one') # check whether the task triggered assert itask.is_manual_submit == should_trigger async def test_preparing_tasks_on_restart(one_conf, flow, scheduler, start): """Preparing tasks should be reset to waiting on restart. This forces preparation to be re-done on restart so that it uses the new configuration. See discussion on: https://github.com/cylc/cylc-flow/pull/4668 """ id_ = flow(one_conf) # start the workflow, reset a task to preparing one = scheduler(id_) async with start(one): itask = one.pool.get_tasks()[0] itask.state.reset(TASK_STATUS_PREPARING) # when we restart the task should have been reset to waiting one = scheduler(id_) async with start(one): itask = one.pool.get_tasks()[0] assert itask.state(TASK_STATUS_WAITING) itask.state.reset(TASK_STATUS_SUCCEEDED) # whereas if we reset the task to succeeded the state is not reset one = scheduler(id_) async with start(one): itask = one.pool.get_tasks()[0] assert itask.state(TASK_STATUS_SUCCEEDED) async def test_reload_stopcp( flow: Callable, scheduler: Callable, start: Callable ): """Test that the task pool stopping point does not revert to the final cycle point on reload.""" cfg = { 'scheduler': { 'allow implicit tasks': True, 'cycle point format': 'CCYY', }, 'scheduling': { 'initial cycle point': 2010, 'stop after cycle point': 2020, 'final cycle point': 2030, 'graph': { 'P1Y': 'anakin' } } } schd: 'Scheduler' = scheduler(flow(cfg)) async with start(schd): assert str(schd.pool.stop_point) == '2020' await commands.run_cmd(commands.reload_workflow(schd)) assert str(schd.pool.stop_point) == '2020' async def test_runahead_after_remove( example_flow: 'Scheduler' ) -> None: """The runahead limit should be recomputed after tasks are removed. """ task_pool = example_flow.pool assert int(task_pool.runahead_limit_point) == 4 # No change after removing an intermediate cycle. await commands.run_cmd(commands.remove_tasks(example_flow, ['3/*'], ["1"])) assert int(task_pool.runahead_limit_point) == 4 # Should update after removing the first point. await commands.run_cmd(commands.remove_tasks(example_flow, ['1/*'], ["1"])) assert int(task_pool.runahead_limit_point) == 5 def list_tasks(schd): """Return a sorted list of task pool tasks. Returns a list in the format: [ (cycle, task, state) ] """ return sorted( (itask.tokens['cycle'], itask.tokens['task'], itask.state.status) for itask in schd.pool.get_tasks() ) async def _test_restart_prereqs_sat(): schd: Scheduler # YIELD: the workflow has now started... schd = yield await schd.update_data_structure() # Release tasks 1/a and 1/b schd.pool.release_runahead_tasks() schd.release_tasks_to_run() assert list_tasks(schd) == [ ('1', 'a', 'running'), ('1', 'b', 'running') ] # Mark both as succeeded and spawn 1/c for itask in schd.pool.get_tasks(): schd.pool.task_events_mgr.process_message(itask, 1, 'succeeded') schd.workflow_db_mgr.put_update_task_outputs(itask) schd.pool.remove_if_complete(itask) schd.workflow_db_mgr.process_queued_ops() assert list_tasks(schd) == [ ('1', 'c', 'waiting') ] # YIELD: the workflow has now restarted or reloaded with the new config... schd = yield await schd.update_data_structure() assert list_tasks(schd) == [ ('1', 'c', 'waiting') ] # Check resulting dependencies of task z task_c = schd.pool.get_tasks()[0] assert sorted( (*key, satisfied) for prereq in task_c.state.prerequisites for key, satisfied in prereq.items() ) == [ ('1', 'a', 'succeeded', 'satisfied naturally'), ('1', 'b', 'succeeded', 'satisfied from database') ] # The prereqs in the data store should have been updated too # await schd.update_data_structure() tasks = ( schd.data_store_mgr.data[schd.data_store_mgr.workflow_id][TASK_PROXIES] ) task_c_prereqs: List[PbPrerequisite] = tasks[ schd.data_store_mgr.id_.duplicate(cycle='1', task='c').id ].prerequisites assert sorted( (condition.task_proxy, condition.satisfied, condition.message) for prereq in task_c_prereqs for condition in prereq.conditions ) == [ ('1/a', True, 'satisfied naturally'), ('1/b', True, 'satisfied from database'), ] # and we're done, yield back control and return yield @pytest.mark.parametrize('do_restart', [True, False]) async def test_graph_change_prereq_satisfaction( flow, scheduler, start, do_restart ): """It should handle graph prerequisites change on reload/restart. If the graph is changed to add a dependency which has been previously satisfied, then Cylc should perform a DB check and mark the prerequisite as satisfied accordingly. See https://github.com/cylc/cylc-flow/pull/5334 """ conf = { 'scheduler': {'allow implicit tasks': 'True'}, 'scheduling': { 'graph': { 'R1': ''' a => c b ''' } } } id_ = flow(conf) schd = scheduler(id_, run_mode='simulation', paused_start=False) test = _test_restart_prereqs_sat() await test.asend(None) if do_restart: async with start(schd): # start the workflow and run part 1 of the tests await test.asend(schd) # shutdown and change the workflow definition conf['scheduling']['graph']['R1'] += '\nb => c' flow(conf, workflow_id=id_) schd = scheduler(id_, run_mode='simulation', paused_start=False) async with start(schd): # restart the workflow and run part 2 of the tests await test.asend(schd) else: async with start(schd): await test.asend(schd) # Modify flow.cylc to add a new dependency on "b" conf['scheduling']['graph']['R1'] += '\nb => c' flow(conf, workflow_id=id_) # Reload the workflow config await commands.run_cmd(commands.reload_workflow(schd)) await test.asend(schd) async def test_runahead_limit_for_sequence_before_start_cycle( flow, scheduler, start, ): """It should obey the runahead limit. Ensure the runahead limit is computed correctly for sequences before the start cycle See https://github.com/cylc/cylc-flow/issues/5603 """ id_ = flow({ 'scheduler': {'allow implicit tasks': 'True'}, 'scheduling': { 'initial cycle point': '2000', 'runahead limit': 'P2Y', 'graph': { 'R1/2000': 'a', 'P1Y': 'b[-P1Y] => b', }, } }) schd = scheduler(id_, startcp='2005') async with start(schd): assert str(schd.pool.runahead_limit_point) == '20070101T0000Z' def list_pool_from_db(schd): """Returns the task pool table as a sorted list.""" db_task_pool = [] schd.workflow_db_mgr.pri_dao.select_task_pool( lambda _, row: db_task_pool.append(row) ) return sorted(db_task_pool) async def test_db_update_on_removal( flow, scheduler, start, ): """It should updated the task_pool table when tasks complete. There was a bug where the task_pool table was only being updated when tasks in the pool were updated. This meant that if a task was removed the DB would not reflect this change and would hold a record of the task in the wrong state. This test ensures that the DB is updated when a task is removed from the pool. See: https://github.com/cylc/cylc-flow/issues/5598 """ id_ = flow({ 'scheduling': { 'graph': { 'R1': 'a', }, }, 'runtime': { 'root': { 'simulation': { # Avoid instant task success for this test 'default run length': 'PT2S', }, }, }, }) schd: Scheduler = scheduler(id_) async with start(schd): task_a = schd.pool.get_tasks()[0] # set the task to running schd.pool.task_events_mgr.process_message(task_a, 1, 'started') await schd._main_loop() # the task should appear in the DB assert list_pool_from_db(schd) == [ ['1', 'a', 'running', 0], ] # mark the task as succeeded and allow it to be removed from the pool schd.pool.task_events_mgr.process_message(task_a, 1, 'succeeded') schd.pool.remove_if_complete(task_a) await schd._main_loop() # the task should be gone from the DB assert list_pool_from_db(schd) == [] async def test_no_flow_tasks_dont_spawn( flow, scheduler, start, ): """Ensure no-flow tasks don't spawn downstreams. No-flow tasks (i.e `--flow=none`) are not attached to any "flow". See https://github.com/cylc/cylc-flow/issues/5613 """ id_ = flow({ 'scheduling': { 'graph': { 'R1': 'a => b => c' } }, }) schd: Scheduler = scheduler(id_) async with start(schd): task_a = schd.pool.get_tasks()[0] # set as no-flow: task_a.flow_nums = set() # Set as completed: should not spawn children. schd.pool.set_prereqs_and_outputs( {task_a.tokens}, [], [], [FLOW_NONE] ) assert not schd.pool.get_tasks() for flow_nums, expected_pool in ( # outputs yielded from a no-flow task should not spawn downstreams (set(), []), # outputs yielded from a task with flow numbers should spawn # downstreams in the same flow ({1}, [('1/b', {1})]), ): # set the flow-nums on 1/a task_a.flow_nums = flow_nums # spawn on the succeeded output schd.pool.spawn_on_output(task_a, TASK_OUTPUT_SUCCEEDED) schd.pool.spawn_on_all_outputs(task_a) # ensure the pool is as expected assert [ (itask.identity, itask.flow_nums) for itask in schd.pool.get_tasks() ] == expected_pool async def test_task_proxy_remove_from_queues( flow, one_conf, scheduler, start, ): """TaskPool.remove should delete task proxies from queues. See https://github.com/cylc/cylc-flow/pull/5573 """ # Set up a scheduler with a non-default queue: one_conf['scheduling'] = { 'queues': {'queue_two': {'members': 'one, control'}}, 'graph': {'R1': 'two & one & control'}, } schd = scheduler(flow(one_conf)) async with start(schd): # Get a list of itasks: itasks = schd.pool.get_tasks() for itask in itasks: id_ = itask.identity # The meat of the test - remove itask from pool if it # doesn't have "control" in the name: if 'control' not in id_: schd.pool.remove(itask) # Look at the queues afterwards: queues_after = { name: [itask.identity for itask in queue.deque] for name, queue in schd.pool.task_queue_mgr.queues.items()} assert queues_after['queue_two'] == ['1/control'] async def test_runahead_offset_start( mod_example_flow_2: 'Scheduler' ) -> None: """Late-start recurrences should not break the runahead limit at start-up. See GitHub #5708 """ task_pool = mod_example_flow_2.pool assert task_pool.runahead_limit_point == ISO8601Point('2004') async def test_detect_incomplete_tasks( flow, scheduler, start, log_filter, ): """Finished but incomplete tasks should be retained as incomplete.""" incomplete_final_task_states = { # final task states that would leave a task with # completion=succeeded incomplete TASK_STATUS_FAILED: TaskEventsManager.EVENT_FAILED, TASK_STATUS_EXPIRED: TaskEventsManager.EVENT_EXPIRED, TASK_STATUS_SUBMIT_FAILED: TaskEventsManager.EVENT_SUBMIT_FAILED } id_ = flow({ 'scheduling': { 'graph': { # a workflow with one task for each of the final task states 'R1': '\n'.join(incomplete_final_task_states.keys()) } } }) schd = scheduler(id_) async with start(schd, level=logging.DEBUG): itasks = schd.pool.get_tasks() for itask in itasks: itask.state_reset(is_queued=False) # spawn the output corresponding to the task schd.pool.task_events_mgr.process_message( itask, 1, incomplete_final_task_states[itask.tdef.name] ) # ensure that it is correctly identified as incomplete assert not itask.state.outputs.is_complete() assert log_filter( contains=( f"[{itask}] did not complete the required outputs:" ), ) # the task should not have been removed assert itask in schd.pool.get_tasks() async def test_trigger_icp_fcp_syntax( flow, scheduler, start, log_filter, ): """It should support the ^/$ syntax for referencing the initial/final cp. See https://github.com/cylc/cylc-flow/issues/6537 """ cfg = { 'scheduling': { 'cycling mode': 'integer', 'initial cycle point': 1, 'final cycle point': 2, 'graph': { 'R1/^': 'start', 'R1/$': 'end', }, }, } # trigger tasks at both the initial and final cycle points id_ = flow(cfg) schd = scheduler(id_) async with start(schd) as log: await commands.run_cmd( commands.force_trigger_tasks(schd, ['^/start', '$/end'], ['1']) ) assert log_filter(contains='[1/start:waiting(queued)] => waiting') assert log_filter(contains='[2/end:waiting(queued)] => waiting') log.clear() # clear the final cycle point del cfg['scheduling']['final cycle point'] del cfg['scheduling']['graph']['R1/$'] # try triggering a task at the (non existent) final cycle point id_ = flow(cfg) schd = scheduler(id_) async with start(schd): await commands.run_cmd( commands.force_trigger_tasks(schd, ['^/start', '$/end'], ['1']) ) assert log_filter(contains='[1/start:waiting(queued)] => waiting') assert not log_filter(contains='[2/end:waiting(queued)] => waiting') assert log_filter( contains='ID references final cycle point, but none is set: $/end' ) async def test_future_trigger_final_point( flow, scheduler, start, log_filter, ): """Check spawning of future-triggered tasks: foo[+P1] => bar. Don't spawn if a prerequisite reaches beyond the final cycle point. """ id_ = flow( { 'scheduling': { 'cycling mode': 'integer', 'initial cycle point': 1, 'final cycle point': 1, 'graph': { 'P1': "foo\n foo[+P1] & bar => baz" } } } ) schd = scheduler(id_) async with start(schd): for itask in schd.pool.get_tasks(): schd.pool.spawn_on_output(itask, "succeeded") await schd._main_loop() assert log_filter( regex=( ".*1/baz.*not spawned: a prerequisite is beyond" r" the workflow stop point \(1\)" ) ) async def test_set_failed_complete( flow, scheduler, start, one_conf, log_filter, db_select: Callable ): """Test manual completion of an incomplete failed task.""" id_ = flow(one_conf) schd: Scheduler = scheduler(id_) async with start(schd, level=logging.DEBUG): one = schd.pool.get_tasks()[0] one.state_reset(is_queued=False) schd.pool.task_events_mgr.process_message(one, 1, "failed") assert log_filter( regex="1/one.* setting implied output: submitted") assert log_filter( regex="1/one.* setting implied output: started") assert log_filter( regex="failed.* did not complete the required outputs") # Set failed task complete via default "set" args. schd.pool.set_prereqs_and_outputs({one.tokens}, [], [], []) assert log_filter( contains=f'[{one}] removed from the n=0 window: completed') db_outputs = db_select( schd, True, 'task_outputs', 'outputs', **{'name': 'one'} ) assert ( sorted(loads((db_outputs[0])[0])) == [ "failed", "started", "submitted", "succeeded" ] ) async def test_set_prereqs( flow, scheduler, start, log_filter, ): """Check manual setting of prerequisites (task and xtrigger). """ id_ = flow( { 'scheduling': { 'initial cycle point': '2040', 'xtriggers': { 'x': 'xrandom(0)' }, 'graph': { 'R1': """ foo & bar & baz => qux", @x => bar """ } }, 'runtime': { 'foo': { 'outputs': { 'a': 'drugs and money', } } } } ) schd: Scheduler = scheduler(id_) async with start(schd): # it should start up with foo, bar, baz assert schd.pool.get_task_ids() == { "20400101T0000Z/bar", "20400101T0000Z/baz", "20400101T0000Z/foo", } # try to set an invalid prereq of qux schd.pool.set_prereqs_and_outputs( {TaskTokens('20400101T0000Z', 'qux')}, [], ["20400101T0000Z/foo:a", "xtrigger/x"], [], ) assert log_filter( contains=( '20400101T0000Z/qux does not depend on "20400101T0000Z/foo:a"' ) ) assert log_filter( contains=( '20400101T0000Z/qux does not depend on xtrigger "x"' ) ) # it should not add 20400101T0000Z/qux to the pool assert schd.pool.get_task_ids() == { "20400101T0000Z/bar", "20400101T0000Z/baz", "20400101T0000Z/foo", } # set an xtrigger (see also test_xtrigger_mgr, and test_data_store_mgr) bar = schd.pool._get_task_by_id('20400101T0000Z/bar') assert bar.state.prerequisites_all_satisfied() assert not bar.state.xtriggers_all_satisfied() schd.pool.set_prereqs_and_outputs( {TaskTokens('20400101T0000Z', 'bar')}, [], ["xtrigger/x:succeeded"], [], ) assert bar.state.xtriggers_all_satisfied() assert log_filter( contains=('prerequisite force-satisfied: x = xrandom(0)')) # set xtrigger in the wrong task schd.pool.set_prereqs_and_outputs( {TaskTokens('20400101T0000Z', 'baz')}, [], ["xtrigger/x:succeeded"], [], ) assert log_filter( contains='20400101T0000Z/baz does not depend on xtrigger "x"') # set one prereq of inactive task 20400101T0000Z/qux schd.pool.set_prereqs_and_outputs( {TaskTokens('20400101T0000Z', 'qux')}, [], ["20400101T0000Z/foo:succeeded"], []) # it should add 20400101T0000Z/qux to the pool assert schd.pool.get_task_ids() == { "20400101T0000Z/bar", "20400101T0000Z/baz", "20400101T0000Z/foo", "20400101T0000Z/qux", } # get the 20400101T0000Z/qux task proxy qux = schd.pool.get_task(ISO8601Point("20400101T0000Z"), "qux") assert not qux.state.prerequisites_all_satisfied() # set its other prereqs (test implicit "succeeded" and "succeed") # and truncated cycle point schd.pool.set_prereqs_and_outputs( {TaskTokens('20400101T0000Z', 'qux')}, [], ["2040/bar", "2040/baz:succeed"], [], ) assert log_filter( contains=('prerequisite force-satisfied: 20400101T0000Z/bar')) assert log_filter( contains=('prerequisite force-satisfied: 20400101T0000Z/baz')) # it should now be fully satisfied assert qux.state.prerequisites_all_satisfied() # set one again schd.pool.set_prereqs_and_outputs( {TaskTokens('20400101T0000Z', 'qux')}, [], ["2040/bar"], []) assert log_filter( contains=('prerequisite already satisfied: 20400101T0000Z/bar')) async def test_set_bad_prereqs( flow, scheduler, start, log_filter, ): """Check manual setting of prerequisites. """ id_ = flow({ 'scheduler': { 'cycle point format': '%Y'}, 'scheduling': { 'initial cycle point': '2040', 'graph': {'R1': "foo => bar"}}, }) schd: Scheduler = scheduler(id_) def set_prereqs(prereqs): """Shorthand so only variable under test given as arg""" schd.pool.set_prereqs_and_outputs( {TaskTokens('2040', 'bar')}, [], prereqs, []) async with start(schd): # Invalid: task name wildcard: set_prereqs(["2040/*"]) assert log_filter(contains='Invalid prerequisite task name') # Invalid: cycle point wildcard. set_prereqs(["*/foo"]) assert log_filter(contains='Invalid prerequisite cycle point') async def test_set_outputs_live( flow, scheduler, start, log_filter, ): """Check manual set outputs in an active (spawned) task. """ id_ = flow( { 'scheduling': { 'graph': { 'R1': """ foo:x => bar foo => baz foo:y """ } }, 'runtime': { 'foo': { 'outputs': { 'x': 'xylophone', 'y': 'yacht' } } } } ) schd: Scheduler = scheduler(id_) async with start(schd): # it should start up with just 1/foo assert schd.pool.get_task_ids() == {"1/foo"} # fake failed foo = schd.pool.get_task(IntegerPoint("1"), "foo") foo.state_reset(is_queued=False) schd.pool.task_events_mgr.process_message(foo, 1, 'failed') # set foo:x: it should spawn bar but not baz schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'foo')}, ["x"], [], [] ) assert schd.pool.get_task_ids() == {"1/bar", "1/foo"} # Foo should have been removed from the queue: assert '1/foo' not in [ i.identity for i in schd.pool.task_queue_mgr.queues['default'].deque ] # set foo:succeed: it should spawn baz but foo remains incomplete. schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'foo')}, ["succeeded"], [], [] ) assert schd.pool.get_task_ids() == {"1/bar", "1/baz", "1/foo"} # it should complete implied outputs (submitted, started) too assert log_filter(contains="setting implied output: submitted") assert log_filter(contains="setting implied output: started") # set foo (default: all required outputs) to complete y. schd.pool.set_prereqs_and_outputs({TaskTokens('1', 'foo')}, [], [], []) assert log_filter(contains="output 1/foo:succeeded completed") assert schd.pool.get_task_ids() == {"1/bar", "1/baz"} async def test_set_outputs_live2( flow, scheduler, start, log_filter, ): """Assert that optional outputs are satisfied before completion outputs to prevent incomplete task warnings. """ id_ = flow( { 'scheduling': {'graph': { 'R1': """ foo:a => apple foo:b => boat """}}, 'runtime': {'foo': {'outputs': { 'a': 'xylophone', 'b': 'yacht'}}} } ) schd: Scheduler = scheduler(id_) async with start(schd): schd.pool.set_prereqs_and_outputs({TaskTokens('1', 'foo')}, [], [], []) assert not log_filter( contains="did not complete required outputs: ['a', 'b']" ) async def test_set_outputs_future( flow, scheduler, start, log_filter, ): """Check manual setting of inactive task outputs. """ id_ = flow( { 'scheduling': { 'cycling mode': 'integer', 'initial cycle point': '1', 'runahead limit': 'P0', 'graph': { 'P1': """ a:x & a:y => b => c a:y => f """ } }, 'runtime': { 'a': { 'outputs': { 'x': 'xylophone', 'y': 'yacht' } } } } ) schd: Scheduler = scheduler(id_) async with start(schd): # It should start up with just 1/a and 2/a. assert schd.pool.get_task_ids() == {"1/a", "2/a"} # Setting inactive future task b succeeded should # spawn c but not b. schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'b')}, ["succeeded"], [], []) assert schd.pool.get_task_ids() == {"1/a", "2/a", "1/c"} # Setting inactive future task f failed should # add it to n=0 as final incomplete. # See https://github.com/cylc/cylc-flow/pull/7248 schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'f')}, ["failed"], [], []) assert schd.pool.get_task_ids() == {"1/a", "2/a", "1/c", "1/f"} # Setting inactive future parentless task a failed should # add it to n=0 as final, and spawn its next instance. # (A failed task has passed runahead release, which is when next # instances are spawned). # See https://github.com/cylc/cylc-flow/pull/7248 schd.pool.set_prereqs_and_outputs( {TaskTokens('5', 'a')}, ["failed"], [], []) assert schd.pool.get_task_ids() == { "1/a", "2/a", "1/c", "1/f", "5/a", "6/a"} schd.pool.set_prereqs_and_outputs( items={TaskTokens('1', 'a')}, outputs=["x", "y", "cheese"], prereqs=[], flow=[] ) assert log_filter(contains="Output 1/a:cheese not found") assert log_filter(contains="completed output x") assert log_filter(contains="completed output y") async def test_set_outputs_from_skip_settings( flow, scheduler, start, log_filter, validate ): """Check working of ``cylc set --out=skip``: 1. --out=skip can be used to set all required outputs. 2. --out=skip,other_output can be used to set other outputs. """ id_ = flow( { 'scheduling': { 'cycling mode': 'integer', 'initial cycle point': 1, 'final cycle point': 2, 'graph': { 'P1': """ a => after_asucceeded a:x => after_ax a:y? => after_ay """ } }, 'runtime': { 'a': { 'outputs': { 'x': 'xebec', 'y': 'yacht' }, 'skip': {'outputs': 'x'} } } } ) validate(id_) schd: Scheduler = scheduler(id_) async with start(schd): # it should start up with just tasks a: assert schd.pool.get_task_ids() == {'1/a', '2/a'} # setting 1/a output to skip should set output x, but not # y (because y is optional). schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'a')}, ['skip'], [], []) assert schd.pool.get_task_ids() == { '1/after_asucceeded', '1/after_ax', '2/a', } # Check that the presence of "skip" in outputs doesn't # trigger a warning: assert not log_filter(level=30) # You should be able to set skip as part of a list of outputs: schd.pool.set_prereqs_and_outputs( {TaskTokens('2', 'a')}, ['skip', 'y'], [], []) assert schd.pool.get_task_ids() == { '1/after_asucceeded', '1/after_ax', '2/after_asucceeded', '2/after_ax', '2/after_ay', } async def test_prereq_satisfaction( flow, scheduler, start, log_filter, ): """Check manual setting of task prerequisites. """ id_ = flow( { 'scheduling': { 'graph': { 'R1': "a:x & a:y => b" } }, 'runtime': { 'a': { 'outputs': { 'x': 'xylophone', 'y': 'yacht' } } } } ) schd: Scheduler = scheduler(id_) async with start(schd): # it should start up with just 1/a assert schd.pool.get_task_ids() == {"1/a"} # spawn b schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'a')}, ["x"], [], [] ) assert schd.pool.get_task_ids() == {"1/a", "1/b"} b = schd.pool.get_task(IntegerPoint("1"), "b") assert not b.prereqs_are_satisfied() # set valid and invalid prerequisites, by label and message. schd.pool.set_prereqs_and_outputs( prereqs=["1/a:xylophone", "1/a:y", "1/a:w", "1/a:z"], items={TaskTokens('1', 'b')}, outputs=[], flow=[], ) assert log_filter(contains="1/a:z not found") assert log_filter(contains="1/a:w not found") # FIXME: testing that something is *not* logged is extremely fragile: assert not log_filter(regex='.*does not depend on.*') assert b.prereqs_are_satisfied() @pytest.mark.parametrize('cycling_mode', ['integer', 'datetime']) @pytest.mark.parametrize('runahead_format', ['P3Y', 'P3']) async def test_compute_runahead( cycling_mode, runahead_format, flow, scheduler, start, ): """Test the calculation of the runahead limit. This test ensures that: * Runahead tasks are excluded from computations see https://github.com/cylc/cylc-flow/issues/5825 * Tasks are initiated with the correct is_runahead status on startup. * Behaviour in compat/regular modes is same unless failed tasks are present * Behaviour is the same for integer/datetime cycling modes. """ if cycling_mode == 'integer': config = { 'scheduler': { 'allow implicit tasks': 'True', }, 'scheduling': { 'initial cycle point': '1', 'cycling mode': 'integer', 'runahead limit': 'P3', 'graph': { 'P1': 'a' }, } } point = lambda point: IntegerPoint(str(int(point))) else: config = { 'scheduler': { 'allow implicit tasks': 'True', 'cycle point format': 'CCYY', }, 'scheduling': { 'initial cycle point': '0001', 'runahead limit': runahead_format, 'graph': { 'P1Y': 'a' }, } } point = ISO8601Point id_ = flow(config) schd = scheduler(id_) async with start(schd): schd.pool.compute_runahead(force=True) assert int(str(schd.pool.runahead_limit_point)) == 4 # ensure task states are initiated with is_runahead status assert schd.pool.get_task(point('0001'), 'a').state(is_runahead=False) assert schd.pool.get_task(point('0005'), 'a').state(is_runahead=True) # mark the first three cycles as running for cycle in range(1, 4): schd.pool.get_task(point(f'{cycle:04}'), 'a').state.reset( TASK_STATUS_RUNNING ) schd.pool.compute_runahead(force=True) assert int(str(schd.pool.runahead_limit_point)) == 4 # no change # any incomplete tasks hold back runahead. schd.pool.get_task(point('0001'), 'a').state.reset( TASK_STATUS_SUBMIT_FAILED ) schd.pool.compute_runahead(force=True) assert int(str(schd.pool.runahead_limit_point)) == 4 # no change schd.pool.get_task(point('0001'), 'a').state.reset( TASK_STATUS_FAILED ) schd.pool.compute_runahead(force=True) assert int(str(schd.pool.runahead_limit_point)) == 4 # no change # mark cycle 1 as complete # (via task message so the task gets removed before runahead compute) schd.task_events_mgr.process_message( schd.pool.get_task(point('0001'), 'a'), logging.INFO, TASK_OUTPUT_SUCCEEDED ) schd.pool.compute_runahead(force=True) assert int(str(schd.pool.runahead_limit_point)) == 5 # +1 async def test_compute_runahead_with_no_tasks(flow, scheduler, run): """It should handle the case of an empty workflow. See https://github.com/cylc/cylc-flow/issues/6225 """ id_ = flow( { 'scheduling': { 'initial cycle point': '2000', 'graph': {'R1': 'foo'}, }, } ) schd = scheduler(id_, startcp='2002', paused_start=False) async with run(schd): assert schd.pool.compute_runahead() is False assert schd.pool.runahead_limit_point is None assert schd.pool.get_tasks() == [] async def test_compute_runahead_with_no_sequences( flow, scheduler, start, run, complete ): """It should handle no sequences within the start-stop cycle range. See https://github.com/cylc/cylc-flow/issues/6154 """ cfg = { 'scheduling': { 'cycling mode': 'integer', 'initial cycle point': '1', 'graph': { 'P1': 'foo[-P1] => foo', }, }, } id_ = flow(cfg) schd = scheduler(id_, paused_start=False) async with run(schd): await complete(schd, '2/foo') cfg['scheduling']['graph']['R1'] = cfg['scheduling']['graph']['P1'] cfg['scheduling']['graph'].pop('P1') flow(cfg, workflow_id=id_) schd = scheduler(id_, paused_start=False) async with start(schd): schd.pool.compute_runahead() assert schd.pool.runahead_limit_point == IntegerPoint('3') @pytest.mark.parametrize('rhlimit', ['P2D', 'P2']) async def test_runahead_future_trigger( flow, scheduler, start, monkeypatch, rhlimit, ): """Equivalent time interval and cycle count runahead limits should yield the same limit point, even if there is a future trigger. See https://github.com/cylc/cylc-flow/pull/5893 """ id_ = flow({ 'scheduler': { 'allow implicit tasks': 'True', 'cycle point format': 'CCYYMMDD', }, 'scheduling': { 'initial cycle point': '2001', 'runahead limit': rhlimit, 'graph': { 'P1D': ''' a a[+P1D] => b ''', }, } }) schd = scheduler(id_,) async with start(schd, level=logging.DEBUG): assert str(schd.pool.runahead_limit_point) == '20010103' schd.pool.release_runahead_tasks() for itask in schd.pool.get_tasks(): schd.pool.spawn_on_output(itask, 'succeeded') # future trigger raises the limit by one cycle point assert str(schd.pool.runahead_limit_point) == '20010104' @pytest.fixture(scope='module') async def mod_blah( mod_flow: Callable, mod_scheduler: Callable, mod_run: Callable ) -> 'Scheduler': """Return a scheduler for interrogating its task pool. This is module-scoped so faster than example_flow, but should only be used where the test does not mutate the state of the scheduler or task pool. """ config = { 'scheduler': { 'allow implicit tasks': 'True', 'cycle point format': '%Y', }, 'scheduling': { 'initial cycle point': '0001', 'runahead limit': 'P1Y', 'graph': { 'P1Y': 'a' }, } } id_ = mod_flow(config) schd: 'Scheduler' = mod_scheduler(id_, paused_start=True) async with mod_run(schd): yield schd async def test_fast_respawn( example_flow: 'Scheduler', caplog: pytest.LogCaptureFixture, ) -> None: """Immediate re-spawn of removed tasks is not allowed. An immediate DB update is required to stop the respawn. https://github.com/cylc/cylc-flow/pull/6067 """ task_pool = example_flow.pool # find task 1/foo in the pool foo = task_pool.get_task(IntegerPoint("1"), "foo") # remove it from the pool task_pool.remove(foo) assert foo not in task_pool.get_tasks() # attempt to spawn it again itask = task_pool.spawn_task("foo", IntegerPoint("1"), {1}) assert itask is None assert "Not respawning 1/foo - task was removed" in caplog.text async def test_remove_active_task( example_flow: 'Scheduler', log_filter: Callable, ) -> None: """Test warning on removing an active task.""" task_pool = example_flow.pool # find task 1/foo in the pool foo = task_pool.get_task(IntegerPoint("1"), "foo") foo.state_reset(TASK_STATUS_RUNNING) task_pool.remove(foo, "request") assert foo not in task_pool.get_tasks() assert log_filter( regex=( "1/foo.*removed from the n=0 window:" " request - active job orphaned" ), level=logging.WARNING ) async def test_remove_by_expire_trigger( flow, validate, scheduler, start, log_filter ): """Test task removal by suicide trigger. * Suicide triggers should remove tasks from the pool. * It should be possible to bring them back by manually triggering them. * Removing a task manually (cylc remove) should work the same. """ def _get_id(b_completion: str = "succeeded"): return flow({ 'scheduler': { 'experimental': { 'expire triggers': 'True', } }, 'scheduling': { 'graph': { 'R1': ''' a? => b a:failed? => !b ''' }, }, 'runtime': { 'b': { 'completion': b_completion } } }) with pytest.raises( WorkflowConfigError, match=re.escape( "This may be due to use of an expire (formerly suicide) trigger" ) ): validate(_get_id()) id_ = _get_id("succeeded or expired") validate(id_) schd: 'Scheduler' = scheduler(id_, paused_start=False) async with start(schd, level=logging.DEBUG) as log: # it should start up with 1/a assert schd.pool.get_task_ids() == {"1/a"} a = schd.pool.get_task(IntegerPoint("1"), "a") # mark 1/a as failed and check that 1/b expires schd.pool.spawn_on_output(a, TASK_OUTPUT_FAILED) assert log_filter(regex="1/b.*=> expired") assert schd.pool.get_task_ids() == {"1/a"} # 1/b should not be resurrected if it becomes ready schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'b')}, [], ["1/a"], ["1"], ) assert log_filter(regex="1/b:expired.* already finished and completed") # but we can still resurrect 1/b by triggering it log.clear() await commands.run_cmd( commands.force_trigger_tasks(schd, ['1/b'], ['1'])) assert log_filter(regex='1/b.*added to the n=0 window') # remove 1/b with "cylc remove"" await commands.run_cmd( commands.remove_tasks(schd, ['1/b'], []) ) assert log_filter( regex='1/b.*removed from the n=0 window: request', ) # and bring 1/b back again by triggering it again log.clear() await commands.run_cmd( commands.force_trigger_tasks(schd, ['1/b'], ['1'])) assert log_filter(regex='1/b.*added to the n=0 window',) async def test_remove_by_suicide( flow, scheduler, start, log_filter ): """Test task removal by suicide trigger. * Suicide triggers should remove tasks from the pool. * It should be possible to bring them back by manually triggering them. * Removing a task manually (cylc remove) should work the same. """ id_ = flow({ 'scheduling': { 'graph': { 'R1': ''' a? & b a:failed? => !b ''' }, } }) schd: 'Scheduler' = scheduler(id_) async with start(schd, level=logging.DEBUG) as log: # it should start up with 1/a and 1/b assert schd.pool.get_task_ids() == {"1/a", "1/b"} a = schd.pool.get_task(IntegerPoint("1"), "a") # mark 1/a as failed and ensure 1/b is removed by suicide trigger schd.pool.spawn_on_output(a, TASK_OUTPUT_FAILED) assert log_filter( regex="1/b.*removed from the n=0 window: suicide trigger" ) assert schd.pool.get_task_ids() == {"1/a"} # ensure that we are able to bring 1/b back by triggering it log.clear() await commands.run_cmd( commands.force_trigger_tasks(schd, ['1/b'], ['1'])) assert log_filter( regex='1/b.*added to the n=0 window', ) # remove 1/b by request (cylc remove) await commands.run_cmd( commands.remove_tasks(schd, ['1/b'], []) ) assert log_filter( regex='1/b.*removed from the n=0 window: request', ) # ensure that we are able to bring 1/b back by triggering it log.clear() await commands.run_cmd( commands.force_trigger_tasks(schd, ['1/b'], ['1'])) assert log_filter(regex='1/b.*added to the n=0 window',) async def test_set_future_flow(flow, scheduler, start, log_filter): """Manually-set outputs for new flow num must be recorded in the DB. See https://github.com/cylc/cylc-flow/pull/6186 To trigger the bug, the flow must be new but the task must have been spawned before in an earlier flow. """ # Scenario: after flow 1, set c1:succeeded in a future flow so # when b succeeds in the new flow it will spawn c2 but not c1. id_ = flow({ 'scheduler': { 'allow implicit tasks': True }, 'scheduling': { 'cycling mode': 'integer', 'graph': { 'R1': 'b => c1 & c2', }, }, }) schd: 'Scheduler' = scheduler(id_) async with start(schd, level=logging.DEBUG): assert schd.pool.get_task(IntegerPoint("1"), "b") is not None, ( '1/b should be spawned on startup' ) # set b, c1, c2 succeeded in flow 1 schd.pool.set_prereqs_and_outputs( { TaskTokens('1', 'b'), TaskTokens('1', 'c1'), TaskTokens('1', 'c2'), }, prereqs=[], outputs=[], flow=['1'], ) schd.workflow_db_mgr.process_queued_ops() # set task c1:succeeded in flow 2 schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'c1')}, prereqs=[], outputs=[], flow=['2'] ) schd.workflow_db_mgr.process_queued_ops() # set b:succeeded in flow 2 and check downstream spawning schd.pool.set_prereqs_and_outputs( {TaskTokens('1', 'b')}, prereqs=[], outputs=[], flow=['2'] ) assert schd.pool.get_task(IntegerPoint("1"), "c1") is None, ( '1/c1 (flow 2) should not be spawned after 1/b:succeeded' ) assert schd.pool.get_task(IntegerPoint("1"), "c2") is not None, ( '1/c2 (flow 2) should be spawned after 1/b:succeeded' ) async def test_trigger_queue(one, run, db_select, complete): """It should handle triggering tasks in the queued state. Triggering a queued task with a new flow number should result in the task running with merged flow numbers. See https://github.com/cylc/cylc-flow/pull/6241 """ async with run(one): # the workflow should start up with one task in the original flow task = one.pool.get_tasks()[0] assert task.state(TASK_STATUS_WAITING, is_queued=True) assert task.flow_nums == {1} # trigger this task even though is already queued in flow 1 await commands.run_cmd( commands.force_trigger_tasks(one, [task.identity], ['2'])) # the merged flow should continue one.resume_workflow() await complete(one, timeout=2) assert db_select(one, False, 'task_outputs', 'flow_nums') == [ ('[1, 2]',), ('[1]',), ] async def test_reload_xtriggers(flow, scheduler, start): """It should rebuild xtriggers when the workflow is reloaded. See https://github.com/cylc/cylc-flow/pull/6263 """ config = { 'scheduling': { 'initial cycle point': '2000', 'graph': { 'R1': ''' @a => foo @b => foo ''' }, 'xtriggers': { 'a': 'wall_clock(offset="P0D")', 'b': 'wall_clock(offset="P5D")', }, } } id_ = flow(config) schd: Scheduler = scheduler(id_) def list_xtrig_mgr(): """List xtrigs from the xtrigger_mgr.""" return { key: repr(value) for key, value in schd.xtrigger_mgr.xtriggers.functx_map.items() } async def list_data_store(): """List xtrigs from the data_store_mgr.""" await schd.update_data_structure() return { value.label: value.id for key, value in schd.data_store_mgr.data[schd.tokens.id][ TASK_PROXIES ][ schd.tokens.duplicate(cycle='20000101T0000Z', task='foo').id ].xtriggers.items() } async with start(schd): # check xtrigs on startup assert list_xtrig_mgr() == { 'a': '', 'b': '', } assert await list_data_store() == { 'a': 'wall_clock(trigger_time=946684800)', 'b': 'wall_clock(trigger_time=947116800)', } # remove @a config['scheduling']['xtriggers'].pop('a') # modify @b config['scheduling']['xtriggers']['b'] = 'wall_clock(offset="PT12H")' # add @c config['scheduling']['xtriggers']['c'] = 'wall_clock(offset="PT1H")' config['scheduling']['graph']['R1'] = config['scheduling']['graph'][ 'R1' ].replace('@a', '@c') # reload flow(config, workflow_id=id_) await commands.run_cmd(commands.reload_workflow(schd)) # check xtrigs post-reload assert list_xtrig_mgr() == { 'b': '', 'c': '', } assert await list_data_store() == { 'b': 'wall_clock(trigger_time=946728000)', 'c': 'wall_clock(trigger_time=946688400)', } @pytest.mark.parametrize( 'expire_type', ['clock-expire', 'manual-expire', 'expire-trigger'] ) async def test_expire_dequeue_with_retries( flow, scheduler, start, expire_type ): """An expired waiting task should be removed from any queues. See https://github.com/cylc/cylc-flow/issues/6284 """ conf = { 'scheduler': { 'experimental': { 'expire triggers': True, }, }, 'scheduling': { 'initial cycle point': '2000', 'graph': { 'R1': 'foo' }, }, 'runtime': { 'foo': { 'execution retry delays': 'PT0S', 'outputs': { 'x': 'xxx', }, }, }, } if expire_type == 'clock-expire': # configure foo to clock-expire conf['scheduling']['special tasks'] = {'clock-expire': 'foo(PT0S)'} # run the clock-expire logic def method(schd): schd.pool.clock_expire_tasks() elif expire_type == 'manual-expire': # run the "cylc set" command to expire "foo" def method(schd): schd.pool.set_prereqs_and_outputs( {TaskTokens('20000101T0000Z', 'foo')}, prereqs=[], outputs=['expired'], flow=['1'], ) else: # configure the task to be expire-triggered once "bar" succeeds conf['scheduling']['graph']['R1'] = ''' foo:x => bar foo:x & bar => !foo ''' # run the "cylc set" command to succeed "bar" def method(schd): schd.pool.set_prereqs_and_outputs( {TaskTokens('20000101T0000Z', 'bar')}, prereqs=[], outputs=['succeeded'], flow=['1'], ) id_ = flow(conf) schd = scheduler(id_, run_mode='live') schd: Scheduler async with start(schd): foo = schd.pool._get_task_by_id('20000101T0000Z/foo') assert foo # fake a real submission failure # NOTE: yes, all of these things are needed for a valid test! # Try removing the "force=True" added in this commit and ensure the # "clock-expire" test fails before changing anything here! foo.submit_num += 1 foo.run_mode = RunMode.LIVE schd.task_job_mgr._set_retry_timers(foo, foo.tdef.rtconfig) schd.task_events_mgr.process_message(foo, 0, 'started') schd.task_events_mgr.process_message(foo, 0, 'xxx') schd.task_events_mgr.process_message(foo, 0, 'failed') schd.task_events_mgr._retry_task(foo, 0) # the task should start as "waiting(queued)" assert foo.state(TASK_STATUS_WAITING, is_queued=True) # expire the task via whichever method we are testing method(schd) # the task should enter the "expired" state assert foo.state(TASK_STATUS_EXPIRED, is_queued=False) # the task should also have been removed from the queue assert not schd.pool.task_queue_mgr.remove_task(foo) async def test_clock_expire_with_sequential_xtriggers( flow, scheduler, run, complete, log_filter, ): """Clock expire should play nicely with sequential xtriggers. See https://github.com/cylc/cylc-flow/issues/7103 """ id_ = flow({ 'scheduler': { 'cycle point format': 'CCYY', }, 'scheduling': { 'initial cycle point': '2000', # v prevent this test passing if sequential spawning gets broken 'runahead limit': 'P2', 'special tasks': { 'clock-expire': 'a', }, 'graph': { 'P1Y': '@wall_clock => a => b => c' }, }, 'runtime': { 'a': { 'completion': 'succeeded or expired', }, }, }) schd = scheduler(id_, paused_start=False) async with run(schd): # each cycle should expire and spawn in turn await complete(schd, '2005/a') for cycle in range(2000, 2006): assert log_filter(regex=rf'{cycle}/a:waiting.*expired') # * the next instance should have been spawned # * it should be the only task in the pool assert { itask.tokens.relative_id for itask in schd.pool.get_tasks() } == {'2006/a'} async def test_downstream_complete_before_upstream( flow, scheduler, start, db_select ): """A downstream task that completes - in the same main loop iteration as its upstream task - does not get re-spawned. See https://github.com/cylc/cylc-flow/issues/6315 """ id_ = flow('a => b') schd: Scheduler = scheduler(id_) async with start(schd): # 1/a should be pre-spawned (parentless) a_1 = schd.pool.get_task(IntegerPoint('1'), 'a') assert a_1 # spawn 1/b (this can happens as the result of request e.g. trigger) b_1 = schd.pool.spawn_task('b', IntegerPoint('1'), {1}) schd.pool.add_to_pool(b_1) assert b_1 # mark 1/b as succeeded schd.task_events_mgr.process_message(b_1, 'INFO', 'succeeded') # 1/b should be removed from the pool (completed) assert schd.pool.get_tasks() == [a_1] # as a side effect the DB should have been updated (not waiting for # the main loop to do it) assert ( TASK_OUTPUT_SUCCEEDED in db_select( schd, # "False" means "do not run the DB update before checking it" False, # do not change this to "True" 'task_outputs', 'outputs', name='b', cycle='1', )[0][0] ) # mark 1/a as succeeded schd.task_events_mgr.process_message(a_1, 'INFO', 'succeeded') # 1/a should be removed from the pool (completed) # 1/b should not be re-spawned by the success of 1/a assert schd.pool.get_tasks() == [] async def test_job_insert_on_crash(one_conf, flow, scheduler, start): """Ensure that a job can be inserted if its config is not known. It is possible, though very difficult, to create the circumstances where the configuration for the latest job is not held in `itask.jobs`. This should not happen under normal circumstances, but should be handled elegantly if it does occur. See https://github.com/cylc/cylc-flow/issues/6314 """ id_ = flow(one_conf) schd: Scheduler = scheduler(id_, run_mode='live') async with start(schd): task_1 = schd.pool.get_tasks()[0] # make it look like the task submitted but without storing the job # config in TaskProxy.jobs task_1.submit_num += 1 task_1.state.reset('preparing') schd.task_events_mgr.process_message( task_1, 'INFO', 'submitted', ) # the task state should be updated correctly assert task_1.state.status == 'submitted' # and a job entry should be added assert len(task_1.jobs) == 1 async def test_start_tasks( flow, scheduler, start, capture_submission, ): """Check starting from "start-tasks" with and without clock-triggers. """ id_ = flow( { 'scheduler': { 'cycle point format': '%Y', }, 'scheduling': { 'initial cycle point': '2040', 'runahead limit': 'P0Y', 'xtriggers': { 'wall_clock_satisfied': "wall_clock(offset='-P100Y')" }, 'graph': { 'P1Y': """ foo @wall_clock => bar @wall_clock_satisfied => baz qux """ } } } ) schd: Scheduler = scheduler( id_, starttask=['2050/foo', '2050/bar', '2050/baz'], paused_start=False ) async with start(schd): # capture any job submissions submitted_tasks = capture_submission(schd) assert submitted_tasks == set() # It should start up with: # - 2050/foo and 2051/foo (spawned to runahead limit) # - 2050/bar waiting on its (unsatisfied) clock-trigger # - 2050/baz waiting on its (satisfied) clock-trigger # - no qux instances (not listed as a start-task) itasks = schd.pool.get_tasks() assert ( set(itask.identity for itask in itasks) == { "2050/foo", "2051/foo", "2050/bar", "2050/baz", } ) await schd._main_loop() # It should submit 2050/foo, 2050/baz # It should not submit 2050/bar (waiting on clock trigger) # It should not submit 2051/foo (runahead limited) assert ( set(itask.identity for itask in submitted_tasks) == { "2050/foo", "2050/baz", } ) async def test_add_new_flow_rows_on_spawn( flow, scheduler, run, complete, db_select, capcall, ) -> None: """Task suicide should not override previously completed outputs. See https://github.com/cylc/cylc-flow/pull/6821 """ # capture all TaskPool.spawn_task() calls spawn_task_calls = capcall( 'cylc.flow.task_pool.TaskPool.spawn_task', TaskPool.spawn_task ) def list_spawn_task_calls(): """Return a list of the names of tasks which have been run through the "spawn_tasks" function so far.""" return [ args[1] for args, _kwargs in spawn_task_calls ] id_ = flow({ 'scheduling': { 'graph': { 'R1': ''' slow:fail? => foo slow? => !foo foo:x => x ''', }, }, 'runtime': { 'foo': { 'outputs': {'x': 'xxx'} }, }, }) schd = scheduler(id_, paused_start=False) async with run(schd): # 1/slow should spawn on startup assert list_spawn_task_calls() == ['slow'] # set foo:x await commands.run_cmd( commands.set_prereqs_and_outputs( schd, ['1/foo'], ['1'], ['x'], None ) ) # 1/foo:x should be recorded in the DB: assert db_select( schd, True, 'task_outputs', 'outputs', cycle='1', name='foo' ) == [('{"x": "(manually completed)"}',)] # and 1/x should spawn: assert list_spawn_task_calls() == ['slow', 'x'] # run the workflow until completion await complete(schd, timeout=5) # 1/foo should spawn as a result of the suicide trigger assert list_spawn_task_calls() == ['slow', 'x', 'foo'] # the manually completed output should not have been overwritten by the # suicide trigger assert db_select( schd, True, 'task_outputs', 'outputs', cycle='1', name='foo' ) == [('{"x": "(manually completed)"}',)] async def test_add_to_pool( flow, scheduler, start, caplog ): """It should log attempts to add the same task again.""" id_ = flow('a') schd = scheduler(id_) async with start(schd): caplog.set_level(logging.DEBUG, CYLC_LOG) # 1/a should be pre-spawned (parentless) a_1 = schd.pool.get_task(IntegerPoint('1'), 'a') assert a_1 # add it again schd.pool.add_to_pool(a_1) assert "1/a not added to n=0: already exists" in caplog.text async def test_parentless_spawning(flow, scheduler, run, complete): """Check that parentless spawning works even if parented in some cycles. Parentless spawning could be blocked if a parented cycle ended up at the runahead limit. See https://github.com/cylc/cylc-flow/pull/7237. """ cfg = { 'scheduling': { 'cycling mode': 'integer', 'initial cycle point': '1', 'runahead limit': 'P0', 'graph': { 'P1': 'b', 'R1/2/P0': 'a => b', } }, } id_ = flow(cfg) schd = scheduler(id_, paused_start=False) async with run(schd): await complete(schd, "2/b") # The task pool should not be empty now. assert len(schd.pool.get_tasks()) cylc-flow-8.7.0/tests/integration/test_reload.py0000664000175000017500000003432515264420307022167 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests for reload behaviour in the scheduler.""" from cylc.flow import ( commands, flags, ) from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.data_store_mgr import TASK_PROXIES from cylc.flow.platforms import get_platform from cylc.flow.scheduler import Scheduler from cylc.flow.task_state import ( TASK_STATUS_PREPARING, TASK_STATUS_SUBMITTED, TASK_STATUS_WAITING, ) async def test_reload_waits_for_pending_tasks( flow, scheduler, start, monkeypatch, log_scan, ): """Reload should flush out preparing tasks and pause the workflow. Reloading a workflow with preparing tasks may be unsafe and is at least confusing. For safety we should pause the workflow and flush out any preparing tasks before attempting reload. See https://github.com/cylc/cylc-flow/issues/5107 """ # speed up the test: monkeypatch.setattr('cylc.flow.scheduler.sleep', lambda *_: None) # a simple workflow with a single task id_ = flow('foo') schd: Scheduler = scheduler(id_, paused_start=False) # we will artificially push the task through these states state_seq = [ # repeat the preparing state a few times to simulate a task # taking multiple main-loop cycles to submit TASK_STATUS_PREPARING, TASK_STATUS_PREPARING, TASK_STATUS_PREPARING, TASK_STATUS_SUBMITTED, ] # start the scheduler async with start(schd) as log: foo = schd.pool.get_tasks()[0] # set the task to go through some state changes def submit_task_jobs(*a, **k): try: foo.state_reset(state_seq.pop(0)) except IndexError: foo.waiting_on_job_prep = False return [foo] monkeypatch.setattr( schd.task_job_mgr, 'submit_task_jobs', submit_task_jobs ) # the task should start as waiting assert foo.state(TASK_STATUS_WAITING) # put the task into the preparing state schd.release_tasks_to_run() assert foo.state(TASK_STATUS_PREPARING) # reload the workflow await commands.run_cmd(commands.reload_workflow(schd)) # the task should end in the submitted state assert foo.state(TASK_STATUS_SUBMITTED) # ensure the order of events was correct log_scan( log, [ # the task should have entered the preparing state before the # reload was requested '[1/foo:waiting] => preparing', # the reload should have put the workflow into the paused state 'Pausing the workflow: Reloading workflow', # reload should have waited for the task to submit '[1/foo/00:preparing] => submitted', # before then reloading the workflow config 'Reloading the workflow definition.', # post-reload the workflow should have been resumed 'RESUMING the workflow now', ], ) async def test_reload_waits_for_preparing_tasks_in_remote_init( flow, scheduler, start, monkeypatch, ): """Reload should wait for preparing tasks stuck in remote-init. Test works by simulating a task taking longer than one reload flush loop to complete remote-init, catching the situation where release_tasks_to_run() returns False (because no tasks are ready for submission) but the task is still waiting_on_job_prep or in the preparing state. The reload flush loop should continue to iterate until the task has actually submitted. https://github.com/cylc/cylc-flow/issues/7296 """ # speed up the test monkeypatch.setattr('cylc.flow.scheduler.sleep', lambda *_: None) id_ = flow('foo') schd: Scheduler = scheduler(id_, paused_start=False) async with start(schd): foo = schd.pool.get_tasks()[0] # Track how many times submit_task_jobs is called to simulate # a task going through remote-init (returns empty list) before # eventually submitting. call_count = 0 def submit_task_jobs(itasks, *a, **k): nonlocal call_count call_count += 1 if call_count <= 3: # Simulate tasks stuck in remote-init: submit_task_jobs # returns an empty list because remote-init hasn't completed # yet, but the task is still waiting_on_job_prep. # On the first call, set state to PREPARING (as the real # prep_submit_task_jobs would do). if not foo.state(TASK_STATUS_PREPARING): foo.submit_num += 1 foo.state_reset(TASK_STATUS_PREPARING) return [] else: # Remote-init done, task actually submits now. foo.waiting_on_job_prep = False foo.state_reset(TASK_STATUS_SUBMITTED) return [foo] monkeypatch.setattr( schd.task_job_mgr, 'submit_task_jobs', submit_task_jobs ) # Put the task into the preparing state schd.release_tasks_to_run() assert foo.state(TASK_STATUS_PREPARING) assert foo.waiting_on_job_prep # Reload the workflow - this should wait for the task to submit await commands.run_cmd(commands.reload_workflow(schd)) # The task should have been flushed through to submitted assert foo.state(TASK_STATUS_SUBMITTED), ( "Reload proceeded before preparing task finished submitting" ) assert not foo.waiting_on_job_prep # submit_task_jobs should have been called multiple times # (not just once before the loop exited) assert call_count == 4 async def test_reload_failure( flow, one_conf, scheduler, start, log_filter, ): """Reload should not crash the workflow on config errors. A warning should be logged along with the error. """ id_ = flow(one_conf) schd = scheduler(id_) async with start(schd): # corrupt the config by removing the scheduling section two_conf = {**one_conf, 'scheduling': {}} flow(two_conf, workflow_id=id_) # reload the workflow await commands.run_cmd(commands.reload_workflow(schd)) # the reload should have failed but the workflow should still be # running assert log_filter( contains=( 'Reload failed - WorkflowConfigError:' ' missing [scheduling][[graph]] section' ) ) # the config should be unchanged assert schd.config.cfg['scheduling']['graph']['R1'] == 'one' async def test_reload_global_platform( flow, one_conf, scheduler, start, log_filter, tmp_path, monkeypatch, ): global_config_path = tmp_path / 'global.cylc' monkeypatch.setenv("CYLC_CONF_PATH", str(global_config_path.parent)) # Original global config file global_config_path.write_text(""" [platforms] [[localhost]] [[[meta]]] x = 1 """) glbl_cfg(reload=True) assert glbl_cfg().get(['platforms', 'localhost', 'meta', 'x']) == '1' id_ = flow(one_conf) schd = scheduler(id_) async with start(schd): # Task platforms reflect the original config rtconf = schd.broadcast_mgr.get_updated_rtconfig( schd.pool.get_tasks()[0] ) platform = get_platform(rtconf) assert platform['meta']['x'] == '1' # Modify the global config file global_config_path.write_text(""" [platforms] [[localhost]] [[[meta]]] x = 2 """) # reload the workflow and global config await commands.run_cmd( commands.reload_workflow(schd, reload_global=True) ) # Global config should have been reloaded assert log_filter(contains=('Reloading the global configuration.')) # Task platforms reflect the new config rtconf = schd.broadcast_mgr.get_updated_rtconfig( schd.pool.get_tasks()[0] ) platform = get_platform(rtconf) assert platform['meta']['x'] == '2' # Modify the global config file with an error global_config_path.write_text(""" [ERROR] [platforms] [[localhost]] [[[meta]]] x = 3 """) # reload the workflow and global config await commands.run_cmd( commands.reload_workflow(schd, reload_global=True) ) # Error is noted in the log assert log_filter( contains=( 'This is probably due to an issue with the new configuration.' ) ) # Task platforms should be the last valid value rtconf = schd.broadcast_mgr.get_updated_rtconfig( schd.pool.get_tasks()[0] ) platform = get_platform(rtconf) assert platform['meta']['x'] == '2' # reload the workflow in verbose mode flags.verbosity = 2 await commands.run_cmd( commands.reload_workflow(schd, reload_global=True) ) # Traceback is shown in the log # (match for ERROR, trace is not captured) assert log_filter(exact_match='ERROR') async def test_reload_global_platform_group( flow, scheduler, start, log_filter, tmp_path, monkeypatch, ): global_config_path = tmp_path / 'global.cylc' monkeypatch.setenv("CYLC_CONF_PATH", str(global_config_path.parent)) # Original global config file global_config_path.write_text(""" [platforms] [[foo]] [[[meta]]] x = 1 [platform groups] [[pg]] platforms = foo """) glbl_cfg(reload=True) # Task using the platform group conf = { 'scheduling': {'graph': {'R1': 'one'}}, 'runtime': { 'one': { 'platform': 'pg', } }, } id_ = flow(conf) schd = scheduler(id_) async with start(schd): # Task platforms reflect the original config rtconf = schd.broadcast_mgr.get_updated_rtconfig( schd.pool.get_tasks()[0] ) platform = get_platform(rtconf) assert platform['meta']['x'] == '1' # Modify the global config file global_config_path.write_text(""" [platforms] [[bar]] [[[meta]]] x = 2 [platform groups] [[pg]] platforms = bar """) # reload the workflow and global config await commands.run_cmd( commands.reload_workflow(schd, reload_global=True) ) # Global config should have been reloaded assert log_filter(contains=('Reloading the global configuration.')) # Task platforms reflect the new config rtconf = schd.broadcast_mgr.get_updated_rtconfig( schd.pool.get_tasks()[0] ) platform = get_platform(rtconf) assert platform['meta']['x'] == '2' async def test_orphan_reload( flow, scheduler, start, log_filter, ): """Reload should not fail because of orphaned tasks. The following aspects of reload-with-orphans are tested: - Broadcast deltas generated after reload. https://github.com/cylc/cylc-flow/issues/6814 - Removal of both xtrigger and associated active/incomplete task. https://github.com/cylc/cylc-flow/issues/6815 (Orphans being active/incomplete tasks removed from reloaded workflow cfg.) """ before = { 'scheduling': { 'initial cycle point': '20010101T0000Z', 'graph': { 'R1': ''' foo => bar @wall_clock => bar ''' } } } after = { 'scheduling': { 'initial cycle point': '20010101T0000Z', 'graph': { 'R1': 'foo' } } } id_ = flow(before) schd = scheduler(id_) async with start(schd): # spawn in bar foo = schd.pool._get_task_by_id('20010101T0000Z/foo') schd.pool.task_events_mgr.process_message(foo, 'INFO', 'succeeded') bar = schd.pool._get_task_by_id('20010101T0000Z/bar') # set bar to failed schd.pool.task_events_mgr.process_message(bar, 'INFO', 'failed') # Save our progress schd.workflow_db_mgr.put_task_pool(schd.pool) # Change workflow to one without bar and xtrigger flow(after, workflow_id=id_) # reload the workflow await commands.run_cmd(commands.reload_workflow(schd)) # test broadcast delta over orphaned task schd.data_store_mgr.delta_broadcast() # the reload should have completed successfully assert log_filter( contains=('Reload completed') ) async def test_data_store_tproxy(flow, scheduler, start): """Check N>0 task proxy in data store has correct info on reload. https://github.com/cylc/cylc-flow/issues/6973 """ schd: Scheduler = scheduler(flow('foo => bar')) def get_ds_tproxy(task): return schd.data_store_mgr.data[schd.id][TASK_PROXIES][ f'{schd.id}//1/{task}' ] async with start(schd): await schd.update_data_structure() assert str(get_ds_tproxy('bar').runtime) await commands.run_cmd(commands.reload_workflow(schd)) assert str(get_ds_tproxy('bar').runtime) cylc-flow-8.7.0/tests/integration/test_platforms.py0000664000175000017500000000416015264420307022722 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Integration testing for platforms functionality.""" from cylc.flow.scheduler import Scheduler async def test_prep_submit_task_tries_multiple_platforms( flow, scheduler, start, mock_glbl_cfg ): """Preparation tries multiple platforms within a group if the task platform setting matches a group, and that after all platforms have been tried that the hosts matching that platform group are cleared. See https://github.com/cylc/cylc-flow/pull/6109 """ global_conf = ''' [platforms] [[myplatform]] hosts = broken [[anotherbad]] hosts = broken2 [platform groups] [[mygroup]] platforms = myplatform, anotherbad''' mock_glbl_cfg('cylc.flow.platforms.glbl_cfg', global_conf) wid = flow({ "scheduling": {"graph": {"R1": "foo"}}, "runtime": {"foo": {"platform": "mygroup"}} }) schd: Scheduler = scheduler(wid, run_mode='live') async with start(schd): itask = schd.pool.get_tasks()[0] itask.submit_num = 1 # simulate failed attempts to contact the job hosts schd.bad_hosts.update({'broken', 'broken2'}) res = schd.task_job_mgr._prep_submit_task_job(itask) assert res is False # ensure the bad hosts have been cleared assert not schd.task_job_mgr.bad_hosts cylc-flow-8.7.0/tests/integration/test_flow_assignment.py0000664000175000017500000001426415264420307024120 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test for flow-assignment in triggered/set tasks.""" import functools import logging import time from typing import Callable import pytest from cylc.flow.flow_mgr import ( FLOW_NEW, FLOW_NONE, ) from cylc.flow.scheduler import Scheduler from cylc.flow.commands import ( run_cmd, force_trigger_tasks, set_prereqs_and_outputs, ) async def test_trigger_no_flows(one, start): """Test triggering a task with no flows present. It should get the flow numbers of the most recent active tasks. """ async with start(one): # Remove the task (flow 1) --> pool empty task = one.pool.get_tasks()[0] one.pool.remove(task) assert len(one.pool.get_tasks()) == 0 # Trigger the task, with new flow nums. time.sleep(2) # The flows need different timestamps! await run_cmd( force_trigger_tasks(one, [task.identity], flow=['5', '9'])) assert len(one.pool.get_tasks()) == 1 # Ensure the new flow is in the db. one.pool.workflow_db_mgr.process_queued_ops() # Remove the task --> pool empty task = one.pool.get_tasks()[0] one.pool.remove(task) assert len(one.pool.get_tasks()) == 0 # Trigger the task; it should get flow nums 5, 9 await run_cmd(force_trigger_tasks(one, [task.identity], [])) assert len(one.pool.get_tasks()) == 1 task = one.pool.get_tasks()[0] assert task.flow_nums == {5, 9} async def test_cli_to_flow_nums(one: Scheduler, start): """Test the flow manager cli_to_flow_nums method.""" async with start(one): # flow 1 is already present task = one.pool.get_tasks()[0] assert one.pool.flow_mgr.cli_to_flow_nums([FLOW_NEW]) == {2} one.pool.merge_flows(task, {2}) # now we have flows {1, 2}: assert one.pool.flow_mgr.cli_to_flow_nums([FLOW_NONE]) == set() assert one.pool.flow_mgr.cli_to_flow_nums([]) == set() assert one.pool.flow_mgr.cli_to_flow_nums([FLOW_NEW]) == {3} assert one.pool.flow_mgr.cli_to_flow_nums(['4', '5']) == {4, 5} @pytest.mark.parametrize('command', ['trigger', 'set']) async def test_flow_assignment( flow, scheduler, start, command: str, log_filter: Callable ): """Test flow assignment when triggering/setting tasks. Active tasks: By default keep existing flows, else merge with requested flows. Inactive tasks: By default assign active flows; else assign requested flows. """ conf = { 'scheduler': { 'allow implicit tasks': 'True' }, 'scheduling': { 'graph': { 'R1': "foo & bar => a & b & c & d & e" } }, 'runtime': { 'foo': { 'outputs': {'x': 'x'} } }, } id_ = flow(conf) schd: Scheduler = scheduler(id_, run_mode='simulation', paused_start=True) async with start(schd): if command == 'set': do_command = functools.partial( set_prereqs_and_outputs, schd, outputs=['x'], prerequisites=[], ) else: do_command = functools.partial( force_trigger_tasks, schd, ) active_1, active_2 = schd.pool.get_tasks() # foo, bar in any order schd.pool.merge_flows( active_2, schd.pool.flow_mgr.cli_to_flow_nums([FLOW_NEW])) assert active_1.flow_nums == {1} assert active_2.flow_nums == {1, 2} # -----(1. Test active tasks)----- await run_cmd(do_command([active_1.identity], flow=[])) if command == "set": # By default active tasks merge existing and active flows. assert active_1.flow_nums == {1, 2} else: # By default active tasks keep existing flows. assert active_1.flow_nums == {1} # Else merge existing and requested flows. await run_cmd(do_command([active_1.identity], flow=['2'])) assert active_1.flow_nums == {1, 2} # (no-flow is ignored for active tasks) await run_cmd(do_command([active_1.identity], flow=[FLOW_NONE])) assert active_1.flow_nums == {1, 2} assert log_filter( contains=("Already active - ignoring"), level=logging.WARNING ) await run_cmd(do_command([active_1.identity], flow=[FLOW_NEW])) assert active_1.flow_nums == {1, 2, 3} # -----(2. Test inactive tasks)----- if command == 'set': do_command = functools.partial( set_prereqs_and_outputs, schd, outputs=[], prerequisites=['all'], ) # By default inactive tasks get all active flows. await run_cmd(do_command(['1/a'], flow=[])) assert schd.pool._get_task_by_id('1/a').flow_nums == {1, 2, 3} # Else assign requested flows. # Run as no-flow: await run_cmd(do_command(['1/b'], flow=[FLOW_NONE])) assert schd.pool._get_task_by_id('1/b').flow_nums == set() await run_cmd(do_command(['1/c'], flow=[FLOW_NEW])) assert schd.pool._get_task_by_id('1/c').flow_nums == {4} await run_cmd(do_command(['1/d'], flow=[])) assert schd.pool._get_task_by_id('1/d').flow_nums == {1, 2, 3, 4} await run_cmd(do_command(['1/e'], flow=["7"])) assert schd.pool._get_task_by_id('1/e').flow_nums == {7} cylc-flow-8.7.0/tests/integration/main_loop/0000775000175000017500000000000015264420307021256 5ustar alastairalastaircylc-flow-8.7.0/tests/integration/main_loop/__init__.py0000664000175000017500000000000015264420307023355 0ustar alastairalastaircylc-flow-8.7.0/tests/integration/main_loop/test_auto_restart.py0000664000175000017500000000351115264420307025403 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio from unittest.mock import Mock import pytest from cylc.flow.scheduler import ( Scheduler, SchedulerError, ) from cylc.flow.workflow_status import AutoRestartMode async def test_no_detach( one_conf, flow, scheduler, run, mock_glbl_cfg, log_filter, monkeypatch: pytest.MonkeyPatch ): """Test that the Scheduler aborts when auto restart tries to happen while in no-detach mode.""" mock_glbl_cfg( 'cylc.flow.scheduler.glbl_cfg', ''' [scheduler] [[main loop]] plugins = auto restart [[[auto restart]]] interval = PT1S ''') monkeypatch.setattr( 'cylc.flow.main_loop.auto_restart._should_auto_restart', Mock(return_value=AutoRestartMode.RESTART_NORMAL) ) id_: str = flow(one_conf) schd: Scheduler = scheduler(id_, paused_start=True, no_detach=True) with pytest.raises(SchedulerError) as exc: async with run(schd): await asyncio.sleep(2) assert log_filter(contains=f"Workflow shutting down - {exc.value}") cylc-flow-8.7.0/tests/integration/main_loop/test_health_check.py0000664000175000017500000000624415264420307025277 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging import os import shutil from unittest.mock import Mock import pytest import cylc.flow.flags from cylc.flow.main_loop import MainLoopPluginException from cylc.flow.main_loop.health_check import HealthCheckFailed from cylc.flow.scheduler import Scheduler from cylc.flow.workflow_files import get_contact_file_path @pytest.fixture(autouse=True) def rapid_health_check(mock_glbl_cfg): mock_glbl_cfg( 'cylc.flow.scheduler.glbl_cfg', ''' [scheduler] [[main loop]] plugins = health check [[[health check]]] interval = PT0S ''') async def test_bad_contact_file(one: Scheduler, run, log_filter): """Test workflow shuts down with error on corrupted contact file.""" with pytest.raises(MainLoopPluginException): async with run(one): with open(get_contact_file_path(one.workflow), 'a') as f: f.write("Haha! I have corrupted the port file!") await one._main_loop() assert log_filter( regex=r"Workflow shutting down - .* contact file corrupted/modified" ) @pytest.mark.parametrize('debug', [True, False]) async def test_no_contact_file( debug, one, run, log_filter, caplog: pytest.LogCaptureFixture, ): """Test workflow shuts down with error on missing contact file.""" if debug: cylc.flow.flags.verbosity = 2 with pytest.raises(HealthCheckFailed): async with run(one): os.unlink(get_contact_file_path(one.workflow)) await one._main_loop() assert log_filter( logging.CRITICAL, ( "Workflow shutting down - health check failed: " "Couldn't load contact file" ), ) assert ("Traceback" in caplog.text) is debug async def test_deleted_run_dir(one: Scheduler, run, log_filter): """Test workflow shuts down with error on missing run directory.""" with pytest.raises(HealthCheckFailed): async with run(one): # Stop the scheduler from trying to access the DB as that will # cause a different abort that we're not testing here one.process_workflow_db_queue = Mock() shutil.rmtree(one.workflow_run_dir) await one._main_loop() assert log_filter( logging.CRITICAL, "Workflow shutting down - health check failed: " "Workflow run directory does not exist", ) cylc-flow-8.7.0/tests/integration/test_subprocctx.py0000664000175000017500000000427515264420307023116 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests involving the Cylc Subprocess Context Object """ from logging import DEBUG from textwrap import dedent async def test_log_xtrigger_stdout( flow, scheduler, run_dir, start, log_filter ): """Output from xtriggers should appear in the scheduler log: (As per the toy example in the Cylc Docs) """ # Setup a workflow: id_ = flow({ 'scheduler': {'allow implicit tasks': True}, 'scheduling': { 'graph': {'R1': '@myxtrigger => foo'}, 'xtriggers': {'myxtrigger': 'myxtrigger()'} } }) # Create an xtrigger: xt_lib = run_dir / id_ / 'lib/python/myxtrigger.py' xt_lib.parent.mkdir(parents=True, exist_ok=True) xt_lib.write_text(dedent(r""" from sys import stderr def myxtrigger(): print('Hello World') print('Hello Hades', file=stderr) return True, {} """)) schd = scheduler(id_) async with start(schd, level=DEBUG): # Set off check for x-trigger: task = schd.pool.get_tasks()[0] schd.xtrigger_mgr.call_xtriggers_async(task) # while not schd.xtrigger_mgr._get_xtrigs(task): while schd.proc_pool.is_not_done(): schd.proc_pool.process() # Assert that both stderr and out from the print statement # in our xtrigger appear in the log. for expected in ['Hello World', 'Hello Hades']: assert log_filter(DEBUG, expected) cylc-flow-8.7.0/tests/integration/test_install.py0000664000175000017500000001043415264420307022362 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test cylc install.""" import pytest from pathlib import Path from typing import Callable, Tuple from cylc.flow.async_util import pipe from cylc.flow.scripts import scan from cylc.flow.workflow_files import WorkflowFiles from cylc.flow.scripts.install import ( InstallOptions, install_cli ) from .network.test_scan import init_flows SRV_DIR = Path(WorkflowFiles.Service.DIRNAME) CONTACT = Path(WorkflowFiles.Service.CONTACT) RUN_N = Path(WorkflowFiles.RUN_N) INSTALL = Path(WorkflowFiles.Install.DIRNAME) INSTALLED_MSG = "INSTALLED {wfrun} from" WF_ACTIVE_MSG = '1 run of "{wf}" is already active:' BAD_CONTACT_MSG = "Bad contact file:" @pytest.fixture() def patch_graphql_query( monkeypatch: pytest.MonkeyPatch ): # Define a mocked graphql_query pipe function. @pipe async def _graphql_query(flow, fields, filters=None): flow.update({"status": "running"}) return flow # Swap out the function that cylc.flow.scripts.scan. monkeypatch.setattr( 'cylc.flow.scripts.scan.graphql_query', _graphql_query, ) @pytest.fixture() def src_run_dirs( mock_glbl_cfg: Callable, monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> Tuple[Path, Path]: """Create some workflow source and run dirs for testing. Source dirs: /w1 /w2 Run dir: /w1/run1 """ tmp_src_path = tmp_path / 'cylc-src' tmp_run_path = tmp_path / 'cylc-run' tmp_src_path.mkdir() tmp_run_path.mkdir() init_flows( tmp_run_path=tmp_run_path, running=('w1/run1',), tmp_src_path=tmp_src_path, src=('w1', 'w2') ) mock_glbl_cfg( 'cylc.flow.install.glbl_cfg', f''' [install] source dirs = {tmp_src_path} ''' ) monkeypatch.setattr('cylc.flow.pathutil._CYLC_RUN_DIR', tmp_run_path) return tmp_src_path, tmp_run_path async def test_install_scan_no_ping( src_run_dirs: Tuple[Path, Path], capsys: pytest.CaptureFixture, caplog: pytest.LogCaptureFixture ) -> None: """At install, running instances should be reported. Ping = False case: don't query schedulers. """ opts = InstallOptions() opts.no_ping = True await install_cli(opts, id_='w1') out = capsys.readouterr().out assert INSTALLED_MSG.format(wfrun='w1/run2') in out assert WF_ACTIVE_MSG.format(wf='w1') in out # Empty contact file faked with "touch": assert f"{BAD_CONTACT_MSG} w1/run1" in caplog.text await install_cli(opts, id_='w2') out = capsys.readouterr().out assert WF_ACTIVE_MSG.format(wf='w2') not in out assert INSTALLED_MSG.format(wfrun='w2/run1') in out async def test_install_scan_ping( src_run_dirs: Tuple[Path, Path], capsys: pytest.CaptureFixture, caplog: pytest.LogCaptureFixture, patch_graphql_query: Callable ) -> None: """At install, running instances should be reported. Ping = True case: but mock scan's scheduler query method. """ opts = InstallOptions() opts.no_ping = False await install_cli(opts, id_='w1') out = capsys.readouterr().out assert INSTALLED_MSG.format(wfrun='w1/run2') in out assert WF_ACTIVE_MSG.format(wf='w1') in out assert scan.FLOW_STATE_SYMBOLS["running"] in out # Empty contact file faked with "touch": assert f"{BAD_CONTACT_MSG} w1/run1" in caplog.text await install_cli(opts, id_='w2') out = capsys.readouterr().out assert INSTALLED_MSG.format(wfrun='w2/run1') in out assert WF_ACTIVE_MSG.format(wf='w2') not in out cylc-flow-8.7.0/tests/conftest.py0000664000175000017500000002254215264420307017162 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import hashlib import logging from pathlib import Path import re from shutil import rmtree import time from typing import Callable import pytest from cylc.flow import LOG, flags from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.cfgspec.globalcfg import SPEC from cylc.flow.graphnode import GraphNodeParser from cylc.flow.parsec.config import ParsecConfig from cylc.flow.parsec.validate import cylc_config_validate @pytest.fixture(autouse=True) def before_each(): """Reset global state before every test.""" flags.verbosity = 0 LOG.setLevel(logging.NOTSET) # Reset graph node parser singleton: GraphNodeParser.get_inst().clear() @pytest.fixture(scope='module') def mod_monkeypatch(): """A module-scoped version of the monkeypatch fixture.""" from _pytest.monkeypatch import MonkeyPatch mpatch = MonkeyPatch() yield mpatch mpatch.undo() @pytest.fixture def mock_glbl_cfg(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): """A Pytest fixture for fiddling global config values. * Hacks the specified `glbl_cfg` object. * Can be called multiple times within a test function. Args: pypath (str): The python-like path to the global configuration object you want to fiddle. E.G. if you want to hack the `glbl_cfg` in `cylc.flow.scheduler` you would provide `cylc.flow.scheduler.glbl_cfg` global_config (str): The global configuration as a multi-line string. Example: Change the value of `UTC mode` in the global config as seen from `the scheduler` module. def test_something(mock_glbl_cfg): mock_glbl_cfg( 'cylc.flow.scheduler.glbl_cfg', ''' [scheduler] UTC mode = True ''' ) """ # TODO: modify Parsec so we can use StringIO rather than a temp file. def _mock_glbl_cfg(pypath: str, global_config: str) -> None: global_config_path = tmp_path / 'global.cylc' global_config_path.write_text(global_config) glbl_cfg = ParsecConfig(SPEC, validator=cylc_config_validate) glbl_cfg.loadcfg(global_config_path) def _inner(cached=False): return glbl_cfg monkeypatch.setattr(pypath, _inner) yield _mock_glbl_cfg rmtree(tmp_path) @pytest.fixture def log_filter(caplog: pytest.LogCaptureFixture): """Filter caplog record_tuples (also discarding the log name entry). Args: level: Filter out records if they aren't at this logging level. contains: Filter out records if this string is not in the message. regex: Filter out records if the message doesn't match this regex. exact_match: Filter out records if the message does not exactly match this string. log: A caplog instance. """ def _log_filter( level: int | None = None, contains: str | None = None, regex: str | None = None, exact_match: str | None = None, log: pytest.LogCaptureFixture | None = None ) -> list[tuple[int, str]]: if log is None: log = caplog return [ (log_level, log_message) for _, log_level, log_message in log.record_tuples if (level is None or level == log_level) and (contains is None or contains in log_message) and (regex is None or re.search(regex, log_message)) and (exact_match is None or exact_match == log_message) ] return _log_filter @pytest.fixture def log_scan(): """Ensure log messages appear in the correct order. TRY TO AVOID DOING THIS! If you are trying to test a sequence of events you are likely better off doing this a different way (e.g. mock the functions you are interested in and test the call arguments/returns later). However, there are some occasions where this might be necessary, e.g. testing a monolithic synchronous function. Args: log: The caplog fixture. items: Iterable of string messages to compare. All are tested by "contains" i.e. "item in string". """ def _log_scan(log, items): records = iter(log.records) record = next(records) for item in items: while item not in record.message: try: record = next(records) except StopIteration: raise Exception(f'Reached end of log looking for: {item}') return _log_scan @pytest.fixture(scope='session') def port_range(): return glbl_cfg().get(['scheduler', 'run hosts', 'ports']) @pytest.fixture def capcall(monkeypatch): """Capture function calls without running the function. Returns a list which is populated with function calls. Args: function_string: The function to replace as it would be specified to monkeypatch.setattr. mock: * If True, the function will be replaced by a "return None". * If False, the original function will be run. * If a Callable is provided, this will be run in place of the original function. Returns: [(args: Tuple, kwargs: Dict), ...] Example: def test_something(capcall): capsys = capcall('sys.exit') sys.exit(1) assert capsys == [(1,), {}] """ def _capcall(function_string: str, mock: bool | Callable = True): calls = [] if mock is True: fcn = lambda *args, **kwargs: None elif mock is False: fcn = import_object_from_string(function_string) else: fcn = mock def _call(*args, **kwargs): calls.append((args, kwargs)) return fcn(*args, **kwargs) monkeypatch.setattr(function_string, _call) return calls return _capcall def import_object_from_string(string): """Import a Python object from a string path. The path may reference a module, function, class, method, whatever. Examples: # import a module >>> import_object_from_string('os') # import a function >>> import_object_from_string('os.path.walk') # import a constant from a namespace package >>> import_object_from_string('cylc.flow.LOG') # import a class >>> import_object_from_string('pathlib.Path') # import a method >>> import_object_from_string('pathlib.Path.exists') """ head = string tail = [] while True: try: # try and import the thing module = __import__(head) except ModuleNotFoundError: # if it's not something we can import, lop the last item off the # end of the string and repeat if '.' in head: head, _tail = head.rsplit('.', 1) tail.append(_tail) else: # we definitely can't import this raise else: # we managed to import something if '(namespace)' in str(module): # with namespace packages you have to pull the module out of # the package yourself for part in head.split('.')[1:]: module = getattr(module, part) break # extract the requested object from the module (if requested) obj = module for part in reversed(tail): obj = getattr(obj, part) return obj @pytest.fixture def set_timezone(monkeypatch): """Fixture to temporarily set a timezone. Will use a very implausible timezone if none is provided. """ def patch(time_zone: str): monkeypatch.setenv('TZ', time_zone) time.tzset() try: yield patch finally: # Reset to the original time zone after the test monkeypatch.undo() time.tzset() @pytest.fixture(scope='module') def mod_tmp_path(tmp_path_factory: pytest.TempPathFactory): """A module scoped version of tmp_path. Yields a temporary directory which will persist for all tests in the same module. Note: Each invovation of this fixture yields a different directory, even if they are in the same module. I.e, the yielded directory will persist within the scope of the module, rather than the yielded directory is consistent within the scope of the module """ yield tmp_path_factory.mktemp(hashlib.sha256().hexdigest()[:10]) cylc-flow-8.7.0/tests/i0000777000175000017500000000000015264420307017451 2integration/ustar alastairalastaircylc-flow-8.7.0/tests/k0000777000175000017500000000000015264420307020321 2flakyfunctional/ustar alastairalastaircylc-flow-8.7.0/tests/README.md0000664000175000017500000000113115264420307016231 0ustar alastairalastair# Tests Cylc-flow has three tiers of tests: | Test Type | Command | Docs | |---|---|---| | Unit Tests | `pytest` | [README](https://github.com/cylc/cylc-flow/blob/master/tests/unit/README.md) | | Integration tests | `pytest tests/integration` | [README](https://github.com/cylc/cylc-flow/blob/master/tests/integration/README.md) | | Functional Tests | `etc/bin/run-functional-tests` | [README](https://github.com/cylc/cylc-flow/blob/master/tests/functional/README.md) | The README files in each directory explain what sort of tests should go into which directory, how to run and debug these tests. cylc-flow-8.7.0/tests/u0000777000175000017500000000000015264420307016121 2unit/ustar alastairalastaircylc-flow-8.7.0/tests/unit/0000775000175000017500000000000015264420307015735 5ustar alastairalastaircylc-flow-8.7.0/tests/unit/test_task_outputs.py0000664000175000017500000003234315264420307022120 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from types import SimpleNamespace from typing import ( Optional, Set, ) import pytest from cylc.flow.task_outputs import ( TASK_OUTPUT_EXPIRED, TASK_OUTPUT_FAILED, TASK_OUTPUT_SUBMIT_FAILED, TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_SUCCEEDED, TASK_OUTPUTS, TaskOutputs, get_completion_expression, get_trigger_completion_variable_maps, ) from cylc.flow.util import sstrip def tdef(required, optional, completion=None): """Stub a task definition. Args: required: Collection of required outputs. optional: Collection of optional outputs. completion: User defined execution completion expression. """ return SimpleNamespace( rtconfig={ 'completion': completion, }, outputs={ output: ( output, ( # output is required: True if output in required # output is optional: else False if output in optional # output is ambiguous (i.e. not referenced in graph): else None ) ) for output in set(TASK_OUTPUTS) | set(required) | set(optional) }, ) def test_completion_implicit(): """It should generate a completion expression when none is provided. The outputs should be considered "complete" according to the logic in proposal point 5: https://cylc.github.io/cylc-admin/proposal-optional-output-extension.html#proposal """ # one required output - succeeded outputs = TaskOutputs(tdef([TASK_OUTPUT_SUCCEEDED], [])) # the completion expression should only contain the one required output assert outputs._completion_expression == 'succeeded' # the outputs should be incomplete - it hasn't run yet assert outputs.is_complete() is False # set the submit-failed output outputs.set_message_complete(TASK_OUTPUT_SUBMIT_FAILED) # the outputs should be incomplete - submited-failed is a "final" output assert outputs.is_complete() is False # set the submitted and succeeded outputs outputs.set_message_complete(TASK_OUTPUT_SUBMITTED) outputs.set_message_complete(TASK_OUTPUT_SUCCEEDED) # the outputs should be complete - it has run an succeedd assert outputs.is_complete() is True # set the expired output outputs.set_message_complete(TASK_OUTPUT_EXPIRED) # the outputs should still be complete - it has run and succeeded assert outputs.is_complete() is True def test_completion_explicit(): """It should use the provided completion expression. The outputs should be considered "complete" according to the logic in proposal point 5: https://cylc.github.io/cylc-admin/proposal-optional-output-extension.html#proposal """ outputs = TaskOutputs(tdef( # no required outputs [], # four optional outputs [ TASK_OUTPUT_SUCCEEDED, TASK_OUTPUT_FAILED, 'x', 'y', ], # one pair must be satisfied for the outputs to be complete completion='(succeeded and x) or (failed and y)', )) # the outputs should be incomplete - it hasn't run yet assert outputs.is_complete() is False # set the succeeded and failed outputs outputs.set_message_complete(TASK_OUTPUT_SUCCEEDED) outputs.set_message_complete(TASK_OUTPUT_FAILED) # the task should be incomplete - it has executed but the completion # expression is not satisfied assert outputs.is_complete() is False # satisfy the (failed and y) pair outputs.set_message_complete('y') assert outputs.is_complete() is True # satisfy the (succeeded and x) pair outputs._completed['y'] = False outputs.set_message_complete('x') assert outputs.is_complete() is True def test_is_message_complete(): """It should report the completion status of individual messages.""" outputs = TaskOutputs(tdef([TASK_OUTPUT_SUCCEEDED], [])) # message exists but hasn't been completed yet assert outputs.is_message_complete(TASK_OUTPUT_SUCCEEDED) is False # message doesn't exist in this task's outputs assert outputs.is_message_complete('no-such-output') is None # message has been completed outputs.set_message_complete(TASK_OUTPUT_SUCCEEDED) assert outputs.is_message_complete(TASK_OUTPUT_SUCCEEDED) is True @pytest.mark.parametrize( 'required, optional, expression', [ pytest.param( {TASK_OUTPUT_SUCCEEDED}, [], 'succeeded', id='0', ), pytest.param( {TASK_OUTPUT_SUCCEEDED, 'x'}, [], '(succeeded and x)', id='1', ), pytest.param( [], {TASK_OUTPUT_SUCCEEDED}, 'succeeded or failed', id='2', ), pytest.param( {TASK_OUTPUT_SUCCEEDED}, {TASK_OUTPUT_EXPIRED}, 'succeeded or expired', id='3', ), pytest.param( [], {TASK_OUTPUT_SUCCEEDED, TASK_OUTPUT_EXPIRED}, 'succeeded or failed or expired', id='4', ), pytest.param( {TASK_OUTPUT_SUCCEEDED}, {TASK_OUTPUT_EXPIRED, TASK_OUTPUT_SUBMITTED}, 'succeeded or submit_failed or expired', id='5', ), pytest.param( {TASK_OUTPUT_SUCCEEDED, TASK_OUTPUT_SUBMITTED}, {TASK_OUTPUT_EXPIRED}, '(submitted and succeeded) or expired', id='6', ), pytest.param( [], {TASK_OUTPUT_SUCCEEDED, TASK_OUTPUT_SUBMIT_FAILED}, 'succeeded or failed or submit_failed', id='7', ), pytest.param( {'x'}, { TASK_OUTPUT_SUCCEEDED, TASK_OUTPUT_SUBMIT_FAILED, TASK_OUTPUT_EXPIRED, }, '(x and succeeded) or failed or submit_failed or expired', id='8', ), ], ) def test_get_completion_expression_implicit(required, optional, expression): """It should generate a completion expression if none is provided.""" assert get_completion_expression(tdef(required, optional)) == expression def test_get_completion_expression_explicit(): """If a completion expression is used, it should be used unmodified.""" assert get_completion_expression(tdef( {'x', 'y'}, {TASK_OUTPUT_SUCCEEDED, TASK_OUTPUT_FAILED, TASK_OUTPUT_EXPIRED}, '((failed and x) or (succeeded and y)) or expired' )) == '((failed and x) or (succeeded and y)) or expired' def test_format_completion_status(): """It should format the completion status of the outputs.""" outputs = TaskOutputs( tdef( {TASK_OUTPUT_SUCCEEDED, 'x', 'y'}, {TASK_OUTPUT_EXPIRED}, ) ) assert outputs.format_completion_status( indent=2, gutter=2 ) == ' ' + sstrip( ''' ┆ ( ⨯ ┆ succeeded ⨯ ┆ and x ⨯ ┆ and y ┆ ) ⨯ ┆ or expired ''' ) outputs.set_message_complete('succeeded') outputs.set_message_complete('x') assert outputs.format_completion_status( indent=2, gutter=2 ) == ' ' + sstrip( ''' ┆ ( ✓ ┆ succeeded ✓ ┆ and x ⨯ ┆ and y ┆ ) ⨯ ┆ or expired ''' ) def test_format_completion_status_nested(): """ It should format nested completion expressions with operators. Catches the case where the opening paren is preceded by an operator. """ outputs = TaskOutputs( tdef( [], [TASK_OUTPUT_SUCCEEDED, TASK_OUTPUT_FAILED, 'x', 'y'], completion='succeeded or (failed and x)', ) ) result = outputs.format_completion_status(indent=2, gutter=2) # the opening paren should be preceded by the 'or' operator assert 'or (' in result def test_format_completion_status_ansimarkup(): """It should colour-code output status when ansimarkup is enabled.""" outputs = TaskOutputs( tdef({TASK_OUTPUT_SUCCEEDED}, {TASK_OUTPUT_EXPIRED}) ) outputs.set_message_complete(TASK_OUTPUT_SUCCEEDED) # ansimarkup=1: only completed outputs get colour result = outputs.format_completion_status(indent=2, gutter=2, ansimarkup=1) assert 'succeeded' in result assert '' not in result # incomplete outputs are NOT coloured # ansimarkup=2: both complete and incomplete get colour result = outputs.format_completion_status(indent=2, gutter=2, ansimarkup=2) assert 'succeeded' in result assert 'expired' in result def test_is_compvar_complete_invalid(): """It should raise KeyError for an unknown completion variable.""" outputs = TaskOutputs(tdef([TASK_OUTPUT_SUCCEEDED], [])) with pytest.raises(KeyError): outputs._is_compvar_complete('no_such_var') @pytest.mark.parametrize( 'required, optional, expected_required, expected_expression', [ # this task has three required outputs and one optional output pytest.param( {TASK_OUTPUT_SUCCEEDED, 'x', 'y'}, {'z'}, {TASK_OUTPUT_SUCCEEDED, 'x', 'y'}, None, id="3-required-1-optional", ), # this task does not have any required outputs (besides the implicitly # required submitted/started outputs) # Note: validation should prevent this at the config level pytest.param( {TASK_OUTPUT_SUCCEEDED, 'x', 'y'}, {TASK_OUTPUT_FAILED}, # task may fail set(), None, id="no-required-outputs", ), # the preconditions expiry/submitted are excluded from this logic when # defined as optional: pytest.param( {TASK_OUTPUT_SUCCEEDED, 'x', 'y'}, {TASK_OUTPUT_EXPIRED}, # task may expire {TASK_OUTPUT_SUCCEEDED, 'x', 'y'}, '(succeeded and x and y) or expired', id="expiry-submitted", ), # NOTE: a required output might not be required! # If success is optional, then apparently-required outputs are made # implicitly optional. See # https://github.com/cylc/cylc-flow/pull/6505#issuecomment-2517781523 pytest.param( {'x'}, {TASK_OUTPUT_SUCCEEDED}, set(), '(x and succeeded) or failed', id="implicit-optional", ), pytest.param( set(), {'x', TASK_OUTPUT_SUCCEEDED}, set(), 'succeeded or failed', id="all-optional", ), ] ) def test_iter_required_outputs( required: Set[str], optional: Set[str], expected_required: Set[str], expected_expression: Optional[str], ): """It should yield required outputs only.""" outputs = TaskOutputs(tdef(required, optional)) if expected_expression: assert outputs._completion_expression == expected_expression assert set(outputs.iter_required_messages()) == expected_required def test_iter_required_outputs__disable(): # Get all outputs required for success path (excluding failure, what # is still required): outputs = TaskOutputs( tdef( {}, {'a', 'succeeded', 'b', 'y', 'failed', 'x'}, '(x and y and failed) or (a and b and succeeded)' ) ) assert set(outputs.iter_required_messages()) == set() # Disabling succeeded leaves us with failure required outputs: assert set( outputs.iter_required_messages(disable=TASK_OUTPUT_SUCCEEDED) ) == { TASK_OUTPUT_FAILED, 'x', 'y', } # Disabling failed leaves us with succeeded required outputs: assert set(outputs.iter_required_messages(disable=TASK_OUTPUT_FAILED)) == { TASK_OUTPUT_SUCCEEDED, 'a', 'b', } # Disabling an arbitrary output leaves us with required outputs # from another branch: assert set(outputs.iter_required_messages(disable='a')) == { TASK_OUTPUT_FAILED, 'x', 'y', } def test_get_trigger_completion_variable_maps(): """It should return a bi-map of triggers to compvars.""" t2c, c2t = get_trigger_completion_variable_maps(('a', 'b-b', 'c-c-c')) assert t2c == {'a': 'a', 'b-b': 'b_b', 'c-c-c': 'c_c_c'} assert c2t == {'a': 'a', 'b_b': 'b-b', 'c_c_c': 'c-c-c'} cylc-flow-8.7.0/tests/unit/test_syntax_generator.py0000664000175000017500000000360615264420307022747 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import importlib from pathlib import Path # Fake up the module. path = Path(__file__).parent.parent.parent / 'etc/bin/syntax_generator.py' loader = importlib.machinery.SourceFileLoader( 'syntaxgenerator', str(path) ) syntax = loader.load_module() def test_get_keywords_from_workflow_cfg(): """It gets a list of config items. Not a thorough check, but ensure type is sensible and some unlikely-to change items are present. """ result = syntax.get_keywords_from_workflow_cfg() assert isinstance(result, list) assert 'meta' in result assert 'scheduling' in result assert 'cycle point format' in result def test_update_cylc_lang_new_section(tmp_path): test_file = tmp_path / 'testfile' test_file.write_text(( "\n" "some stuff" "" )) syntax.update_cylc_lang( ['gamma', 'nu'], test_file, '#kword#{word}#kword# - ' ) assert test_file.read_text() == ( "\n#kword#gamma#kword#" " - #kword#nu#kword# - " "" ) cylc-flow-8.7.0/tests/unit/test_filter.py0000664000175000017500000000606515264420307020642 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test set for the filter_keys function. Tests a series of different keys to evaluate whether they get the correct possible output. """ import pytest from cylc.flow.parsec.util import filter_keys # A real example of a list of 40 possible keys REAL_TEST_DATA: list[str] = ['completion', 'platform', 'inherit', 'script', 'init-script', 'env-script', 'err-script', 'exit-script', 'pre-script', 'post-script', 'work sub-directory', 'execution polling intervals', 'execution retry delays', 'execution time limit', 'submission polling intervals', 'submission retry delays', 'run mode', 'meta', 'skip', 'simulation', 'environment filter', 'job', 'remote', 'events', 'mail', 'workflow state polling', 'environment', 'directives', 'outputs', 'parameter environment templates'] # The list of different key inputs and expected outputs for that key. @pytest.mark.parametrize( 'key, expected', [("execution retry delays", ['execution retry delays']), ("retry delays", ['execution retry delays', 'submission retry delays', 'parameter environment templates']), ("execution retry SHED delays", ['execution retry delays']), ("execution retry delays SHED", ['execution retry delays']), ("execution delays retry", ['execution retry delays']), ("execution retrydelays", ['execution retry delays']), ("", []), ("execution retry delays", ['execution retry delays']), ("execution retry delays", ['execution retry delays']), ("executionee retry edelays", ['execution retry delays']), ("execution", ['execution polling intervals', 'execution retry delays', 'execution time limit']), ("j", ['job']), ("retr delays", ['execution retry delays', 'submission retry delays', 'parameter environment templates']), ] ) def test_filter_key(key, expected): filtered_keys = filter_keys(REAL_TEST_DATA, key) assert filtered_keys == expected cylc-flow-8.7.0/tests/unit/scripts/0000775000175000017500000000000015264420307017424 5ustar alastairalastaircylc-flow-8.7.0/tests/unit/scripts/test_lint.py0000664000175000017500000005166515264420307022020 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests `cylc lint` CLI Utility.""" from collections import Counter import logging from pathlib import Path from pprint import pformat import re from textwrap import dedent from types import SimpleNamespace import pytest from pytest import param from cylc.flow.scripts.lint import ( LINT_SECTION, MANUAL_DEPRECATIONS, check_lowercase_family_names, get_cylc_files, get_pyproject_toml, get_reference, get_upgrader_info, lint, _merge_cli_with_tomldata, parse_checks, validate_toml_items ) from cylc.flow.exceptions import CylcError STYLE_CHECKS = parse_checks(['style']) UPG_CHECKS = parse_checks(['728']) TEST_FILE = ''' [visualization] [cylc] include at start-up = foo exclude at start-up = bar reset timer = false log resolved dependencies = True required run mode = False health check interval = PT10M abort if any task fails = true suite definition directory = '/woo' disable automatic shutdown = false spawn to max active cycle points = false [[reference test]] allow task failures = true [[simulation]] disable suite event handlers = true [[authentication]] [[environment]] force run mode = dummy [[events]] reset inactivity timer = 42 abort on stalled = True abort on timeout = False abort if startup handler fails= True # deliberately not added a space. abort if shutdown handler fails= True abort if timeout handler fails = True abort if stalled handler fails = True abort if inactivity handler fails = False aborted handler = woo stalled handler = bar timeout handler = bas shutdown handler = qux startup handler = now inactivity handler = bored mail to = eleanor.rigby@beatles.lv mail from = fr.mckenzie@beatles.lv mail footer = "Collecting The Rice" mail smtp = 123.456.789.10 timeout = 30 inactivity = 30 abort on inactivity = 30 [[parameters]] [[parameter templates]] [[mail]] task event mail interval = PT4M # deliberately added lots of spaces. [scheduling] max active cycle points = 5 hold after point = 20220101T0000Z [[dependencies]] [[[R1]]] graph = """ MyFaM:finish-all => remote => !mash_theme a & \\ b => c c | \\ d => e """ [runtime] [[root]] [[[environment]]] CYLC_VERSION={{CYLC_VERSION}} ROSE_VERSION = {{ROSE_VERSION }} FCM_VERSION = {{ FCM_VERSION }} [[MyFaM]] extra log files = True {% from 'cylc.flow' import LOG %} pre-script = "echo ${CYLC_SUITE_DEF_PATH}" script = {{HELLOWORLD}} post-script = "echo ${CYLC_SUITE_INITIAL_CYCLE_TIME}" env-script = POINT=$(rose date 2059 --offset P1M) [[[suite state polling]]] template = and [[[remote]]] host = parasite suite definition directory = '/home/bar' [[[job]]] batch system = slurm shell = fish [[[events]]] mail retry delays = PT30S warning handler = frr.sh submission timeout handler = faiuhdf submission retry handler = vhbayhrbfgau submission failed handler = giaSEHFUIHJ failed handler = woo execution timeout handler = sdfghjkl expired handler = %(suite_uuid)s %(user@host)s late handler = dafuhj submitted handler = dafuhj started handler = dafuhj succeeded handler = dafuhj custom handler = efrgh critical handler = fgjdsfs retry handler = dfaiuhfrgpa submission handler = fas9hrfgaiuph # Shouldn't object to a comment, unlike the terrible indents below: [[bad indent]] inherit = MyFaM [[remote]] platform = $(rose host-select parasite) script = "cylc nudge" post-script = "rose suite-hook" [meta] [[and_another_thing]] [[[remote]]] host = `rose host-select thingy` %include foo.cylc ''' LINT_TEST_FILE = ''' \t[scheduler] [scheduler] [[dependencies]] {% set N = 009 %} {% foo %} {{foo}} # {{quix}} R1 = """ foo & \\ bar => \\ baz """ [runtime] [[this_is_ok]] script = echo "this is incorrectly indented" [[foo]] inherit = hello [[[job]]] something\t [[bar]] platform = $(some-script foo) [[[directives]]] -l walltime = 666 [[baz]] run mode = skip platform = `no backticks` [[[skip]]] outputs = succeeded, failed ''' + ( '\nscript = the quick brown fox jumps over the lazy dog until it becomes ' 'clear that this line is longer than the default 130 character limit.' ) def lint_text(text, checks, ignores=None, modify=False): checks = parse_checks(checks, ignores) counter = Counter() messages = [] outlines = list( lint( Path('flow.cylc'), iter(text.splitlines()), checks, counter, modify=modify, write=messages.append, ) ) return SimpleNamespace( counter=counter, messages=messages, outlines=outlines ) def filter_strings(items, contains): """Return only items which contain a given string.""" return [ message for message in items if contains in message ] def assert_contains(items, contains, instances=None): """Pass if at least one item contains a given string.""" filtered = filter_strings(items, contains) if not filtered: raise Exception( f'Could not find: "{contains}" in:\n' + pformat(items)) elif instances and len(filtered) != instances: raise Exception( f'Expected "{contains}" to appear {instances} times' f', got it {len(filtered)} times.') EXPECT_INSTANCES_OF_ERR = { 16: 3, } @pytest.mark.parametrize( # 11 won't be tested because there is no jinja2 shebang 'number', set(range(1, len(MANUAL_DEPRECATIONS) + 1)) - {11} ) def test_check_cylc_file_7to8(number): """TEST File has one of each manual deprecation;""" lint = lint_text(TEST_FILE, ['728']) instances = EXPECT_INSTANCES_OF_ERR.get(number, None) # U0008 Has been removed, but we don't want to change the numbering if number != 8: assert_contains(lint.messages, f'[U{number:03d}]', instances) def test_check_cylc_file_7to8_has_shebang(): """Jinja2 code comments will not be added if shebang present""" lint = lint_text('#!jinja2\n{{FOO}}', '', '[scheduler]') assert not lint.counter def test_check_cylc_file_line_no(): """It prints the correct line numbers""" lint = lint_text(TEST_FILE, ['728']) # the first message should be for line number 2 (line is a shebang) assert ':2:' in lint.messages[0] @pytest.mark.parametrize( 'line', [ # lowercase family names are not permitted 'inherit = g', 'inherit = FOO, bar', 'inherit = None, bar', 'inherit = A, b, C', 'inherit = "A", "b"', "inherit = 'A', 'b'", 'inherit = FOO_BAr', # whitespace & trailing commas ' inherit = a , ', # parameters, templating code should be ignored # but any lowercase chars before or after should not 'inherit = Az', 'inherit = A{{ x }}z', 'inherit = N{# #}one', 'inherit = A@( x )z', ] ) def test_check_lowercase_family_names__true(line): assert check_lowercase_family_names(line) is True @pytest.mark.parametrize( 'line', [ # undefined values are ok 'inherit =', 'inherit = ', # none, None and root are ok 'inherit = none', 'inherit = None', 'inherit = root', # whitespace & trailing commas 'inherit = None,', 'inherit = None, ', ' inherit = None , ', # uppercase family names are ok 'inherit = None, FOO, BAR', 'inherit = FOO', 'inherit = FOO_BAR_0', # parameters should be ignored 'inherit = AZ', 'inherit = ', # jinja2 should be ignored param( 'inherit = A{{ a }}Z, {% for x in range(5) %}' 'A{{ x }}, {% endfor %}', id='jinja2-long' ), # trailing comments should be ignored 'inherit = A, B # no, comment', 'inherit = # a', # quotes are ok 'inherit = "A", "B"', "inherit = 'A', 'B'", 'inherit = "None", B', 'inherit = ', # one really awkward, but valid example param( 'inherit = none, FOO_BAR_0, "", AZ, A{{a}}Z', id='awkward' ), ] ) def test_check_lowercase_family_names__false(line): assert check_lowercase_family_names(line) is False def test_inherit_lowercase_matches(): lint = lint_text('inherit = a', ['style']) assert any('S007' in msg for msg in lint.messages) @pytest.mark.parametrize( # 8 and 11 won't be tested because there is no jinja2 shebang 'number', set(range(1, len(STYLE_CHECKS) + 1)) - {8, 11} ) def test_check_cylc_file_lint(number): lint = lint_text(LINT_TEST_FILE, ['style']) assert_contains(lint.messages, f'S{number:03d}') @pytest.mark.parametrize('code', STYLE_CHECKS.keys()) def test_check_exclusions(code): """It does not report any items excluded.""" lint = lint_text(LINT_TEST_FILE, ['style'], [code]) assert not filter_strings(lint.messages, code) def test_check_cylc_file_jinja2_comments(): """Jinja2 inside a Jinja2 comment should not warn""" lint = lint_text('#!jinja2\n{# {{ foo }} #}', ['style']) assert not any('S011' in msg for msg in lint.messages) def test_check_cylc_file_jinja2_comments_shell_arithmetic_not_warned(): """Jinja2 after a $((10#$variable)) should not warn""" lint = lint_text('#!jinja2\na = b$((10#$foo+5)) {{ BAR }}', ['style']) assert not any('S011' in msg for msg in lint.messages) @pytest.mark.parametrize( # 11 won't be tested because there is no jinja2 shebang 'number', set(range(1, len(MANUAL_DEPRECATIONS) + 1)) - {11} ) def test_check_cylc_file_inplace(number): lint = lint_text(TEST_FILE, ['728', 'style'], modify=True) # U0008 Has been removed, but we don't want to change the numbering if number != 8: assert_contains(lint.outlines, f'[U{number:03d}]') def test_get_cylc_files_get_all_rcs(tmp_path): """It returns all paths except `log/**`. """ expect = [('etc', 'foo.rc'), ('bin', 'foo.rc'), ('an_other', 'foo.rc')] # Create a fake run directory, including the log file which should not # be searched: dirs = ['etc', 'bin', 'log', 'an_other', 'log/skarloey/'] for path in dirs: thispath = tmp_path / path thispath.mkdir(parents=True) (thispath / 'foo.rc').touch() # Run the test result = [(i.parent.name, i.name) for i in get_cylc_files(tmp_path)] assert sorted(result) == sorted(expect) def mock_parse_checks(*args, **kwargs): return { 'U042': { 'short': 'section `[vizualization]` has been removed.', 'url': 'some url or other', 'purpose': 'U', 'rst': 'section ``[vizualization]`` has been removed.', 'function': re.compile('not a regex') }, } def test_get_reference_rst(monkeypatch): """It produces a reference file for our linting.""" monkeypatch.setattr( 'cylc.flow.scripts.lint.parse_checks', mock_parse_checks ) ref = get_reference('all', 'rst') expect = ( '\n7 to 8 upgrades\n---------------\n\n' '`U042 `_' f'\n{"^" * 78}' '\nsection ``[vizualization]`` has been ' 'removed.\n\n\n' ) assert ref == expect def test_get_reference_text(monkeypatch): """It produces a reference file for our linting.""" monkeypatch.setattr( 'cylc.flow.scripts.lint.parse_checks', mock_parse_checks ) ref = get_reference('all', 'text') expect = ( '\n7 to 8 upgrades\n---------------\n\n' 'U042:\n section `[vizualization]` has been ' 'removed.' '\n https://cylc.github.io/cylc-doc/stable/html/7-to-8/some' ' url or other\n\n\n' ) assert ref == expect @pytest.fixture() def fixture_get_deprecations(): """Get the deprections list for cylc.flow.cfgspec.workflow""" deprecations = get_upgrader_info() return deprecations @pytest.mark.parametrize( 'findme', [ pytest.param( 'abort if any task fails =', id='Item not available at Cylc 8' ), pytest.param( 'timeout =', id='Item renamed at Cylc 8' ), pytest.param( '!execution retry delays', id='Item moved, name unchanged at Cylc 8' ), pytest.param( '[cylc]', id='Section changed at Cylc 8' ), ] ) def test_get_upg_info(fixture_get_deprecations, findme): """It correctly scrapes the Cylc upgrader object. n.b this is just sampling to ensure that the test it getting items. """ if findme.startswith('!'): assert not any( i['function'](findme) for i in fixture_get_deprecations.values() ) else: assert any( i['function'](findme) for i in fixture_get_deprecations.values() ) is True @pytest.mark.parametrize( 'settings, expected', [ param( """ rulesets = ['style'] ignore = ['S004'] exclude = ['sites/*.cylc'] """, { 'rulesets': ['style'], 'ignore': ['S004'], 'exclude': ['sites/*.cylc'], 'max-line-length': None, }, id="returns what we want" ), param( """ northgate = ['sites/*.cylc'] mons-meg = 42 """, (CylcError, ".*northgate"), id="invalid settings fail validation" ), param( "max-line-length = 22", { 'exclude': [], 'ignore': [], 'rulesets': [], 'max-line-length': 22, }, id='sets max line length' ) ] ) def test_get_pyproject_toml(tmp_path, settings, expected): """It returns only the lists we want from the toml file.""" tomlcontent = "[tool.cylc.lint]\n" + dedent(settings) (tmp_path / 'pyproject.toml').write_text(tomlcontent) if isinstance(expected, tuple): exc, match = expected with pytest.raises(exc, match=match): get_pyproject_toml(tmp_path) else: assert get_pyproject_toml(tmp_path) == expected @pytest.mark.parametrize( 'tomlfile', [None, '', '[tool.cylc.lint]', '[cylc-lint]'] ) def test_get_pyproject_toml_returns_blank(tomlfile, tmp_path): if tomlfile is not None: tfile = (tmp_path / 'pyproject.toml') tfile.write_text(tomlfile) expect = { 'exclude': [], 'ignore': [], 'max-line-length': None, 'rulesets': [] } assert get_pyproject_toml(tmp_path) == expect def test_get_pyproject_toml__depr( tmp_path: Path, caplog: pytest.LogCaptureFixture ): """It warns if the section is deprecated.""" file = tmp_path / 'pyproject.toml' caplog.set_level(logging.WARNING) file.write_text(f'[{LINT_SECTION}]\nmax-line-length=14') assert get_pyproject_toml(tmp_path)['max-line-length'] == 14 assert not caplog.text file.write_text('[cylc-lint]\nmax-line-length=17') assert get_pyproject_toml(tmp_path)['max-line-length'] == 17 assert "[cylc-lint] section in pyproject.toml is deprecated" in caplog.text @pytest.mark.parametrize( 'input_, output', [ param( {'exclude': ['hey', 'there', 'Delilah']}, True, id='it works' ), param( {'foo': ['hey', 'there', 'Delilah', 42]}, 'allowed', id='it fails with illegal section name' ), param( {'exclude': 'woo!'}, 'should be a list, but', id='it fails with illegal section type' ), param( {'exclude': ['hey', 'there', 'Delilah', 42]}, 'should be a string', id='it fails with illegal value name' ), param( {'rulesets': ['hey']}, 'hey not valid: Rulesets can be', id='it fails with illegal ruleset' ), param( {'ignore': ['hey']}, 'hey not valid: Ignore codes', id='it fails with illegal ignores' ), param( {'ignore': ['R999']}, 'R999 is a not a known linter code.', id='it fails with non-existent checks ignored' ), param( {'ignore': ['U008']}, 'warn', id='valid, but deprecated linter code' ), ] ) def test_validate_toml_items(input_, output, caplog): """It chucks out the wrong sort of items.""" if output not in [True, 'warn']: with pytest.raises(CylcError, match=output): validate_toml_items(input_) elif output is True: assert validate_toml_items(input_) is output elif output == 'warn': assert validate_toml_items(input_) is True assert 'U008 is a deprecated linter code' in caplog.messages @pytest.mark.parametrize( 'clidata, tomldata, expect', [ param( { 'rulesets': ['foo', 'bar'], 'ignore': ['R101'], }, { 'rulesets': ['baz'], 'ignore': ['R100'], 'exclude': ['not_me-*.cylc'] }, { 'rulesets': ['foo', 'bar'], 'ignore': ['R100', 'R101'], 'exclude': ['not_me-*.cylc'], 'max-line-length': None }, id='It works with good path' ), ] ) def test_merge_cli_with_tomldata(clidata, tomldata, expect): """It merges each of the three sections correctly: see function.__doc__""" assert _merge_cli_with_tomldata(clidata, tomldata) == expect def test_invalid_tomlfile(tmp_path): """It fails nicely if pyproject.toml is malformed""" tomlfile = (tmp_path / 'pyproject.toml') tomlfile.write_text('foo :{}') expected_msg = 'pyproject.toml did not load:' with pytest.raises(CylcError, match=expected_msg): get_pyproject_toml(tmp_path) @pytest.mark.parametrize( 'ref, expect', [ [True, 'line > ```` characters'], [False, 'line > 42 characters'] ] ) def test_parse_checks_reference_mode(ref, expect): """Add extra explanation of max line length setting in reference mode. """ result = parse_checks(['style'], reference=ref, max_line_len=42) value = result['S012'] assert expect in value['short'] @pytest.mark.parametrize( 'spaces, expect', ( (0, 'S002'), (1, 'S013'), (2, 'S013'), (3, 'S013'), (4, None), (5, 'S013'), (6, 'S013'), (7, 'S013'), (8, None), (9, 'S013') ) ) def test_indents(spaces, expect): """Test different wrong indentations Parameterization deliberately over-obvious to avoid replicating arithmetic logic from code. Dangerously close to re-testing ``%`` builtin. """ result = lint_text( f"{' ' * spaces}foo = 42", ['style'] ) result = ''.join(result.messages) if expect: assert expect in result else: assert not result def test_noqa(): """Comments turn of checks. """ output = lint_text( 'foo = bar#noqa\n' 'qux = baz # noqa: S002\n' 'buzz = food # noqa: S007\n' 'quixotic = foolish # noqa: S007, S992 S002\n', ['style'] ) assert len(output.messages) == 1 assert 'flow.cylc:3' in output.messages[0] cylc-flow-8.7.0/tests/unit/scripts/test_validate_reinstall_units.py0000664000175000017500000000334315264420307026130 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests for cylc.flow.scripts.validate_reinstall.py """ import pytest from cylc.flow.scripts.validate_reinstall import ( check_tvars_and_workflow_stopped) @pytest.mark.parametrize( 'is_running, tvars, tvars_file, expect', [ (True, [], None, True), (True, ['FOO="Bar"'], None, False), (True, [], ['bar.txt'], False), (True, ['FOO="Bar"'], ['bar.txt'], False), (False, [], None, True), (False, ['FOO="Bar"'], ['bar.txt'], True), (False, [], ['bar.txt'], True), (False, ['FOO="Bar"'], ['bar.txt'], True), ] ) def test_check_tvars_and_workflow_stopped( caplog, is_running, tvars, tvars_file, expect ): """It returns true if workflow is running and tvars or tvars_file is set. """ result = check_tvars_and_workflow_stopped(is_running, tvars, tvars_file) assert result == expect if expect is False: warn = 'can only be changed if' assert warn in caplog.records[0].msg cylc-flow-8.7.0/tests/unit/scripts/test_stop.py0000664000175000017500000000514415264420307022026 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test logic in cylc-stop script.""" import pytest from typing import TYPE_CHECKING, Optional, Tuple, Type from cylc.flow.exceptions import InputError from cylc.flow.option_parsers import Options from cylc.flow.scripts.stop import get_option_parser, _validate if TYPE_CHECKING: from optparse import Values Opts = Options(get_option_parser()) @pytest.mark.parametrize( 'options, stop_task, stop_cycle, globs, expected_err', [ ( Opts(), None, None, None, None, ), ( Opts(kill=True), None, '10', None, (InputError, "--kill is not compatible with stop-cycle") ), ( Opts(), '10/foo', '10', None, (InputError, "stop-task is not compatible with stop-cycle") ), ( Opts(kill=True, now=True), None, None, None, (InputError, "--kill is not compatible with --now") ), ( Opts(flow_num=2, max_polls=2), None, None, None, (InputError, "--flow is not compatible with --max-polls") ), ( Opts(flow_num=2), None, None, '*', (InputError, "--flow is not compatible with task IDs") ), ] ) def test_validate( options: 'Values', stop_task: str, stop_cycle: str, globs: str, expected_err: Optional[Tuple[Type[Exception], str]]): if expected_err: err, msg = expected_err with pytest.raises(err) as exc: _validate(options, stop_task, stop_cycle, globs) assert msg in str(exc.value) else: _validate(options, stop_task, stop_cycle, globs) cylc-flow-8.7.0/tests/unit/scripts/test_reinstall.py0000664000175000017500000000437015264420307023036 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from textwrap import dedent from ansimarkup import parse as cparse import pytest from cylc.flow.scripts.reinstall import format_reinstall_output from cylc.flow.terminal import DIM @pytest.mark.parametrize('verbose', [True, False]) def test_format_reinstall_output(verbose, monkeypatch: pytest.MonkeyPatch): """It should: - colorize the output - remove the itemized changes summary if not in verbose mode - remove the "cannot delete non-empty directory" message """ output = dedent(""" *deleting del. Cloud.jpg >f+++++++++ send cloud.jpg .f...p..... send foo >fcsTp..... send bar cannot delete non-empty directory: scarf >f+++++++++ send meow.txt cL+++++++++ send garage -> foo """).strip() expected = [ f"<{DIM}>*deleting del. Cloud.jpg", f"<{DIM}>>f+++++++++ send cloud.jpg", f"<{DIM}>.f...p..... send foo", f"<{DIM}>>fcsTp..... send bar", f"<{DIM}>>f+++++++++ send meow.txt", f"<{DIM}>cL+++++++++ send garage -> foo", ] if verbose: monkeypatch.setattr('cylc.flow.flags.verbosity', 1) else: # itemized changes summary should not be in output shift = len(f'<{DIM}> ') + 11 expected = [i[shift:] for i in expected] assert format_reinstall_output(output) == [cparse(i) for i in expected] cylc-flow-8.7.0/tests/unit/scripts/test_validate.py0000664000175000017500000000000015264420307022614 0ustar alastairalastaircylc-flow-8.7.0/tests/unit/scripts/test_completion_server.py0000664000175000017500000005503115264420307024600 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from types import SimpleNamespace from cylc.flow.async_util import pipe from cylc.flow.id import Tokens from cylc.flow.network.scan import scan from cylc.flow.scripts.completion_server import ( _list_prereqs_and_outputs, server, complete_cylc, complete_command, complete_option, complete_option_value, complete_argument, list_cylc_id, list_options, list_option_values, list_workflows, list_src_workflows, list_in_workflow, list_resources, list_dir, list_flows, list_colours, cli_detokenise, get_completion_script_file, get_current_completion_script_version, check_completion_script_compatibility, COMMANDS, ) from cylc.flow.scripts.trigger import get_option_parser import pytest def setify(coro): """Cast returned lists to sets for coroutines. Convenience function to use when you want to test output not order. """ async def _coro(*args, **kwargs): ret = await coro(*args, **kwargs) if isinstance(ret, list): return set(ret) return ret return _coro @pytest.fixture def dummy_workflow(tmp_path, monkeypatch, mock_glbl_cfg): """A simple workflow run dir with some job directories to inspect. This patches the relevant interfaces so that this workflow will show up e.g. in the scan interface. """ install_dir = tmp_path / 'foo' install_dir.mkdir() (install_dir / 'run1').mkdir() run_dir = install_dir / 'run2' run_dir.mkdir() (install_dir / 'runN').symlink_to('run2', target_is_directory=True) (run_dir / 'flow.cylc').touch() job_log_dir = (run_dir / 'log') / 'job' job_log_dir.mkdir(parents=True) for cycle in ('1', '2', '3'): for task in ('foo', 'bar', 'baz'): for job in ('01', 'NN'): ((job_log_dir / cycle) / task / job).mkdir(parents=True) # patch scan for list_workflows @pipe async def _scan(*args, **kwargs): kwargs['run_dir'] = tmp_path async for flow in scan(*args, **kwargs): yield flow monkeypatch.setattr( 'cylc.flow.scripts.scan.scan', _scan, ) # patch scan for list_src_workflows mock_glbl_cfg( 'cylc.flow.scripts.completion_server.glbl_cfg', f''' [install] source dirs = {tmp_path} ''' ) # patch get_workflow_run_job_dir for list_in_workflow monkeypatch.setattr( 'cylc.flow.pathutil._CYLC_RUN_DIR', tmp_path, ) async def test_server(): """Test the request/response server.""" def _listener(timeout=None): """The listener yields requests.""" yield 'cylc|trigger|' yield 'cylc|trigger|one|' async def _responder(*parts): """The responder computes responses.""" return [f'x{part}' for part in parts] # in "once" mode the server shuts down after returning the first response ret = [] await server(_listener, _responder, once=True, write=ret.append) assert ret == [ 'xcylc xtrigger' ] # otherwise the server stays up until the listener returns ret = [] await server(_listener, _responder, once=False, write=ret.append) assert ret == [ 'xcylc xtrigger', 'xcylc xone xtrigger', # the server sorts the responses ] # if *anything* goes wrong the server should send an empty response # rather than crashing async def _responder(*parts): raise Exception() ret = [] await server(_listener, _responder, once=True, write=ret.append) assert ret == [''] async def test_complete_cylc(dummy_workflow): """Test the completion for everything Cylc. Each individual completion function is tested individually too. This test is to ensure they work together correctly. """ _complete_cylc = setify(complete_cylc) # results are un-ordered # $ cylc assert 'trigger' in await _complete_cylc('cylc') assert 'help' in await _complete_cylc('cylc') # $ cylc tri assert 'trigger' in await _complete_cylc('cylc', 'tri') # $ cylc trigger assert await _complete_cylc('cylc', 'trigger') == { 'trigger', } # $ cylc trigger assert await _complete_cylc('cylc', 'trigger', '') == { 'foo/run2//', } # $ cylc trigger f assert await _complete_cylc('cylc', 'trigger', 'f') == { 'foo/run2//', } # $ cylc trigger foo/run2// assert await _complete_cylc('cylc', 'trigger', 'foo/run2//') == { 'foo/run2//1/', 'foo/run2//2/', 'foo/run2//3/', } # $ cylc trigger foo/run2//1 assert await _complete_cylc('cylc', 'trigger', 'foo/run2//1') == { 'foo/run2//1/', } # $ cylc trigger foo/run2//1/ assert set(await _complete_cylc('cylc', 'trigger', 'foo/run2//1/')) == { 'foo/run2//1/foo/', 'foo/run2//1/bar/', 'foo/run2//1/baz/', } # $ cylc trigger foo/run2//1/f assert await _complete_cylc('cylc', 'trigger', 'foo/run2//1/f') == { 'foo/run2//1/foo/', } # $ cylc trigger foo/run2//1/foo/ assert await _complete_cylc('cylc', 'trigger', 'foo/run2//1/foo/') == { 'foo/run2//1/foo/01/', 'foo/run2//1/foo/NN/', } # $ cylc trigger foo/run2//1/foo/N assert await _complete_cylc('cylc', 'trigger', 'foo/run2//1/foo/N') == { 'foo/run2//1/foo/NN/', } # $ cylc trigger - assert '--flow' in await _complete_cylc('cylc', 'trigger', '-') # $ cylc trigger --flow assert await _complete_cylc('cylc', 'trigger', '--flow', '') == { 'none', 'new', } # $ cylc trigger --flow assert await _complete_cylc('cylc', 'trigger', '--flow', '', '') == { 'foo/run2//' } # $ cylc trigger --flow=none assert await _complete_cylc('cylc', 'trigger', '--flow=none', '') == { 'foo/run2//' } # $ cylc trigger --flow= assert await _complete_cylc('cylc', 'trigger', '--flow=') == { '--flow=none', '--flow=new', } # $ cylc trigger --flow=1 assert await _complete_cylc('cylc', 'trigger', '--flow=1', '') == { 'foo/run2//' } # $ cylc trigger --62656566 assert await _complete_cylc('cylc', 'trigger', '--62656566') == set() # $ cylc trigger 62656566 --77656C6C696E67746F6E assert await _complete_cylc( 'cylc', '62656566', '--77656C6C696E67746F6E=' ) == set() # $ cylc cat-log f assert await _complete_cylc('cylc', 'cat-log', 'f') == {'foo/run2//'} # $ cylc log f # NOTE: "log" is an alias for "cat-log" assert await _complete_cylc('cylc', 'log', 'f') == {'foo/run2//'} # $ cylc help assert 'all' in await _complete_cylc('cylc', 'help', '') # $ cylc version assert '--long' in await _complete_cylc('cylc', 'version', '') async def test_complete_command(): """Test completion for Cylc commands.""" ret = await complete_command('t') assert 'tui' in ret assert 'trigger' in ret ret = await complete_command('tri') assert 'tui' not in ret assert 'trigger' in ret # we should get an empty list for a non-existent command ret = await complete_command('626565662077656C6C696E67746F6E') assert ret == [] async def test_complete_option(): """Test completion for --options of Cylc commands.""" ret = await complete_option('trigger') assert all( item.startswith('-') for item in ret ) assert '--flow' in ret # we should get an empty list for a non-existent options ret = await complete_option('626565662077656C6C696E67746F6E') assert ret == [] assert await complete_option('trigger', '--flow=no') == ['--flow=none'] assert await complete_option('trigger', '--flow=ne') == ['--flow=new'] # we should get None for no existent options, this enables use to fail # over to other completion methods assert await complete_option('trigger', '--float=a') is None async def test_option_value(): """Test completion for values of --options of Cylc commands.""" ret = await complete_option_value('trigger', '--flow') assert 'none' in ret ret = await complete_option_value('trigger', '--flow', 'a') assert 'none' not in ret # we should get None for no existent values, this enables use to fail # over to other completion methods ret = await complete_option_value('626565662077656C6C696E67746F6E', 'x') assert ret is None ret = await complete_option_value('trigger', '626565662077656C6C696E6774') assert ret is None async def test_complete_argument(monkeypatch): """Test completions for positional arguments of Cylc commands.""" # register two fake commands with their own special completions def _complete_arg(x): async def __complete_arg(*args): return x return __complete_arg monkeypatch.setattr( 'cylc.flow.scripts.completion_server.COMMAND_MAP', { 'foo': _complete_arg(['aaa', 'bbb']), 'bar': _complete_arg(['ccc', 'ddd']), 'baz': None, } ) # patch the default ID listing completion async def _list_cylc_id(*args): return ['eee', 'fff'] monkeypatch.setattr( 'cylc.flow.scripts.completion_server.list_cylc_id', _list_cylc_id ) # the foo command should provide foo-specific completions ret = await complete_argument('foo') assert ret == ['aaa', 'bbb'] ret = await complete_argument('foo', 'a') assert ret == ['aaa'] # the bar command should provide bar-specific completions ret = await complete_argument('bar') assert ret == ['ccc', 'ddd'] # the bax command should not provide argument completions ret = await complete_argument('baz') assert ret == [] # all other commands should fallback to the default completions ret = await complete_argument('pub') assert ret == ['eee', 'fff'] async def test_list_cylc_id(monkeypatch): """Test listing Cylc IDs. This test ensures that list_cylc_ids is calling the right interfaces. """ _list_cylc_id = setify(list_cylc_id) async def _list_workflows(): return ['abc', 'bcd', 'cde'] async def _list_in_workflow(tokens): if tokens.workflow_id == 'abc': return ['1', '2'] return ['2', '3'] monkeypatch.setattr( 'cylc.flow.scripts.completion_server.list_workflows', _list_workflows, ) monkeypatch.setattr( 'cylc.flow.scripts.completion_server.list_in_workflow', _list_in_workflow, ) assert await _list_cylc_id(None) == {'abc', 'bcd', 'cde'} assert await _list_cylc_id('a') == {'abc', 'bcd', 'cde'} assert await _list_cylc_id('abc//') == {'1', '2'} assert await _list_cylc_id('bcd//') == {'2', '3'} def test_list_options(monkeypatch): """Test listing of command --options.""" assert '--flow' in list_options('trigger') assert '--color' in list_options('trigger') # we should get an empty list if anything goes wrong assert list_options('zz9+za') == [] # patch the logic to turn off the auto_add behaviour of CylcOptionParser class EntryPoint: def load(self): def _parser_function(): parser = get_option_parser() del parser.auto_add return parser return SimpleNamespace(parser_function=_parser_function) monkeypatch.setitem( COMMANDS, 'trigger', EntryPoint(), ) # with auto_add turned off the --color option should be absent assert '--color' not in list_options('trigger') async def test_list_option_values(monkeypatch): """Test listing of --option values.""" _list_option_values = setify(list_option_values) async def _list_a_options(*args): return ['foo', 'bar', 'baz'] # register two options monkeypatch.setattr( 'cylc.flow.scripts.completion_server.OPTION_MAP', { # --a has a registered completion '--a': _list_a_options, # --b has completions explicitly turned off '--b': None, # --c is not registered }, ) assert await _list_option_values(None, '--a', None) == { 'foo', 'bar', 'baz' } assert await _list_option_values(None, '--b', None) == set() assert await _list_option_values(None, '--c', None) is None async def test_list_workflows(dummy_workflow): """Test listing workflows (via "scan").""" # test list_workflows assert await list_workflows() == ['foo/run2//'] assert await list_workflows(states={'running'}) == [] # test list_src_workflows assert await list_src_workflows(None) == ['foo/run2'] async def test_list_in_workflow(dummy_workflow): """Test listing of "things" within workflows. Things i.e. cycles/tasks/jobs. """ _list_in_workflow = setify(list_in_workflow) # workflow => list cycles assert await _list_in_workflow(Tokens('foo/run2//')) == { 'foo/run2//1/', 'foo/run2//2/', 'foo/run2//3/', } # cycle => list tasks assert await _list_in_workflow(Tokens('foo/run2//1')) == { 'foo/run2//1/foo/', 'foo/run2//1/bar/', 'foo/run2//1/baz/', } # task => list jobs assert await _list_in_workflow(Tokens('foo/run2//1/foo/')) == { 'foo/run2//1/foo/01/', 'foo/run2//1/foo/NN/', } # jobs => nothing to do assert await _list_in_workflow( Tokens('foo/run2//1/foo/01'), ) == set() # no tokens => nothing to list assert await _list_in_workflow(Tokens()) == set() # non-existent workflow => nothing to list assert await _list_in_workflow( Tokens('forty-two'), # set infer_run to false as this workflow does not exist so will raise # an exception # (note exceptions are fine, they get caught and ignored at the top # level) infer_run=False ) == set() # non-existent cycle => nothing to list assert await _list_in_workflow(Tokens('foo/run2//4')) == set() # non-existent task => nothing to list assert await _list_in_workflow(Tokens('foo/run2//4/foo')) == set() # non-existent job => nothing to list assert await _list_in_workflow(Tokens('foo/run2//4/foo/02')) == set() async def test_list_in_workflow_inference(dummy_workflow): """It should infer the latest run when appropriate.""" _list_in_workflow = setify(list_in_workflow) assert await _list_in_workflow(Tokens('foo/run2//')) == { 'foo/run2//1/', 'foo/run2//2/', 'foo/run2//3/', } assert await _list_in_workflow(Tokens('foo//')) == { 'foo//1/', 'foo//2/', 'foo//3/', } async def test_list_resources(): """Test listing of "resources. Resources i.e. things provided by `cylc get-resources`. """ assert 'cylc-completion.bash' in await list_resources(None) async def test_list_dir(tmp_path, monkeypatch): """Test directory listing.""" (tmp_path / 'x').mkdir() ((tmp_path / 'x') / 'y').mkdir() ((tmp_path / 'x') / 'z').touch() monkeypatch.chdir(tmp_path) _list_dir = setify(list_dir) # --- relative paths --- # no "partial" # => list $PWD assert { str(path) for path in await _list_dir(None) } == {'x/'} # no trailing `/` at the end of the path # (i.e. an incomplete path) # => list the parent assert { str(path) for path in await _list_dir('x') } == {'x/'} # # trailing `/` at the end of the path # # (i.e. complete path) # # => list dir path assert { str(path) for path in await _list_dir('x/') } == {'x/y/', 'x/z'} # "y" is a dir, "z" is a file # listing a file # => noting to list, just return the file assert { str(path) for path in await _list_dir('x/z/') } == {'x/z'} # --- absolute paths --- # no trailing `/` at the end of the path # (i.e. an incomplete path) # => list the parent assert { # '/'.join(path.rsplit('/', 2)[-2:]) path.replace(str(tmp_path), '') for path in await _list_dir(str(tmp_path / 'x')) } == {'/x/'} # trailing `/` at the end of the path # (i.e. complete path) # => list dir path assert { path.replace(str(tmp_path), '') for path in await _list_dir(str(tmp_path / 'x') + '/') } == {'/x/y/', '/x/z'} # "y" is a dir, "z" is a file # listing a file # => noting to list, just return the file assert { path.replace(str(tmp_path), '') for path in await _list_dir(str(tmp_path / 'x' / 'z') + '/') } == {'/x/z'} async def test_list_flows(): """Test listing values for the --flow option. Currently this only provides the textual options i.e. it doesn't list "flows" running in a workflow, yet... """ assert 'none' in await list_flows(None) async def test_list_colours(): """Test listing values for the --color option.""" assert 'always' in await list_colours(None) async def test_cli_detokenise(): """Test that Cylc IDs are detokenised with a trailing slash. Cylc completion used the trailing slash to determine that the previous part of the ID has been completed the same way as regular directory completion does. """ assert cli_detokenise(Tokens()) == '' assert cli_detokenise(Tokens('~u/w')) == '~u/w//' assert cli_detokenise(Tokens('~u/w//c')) == '~u/w//c/' assert cli_detokenise(Tokens('~u/w//c/t')) == '~u/w//c/t/' assert cli_detokenise(Tokens('~u/w//c/t/01')) == '~u/w//c/t/01/' def test_get_completion_script_file(): """Test retrieving the completion script file.""" assert get_completion_script_file('bash').exists() # if the requested language is not supported it should return None assert get_completion_script_file('ksh') is None def test_get_current_completion_script_version(tmp_path): """Test extracting the completion script version from the script file. We do this to determine the version of the completion script bundled with this version of Cylc (in order to inform users of upgrades). """ completion_script = get_completion_script_file('bash') # it should extract the version for bash assert get_current_completion_script_version( completion_script, 'bash', ) is not None # it should return None for ksh assert get_current_completion_script_version( completion_script, 'ksh', ) is None # it should return None if it can't find the version completion_script = (tmp_path / 'foo') completion_script.touch() assert get_current_completion_script_version( completion_script, 'bash', ) is None def test_check_completion_script_compatibility(monkeypatch, capsys): """Test whether a completion script is compatible with the server. In case the server interface changes at a later date this will allow us to exit gracefully rather than crashing in a horrible way. """ # set the bash completion script version to 1.0.1 def _get_current_completion_script_version(_script, lang): if lang == 'bash': return '1.0.1' return None # set the completion script compatibility range to >=1.0.0, <2.0.0 monkeypatch.setattr( 'cylc.flow.scripts.completion_server.REQUIRED_SCRIPT_VERSION', '>=1.0.0, <2.0.0', ) monkeypatch.setattr( 'cylc.flow.scripts.completion_server' '.get_current_completion_script_version', _get_current_completion_script_version ) # versions which match ">=1.0.0, <2.0.0" should be valid assert check_completion_script_compatibility('bash', '0.9.9') is False assert check_completion_script_compatibility('bash', '1.0.0') is True assert check_completion_script_compatibility('bash', '1.0.1') is True assert check_completion_script_compatibility('bash', '1.0.2') is True assert check_completion_script_compatibility('bash', '2.0.0') is False # all versions should be invalid (because we don't offer ksh support) assert check_completion_script_compatibility('ksh', '0.9.9') is False assert check_completion_script_compatibility('ksh', '1.0.0') is False assert check_completion_script_compatibility('ksh', '1.0.1') is False assert check_completion_script_compatibility('ksh', '1.0.2') is False assert check_completion_script_compatibility('ksh', '2.0.0') is False # it should tell the user when a new version of the script is available capsys.readouterr() # clear assert check_completion_script_compatibility('bash', '1.0.0') is True out, err = capsys.readouterr() assert not out # never write to stdout assert 'A new version of the Cylc bash script is available' in err # it should tell the user if the script is incompatible capsys.readouterr() # clear assert check_completion_script_compatibility('bash', '0.9.9') is False out, err = capsys.readouterr() assert not out # never write to stdout assert 'The Cylc bash script needs to be updated' in err # it shouldn't say anything unless necessary capsys.readouterr() # clear assert check_completion_script_compatibility('bash', '1.0.1') is True out, err = capsys.readouterr() assert not out # never write to stdout assert not err async def test_prereqs_and_outputs(): """Test the error cases for listing task prereqs/outputs. The success cases are tested in an integration test (requires a running scheduler). """ # if no tokens are provided, no prereqs or outputs are returned assert await _list_prereqs_and_outputs([]) == ([], []) # if an invalid workflow is provided, we can't list anything assert await _list_prereqs_and_outputs( [Tokens(workflow='no-such-workflow')] ) == ([], []) cylc-flow-8.7.0/tests/unit/scripts/__init__.py0000664000175000017500000000140715264420307021537 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . cylc-flow-8.7.0/tests/unit/scripts/test_config.py0000664000175000017500000001506515264420307022311 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio import os from pathlib import Path from textwrap import dedent from typing import Any, Optional, List import pytest from cylc.flow.cfgspec.globalcfg import GlobalConfig from cylc.flow.option_parsers import Options from cylc.flow.scripts.config import ( _main, get_config_file_hierarchy, get_option_parser, ) from cylc.flow.workflow_files import WorkflowFiles Fixture = Any HOME: str = str(Path('~').expanduser()) @pytest.fixture def conf_env(monkeypatch): """Clear any env vars that affect which conf files get loaded. Return a convenience function for setting environment variables. """ # wipe any cached config monkeypatch.setattr( GlobalConfig, '_DEFAULT', None, ) for envvar in ('CYLC_SITE_CONF_PATH', 'CYLC_CONF_PATH'): if envvar in os.environ: monkeypatch.delenv(envvar) def _set_env(key, value): if value: monkeypatch.setenv(key, value) return _set_env @pytest.fixture def dummy_version_hierarchy(monkeypatch): """Set the config version hierarchy.""" monkeypatch.setattr( 'cylc.flow.cfgspec.globalcfg.GlobalConfig.VERSION_HIERARCHY', ['', '1', '1.0'] ) @pytest.fixture def capload(monkeypatch): """Capture configuration load events. This prevents actual file loading. If the file name contains the string "invalid" it will not appear in the results as if it diddn't exist on the filesystem. """ files = [] def _capload(glblcfg, fname, _): if 'invalid' not in fname: # if the file is called invalid skip it # this is to replicate the behaviour of skipping files that # don't exist files.append(fname.replace(HOME, '~')) monkeypatch.setattr( GlobalConfig, '_load', _capload ) return files def test_get_config_file_hierarchy_global( monkeypatch: Fixture, conf_env: Fixture, capload: Fixture, dummy_version_hierarchy: Fixture ): """Test get_config_file_hierarchy() for the global hierarchy only.""" assert [ path.replace(HOME, '~') for path in get_config_file_hierarchy() ] == [ '/etc/cylc/flow/global.cylc', '/etc/cylc/flow/1/global.cylc', '/etc/cylc/flow/1.0/global.cylc', '~/.cylc/flow/global.cylc', '~/.cylc/flow/1/global.cylc', '~/.cylc/flow/1.0/global.cylc' ] @pytest.mark.parametrize( 'conf_path,site_conf_path,files', [ pytest.param( None, None, [ '/etc/cylc/flow/global.cylc', '/etc/cylc/flow/1/global.cylc', '/etc/cylc/flow/1.0/global.cylc', '~/.cylc/flow/global.cylc', '~/.cylc/flow/1/global.cylc', '~/.cylc/flow/1.0/global.cylc', ], id='(default)' ), pytest.param( None, '', [ '/flow/global.cylc', '/flow/1/global.cylc', '/flow/1.0/global.cylc', '~/.cylc/flow/global.cylc', '~/.cylc/flow/1/global.cylc', '~/.cylc/flow/1.0/global.cylc', ], id='CYLC_SITE_CONF_PATH=valid' ), pytest.param( None, 'invalid', [ '~/.cylc/flow/global.cylc', '~/.cylc/flow/1/global.cylc', '~/.cylc/flow/1.0/global.cylc', ], id='CYLC_SITE_CONF_PATH=invalid' ), pytest.param( '', None, ['/global.cylc'], id='CYLC_CONF_PATH=valid' ), pytest.param( 'invalid', None, [], id='CYLC_CONF_PATH=invalid' ), pytest.param( '', '', ['/global.cylc'], # should ignore CYLC_SITE_CONF_PATH id='CYLC_CONF_PATH=valid, CYLC_SITE_CONF_PATH=valid' ), pytest.param( 'invalid', '', [], id='CYLC_CONF_PATH=invalid, CYLC_SITE_CONF_PATH=valid' ), ] ) def test_cylc_site_conf_path_env_var( monkeypatch: Fixture, conf_env: Fixture, capload: Fixture, dummy_version_hierarchy: Fixture, conf_path: Optional[str], site_conf_path: Optional[str], files: List[str], ): """Test that the right files are loaded according to env vars.""" # set the relevant environment variables conf_env('CYLC_CONF_PATH', conf_path) conf_env('CYLC_SITE_CONF_PATH', site_conf_path) # load the global config GlobalConfig.get_inst() assert capload == files def test_cylc_config_xtriggers(tmp_run_dir, capsys: pytest.CaptureFixture): """Test `cylc config` outputs any xtriggers properly""" run_dir: Path = tmp_run_dir('constellation') flow_file = run_dir / WorkflowFiles.FLOW_FILE flow_file.write_text(dedent(""" [scheduler] allow implicit tasks = True [scheduling] initial cycle point = 2020-05-05 [[xtriggers]] clock_1 = wall_clock(offset=PT1H):PT4S rotund = xrandom(90, 2) [[graph]] R1 = @rotund => foo """)) option_parser = get_option_parser() asyncio.run( _main(Options(option_parser)(), 'constellation') ) assert capsys.readouterr().out == dedent("""\ [scheduler] allow implicit tasks = True [scheduling] initial cycle point = 2020-05-05 [[xtriggers]] clock_1 = wall_clock(offset=PT1H):4.0 rotund = xrandom(90, 2):10.0 [[graph]] R1 = @rotund => foo [runtime] [[root]] [[foo]] completion = succeeded """) cylc-flow-8.7.0/tests/unit/scripts/test_cylc.py0000664000175000017500000001131515264420307021770 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import os import sys from types import SimpleNamespace from typing import Callable from unittest.mock import Mock import pytest from cylc.flow.scripts.cylc import iter_commands, pythonpath_manip @pytest.fixture def mock_entry_points(monkeypatch: pytest.MonkeyPatch): """Mock a range of entry points.""" def _load_fail(*args, **kwargs): raise ModuleNotFoundError('foo') def _resolve_ok(*args, **kwargs): return Mock() def _require_ok(*args, **kwargs): return def _mocked_entry_points(include_bad: bool = False): commands = { # an entry point with all dependencies installed: 'good': SimpleNamespace( name='good', module='os.path', load=_resolve_ok, extras=[], dist=SimpleNamespace(name='a'), ), # an entry point with optional dependencies missing: 'missing': SimpleNamespace( name='missing', module='not.a.python.module', # force an import error load=_load_fail, extras=[], dist=SimpleNamespace(name='foo'), ), } if include_bad: # an entry point with non-optional dependencies unexpectedly # missing: commands['bad'] = SimpleNamespace( name='bad', module='not.a.python.module', load=_load_fail, require=_require_ok, extras=[], dist=SimpleNamespace(name='d'), ) monkeypatch.setattr('cylc.flow.scripts.cylc.COMMANDS', commands) return _mocked_entry_points def test_iter_commands(mock_entry_points): """Test listing commands works ok. It should exclude commands with missing optional dependencies. """ mock_entry_points() commands = list(iter_commands()) assert [i[0] for i in commands] == ['good'] def test_iter_commands_bad(mock_entry_points): """Test listing commands doesn't fail on import error.""" mock_entry_points(include_bad=True) list(iter_commands()) def test_execute_cmd( mock_entry_points, capsys: pytest.CaptureFixture, ): """It should fail with a warning for commands with missing dependencies.""" # (stop IDEs reporting code as unreachable in this test) execute_cmd: Callable from cylc.flow.scripts.cylc import execute_cmd mock_entry_points(include_bad=True) # the "good" entry point should exit 0 (exit with no args) assert execute_cmd('good') == 0 assert capsys.readouterr().err == '' # the "missing" entry point should exit 1 with a warning to stderr assert execute_cmd('missing') == 1 assert capsys.readouterr().err.strip() == ( '"cylc missing" requires "foo"\n\nModuleNotFoundError: foo' ) # the "bad" entry point should log an error assert execute_cmd('bad') == 1 stderr = capsys.readouterr().err.strip() assert '"cylc bad" requires "d"' in stderr assert 'ModuleNotFoundError: foo' in stderr def test_pythonpath_manip(monkeypatch): """pythonpath_manip removes items in PYTHONPATH from sys.path and adds items from CYLC_PYTHONPATH """ # Local CYLC_PYTHONPATH can mess with this test. monkeypatch.delenv('CYLC_PYTHONPATH', raising=False) monkeypatch.setenv('PYTHONPATH', '/remove1:/remove2') monkeypatch.setattr('sys.path', ['/leave-alone', '/remove1', '/remove2']) pythonpath_manip() # ... we don't change PYTHONPATH assert os.environ['PYTHONPATH'] == '/remove1:/remove2' # ... but we do remove PYTHONPATH items from sys.path, and don't remove # items there not in PYTHONPATH assert sys.path == ['/leave-alone'] # If CYLC_PYTHONPATH is set we retrieve its contents and # add them to the sys.path: monkeypatch.setenv('CYLC_PYTHONPATH', '/add1:/add2') pythonpath_manip() assert sys.path == ['/add1', '/add2', '/leave-alone'] cylc-flow-8.7.0/tests/unit/scripts/test_lint_checkers.py0000664000175000017500000000614315264420307023656 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test check functions in the `cylc lint` CLI Utility.""" import doctest import json import pytest import re from cylc.flow.scripts import lint VARS = re.compile(r'\{(.*)\}') # Functions in Cylc Lint defined with "check_* CHECKERS = [ getattr(lint, i) for i in lint.__dir__() if i.startswith('check_')] # List of checks defined as checks by Cylc Lint ALL_CHECKS = [ *lint.MANUAL_DEPRECATIONS.values(), *lint.STYLE_CHECKS.values(), ] finder = doctest.DocTestFinder() @pytest.mark.parametrize( 'check', # Those checks that have custom checker functions # and a short message with variables to insert: [ pytest.param(c, id=c.get('function').__name__) for c in ALL_CHECKS if c.get('function') in CHECKERS ] ) def test_custom_checker_doctests(check): """All check functions have at least one failure doctest By forcing each check function to have valid doctests for the case that linting has failed we are able to check that the function outputs the correct information for formatting the short formats. """ doctests = finder.find(check['function'])[0] msg = f'{check["function"].__name__}: No failure examples in doctest' assert any(i.want for i in doctests.examples if i.want), msg @pytest.mark.parametrize( 'ref, check', # Those checks that have custom checker functions # and a short message with variables to insert: [ (c.get('function'), c) for c in ALL_CHECKS if c.get('function') in CHECKERS and VARS.findall(c['short']) ] ) def test_custom_checkers_short_formatters(ref, check): """If a check message has a format string assert that the checker function will return a dict to be used in ``check['short'].format(**kwargs)``, based on doctest output. ref is useful to allow us to identify the check, even though not used in the test. """ doctests = finder.find(check['function'])[0] # Filter doctest examples for cases where there is a json parsable # want. examples = [ eg for eg in [ json.loads(e.want.replace("'", '"')) for e in doctests.examples if e.want ] if eg ] # Formatting using the example output changes the check short text: for example in examples: assert check['short'].format(**example) != check['short'] cylc-flow-8.7.0/tests/unit/scripts/test_graph.py0000664000175000017500000002021415264420307022135 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from types import SimpleNamespace import typing as t import pytest from cylc.flow.scripts.graph import ( Edge, Node, format_cylc_reference, format_graphviz, get_nodes_and_edges, ) @pytest.fixture def example_graph(): """Example workflow graph with inter-cycle dependencies.""" nodes: t.List[Node] = [ '1/a', '1/b', '1/c', '2/a', '2/b', '2/c', ] edges: t.List[Edge] = [ ('1/a', '1/b'), ('1/b', '1/c'), ('2/a', '2/b'), ('2/b', '2/c'), ('1/b', '2/b'), ] return nodes, edges @pytest.fixture def example_namespace_graph(): """Example namespace graph with inheritance.""" nodes: t.List[Node] = [ 'A', 'a1', 'a2', 'B', 'B1', 'b11', 'B2', 'b22', ] edges: t.List[Edge] = [ ('A', 'a1'), ('A', 'a2'), ('B', 'B1'), ('B', 'B2'), ('B1', 'b11'), ('B2', 'b22'), ] return nodes, edges @pytest.fixture def null_config(monkeypatch): """Patch the config loader to return a workflow with no nodes or edges.""" def _get_graph_raw(*args, **kwargs): return None def _get_parents_lists(*args, **kwargs): return {} config = SimpleNamespace( get_graph_raw=_get_graph_raw, get_parent_lists=_get_parents_lists, ) monkeypatch.setattr( 'cylc.flow.scripts.graph.get_config', lambda x, y, z: config ) def test_format_graphviz_normal(example_graph): """Test graphviz output for default options. Tests both orientations (--transpose). """ nodes, edges = example_graph # format the graph in regular orientation opts = SimpleNamespace(transpose=False, namespaces=False, cycles=False) lines = format_graphviz(opts, nodes, edges) assert lines == [ 'digraph {', ' graph [fontname="sans" fontsize="25"]', ' node [fontname="sans"]', '', ' "1/a" [label="a\\n1"]', ' "1/b" [label="b\\n1"]', ' "1/c" [label="c\\n1"]', '', ' "2/a" [label="a\\n2"]', ' "2/b" [label="b\\n2"]', ' "2/c" [label="c\\n2"]', '', ' "1/a" -> "1/b"', ' "1/b" -> "1/c"', ' "2/a" -> "2/b"', ' "2/b" -> "2/c"', ' "1/b" -> "2/b"', '}', ] # format the graph in transposed orientation opts = SimpleNamespace(transpose=True, namespaces=False, cycles=False) transposed_lines = format_graphviz(opts, nodes, edges) # the transposed graph should be the same except for one line... assert [ line for line in transposed_lines if line not in lines ] == [ # ...the one which sets the orientation ' rankdir="LR"', ] def test_format_graphviz_namespace(example_namespace_graph): """Test graphviz output for a namespace graph. Tests both orientations (--transpose). """ nodes, edges = example_namespace_graph # format the graph in regular orientation opts = SimpleNamespace(transpose=False, namespaces=True, cycles=False) lines = format_graphviz(opts, nodes, edges) assert lines == [ 'digraph {', ' graph [fontname="sans" fontsize="25"]', ' node [fontname="sans"]', ' node [shape="rect"]', '', ' "A"', ' "a1"', ' "a2"', ' "B"', ' "B1"', ' "b11"', ' "B2"', ' "b22"', '', ' "A" -> "a1"', ' "A" -> "a2"', ' "B" -> "B1"', ' "B" -> "B2"', ' "B1" -> "b11"', ' "B2" -> "b22"', '}', ] # format the graph in transposed orientation opts = SimpleNamespace(transpose=True, namespaces=True, cycles=False) transposed_lines = format_graphviz(opts, nodes, edges) # the transposed graph should be the same except for one line... assert [ line for line in transposed_lines if line not in lines ] == [ # ...the one which sets the orientation ' rankdir="LR"', ] def test_format_graphviz_cycles(example_graph): """Test graphviz format for the --cycles option (group by cycle). Note: There is no difference between iso8601 and integer cycle points here, the graph logic is cycle point format agnostic. Sorting is not performed in this function. """ nodes, edges = example_graph opts = SimpleNamespace(transpose=False, namespaces=False, cycles=True) lines = format_graphviz(opts, nodes, edges) assert lines == [ 'digraph {', ' graph [fontname="sans" fontsize="25"]', ' node [fontname="sans"]', '', ' subgraph "cluster_1" { ', ' label="1"', ' style="dashed"', ' "1/a" [label="a\\n1"]', ' "1/b" [label="b\\n1"]', ' "1/c" [label="c\\n1"]', ' }', '', ' subgraph "cluster_2" { ', ' label="2"', ' style="dashed"', ' "2/a" [label="a\\n2"]', ' "2/b" [label="b\\n2"]', ' "2/c" [label="c\\n2"]', ' }', '', ' "1/a" -> "1/b"', ' "1/b" -> "1/c"', ' "2/a" -> "2/b"', ' "2/b" -> "2/c"', ' "1/b" -> "2/b"', '}', ] def test_format_cylc_reference_normal(example_graph): """Test Cylc "reference" format (used by the test battery). Note: There is no difference between iso8601 and integer cycle points here, the graph logic is cycle point format agnostic. Sorting is not performed in this function. Note: There is no transpose mode for reference graphs. """ nodes, edges = example_graph opts = SimpleNamespace(namespaces=False) lines = format_cylc_reference(opts, nodes, edges) assert lines == [ 'edge "1/a" "1/b"', 'edge "1/b" "1/c"', 'edge "2/a" "2/b"', 'edge "2/b" "2/c"', 'edge "1/b" "2/b"', 'graph', 'node "1/a" "a\\n1"', 'node "1/b" "b\\n1"', 'node "1/c" "c\\n1"', 'node "2/a" "a\\n2"', 'node "2/b" "b\\n2"', 'node "2/c" "c\\n2"', 'stop', ] def test_format_cylc_reference_namespace(example_namespace_graph): """Test Cylc "reference" format for namespace graphs. Note: There is no transpose mode for reference graphs. """ nodes, edges = example_namespace_graph opts = SimpleNamespace(namespaces=True) lines = format_cylc_reference(opts, nodes, edges) assert lines == [ 'edge "A" "a1"', 'edge "A" "a2"', 'edge "B" "B1"', 'edge "B" "B2"', 'edge "B1" "b11"', 'edge "B2" "b22"', 'graph', 'node "A" "A"', 'node "a1" "a1"', 'node "a2" "a2"', 'node "B" "B"', 'node "B1" "B1"', 'node "b11" "b11"', 'node "B2" "B2"', 'node "b22" "b22"', 'stop', ] def test_null(null_config): """Ensure that an empty graph is handled elegantly.""" opts = SimpleNamespace( namespaces=False, grouping=False, show_suicide=False, flatten_icp_dependence=False, ) assert get_nodes_and_edges(opts, None, 1, 2, '') == ([], []) opts = SimpleNamespace( namespaces=True, grouping=False, show_suicide=False, flatten_icp_dependence=False, ) assert get_nodes_and_edges(opts, None, 1, 2, '') == ([], []) cylc-flow-8.7.0/tests/unit/scripts/test_release.py0000664000175000017500000000352115264420307022456 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test logic in cylc-release script.""" from optparse import Values import pytest from typing import Iterable, Optional, Tuple, Type from cylc.flow.exceptions import InputError from cylc.flow.option_parsers import Options from cylc.flow.scripts.release import get_option_parser, _validate Opts = Options(get_option_parser()) @pytest.mark.parametrize( 'opts, task_globs, expected_err', [ (Opts(), ['*'], None), (Opts(release_all=True), [], None), ( Opts(release_all=True), ['*'], (InputError, "Cannot combine --all with Cycle/Task IDs") ), ( Opts(), [], (InputError, "Must define Cycles/Tasks") ), ] ) def test_validate( opts: Values, task_globs: Iterable[str], expected_err: Optional[Tuple[Type[Exception], str]]): if expected_err: err, msg = expected_err with pytest.raises(err) as exc: _validate(opts, *task_globs) assert msg in str(exc.value) else: _validate(opts, *task_globs) cylc-flow-8.7.0/tests/unit/scripts/test_profiler.py0000664000175000017500000003500115264420307022656 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) NIWA & British Crown (Met Office) & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # Tests for functions contained in cylc.flow.scripts.profiler from pathlib import Path from unittest import mock import pytest from cylc.flow.exceptions import CylcProfilerError from cylc.flow.scripts.profiler import ( Process, _main, get_cgroup_name, get_cgroup_paths, get_cgroup_version, get_profiler_data, parse_cpu_file, parse_memory_allocated, parse_memory_file, profile, report_to_scheduler, ) async def test_report_to_scheduler(monkeypatch, tmpdir): """It should capture and record profiler data.""" monkeypatch.setenv('CYLC_WORKFLOW_ID', "test_value") monkeypatch.setenv('CYLC_TASK_JOB', "test_task_job") # capture record_messages calls record_messages = mock.AsyncMock() monkeypatch.setattr( 'cylc.flow.scripts.profiler.record_messages', record_messages ) # mock the cGroup filesystem mem_file = tmpdir.join("memory_file.txt") mem_file.write('total_rss=1234') cpu_file = tmpdir.join("cpu_file.txt") cpu_file.write('5678000') # NOTE: memory allocated is not available for cGroups v1 mem_allocated_file = tmpdir.join("memory_allocated.txt") mem_allocated_file.write('99999') process_object = Process( cgroup_memory_path=mem_file, cgroup_cpu_path=cpu_file, memory_allocated_path=mem_allocated_file, cgroup_version=1, max_rss=42, ) await report_to_scheduler(process_object, 1) # the profiler should record a "cylc message" with the profiler data assert record_messages.call_args_list == [ mock.call( 'test_value', 'test_task_job', [ [ 'DEBUG', '_cylc_profiler:' ' {"max_rss": 42, "cpu_time": 6, "memory_allocated": 0}', ] ], comms_timeout=1, ) ] def test_get_resource_usage(): """It should return 0 if cGroup information is not provided.""" process_object = Process( cgroup_memory_path=None, cgroup_cpu_path=None, memory_allocated_path=None, cgroup_version=1, max_rss=0) assert get_profiler_data(process_object) == { 'max_rss': 0, 'cpu_time': 0, 'memory_allocated': 0, } def test_parse_memory_file(tmpdir): """It should return the memory usage of the process.""" mem_file_v1 = tmpdir.join("memory_file_v1.txt") mem_file_v1.write('total_rss=1024') mem_file_v2 = tmpdir.join("memory_file_v2.txt") mem_file_v2.write('anon=666') cpu_file = tmpdir.join("cpu_file.txt") cpu_file.write('5678') mem_allocated_file = tmpdir.join("memory_allocated.txt") mem_allocated_file.write('99999') good_process_object_v1 = Process( cgroup_memory_path=mem_file_v1, cgroup_cpu_path=cpu_file, memory_allocated_path=mem_allocated_file, cgroup_version=1, max_rss=0) good_process_object_v2 = Process( cgroup_memory_path=mem_file_v2, cgroup_cpu_path=cpu_file, memory_allocated_path=mem_allocated_file, cgroup_version=2, max_rss=0) bad_process_object = Process( cgroup_memory_path='', cgroup_cpu_path='', memory_allocated_path='', cgroup_version=1, max_rss=0) with pytest.raises(CylcProfilerError) as excinfo: parse_memory_file(bad_process_object) assert "Unable to find memory usage data" in str(excinfo.value) # Test the parse_memory_file function assert parse_memory_file(good_process_object_v1) == 1024 assert parse_memory_file(good_process_object_v2) == 666 def test_parse_cpu_file(tmpdir): """It should return the cpu usage of the process.""" mem_file = tmpdir.join("memory_file.txt") mem_file.write('1024') cpu_file_v1_good = tmpdir.join("cpu_file_v1_good.txt") cpu_file_v1_good.write('1234567890') cpu_file_v1_bad = tmpdir.join("cpu_file_v1_bad.txt") cpu_file_v1_bad.write("I'm your dream, mind ashtray") cpu_file_v2_good = tmpdir.join("cpu_file_v2_good.txt") cpu_file_v2_good.write('usage_usec 1234567890') cpu_file_v2_bad = tmpdir.join("cpu_file_v2_bad.txt") cpu_file_v2_bad.write('Give me fuel, give me fire, ' 'give me that which I desire') mem_allocated_file = tmpdir.join("memory_allocated.txt") mem_allocated_file.write('99999') good_process_object_v1 = Process( cgroup_memory_path=mem_file, cgroup_cpu_path=cpu_file_v1_good, memory_allocated_path=mem_allocated_file, cgroup_version=1, max_rss=0) good_process_object_v2 = Process( cgroup_memory_path=mem_file, cgroup_cpu_path=cpu_file_v2_good, memory_allocated_path=mem_allocated_file, cgroup_version=2, max_rss=0) bad_process_object_v1_1 = Process( cgroup_memory_path='', cgroup_cpu_path='', memory_allocated_path='', cgroup_version=1, max_rss=0) bad_process_object_v1_2 = Process( cgroup_memory_path=mem_file, cgroup_cpu_path=cpu_file_v1_bad, memory_allocated_path=mem_allocated_file, cgroup_version=1, max_rss=0) bad_process_object_v2 = Process( cgroup_memory_path=mem_file, cgroup_cpu_path=cpu_file_v2_bad, memory_allocated_path=mem_allocated_file, cgroup_version=2, max_rss=0) assert parse_cpu_file(good_process_object_v1) == 1235 assert parse_cpu_file(good_process_object_v2) == 1234568 with pytest.raises(CylcProfilerError) as excinfo: parse_cpu_file(bad_process_object_v1_1) assert "Unable to find cpu usage data" in str(excinfo.value) with pytest.raises(CylcProfilerError) as excinfo: parse_cpu_file(bad_process_object_v1_2) assert "Unable to find cpu usage data" in str(excinfo.value) with pytest.raises(CylcProfilerError) as excinfo: parse_cpu_file(bad_process_object_v2) assert "Unable to find cpu usage data" in str(excinfo.value) def test_get_cgroup_name(mocker): """It should return the cgroup name of the process.""" mocker.patch.object( Path, 'read_text', return_value="0::bad/test/cgroup/place" ) with pytest.raises(CylcProfilerError): get_cgroup_name() mocker.patch.object( Path, 'read_text', return_value="0::good/cgroup/place/2222222" ) assert get_cgroup_name() == "good/cgroup/place/2222222" def test_parse_memory_allocated(tmp_path_factory): """It should return the memory allocated to the process.""" good_mem_dir = tmp_path_factory.mktemp("mem_dir") mem_allocated_file = good_mem_dir / "memory.max" mem_allocated_file.write_text('99999') # We currently do not track memory allocated for cgroups v1 good_process_object_v1 = Process( cgroup_memory_path='', cgroup_cpu_path='', memory_allocated_path=good_mem_dir, cgroup_version=1, max_rss=0) good_process_object_v2 = Process( cgroup_memory_path='', cgroup_cpu_path='', memory_allocated_path=good_mem_dir, cgroup_version=2, max_rss=0) bad_process_object_v2_1 = Process( cgroup_memory_path='', cgroup_cpu_path='', memory_allocated_path=Path('/'), cgroup_version=2, max_rss=0) assert parse_memory_allocated(good_process_object_v1) == 0 assert parse_memory_allocated(good_process_object_v2) == 99999 with pytest.raises(CylcProfilerError) as excinfo: parse_memory_file(bad_process_object_v2_1) assert "Unable to find memory usage data" in str(excinfo.value) # Nested directories with 'max' value base_dir = tmp_path_factory.mktemp("base") dir_1 = base_dir / "dir_1" dir_1.mkdir() mem_file_1 = dir_1 / "memory.max" mem_file_1.write_text("max") dir_2 = dir_1 / "dir_2" dir_2.mkdir() mem_file_2 = dir_2 / "memory.max" mem_file_2.write_text("max") dir_3 = dir_2 / "dir_3" dir_3.mkdir() mem_file_3 = dir_3 / "memory.max" mem_file_3.write_text("max") dir_4 = dir_3 / "dir_4" dir_4.mkdir() mem_file_4 = dir_4 / "memory.max" mem_file_4.write_text("max") dir_5 = dir_4 / "dir_5" dir_5.mkdir() mem_file_5 = dir_5 / "memory.max" mem_file_5.write_text("max") bad_process_object_v2_2 = Process( cgroup_memory_path='', cgroup_cpu_path='', memory_allocated_path=dir_5, cgroup_version=2, max_rss=0) # The function should return 0 if it cannot find a memory.max file with # a value assert parse_memory_allocated(bad_process_object_v2_2) == 0 # Add a memory.max file with a value to the top level directory # and check it is read mem_file_1.write_text("99999") assert parse_memory_allocated(bad_process_object_v2_2) == 99999 def test_get_cgroup_name_file_not_found(mocker): """It should raise an error if the cgroup file is not found.""" def mock_os_pid(): return 'The Thing That Should Not Be' mocker.patch("os.getpid", mock_os_pid) with pytest.raises(CylcProfilerError) as excinfo: get_cgroup_name() assert "/cgroup not found" in str(excinfo.value) def test_get_cgroup_version(mocker): """It should return the cgroup version of the process.""" # Mock the Path.exists function call to return True mocker.patch("pathlib.Path.exists", return_value=True) assert get_cgroup_version(Path('stuff/in/place'), 'more_stuff') == 2 with mock.patch('pathlib.Path.exists', side_effect=[False, True]): assert get_cgroup_version(Path('stuff/in/place'), 'more_stuff') == 1 # Mock the Path.exists function call to return False mocker.patch("pathlib.Path.exists", return_value=False) with pytest.raises(CylcProfilerError) as excinfo: get_cgroup_version(Path('stuff/in/other/place'), 'things') assert "Cgroup not found" in str(excinfo.value) def test_get_cgroup_paths(mocker): """It should return the cgroup paths of the process.""" mocker.patch("cylc.flow.scripts.profiler.get_cgroup_name", return_value='test_name') mocker.patch("cylc.flow.scripts.profiler.get_cgroup_version", return_value=2) process = get_cgroup_paths(Path("test_location/")) assert (process.cgroup_memory_path == Path("test_location/test_name/memory.stat")) assert process.cgroup_cpu_path == Path("test_location/test_name/cpu.stat") mocker.patch("cylc.flow.scripts.profiler.get_cgroup_version", return_value=1) process = get_cgroup_paths(Path("test_location/")) assert (process.cgroup_memory_path == Path("test_location/memory/test_name/memory.stat")) assert (process.cgroup_cpu_path == Path("test_location/cpu/test_name/cpuacct.usage")) mocker.patch("cylc.flow.scripts.profiler.get_cgroup_version", return_value=3) with pytest.raises(CylcProfilerError) as excinfo: get_cgroup_paths(Path("test_location/")) assert "Unable to determine cgroup version" in str(excinfo.value) async def test_profile_data(mocker): """Test the profile function with mocked data to ensure it calls the parse functions and handles the data correctly.""" mocker.patch("cylc.flow.scripts.profiler.get_cgroup_name", return_value='test_name') mocker.patch("cylc.flow.scripts.profiler.get_cgroup_version", return_value=2) process = get_cgroup_paths(Path("test_location/")) mock_file = mocker.mock_open(read_data="") mocker.patch("builtins.open", mock_file) mocker.patch("cylc.flow.scripts.profiler.parse_memory_file", side_effect=[100, 200, 150]) mocker.patch("cylc.flow.scripts.profiler.parse_cpu_file", return_value=2048) # The profile function runs until the callable returns False. run_count = 0 def run_four_times(): nonlocal run_count run_count += 1 return run_count < 4 await profile(process, 0.1, run_four_times) # It should have been called 4 times before run_once returned False assert run_count == 4 # The max_rss should be the highest value from parse_memory_file assert process.max_rss == 200 @pytest.fixture def options(mocker): opts = mocker.Mock() opts.cgroup_location = "/fake/path" opts.cgroup_memory_path = "/another/fake/path" opts.comms_timeout = 10 opts.delay = "PT1S" return opts async def test_main(mocker, options, monkeypatch): """It should run the profiler and watch and kill functions concurrently.""" # Mock Cylc env vars monkeypatch.setenv('CYLC_WORKFLOW_ID', "Exit Light") monkeypatch.setenv('CYLC_TASK_JOB', "Enter Night") # Mock the gets and parse functions to return something sensible # without needing actual files mocker.patch("cylc.flow.scripts.profiler.get_cgroup_paths", return_value=Process( cgroup_memory_path=Path("/some/place/memory.stat"), cgroup_cpu_path=Path("/some/place/cpu.stat"), memory_allocated_path=Path("/some/place"), cgroup_version=2, max_rss=0,)) mocker.patch("cylc.flow.scripts.profiler.parse_memory_file", return_value=1234) mocker.patch("cylc.flow.scripts.profiler.parse_cpu_file", return_value=5678) mocker.patch("cylc.flow.scripts.profiler.parse_memory_allocated", return_value=90) mock_profile = mocker.patch("cylc.flow.scripts.profiler.profile") mock_watch_and_kill = mocker.patch( "cylc.flow.scripts.profiler.watch_and_kill" ) await _main(options) # Ensure the profiler and watch and kill functions are called by # asyncio.gather mock_profile.assert_called_once() mock_watch_and_kill.assert_called_once() cylc-flow-8.7.0/tests/unit/scripts/test_check_versions.py0000664000175000017500000000346515264420307024052 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.exceptions import NoHostsError from cylc.flow.scripts.check_versions import check_versions @pytest.fixture def break_host_selection(monkeypatch): """Make host selection for any platform fail with NoHostsError.""" def _get_host_from_platform(platform, *args, **kwargs): raise NoHostsError(platform) monkeypatch.setattr( 'cylc.flow.scripts.check_versions.get_host_from_platform', _get_host_from_platform, ) def _get_platform(platform_name, *args, **kwargs): return {'name': platform_name} monkeypatch.setattr( 'cylc.flow.scripts.check_versions.get_platform', _get_platform, ) def test_no_hosts_error(break_host_selection, capsys): """It should handle NoHostsError events.""" versions = check_versions(['buggered'], True) # the broken platform should be skipped (so no returned versions) assert not versions # a warning should have been logged to stderr out, err = capsys.readouterr() assert 'Could not connect to buggered' in err cylc-flow-8.7.0/tests/unit/scripts/test_hold.py0000664000175000017500000000352515264420307021770 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test logic in cylc-hold script.""" from optparse import Values import pytest from typing import Iterable, Optional, Tuple, Type from cylc.flow.exceptions import InputError from cylc.flow.option_parsers import Options from cylc.flow.scripts.hold import get_option_parser, _validate Opts = Options(get_option_parser()) @pytest.mark.parametrize( 'opts, task_globs, expected_err', [ (Opts(), ['*'], None), (Opts(hold_point_string='2'), [], None), ( Opts(hold_point_string='2'), ['*'], (InputError, "Cannot combine --after with Cylc/Task ID") ), ( Opts(), [], (InputError, "Must define Cycles/Tasks") ), ] ) def test_validate( opts: Values, task_globs: Iterable[str], expected_err: Optional[Tuple[Type[Exception], str]]): if expected_err: err, msg = expected_err with pytest.raises(err) as exc: _validate(opts, *task_globs) assert msg in str(exc.value) else: _validate(opts, *task_globs) cylc-flow-8.7.0/tests/unit/scripts/test_clean.py0000664000175000017500000001002415264420307022114 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from typing import ( Callable, Type, Union, ) import pytest from cylc.flow.exceptions import InputError from cylc.flow.scripts.clean import ( CleanOptions, _main, parse_timeout, run, scan, ) async def test_scan(tmp_run_dir): """It should scan the filesystem to expand partial IDs.""" # regular workflows pass straight through tmp_run_dir('foo') workflows, multi_mode = await scan(['foo'], False) assert workflows == ['foo'] assert multi_mode is False # hierarchies, however, get expanded tmp_run_dir('bar/run1') workflows, multi_mode = await scan(['bar'], False) assert workflows == ['bar/run1'] assert multi_mode is True # because an expansion has happened # multiple results are returned sorted. tmp_run_dir('bar/run2') tmp_run_dir('bar/run10') workflows, multi_mode = await scan(['bar'], False) assert workflows == ['bar/run1', 'bar/run2', 'bar/run10'] assert multi_mode is True @pytest.fixture def mute(monkeypatch: pytest.MonkeyPatch) -> list[str]: """Stop cylc clean from doing anything and log all init_clean calls.""" items = [] async def _clean(id_, *_): items.append(id_) monkeypatch.setattr('cylc.flow.scripts.clean.init_clean', _clean) monkeypatch.setattr('cylc.flow.scripts.clean.prompt', lambda x: None) return items async def test_multi(tmp_run_dir: Callable, mute: list[str]): """It supports cleaning multiple workflows.""" # cli opts opts = CleanOptions() # create three dummy workflows tmp_run_dir('bar/pub/beer') tmp_run_dir('baz/run1') tmp_run_dir('foo') # an explicit workflow ID goes straight through mute.clear() await run('foo', opts=opts) assert mute == ['foo'] # a partial hierarchical ID gets expanded to all workflows contained # in the hierarchy (note runs are a special case of hierarchical ID) mute.clear() await run('bar', opts=opts) assert mute == ['bar/pub/beer'] # test a mixture of explicit and partial IDs mute.clear() await run('bar', 'baz', 'foo', opts=opts) assert mute == ['bar/pub/beer', 'baz/run1', 'foo'] # test a glob mute.clear() await run('*', opts=opts) assert mute == ['bar/pub/beer', 'baz/run1', 'foo'] @pytest.mark.parametrize( 'timeout, expected', [('100', '100'), ('PT1M2S', '62'), ('', ''), ('oopsie', InputError), (' ', InputError)] ) def test_parse_timeout( timeout: str, expected: Union[str, Type[InputError]] ): """It should accept ISO 8601 format or number of seconds.""" opts = CleanOptions(remote_timeout=timeout) if expected is InputError: with pytest.raises(expected): parse_timeout(opts) else: parse_timeout(opts) assert opts.remote_timeout == expected @pytest.mark.parametrize( 'opts, expected_msg', [ ({'local_only': True, 'remote_only': True}, "mutually exclusive"), ({'remote_timeout': 'oops'}, "Invalid timeout"), ] ) def test_bad_user_input(opts: dict, expected_msg: str, mute): """It should raise an InputError for bad user input.""" with pytest.raises(InputError) as exc_info: _main(CleanOptions(**opts), 'blah') assert expected_msg in str(exc_info.value) cylc-flow-8.7.0/tests/unit/scripts/test_cat_log.py0000664000175000017500000001775415264420307022463 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from subprocess import Popen, PIPE from ansimarkup import parse as cparse from colorama import Style import pytest import shlex from cylc.flow.exceptions import InputError from cylc.flow.option_parsers import Options from cylc.flow.loggingutil import CylcLogFormatter from cylc.flow.scripts.cat_log import ( colorise_cat_log, TAIL, TAIL_END, _main as cat_log, _get_remote_log, get_option_parser as cat_log_gop, get_tail_lines, view_log, ) TAILER_PLATFORM = { 'tail command template': 'tail -n %(lines)s --follow=name %(filename)s', } @pytest.fixture def log_file(tmp_path): _log_file = tmp_path / 'log' with open(_log_file, 'w+') as fh: fh.write('DEBUG - 1\n') fh.write('INFO - 2\n') fh.write('WARNING - 3\n') fh.write('ERROR - 4\n') fh.write('CRITICAL - 5\n') return _log_file def test_colorise_cat_log_plain(log_file): """It should not colourise logs when color=False.""" # command for colorise_cat_log to colourise cat_proc = Popen( ['cat', str(log_file)], stdout=PIPE, ) colorise_cat_log(cat_proc, color=False) assert cat_proc.communicate()[0].decode().splitlines() == [ # there should not be any ansii color characters here 'DEBUG - 1', 'INFO - 2', 'WARNING - 3', 'ERROR - 4', 'CRITICAL - 5', ] def test_colorise_cat_log_colour(log_file): """It should colourise logs when color=True.""" # command for colorise_cat_log to colourise cat_proc = Popen( ['cat', str(log_file)], stdout=PIPE, ) out, err = colorise_cat_log(cat_proc, color=True, stdout=PIPE) # strip the line breaks (because tags can come before or after them) # strip the reset tags (because they might not be needed if redeclared) out = ''.join( line.replace(Style.RESET_ALL, '') for line in out.decode().splitlines() ) col = CylcLogFormatter.COLORS assert out == ( ''.join([ # strip the reset tags cparse(line).replace(Style.RESET_ALL, '') for line in [ col['DEBUG'].format('DEBUG - 1'), 'INFO - 2', col['WARNING'].format('WARNING - 3'), col['ERROR'].format('ERROR - 4'), col['CRITICAL'].format('CRITICAL - 5'), '' ] ]) ) class TestGetTailLines: """Tests for the get_tail_lines function.""" @pytest.mark.parametrize( 'mode, expected', [ (TAIL, '+1'), (TAIL_END, '100'), ('unknown_mode', '+1'), ], ) def test_modes(self, mode, expected): """tail-end starts from the end; all other modes from the start.""" assert get_tail_lines(mode, 100) == expected async def test_get_remote_log_bakes_tail_lines_for_tail_end(monkeypatch): """TAIL_END bakes the line count into the forwarded tail command. The line count is substituted locally so the remote cat-log needs no knowledge of --tail-lines; the mode is forwarded unchanged so the remote end still knows it is a tail-end view. """ captured = {} async def mock_remote_cylc_cmd(cmd, platform, **kwargs): captured['cmd'] = cmd captured['kwargs'] = kwargs return 0 monkeypatch.setattr( 'cylc.flow.scripts.cat_log.remote_cylc_cmd', mock_remote_cylc_cmd ) monkeypatch.setattr( 'cylc.flow.scripts.cat_log.get_remote_workflow_run_job_dir', lambda *a, **k: '/remote/workflow/log/job.out', ) monkeypatch.setattr( 'cylc.flow.scripts.cat_log.verbosity_to_opts', lambda *a, **k: [] ) workflow_id = 'workflow' await _get_remote_log( workflow_id, TAILER_PLATFORM, point='1', task='foo', submit_num='NN', filename='job.out', mode=TAIL_END, tail_lines=42, ) assert ( f"--remote-arg={shlex.quote('tail -n 42 --follow=name %(filename)s')}" in captured['cmd'] ) assert f'--remote-arg={TAIL_END}' in captured['cmd'] assert captured['kwargs']['manage'] is True async def test_view_log_tail_vs_tail_end(tmp_path, capfd): """TAIL reads from the start; TAIL_END reads from the end.""" logpath = tmp_path / 'job.out' lines = [ 'line-1', 'line-2', 'line-3', 'line-4', ] logpath.write_text('\n'.join(lines) + '\n') await view_log( logpath, TAIL, 'tail -n %(lines)s %(filename)s', ) out = capfd.readouterr().out.splitlines() assert out == lines await view_log( logpath, TAIL_END, 'tail -n %(lines)s %(filename)s', tail_lines=2, ) out = capfd.readouterr().out.splitlines() assert out == lines[-2:] await view_log( logpath, TAIL, 'tail -n +1 --follow=name %(filename)s', batchview_cmd=f'cat {logpath}', ) out = capfd.readouterr().out.splitlines() assert out == lines @pytest.mark.parametrize('tail_lines', [0, -1, -100]) async def test_bad_tail_lines(tail_lines): """Non-positive --tail-lines values should be rejected.""" parser = cat_log_gop() with pytest.raises(InputError, match='--tail-lines must be a positive'): await cat_log( parser, Options(parser)(tail_lines=tail_lines), 'workflow//1/foo', ) async def test_bad_submit_number(monkeypatch, capsys): """Illegal submit numbers should be rejected before log lookup.""" parser = cat_log_gop() async def mock_parse_id_async(*args, **kwargs): return 'workflow', {'task': 'foo', 'cycle': '1'}, None monkeypatch.setattr( 'cylc.flow.scripts.cat_log.parse_id_async', mock_parse_id_async, ) with pytest.raises(SystemExit) as exc_info: await cat_log( parser, Options(parser)(submit_num='not-a-number'), 'workflow//1/foo', ) # The SystemExit is raised from within the `except ValueError` handler, # so its context should be the underlying ValueError. assert isinstance(exc_info.value.__context__, ValueError) assert 'Illegal submit number: not-a-number' in capsys.readouterr().err async def test_good_submit_number(monkeypatch): """A valid submit number should be zero-padded and passed through.""" parser = cat_log_gop() async def mock_parse_id_async(*args, **kwargs): return 'workflow', {'task': 'foo', 'cycle': '1'}, None monkeypatch.setattr( 'cylc.flow.scripts.cat_log.parse_id_async', mock_parse_id_async, ) captured = {} class StopHere(Exception): pass def mock_get_task_job_attrs(workflow_id, point, task, submit_num): captured['submit_num'] = submit_num raise StopHere monkeypatch.setattr( 'cylc.flow.scripts.cat_log.get_task_job_attrs', mock_get_task_job_attrs, ) # Stop execution once the submit number has been processed. with pytest.raises(StopHere): await cat_log( parser, Options(parser)(submit_num='1'), 'workflow//1/foo', ) # The submit number should have been zero-padded to two digits. assert captured['submit_num'] == '01' cylc-flow-8.7.0/tests/unit/scripts/test_scan.py0000664000175000017500000000553515264420307021771 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from pathlib import Path from cylc.flow.scripts.scan import ( ScanOptions, get_pipe, _format_plain, _construct_tree, FLOW_STATES, BAD_CONTACT_FILE_MSG ) def test_no_connection(): """Ensure scan uses the filesystem where possible.""" pipe = get_pipe(ScanOptions(states=FLOW_STATES), _format_plain) assert 'graphql_query' not in repr(pipe) def test_ping_connection(): """Ensure scan always connects to the flow when requested via --ping.""" pipe = get_pipe(ScanOptions(states=FLOW_STATES, ping=True), _format_plain) assert 'graphql_query' in repr(pipe) def test_good_contact_info() -> None: """Check correct reporting of workflow contact information.""" port = 8888 host = "wizard" name = "blargh/run1" pid = 12345 res = _format_plain( { "name": name, "contact": Path(f"/path/to/{name}"), "CYLC_WORKFLOW_HOST": host, "CYLC_WORKFLOW_PORT": port, "CYLC_WORKFLOW_PID": pid, }, None ) assert name in res assert f"{host}:{port} {pid}" in res def test_bad_contact_info(caplog: pytest.LogCaptureFixture) -> None: """Check correct reporting of bad workflow contact information. Missing contact keys should result in a warning. """ name = "blargh/run1" _format_plain( { "name": name, "contact": Path(f"/path/to/{name}"), }, None ) assert BAD_CONTACT_FILE_MSG.format(flow_name=name) in caplog.text def test_bad_contact_info_tree(caplog: pytest.LogCaptureFixture) -> None: """Check correct reporting of bad workflow contact information. Missing contact keys should result in a warning. """ name = "blargh/run1" flows = [{ "name": name, "contact": Path(f"/path/to/{name}"), }] tree = {} _construct_tree(flows, tree, _format_plain, None, None) # Error during tree formatting: reports only the last name component. assert ( BAD_CONTACT_FILE_MSG.format(flow_name=f"{Path(name).name}") in caplog.text ) cylc-flow-8.7.0/tests/unit/scripts/test_install.py0000664000175000017500000000431015264420307022501 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import os.path from pathlib import Path from typing import Optional import pytest from cylc.flow.scripts.install import get_source_location @pytest.mark.parametrize( 'path, expected', [ pytest.param( 'isla/nublar', '{cylc_src}/isla/nublar', id="implicit relative" ), pytest.param( './isla/nublar', '{cwd}/isla/nublar', id="explicit relative" ), pytest.param( '/welcome/to/jurassic/park', '/welcome/to/jurassic/park', id="absolute" ), pytest.param( None, '{cwd}', id="None" ), pytest.param( '.', '{cwd}', id="dot" ), pytest.param( '$GENNARO/coupon-day', '{env_var}/coupon-day', id="env var expanded" ), ] ) def test_get_source_location( path: Optional[str], expected: str, monkeypatch: pytest.MonkeyPatch ): # Setup mock_cylc_src = '/ingen/cylc-src' monkeypatch.setattr( 'cylc.flow.scripts.install.search_install_source_dirs', lambda x: Path(mock_cylc_src, x) ) mock_env_var = '/donald/gennaro' monkeypatch.setenv('GENNARO', mock_env_var) expected = expected.format( cwd=Path.cwd(), cylc_src=mock_cylc_src, env_var=mock_env_var, ) # Test assert get_source_location(path) == Path(expected) assert os.path.isabs(expected) cylc-flow-8.7.0/tests/unit/test_workflow_status.py0000664000175000017500000001203515264420307022624 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from types import SimpleNamespace import pytest from metomi.isodatetime.data import TimePoint from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.workflow_status import ( WORKFLOW_STATUS_RUNNING_TO_HOLD, WORKFLOW_STATUS_RUNNING_TO_STOP, StopMode, WorkflowStatus, get_workflow_status, get_workflow_status_msg, ) STOP_TIME = TimePoint(year=2006).to_local_time_zone() def schd( final_point=None, hold_point=None, is_paused=False, is_stalled=None, stop_clock_time=None, stop_mode=None, stop_point=None, stop_task_id=None, reload_pending=False, ): return SimpleNamespace( is_paused=is_paused, is_stalled=is_stalled, stop_clock_time=stop_clock_time, stop_mode=stop_mode, reload_pending=reload_pending, pool=SimpleNamespace( hold_point=hold_point, stop_point=stop_point, stop_task_id=stop_task_id, ), config=SimpleNamespace(final_point=final_point), options=SimpleNamespace(utc_mode=True), ) @pytest.mark.parametrize( 'kwargs, state, message', [ # test each of the states ( {'is_paused': True}, WorkflowStatus.PAUSED, 'paused' ), ( {'reload_pending': 'message'}, WorkflowStatus.PAUSED, 'reloading: message' ), ( {'stop_mode': StopMode.AUTO}, WorkflowStatus.STOPPING, 'stopping: waiting for active jobs to complete' ), ( {'hold_point': 2}, WorkflowStatus.RUNNING, WORKFLOW_STATUS_RUNNING_TO_HOLD % 2 ), ( {'stop_point': 4}, WorkflowStatus.RUNNING, WORKFLOW_STATUS_RUNNING_TO_STOP % 4 ), ( {'stop_clock_time': STOP_TIME.seconds_since_unix_epoch}, WorkflowStatus.RUNNING, WORKFLOW_STATUS_RUNNING_TO_STOP % str(STOP_TIME) ), ( {'stop_task_id': '6/foo'}, WorkflowStatus.RUNNING, WORKFLOW_STATUS_RUNNING_TO_STOP % '6/foo' ), ( {'final_point': 8}, WorkflowStatus.RUNNING, WORKFLOW_STATUS_RUNNING_TO_STOP % 8 ), ( {'is_stalled': True}, WorkflowStatus.RUNNING, 'stalled' ), ( {}, WorkflowStatus.RUNNING, 'running' ), # test combinations ( # stopping should trump stalled, paused & running { 'stop_mode': StopMode.REQUEST_NOW, 'is_stalled': True, 'is_paused': True }, WorkflowStatus.STOPPING, 'stopping: shutting down' ), ( {'is_stalled': True, 'is_paused': True}, WorkflowStatus.PAUSED, 'stalled and paused', ), ( # earliest of stop point, hold point and stop task id { 'stop_point': IntegerPoint(4), 'hold_point': IntegerPoint(2), 'stop_task_id': '6/foo', }, WorkflowStatus.RUNNING, WORKFLOW_STATUS_RUNNING_TO_HOLD % 2, ), ( { 'stop_point': IntegerPoint(11), 'hold_point': IntegerPoint(15), 'stop_task_id': '9/bar', }, WorkflowStatus.RUNNING, WORKFLOW_STATUS_RUNNING_TO_STOP % '9/bar', ), ( { 'stop_point': IntegerPoint(3), 'hold_point': IntegerPoint(3), }, WorkflowStatus.RUNNING, WORKFLOW_STATUS_RUNNING_TO_STOP % 3, ), ( # stop point trumps final point { 'stop_point': IntegerPoint(1), 'final_point': IntegerPoint(2), }, WorkflowStatus.RUNNING, WORKFLOW_STATUS_RUNNING_TO_STOP % 1, ), ] ) def test_get_workflow_status(kwargs, state, message, set_cycling_type): set_cycling_type() scheduler = schd(**kwargs) assert get_workflow_status(scheduler) == state assert get_workflow_status_msg(scheduler) == message cylc-flow-8.7.0/tests/unit/test_remote.py0000664000175000017500000000770115264420307020646 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test the cylc.flow.remote module.""" import os import pytest import cylc.flow from cylc.flow.remote import ( construct_rsync_over_ssh_cmd, construct_ssh_cmd, get_proc_ancestors, run_cmd, ) async def test_run_cmd_stdin_str(): """Test passing stdin as a string.""" proc = await run_cmd( ['sed', 's/foo/bar/'], stdin_str='1foo2', capture_process=True ) assert [s.strip() for s in proc.communicate()] == [ '1bar2', '' ] async def test_run_cmd_stdin_file(tmp_path): """Test passing stdin as a file.""" tmp_path = tmp_path / 'stdin' with tmp_path.open('w+') as tmp_file: tmp_file.write('1foo2') tmp_file = tmp_path.open('rb') proc = await run_cmd( ['sed', 's/foo/bar/'], stdin=tmp_file, capture_process=True ) assert [s.strip() for s in proc.communicate()] == [ '1bar2', '' ] def test_construct_rsync_over_ssh_cmd(): """Function against known good output. """ cmd, host = construct_rsync_over_ssh_cmd( '/foo', '/bar', { 'rsync command': 'rsync command', 'hosts': ['miklegard'], 'ssh command': 'strange_ssh', 'selection': {'method': 'definition order'}, 'name': 'testplat' } ) assert host == 'miklegard' assert cmd == [ 'rsync', 'command', '--delete', '--rsh=strange_ssh', '--include=/.service/', '--include=/.service/server.key', '-a', '--checksum', '--out-format=%o %n%L', '--no-t', '--exclude=/log', '--exclude=/share', '--exclude=/work', '--include=/ana/***', '--include=/app/***', '--include=/bin/***', '--include=/etc/***', '--include=/lib/***', '--exclude=*', '/foo/', 'miklegard:/bar/', ] def test_construct_ssh_cmd_forward_env(monkeypatch: pytest.MonkeyPatch): """ Test for 'ssh forward environment variables' """ # Clear CYLC_* env vars as these will show up in the command for env_var in os.environ: if env_var.startswith('CYLC'): monkeypatch.delenv(env_var) host = 'example.com' config = { 'ssh command': 'ssh', 'use login shell': None, 'cylc path': None, 'ssh forward environment variables': ['FOO', 'BAZ'], } # Variable isn't set, no change to command expect = [ 'ssh', host, 'env', f'CYLC_VERSION={cylc.flow.__version__}', 'cylc', 'play', ] cmd = construct_ssh_cmd(['play'], config, host) assert cmd == expect # Variable is set, appears in `env` list monkeypatch.setenv('FOO', 'BAR') expect = [ 'ssh', host, 'env', f'CYLC_VERSION={cylc.flow.__version__}', 'FOO=BAR', 'cylc', 'play', ] cmd = construct_ssh_cmd(['play'], config, host) assert cmd == expect def test_get_proc_ancestors__basic(): """It should return a list of ancestor PIDs, starting with the parent.""" gpa = get_proc_ancestors() assert len(gpa) > 1 assert gpa[0] == os.getppid() cylc-flow-8.7.0/tests/unit/test_links.py0000664000175000017500000000621615264420307020473 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Check links inserted into internal documentation. Reason for doing this here: - Some links don't appear to be being picked up by Cylc-doc linkcheck. - As we have more links it's worth checking them here, rather than waiting for them to show up in Cylc. """ import fnmatch import os from pathlib import Path import re from time import sleep from urllib import request from urllib.error import HTTPError import pytest class LinkCheckError(Exception): """A link has failed to validate.""" EXCLUDE = [ r'*//www.gnu.org/licenses/', r'*//my-site.com/*', r'*//ahost/%(owner)s/notes/%(workflow)s', r'*//www.h2g2.com/*', r'*//web.archive.org/*' ] def get_links(): searchdir = Path(__file__).parent.parent.parent / 'cylc' / 'flow' return sorted({ url for file_ in searchdir.rglob('*.py') for url in re.findall( r'(https?:\/\/.*?)[\n\s\>`"\',]', file_.read_text() ) if not any( fnmatch.fnmatch(url, pattern) for pattern in EXCLUDE ) }) def make_request(link): """Make an HTTP request, using GITHUB_TOKEN for GitHub URLs if available. The GITHUB_TOKEN environment variable contains a GitHub Actions token. This is used to authenticate the workflow requests to github.com, which helps avoid rate limiting (unauthenticated requests are limited to 60/hour, authenticated to 5000/hour). """ req = request.Request(link) github_token = os.environ.get('GITHUB_TOKEN') if github_token and 'github.com' in link: req.add_header('Authorization', f'token {github_token}') return request.urlopen(req).getcode() @pytest.mark.linkcheck @pytest.mark.parametrize('link', get_links()) def test_embedded_url(link): """Check links in the source code are not broken. TIP: use `--dist=load` when running pytest to enable parametrized tests to run in parallel """ try: make_request(link) except HTTPError as exc: # Allowing 403 (forbidden), 429 (rate-limited), and 503 # (service unavailable) as the link is probably valid, # but we are blocked. if exc.code in {403, 429, 503}: pytest.skip(f'{exc} | {link}') # Sleep and retry to reduce risk of flakiness: sleep(10) try: make_request(link) except HTTPError as exc: raise LinkCheckError(f'{exc} | {link}') from None cylc-flow-8.7.0/tests/unit/test_psutil.py0000664000175000017500000000317015264420307020667 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import json import os from psutil import Process from cylc.flow.scripts.psutil import _psutil def test_psutil_basic(): """It should return measurements.""" # obtain memory reading ret = _psutil('[["virtual_memory"]]') # we asked for one thing so we should get one response assert len(ret) == 1 # the result should be dict-like and serialise to json mem = ret[0] dict(mem) json.dumps(mem) for key in ('total', 'available', 'used', 'free'): # it should have multiple fields assert key in mem # all of which should be integers assert isinstance(mem[key], int) def test_psutil_object(): """It should call object methods.""" # obtain the commandline for this test process ret = _psutil(f'[["Process.cmdline", {os.getpid()}]]') assert ret[0] == Process().cmdline() cylc-flow-8.7.0/tests/unit/test_indep_task_queues.py0000664000175000017500000000567715264420307023075 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # Tests for the task queue manager module from collections import Counter from unittest.mock import Mock import pytest from cylc.flow.task_proxy import TaskProxy from cylc.flow.task_queues.independent import IndepQueueManager MEMBERS = {"a", "b", "c", "d", "e", "f"} ACTIVE = Counter(["a", "a", "d"]) ALL_TASK_NAMES = [ "o1", "o2", "o3", "o4", "o5", "o6", "o7", "s1", "s2", "s3", "s4", "s5", "b1", "b2", "b3", "b4", "b5", "foo" ] DESCENDANTS = { "root": ALL_TASK_NAMES + ["BIG", "SML", "OTH", "foo"], "BIG": ["b1", "b2", "b3", "b4", "b5"], "SML": ["s1", "s2", "s3", "s4", "s5"], "OTH": ["o1", "o2", "o3", "o4", "o5", "o6", "o7"] } QCONFIG = { "default": { "limit": 6, "members": [] # (auto: all task names) }, "big": { "members": ["BIG", "foo"], "limit": 2 }, "sml": { "members": ["SML", "foo"], "limit": 3 } } READY_TASK_NAMES = ["b3", "s4", "o2", "s3", "b4", "o3", "o4", "o5", "o6", "o7"] @pytest.mark.parametrize( "active, expected_released, expected_foo_groups", [ ( Counter(["b1", "b2", "s1", "o1"]), ["s4", "o2", "s3", "o3", "o4", "o5", "o6"], ["sml"] ) ] ) def test_queue_and_release( active, expected_released, expected_foo_groups ): """Test task queue and release.""" # configure the queue queue_mgr = IndepQueueManager(QCONFIG, ALL_TASK_NAMES, DESCENDANTS) # add newly ready tasks to the queue for name in READY_TASK_NAMES: itask = Mock(spec=TaskProxy) itask.tdef.name = name itask.state.is_held = False queue_mgr.push_task(itask) # release tasks, given current active task counter released = queue_mgr.release_tasks(active) assert sorted(r.tdef.name for r in released) == sorted(expected_released) # check that adopted orphans end up in the default queue orphans = ["orphan1", "orphan2"] queue_mgr.adopt_tasks(orphans) for orphan in orphans: assert orphan in queue_mgr.queues["default"].members # check second assignment overrides first for group in expected_foo_groups: assert "foo" in queue_mgr.queues[group].members cylc-flow-8.7.0/tests/unit/test_host_select.py0000664000175000017500000001634215264420307021670 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test the cylc.flow.host_select module. NOTE: these are functional tests, for unit tests see the docstrings in the host_select module. """ import logging from secrets import token_hex import socket import pytest from cylc.flow import CYLC_LOG from cylc.flow.exceptions import HostSelectException from cylc.flow.host_select import ( _get_metrics, select_host, select_workflow_host, ) from cylc.flow.hostuserutil import get_fqdn_by_host from cylc.flow.parsec.exceptions import ListValueError localhost, localhost_aliases, _ = socket.gethostbyname_ex('localhost') localhost_fqdn = get_fqdn_by_host(localhost) # NOTE: ensure that all localhost aliases are actually aliases of localhost, # it would appear that this is not always the case # on Travis-CI on of the aliases has a different fqdn from the fqdn # of the host it is an alias of localhost_aliases = [ alias for alias in localhost_aliases if get_fqdn_by_host(alias) == localhost_fqdn ] async def test_localhost(): """Basic test with one host to choose from.""" assert await select_host([localhost]) == ( localhost, localhost_fqdn ) async def test_unique(): """Basic test choosing from multiple forms of localhost""" name, fqdn = await select_host( localhost_aliases + [localhost] ) assert name in localhost_aliases + [localhost] assert fqdn == localhost_fqdn async def test_filter(): """Test that hosts are filtered out if specified.""" message = 'Localhost not allowed' with pytest.raises(HostSelectException) as excinfo: await select_host( [localhost], blacklist=[localhost], blacklist_name=message ) assert message in str(excinfo.value) async def test_filter_invalid_blacklist(log_filter): """Test that invalid blacklist doesn't bring down the program.""" result = await select_host( [localhost], blacklist=[f'non_exist_{token_hex(4)}'] ) assert result[0] == localhost assert log_filter(logging.WARNING, "Could not resolve blacklisted host") async def test_rankings(): """Positive test that rankings are evaluated. (doesn't prove anything by itself hence test_unreasonable_rankings) """ assert await select_host( [localhost], ranking_string=''' # if this test fails due to race conditions # then you have bigger issues than a test failure virtual_memory().available > 1 getloadavg()[0] < 500 cpu_count() > 1 disk_usage('/').free > 1 ''' ) == (localhost, localhost_fqdn) async def test_unreasonable_rankings(): """Negative test that rankings are evaluated. (doesn't prove anything by itself hence test_rankings) """ with pytest.raises(HostSelectException) as excinfo: await select_host( [localhost], ranking_string=''' # if this test fails due to race conditions # then you are very lucky virtual_memory().available > 123456789123456789 getloadavg()[0] < 1 cpu_count() > 512 disk_usage('/').free > 123456789123456789 ''' ) assert ( 'virtual_memory().available > 123456789123456789: False' ) in str(excinfo.value) async def test_metric_command_failure(): """If the psutil command (or SSH) fails ensure the host is excluded.""" with pytest.raises(HostSelectException) as excinfo: await select_host( [localhost], ranking_string=''' # elephant is not a psutil attribute # so will cause the command to fail elephant ''' ) # we should expect the special (2) returncode # (i.e. the command ran fine but there was something wrong with the # provided expression) assert excinfo.value.data[localhost_fqdn]['returncode'] == 2 async def test_workflow_host_select(mock_glbl_cfg): """Run the workflow_host_select mechanism.""" mock_glbl_cfg( 'cylc.flow.host_select.glbl_cfg', f''' [scheduler] [[run hosts]] available= {localhost} ''' ) assert await select_workflow_host() == (localhost, localhost_fqdn) async def test_workflow_host_select_default(mock_glbl_cfg): """Ensure "localhost" is provided as a default host.""" mock_glbl_cfg( 'cylc.flow.host_select.glbl_cfg', ''' [scheduler] [[run hosts]] available = ''' ) hostname, host_fqdn = await select_workflow_host() assert hostname in localhost_aliases + [localhost] assert host_fqdn == localhost_fqdn # NOTE: on Travis-CI the fqdn of `localhost` is `localhost` @pytest.mark.skipif( localhost == localhost_fqdn, reason='Cannot condemn a host unless is has a safe unique fqdn.' ) async def test_workflow_host_select_condemned(mock_glbl_cfg): """Ensure condemned hosts are filtered out.""" mock_glbl_cfg( 'cylc.flow.host_select.glbl_cfg', f''' [scheduler] [[run hosts]] available = {localhost} condemned = {localhost_fqdn} ''' ) with pytest.raises(HostSelectException) as excinfo: await select_workflow_host() assert 'blacklisted' in str(excinfo.value) assert 'condemned host' in str(excinfo.value) def test_condemned_host_ambiguous(mock_glbl_cfg): """Test the [scheduler]condemend host coercer Not actually host_select code but related functionality. """ with pytest.raises(ListValueError) as excinfo: mock_glbl_cfg( 'cylc.flow.host_select.glbl_cfg', f''' [scheduler] [[run hosts]] available = {localhost} condemned = {localhost} ''' ) assert 'ambiguous host' in excinfo.value.msg async def test_get_metrics_no_hosts_error(caplog): """It should handle SSH errors. If a host is not contactable then it should be shipped. """ caplog.set_level(logging.WARN, CYLC_LOG) data = {} host_stats = await _get_metrics(['not-a-host'], None, data) # a warning should be logged assert len(caplog.records) == 1 # no data for the host should be returned assert not host_stats # the return code should be recorded assert data == {'not-a-host': {'returncode': 255}} cylc-flow-8.7.0/tests/unit/test_exceptions.py0000664000175000017500000000714615264420307021537 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from textwrap import dedent import pytest from cylc.flow.exceptions import ( CylcError, HostSelectException, CommandFailedError, FileRemovalError, CylcVersionError, ) def test_cylc_error_str(): error = CylcError("abcd") assert str(error) == "abcd" def test_host_select_exception(): """Test exception used for host selection failures. * Could not connect to hosts (e.g. SSH failure). * Commands failed (e.g. configuration error). * Could not obtain metrics (e.g. host ranking expression error). * No available hosts (e.g. no hosts met ranking thresholds). """ # it should format the selection results nicely exc = HostSelectException( { 'host-1': {'returncode': 1}, 'host-2': {'returncode': 1}, }, ranking='virtual_memory().available > 1', ) assert str(exc) == dedent(''' Could not select host from: host-1: returncode: 1 host-2: returncode: 1 ''').strip() @pytest.mark.parametrize( 'ret_code, expect', [ # it should give a useful hint for exit code "2" # (error in the selection ranking expression) (2, 'This is likely an error in the ranking expression'), # it should give a useful hint for the exit code "255" # (ssh error) (255, 'Cylc could not establish SSH connection to the run hosts.') ] ) def test_host_select_exception_returncodes(ret_code, expect): assert expect in str( HostSelectException( { 'host-1': {'returncode': ret_code}, 'host-2': {'returncode': ret_code}, }, ranking='virtual_memory().available > 1', ) ) def test_command_failed_exception(): """Test exception used for command failures.""" # when the value is a string, it should return it directly exc = CommandFailedError("something broke") assert str(exc) == "something broke" # when the value is an exception, it should format the type and message exc = CommandFailedError(ValueError("bad value")) assert str(exc) == "ValueError: bad value" def test_file_removal_error(): """Test exception used for file removal failures.""" exc = FileRemovalError(OSError("error removing file")) assert str(exc) == ( "error removing file. This is probably a temporary " "issue with the filesystem, " "not a problem with Cylc." ) def test_cylc_version_error(): """Test exception used for incompatible Cylc versions.""" # when version is provided exc = CylcVersionError(version='7.0.0') assert str(exc) == ( 'Installed Cylc 7.0.0 workflow is not compatible with Cylc 8.' ) # when version is None exc = CylcVersionError() assert str(exc) == "Installed workflow is not compatible with Cylc 8." cylc-flow-8.7.0/tests/unit/test_wallclock.py0000664000175000017500000000720315264420307021323 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from datetime import ( datetime, timedelta, timezone, ) from metomi.isodatetime.data import CALENDAR import pytest from pytest import param from cylc.flow.wallclock import ( get_current_time_string, get_time_string, get_unix_time_from_time_string, ) @pytest.mark.parametrize( 'time_str,time_sec', [ ('2016-09-08T09:09:00+01', 1473322140), ('2016-09-08T08:09:00Z', 1473322140), ('2016-09-07T20:09:00-12', 1473322140), ] ) def test_get_unix_time_from_time_string(time_str, time_sec): assert get_unix_time_from_time_string(time_str) == time_sec @pytest.mark.parametrize( 'time_str,time_sec', [ ('2016-09-08T09:09:00+01', 1473322140), ('2016-09-08T08:09:00Z', 1473322140), ('2016-09-07T20:09:00-12', 1473322140), ('2016-08-31T18:09:00+01', 1472663340), ] ) def test_get_unix_time_from_time_string_360(time_str, time_sec): mode = CALENDAR.mode CALENDAR.set_mode(CALENDAR.MODE_360) try: assert get_unix_time_from_time_string(time_str) == time_sec finally: CALENDAR.set_mode(mode) @pytest.mark.parametrize( 'value,error', [ (None, TypeError), (42, TypeError) ] ) def test_get_unix_time_from_time_string_error(value, error): with pytest.raises(error): get_unix_time_from_time_string(value) @pytest.mark.parametrize('tz_info', [ pytest.param(None, id="naive"), pytest.param(timezone.utc, id="utc-tz-aware"), pytest.param(timezone(timedelta(hours=5)), id="custom-tz-aware"), ]) def test_get_time_string_tzinfo(tz_info, set_timezone): """Basic check it handles naive and timezone-aware datetime objects. Currently we just ignore the timezone information in the datetime object. """ set_timezone('UTC') assert get_time_string( datetime(2077, 2, 8, 13, 42, 39, 123456, tz_info) ) == '2077-02-08T13:42:39Z' def test_get_current_time_string(set_timezone): """It reacts to local time zone changes. https://github.com/cylc/cylc-flow/issues/6701 """ set_timezone('XXX-19:17') res = get_current_time_string() assert res[-6:] == '+19:17' @pytest.mark.parametrize( 'arg, kwargs, expect', ( param( datetime(2000, 12, 13, 15, 30, 12, 123456), {}, '2000-12-14T10:47:12+19:17', id='good', ), param( datetime(2000, 12, 13, 15, 30, 12, 123456), {'date_time_is_local': True}, '2000-12-13T15:30:12+19:17', id='dt_is_local', ), param( datetime(2000, 12, 13, 15, 30, 12, 123456), {'use_basic_format': True}, '20001214T104712+1917', id='basic_format', ), ), ) def test_get_time_string(set_timezone, arg, kwargs, expect): set_timezone('XXX-19:17') assert get_time_string(arg, **kwargs) == expect cylc-flow-8.7.0/tests/unit/test_task_state.py0000664000175000017500000001325615264420307021517 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from unittest.mock import MagicMock import pytest from types import SimpleNamespace from cylc.flow.prerequisite import Prerequisite from cylc.flow.taskdef import TaskDef from cylc.flow.cycling.integer import IntegerSequence, IntegerPoint from cylc.flow.run_modes import RunMode, disable_task_event_handlers from cylc.flow.task_trigger import Dependency, TaskTrigger from cylc.flow.task_state import ( TaskState, TASK_STATUS_PREPARING, TASK_STATUS_SUBMIT_FAILED, TASK_STATUS_SUBMITTED, TASK_STATUS_SUCCEEDED, TASK_STATUS_WAITING, TASK_STATUS_RUNNING, ) @pytest.mark.parametrize( 'state,is_held', [ (TASK_STATUS_WAITING, True), (TASK_STATUS_SUCCEEDED, False) ] ) def test_state_comparison(state, is_held): """Test the __call__ method.""" tdef = TaskDef('foo', {}, '123', '123') tstate = TaskState(tdef, '123', state, is_held) assert tstate(state, is_held=is_held) assert tstate(state) assert tstate(is_held=is_held) assert tstate(state, 'of', 'flux') assert tstate(state, 'of', 'flux', is_held=is_held) assert not tstate(state + 'x', is_held=not is_held) assert not tstate(state, is_held=not is_held) assert not tstate(state + 'x', is_held=is_held) assert not tstate(state + 'x') assert not tstate(is_held=not is_held) assert not tstate(state + 'x', 'of', 'flux') @pytest.mark.parametrize( 'state,is_held,should_reset', [ (None, None, False), (TASK_STATUS_WAITING, None, False), (None, True, False), (TASK_STATUS_WAITING, True, False), (TASK_STATUS_SUCCEEDED, None, True), (None, False, True), (TASK_STATUS_WAITING, False, True), ] ) def test_reset(state, is_held, should_reset): """Test that tasks do or don't have their state changed.""" tdef = TaskDef('foo', {}, '123', '123') # create task state: # * status: waiting # * is_held: true tstate = TaskState(tdef, '123', TASK_STATUS_WAITING, True) assert tstate.reset(state, is_held) == should_reset if is_held is not None: assert tstate.is_held == is_held if state is not None: assert tstate.status == state def test_task_prereq_duplicates(set_cycling_type): """Test prerequisite duplicates from multiple recurrences are discarded.""" set_cycling_type() seq1 = IntegerSequence('R1', "1") seq2 = IntegerSequence('R/1/P1', "1") trig = TaskTrigger('a', "1", 'succeeded', None, None, None, None) dep = Dependency([trig], [trig], False) tdef = TaskDef('foo', {}, IntegerPoint("1"), IntegerPoint("1")) tdef.add_dependency(dep, seq1) tdef.add_dependency(dep, seq2) # duplicate! tstate = TaskState(tdef, IntegerPoint("1"), TASK_STATUS_WAITING, False) prereqs = [p._satisfied for p in tstate.prerequisites] assert prereqs == [{("1", "a", "succeeded"): False}] def test_task_state_order(): """Test is_gt and is_gte methods.""" tdef = TaskDef('foo', {}, IntegerPoint("1"), IntegerPoint("1")) tstate = TaskState(tdef, IntegerPoint("1"), TASK_STATUS_SUBMITTED, False) assert tstate.is_gt(TASK_STATUS_WAITING) assert tstate.is_gt(TASK_STATUS_PREPARING) assert tstate.is_gt(TASK_STATUS_SUBMIT_FAILED) assert not tstate.is_gt(TASK_STATUS_SUBMITTED) assert tstate.is_gte(TASK_STATUS_SUBMITTED) assert not tstate.is_gt(TASK_STATUS_RUNNING) assert not tstate.is_gte(TASK_STATUS_RUNNING) def test_get_resolved_dependencies(): prereq1 = Prerequisite(IntegerPoint('2')) prereq1[('1', 'a', 'x')] = True prereq1[('1', 'b', 'x')] = False prereq1[('1', 'c', 'x')] = 'satisfied from database' prereq1[('1', 'd', 'x')] = 'force satisfied' prereq2 = Prerequisite(IntegerPoint('2')) prereq2[('1', 'e', 'succeeded')] = False prereq2[('1', 'e', 'failed')] = True task_state = TaskState( MagicMock(), IntegerPoint('2'), TASK_STATUS_WAITING, False ) task_state.prerequisites = [prereq1, prereq2] assert task_state.get_resolved_dependencies() == [ '1/a', '1/c', '1/d', '1/e', ] @pytest.mark.parametrize( 'itask_run_mode, disable_handlers, expect', ( ('live', True, False), ('live', False, False), ('dummy', True, False), ('dummy', False, False), ('simulation', True, True), ('simulation', False, True), ('skip', True, True), ('skip', False, False), ) ) def test_disable_task_event_handlers(itask_run_mode, disable_handlers, expect): """Conditions under which task event handlers should not be used. """ # Construct a fake itask object: itask = SimpleNamespace( run_mode=RunMode(itask_run_mode), platform={'disable task event handlers': disable_handlers}, tdef=SimpleNamespace( rtconfig={ 'skip': {'disable task event handlers': disable_handlers}}) ) # Check method: assert disable_task_event_handlers(itask) is expect cylc-flow-8.7.0/tests/unit/test_dbstatecheck.py0000664000175000017500000002451515264420307022001 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import sqlite3 import pytest from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.cycling.iso8601 import ISO8601Point from cylc.flow.dbstatecheck import CylcWorkflowDBChecker, check_polling_config from cylc.flow.exceptions import InputError def test_check_polling_config(): """It should reject invalid or unreliable polling configurations. See https://github.com/cylc/cylc-flow/issues/6157 """ # invalid polling use cases with pytest.raises(InputError, match='No such task state'): check_polling_config('elephant', False, False) with pytest.raises(InputError, match='Cannot poll for'): check_polling_config('waiting', False, False) with pytest.raises(InputError, match='is not reliable'): check_polling_config('running', False, False) # valid polling use cases check_polling_config('started', True, False) check_polling_config('started', False, True) # valid query use cases check_polling_config(None, False, True) check_polling_config(None, False, False) class MockConn(sqlite3.Connection): """Mock Connection that allows asserting inputs and providing outputs.""" def __init__( self, rows: list[tuple], expected_statement: str | None = None, expected_parameters: tuple | None = None, ) -> None: self.rows = rows self.expected_stmt = expected_statement self.expected_parameters = expected_parameters def execute(self, sql: str, parameters=None): # type: ignore if self.expected_stmt is not None: assert self.expected_stmt == sql assert self.expected_parameters == parameters return self def fetchone(self): return self.rows[0] if len(self.rows) > 0 else None def fetchall(self): return self.rows class MockDBChecker(CylcWorkflowDBChecker): """Mock CylcWorkflowDBChecker that doesn't need a real database.""" def __init__(self, conn: MockConn | None, db_point_fmt: str | None = None): self.conn = conn if conn is not None else MockConn([]) self.db_point_fmt = db_point_fmt class TestCylcWorkflowDBChecker: @pytest.mark.parametrize( "cycle_string, db_point_fmt, expected", [ # IntegerPoint produced. ("5", None, IntegerPoint("5")), # ISO8601Point produced. ( "20260925T0000Z", "CCYYMMDDThhmmZ", ISO8601Point("20260925T0000Z"), ), ], ) def test_str_to_point(self, cycle_string, db_point_fmt, expected): """String produces expected Point object.""" point = MockDBChecker(None, db_point_fmt)._str_to_point(cycle_string) assert isinstance(point, expected.__class__) assert point == expected def test_str_to_point_invalid_integer_format(self): """An improperly formatted integer cycle point raises an error.""" with pytest.raises( InputError, match=('Cycle point "not an int" is not a valid integer point'), ): MockDBChecker(None, None)._str_to_point("not an int") def test_str_to_point_invalid_datetime_format(self): """An improperly formatted ISO8601 cycle point raises an error.""" with pytest.raises( InputError, match=( 'Cycle point "not a date" is not compatible ' 'with DB point format "CCYYMMDDThhmmZ"' ), ): MockDBChecker(None, "CCYYMMDDThhmmZ")._str_to_point("not a date") @pytest.mark.parametrize( "cycle, offset, db_point_fmt, expected", [ # None cycles are returned unchanged. (None, None, None, None), # Integer cycles without offsets are returned unchanged. ("1", None, None, "1"), # Offsets are applied to integer offsets. ("2", "-P1", None, "1"), ("2", "P1", None, "3"), # Date cycles are normalised. ("2026-09-25T00:00Z", None, "CCYYMMDDThhmmZ", "20260925T0000Z"), ("20510101T0000Z", None, "CCYY", "2051"), # Date cycles have offsets applied. ("20260925T0000Z", "-P1D", "CCYYMMDDThhmmZ", "20260924T0000Z"), ("20260925T0000Z", "+P1D", "CCYYMMDDThhmmZ", "20260926T0000Z"), ("20510101T0000Z", "P1Y", "CCYY", "2052"), ("2026", "P1Y", "CCYY", "2027"), ], ) def test_adjust_point_to_db(self, cycle, offset, db_point_fmt, expected): """Cycle point is offset and normalised.""" db_checker = MockDBChecker(None, db_point_fmt) normalised_cycle = db_checker.adjust_point_to_db(cycle, offset) assert normalised_cycle == expected @pytest.mark.parametrize( "rows, expected", [ # None returned when no key match in database. ([], None), # None if blank in database. ([("",)], None), # Rows correctly parsed to return value. ([("1",)], "1"), ], ) def test_get_db_param(self, rows, expected): """Database param values are parsed out of the workflow database.""" db_checker = MockDBChecker( MockConn(rows, expected_parameters=("startcp",)) ) start_cycle_point = db_checker._get_db_param("startcp") assert start_cycle_point == expected @pytest.mark.parametrize( "fmt, icp, scp, tcp, expected, warns", [ # Not before an unspecified start cycle point. (None, None, None, "1", False, False), # Non-specific target cycle point is not before. (None, "5", "5", "*", False, False), (None, "5", "5", "%", False, False), ( "CCYYMMDDThhmmZ", "20260925T0000Z", "20260925T0000Z", "*", False, False, ), ( "CCYYMMDDThhmmZ", "20260925T0000Z", "20260925T0000Z", "%", False, False, ), # Target is equal or after start cycle point. (None, "5", "5", "5", False, False), (None, "5", "5", "6", False, False), # Target is before start cycle point but after initial. (None, "1", "5", "2", True, False), # Target is before initial cycle point. (None, "2", "2", "1", False, True), (None, "2", "5", "1", False, True), # Target is before initial cycle point and no start. (None, "2", None, "1", False, True), # ISO8601Point is equal or after start cycle point. ( "CCYYMMDDThhmmZ", "20260925T0000Z", "20260925T0000Z", "20260925T0000Z", False, False, ), ( "CCYYMMDDThhmmZ", "20260925T0000Z", "20260925T0000Z", "20260930T0000Z", False, False, ), # ISO8601Point is before start cycle point but after initial. ( "CCYYMMDDThhmmZ", "20260801T0000Z", "20260925T0000Z", "20260901T0000Z", True, False, ), # ISO8601Point is before initial cycle point. ("CCYY", "2020", "2026", "2000", False, True), ], ) def test_is_before_start_cycle_point( self, fmt, icp, scp, tcp, expected, warns, capsys ): """Whether the current cycle point is before the start cycle point.""" db_checker = MockDBChecker(None, fmt) db_checker.start_cycle_point = ( db_checker._str_to_point(scp) if isinstance(scp, str) else scp ) db_checker.initial_cycle_point = ( db_checker._str_to_point(icp) if isinstance(icp, str) else icp ) before_start_cycle_point = db_checker._is_before_start_cycle_point(tcp) assert before_start_cycle_point == expected captured = capsys.readouterr() assert warns == ("WARNING" in captured.err) @pytest.mark.parametrize( "task, cycle, selector, flow_num, is_output_query, expected", [ # Test defaults. ( None, "1", None, None, None, ["", "1", "succeeded"], ), # Test task name, alternative succeeded, and cycle. ( "mytask", "2", "succeeded", None, False, ["mytask", "2", "succeeded"], ), # Test date cycle and flow number. ( "mytask", "20260926T1200Z", "succeeded", 3, False, ["mytask", "20260926T1200Z", "succeeded", "3"], ), # Test an output query. ( "mytask", "1", "custom", None, True, ["mytask", "1", '{"custom": "before start cycle point"}'], ), ], ) def test_dummy_result( self, task, cycle, selector, flow_num, is_output_query, expected ): """It should produce the expected result.""" result = CylcWorkflowDBChecker._dummy_result( task, cycle, selector, flow_num, is_output_query ) assert result == expected cylc-flow-8.7.0/tests/unit/test_pipe_poller.py0000664000175000017500000000241215264420307021657 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from subprocess import Popen, PIPE from cylc.flow.pipe_poller import pipe_poller def test_pipe_poller_str(): proc = Popen(['echo', 'Hello World!'], stdout=PIPE, text=True) (stdout,) = pipe_poller(proc, proc.stdout) assert proc.returncode == 0 assert stdout == 'Hello World!\n' def test_pipe_poller_bytes(): proc = Popen(['echo', 'Hello World!'], stdout=PIPE, text=False) (stdout,) = pipe_poller(proc, proc.stdout) assert proc.returncode == 0 assert stdout == b'Hello World!\n' cylc-flow-8.7.0/tests/unit/test_data_store_mgr.py0000664000175000017500000000361215264420307022342 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from copy import deepcopy from time import time from cylc.flow.data_store_mgr import ( task_mean_elapsed_time, apply_delta, WORKFLOW, DELTAS_MAP, ALL_DELTAS, DATA_TEMPLATE ) def int_id(): return '20130808T00/foo/03' class FakeTDef: elapsed_times = (0.0, 10.0) def test_task_mean_elapsed_time(): tdef = FakeTDef() result = task_mean_elapsed_time(tdef) assert result == 5 assert isinstance(result, int) def test_apply_delta(): """Test delta application. Some functionality is not used at the Scheduler, so is not covered by integration testing. """ w_id = 'workflow_id' delta = DELTAS_MAP[ALL_DELTAS]() delta.workflow.time = time() flow = delta.workflow.updated flow.id = 'workflow_id' flow.stamp = f'{w_id}@{delta.workflow.time}' delta.workflow.pruned = w_id data = deepcopy(DATA_TEMPLATE) assert data[WORKFLOW].id != w_id assert data[WORKFLOW].pruned is False for field, sub_delta in delta.ListFields(): apply_delta(field.name, sub_delta, data) assert data[WORKFLOW].id == w_id assert data[WORKFLOW].pruned is True cylc-flow-8.7.0/tests/unit/test_platforms_get_host.py0000664000175000017500000000757115264420307023263 0ustar alastairalastair# Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # Tests for the get_host_from_platform lookup. import pytest from cylc.flow.exceptions import NoHostsError, NoPlatformsError from cylc.flow.platforms import get_host_from_platform, get_platform_from_group from cylc.flow.exceptions import CylcError TEST_PLATFORM = { 'name': 'Elephant', 'hosts': ['nellie', 'dumbo', 'jumbo'], 'selection': {'method': 'definition order'} } TEST_GROUP = { 'platforms': ['one', 'two', 'three'], 'selection': {'method': 'definition order'} } @pytest.mark.parametrize( 'badhosts, expect', [ pytest.param(None, 'nellie'), pytest.param({'nellie', 'dumbo'}, 'jumbo') ] ) def test_get_host_from_platform(badhosts, expect): platform = TEST_PLATFORM assert get_host_from_platform(platform, badhosts) == expect def test_get_host_from_platform_fails_no_goodhosts(): platform = TEST_PLATFORM with pytest.raises(NoHostsError) as err: get_host_from_platform(platform, {'nellie', 'dumbo', 'jumbo'}) assert err.exconly() == ( 'cylc.flow.exceptions.NoHostsError: ' 'Unable to find contactable host for platform: Elephant' ) def test_get_host_from_platform_fails_bad_method(): platform = TEST_PLATFORM.copy() platform['selection']['method'] = 'roulette' with pytest.raises(CylcError) as err: get_host_from_platform(platform, {'Elephant'}) assert err.exconly() == ( 'cylc.flow.exceptions.CylcError: method "roulette" is not a ' 'supported host selection method.' ) @pytest.mark.parametrize( 'expect, bad_hosts', [ pytest.param( 'one', None, id='No bad_hosts get platform one'), pytest.param( 'three', {'foo', 'bar'}, id='All platforms bad except platform three') ] ) def test_get_platform_from_group(monkeypatch, expect, bad_hosts): def get_plat(name): if name == 'three': return {'hosts': {'foo', 'bar', 'baz'}} else: return {'hosts': {'foo', 'bar'}} monkeypatch.setattr('cylc.flow.platforms.platform_from_name', get_plat) platform = get_platform_from_group(TEST_GROUP, 'mygroup_name', bad_hosts) assert platform == expect def test_get_platform_from_group_fails_no_goodhosts(monkeypatch): monkeypatch.setattr( 'cylc.flow.platforms.platform_from_name', lambda _: {'hosts': {'foo', 'bar', 'baz'}}, ) with pytest.raises(NoPlatformsError) as err: get_platform_from_group( TEST_GROUP, 'mygroup_name', {'foo', 'bar', 'baz'} ) assert err.exconly() == ( 'cylc.flow.exceptions.NoPlatformsError: ' 'Unable to find a contactable platform from group mygroup_name' ) def test_get_platform_from_group_fails_bad_method(monkeypatch): monkeypatch.setattr( 'cylc.flow.platforms.platform_from_name', lambda _: {'hosts': {'foo', 'bar', 'baz'}}, ) group = TEST_GROUP.copy() group['selection']['method'] = 'roulette' with pytest.raises(CylcError) as err: get_platform_from_group(group, {'foo'}) assert err.exconly() == ( 'cylc.flow.exceptions.CylcError: "roulette" is not a ' 'supported platform selection method.' ) cylc-flow-8.7.0/tests/unit/test_workflow_db_mgr.py0000664000175000017500000000540715264420307022540 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from pathlib import Path from typing import ( List, Set, ) from unittest.mock import Mock import pytest from pytest import param from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.flow_mgr import FlowNums from cylc.flow.id import Tokens from cylc.flow.task_proxy import TaskProxy from cylc.flow.taskdef import TaskDef from cylc.flow.util import serialise_set from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager @pytest.mark.parametrize('flow_nums, expected_removed', [ param(set(), {1, 2, 5}, id='all'), param({1}, {1}, id='subset'), param({1, 2, 5}, {1, 2, 5}, id='complete-set'), param({1, 3, 5}, {1, 5}, id='intersect'), param({3, 4}, set(), id='disjoint'), ]) def test_remove_task_from_flows( tmp_path: Path, flow_nums: FlowNums, expected_removed: FlowNums ): db_flows: List[FlowNums] = [ {1, 2}, {5}, set(), # FLOW_NONE ] expected_remaining = { serialise_set(flow - expected_removed) for flow in db_flows } db_mgr = WorkflowDatabaseManager(tmp_path) schd_tokens = Tokens('~asterix/gaul') tdef = TaskDef('a', rtcfg={}, start_point=None, initial_point=None) with db_mgr.get_pri_dao() as dao: db_mgr.pri_dao = dao db_mgr.pub_dao = Mock() for flow in db_flows: itask = TaskProxy( schd_tokens, tdef, IntegerPoint('1'), flow_nums=flow ) db_mgr.put_insert_task_states(itask) db_mgr.put_insert_task_outputs(itask) db_mgr.process_queued_ops() removed_fnums = db_mgr.remove_task_from_flows('1', 'a', flow_nums) assert removed_fnums == expected_removed db_mgr.process_queued_ops() for table in ('task_states', 'task_outputs'): remaining_fnums: Set[str] = { fnums_str for fnums_str, *_ in dao.connect().execute( f'SELECT flow_nums FROM {table}' ) } assert remaining_fnums == expected_remaining cylc-flow-8.7.0/tests/unit/parsec/0000775000175000017500000000000015264420307017212 5ustar alastairalastaircylc-flow-8.7.0/tests/unit/parsec/test_types.py0000664000175000017500000001115115264420307021766 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test parsec type parsing. .. note:: This test is a pytest port of the old ``tests/parsec/synonyms`` test battery. """ import pytest from cylc.flow.parsec.config import ( ConfigNode as Conf ) from cylc.flow.parsec.validate import ( CylcConfigValidator as VDR ) @pytest.fixture def generate_spec(): def _inner(typ, validator): """Return a sample spec for the given data type. Args: typ (str): An arbitrary name for this type. validator (str): A parsec validator. Returns: cylc.flow.parsec.config.ConfigNode """ with Conf('/') as myconf: with Conf(typ): Conf('', validator) return myconf return _inner @pytest.fixture def generate_config(): def _inner(typ, value): """Return a sample config for the given data type. The aim is to cover every facetious combination of quotes newlines and comments. Args: typ (str): An arbitrary name for this type. value (object): A stringable value to dump in the conf. For list types provide a list of stringable values. """ if isinstance(value, list): return f''' [{typ}] plain = {','.join(value)} # comment spaced = {', '.join(value)} # comment badly spaced = {' , '.join(value)} # comment single quoted = {', '.join((f"'{x}'" for x in value))} double quoted = {', '.join((f'"{x}"' for x in value))} multi line = {value[0]}, \\ {', '.join(value[1:])} # comment ''' else: return f''' [{typ}] plain1 = {value} # comment single quoted = '{value}' # comment double quoted = "{value}" # comment triple single quoted = \'\'\'{value}\'\'\' # comment triple double quoted = """{value}""" # comment triple single quoted multi = \'\'\' {value} \'\'\' # comment triple double quoted multi = """ {value} """ # comment ''' return _inner def test_types(generate_spec, generate_config, config): """Test type parsing. Test every facetious combination of: * Data types. * Quotation. * Inline-commenting. """ for typ, validator, string_repr, parsed_value in [ ('boolean', VDR.V_BOOLEAN, 'true', True), ('boolean', VDR.V_BOOLEAN, 'True', True), ('boolean', VDR.V_BOOLEAN, 'false', False), ('boolean', VDR.V_BOOLEAN, 'False', False), ('integer', VDR.V_INTEGER, '42', 42), ('float', VDR.V_FLOAT, '9.9', 9.9), ('string', VDR.V_STRING, 'the quick brown fox', 'the quick brown fox'), ( 'integer_list', VDR.V_INTEGER_LIST, ['1', '2', '3', '4', '5'], [1, 2, 3, 4, 5] ), ( 'float_list', VDR.V_FLOAT_LIST, ['1.1', '2.2', '3.3', '4.4', '5.5'], [1.1, 2.2, 3.3, 4.4, 5.5] ), ( 'string_list', VDR.V_STRING_LIST, ['be', 'ef', 'we', 'll', 'in', 'gt', 'on'], ['be', 'ef', 'we', 'll', 'in', 'gt', 'on'] ), ]: spec = generate_spec(typ, validator) conf = generate_config(typ, string_repr) cfg = config(spec, conf) assert all(( value == parsed_value for value in cfg.get()[typ].values() )) cylc-flow-8.7.0/tests/unit/parsec/test_ordered_dict.py0000664000175000017500000001067415264420307023262 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import unittest from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults class TestOrderedDict(unittest.TestCase): def test_getitem(self): d = OrderedDictWithDefaults() d['name'] = 'Joseph' d.defaults_ = { 'surname': 'Wyndham' } self.assertEqual('Joseph', d['name']) self.assertEqual('Wyndham', d['surname']) def test_setitem(self): d = OrderedDictWithDefaults() d['name'] = 'Matthews' self.assertEqual('Matthews', d['name']) d['name'] = 'Zaccharias' self.assertEqual('Zaccharias', d['name']) def test_keys(self): d = OrderedDictWithDefaults() d['name'] = 'Andrew' d['surname'] = 'Gray' d.defaults_ = { 'address': 'N/A' } keys = list(d.keys()) self.assertTrue(len(keys) == 3) self.assertTrue('name' in keys) self.assertTrue('surname' in keys) self.assertTrue('address' in keys) def test_values(self): d = OrderedDictWithDefaults() d['name'] = 'Paul' d['color'] = 'Green' values = list(d.values()) self.assertTrue(len(values) == 2) self.assertTrue('Paul' in values) self.assertTrue('Green' in values) def test_items(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d['len'] = '89' for _, v in list(d.items()): self.assertTrue(v in ['Birds', '89']) def test_iterkeys(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d['len'] = '89' d.defaults_ = { 'surname': 'Wyndham' } count = 0 for k in d.keys(): self.assertTrue(k in ['key', 'len', 'surname']) count += 1 self.assertEqual(3, count) def test_itervalues(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d['len'] = '89' d.defaults_ = { 'surname': 'Wyndham' } count = 0 for k in d.values(): self.assertTrue(k in ['Birds', '89', 'Wyndham']) count += 1 self.assertEqual(3, count) def test_iteritems(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d['len'] = '89' d.defaults_ = { 'surname': 'Wyndham' } count = 0 for k, v in d.items(): self.assertTrue(k in ['key', 'len', 'surname']) self.assertTrue(v in ['Birds', '89', 'Wyndham']) count += 1 self.assertEqual(3, count) def test_contains(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d.defaults_ = { 'value': '10' } self.assertTrue('key' in d) self.assertTrue('value' in d) self.assertFalse('test' in d) def test_nonzero(self): d = OrderedDictWithDefaults() self.assertFalse(d) d['value'] = 10 self.assertTrue(d) def test_prepend(self): d = OrderedDictWithDefaults() d['key'] = 'Birds' d.prepend('year', 1980) d.prepend('key', 2000) iterator = iter(d.keys()) self.assertEqual('key', next(iterator)) self.assertEqual('year', next(iterator)) d = OrderedDictWithDefaults() d['key'] = 'Birds' d.prepend('year', 1980) iterator = iter(d.keys()) self.assertEqual('year', next(iterator)) self.assertEqual('key', next(iterator)) cylc-flow-8.7.0/tests/unit/parsec/test_config_node.py0000664000175000017500000001025015264420307023073 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.parsec.config import ConfigNode as Conf @pytest.fixture(scope='module') def basic_config(): """A basic config with a file, section and setting.""" with Conf('file.cylc') as file_: with Conf('section') as section: setting = Conf('setting') return (file_, section, setting) def test_config_node(basic_config): """It should associate parents & children in a tree.""" file_, section, setting = basic_config assert file_.name == 'file.cylc' assert file_._parent is None assert file_._children == {'section': section} assert section.name == 'section' assert section._parent == file_ assert section._children == {'setting': setting} assert setting.name == 'setting' assert setting._parent == section assert setting._children is None def test_config_str(basic_config): """A node should str as a relative path from its parent node..""" file_, section, setting = basic_config assert str(file_) == 'file.cylc' assert str(section) == '[section]' assert str(setting) == 'setting' def test_config_repr(basic_config): """A node should repr as a full path.""" file_, section, setting = basic_config assert repr(file_) == 'file.cylc' assert repr(section) == 'file.cylc[section]' assert repr(setting) == 'file.cylc[section]setting' @pytest.fixture(scope='module') def many_setting(): """A config containing a user-definable setting.""" with Conf('file.cylc') as file_: Conf('') # __MANY__ return file_ def test_many_setting(many_setting): """It should recognise this is a user-definable setting.""" setting = list(many_setting)[0] assert setting.name == '__MANY__' assert setting.display_name == '' assert str(setting) == '' assert repr(setting) == 'file.cylc|' @pytest.fixture(scope='module') def many_section(): """A config containing a user-definable section.""" with Conf('file.cylc') as file_: with Conf('
'): Conf('setting') return file_ def test_many_section(many_section): """It should recognise this is a user-definable section.""" section = list(many_section)[0] assert section.name == '__MANY__' assert section.display_name == '
' assert str(section) == '[
]' assert repr(section) == 'file.cylc[
]' setting = list(section)[0] assert str(setting) == 'setting' assert repr(setting) == 'file.cylc[
]setting' @pytest.fixture(scope='module') def meta_conf(): """A config with an inherited section.""" with Conf('Foo') as spec: with Conf('') as template: Conf('a', default='a') Conf('b', default='b') with Conf('y', meta=template) as copy: Conf('a', default='c') return spec, template, copy def test_meta(meta_conf): """It should inherit sections using the meta kwarg.""" spec, template, copy = meta_conf assert template.meta is None assert copy.meta == template # make sure the template is unaffected assert template['a'].default == 'a' assert template['b'].default == 'b' # make sure the copy is affected assert copy['a'].default == 'c' assert copy['b'].default == 'b' # make sure inherited configurations are marked accordingly assert copy['a'].meta is None # not inherited assert copy['b'].meta is True # inherited cylc-flow-8.7.0/tests/unit/parsec/test_validate.py0000664000175000017500000005645215264420307022430 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Unit Tests for cylc.flow.parsec.validate.ParsecValidator.coerce methods.""" import logging from typing import List from cylc.flow import CYLC_LOG import pytest from pytest import approx, param from cylc.flow.parsec.config import ConfigNode as Conf from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults from cylc.flow.parsec.exceptions import IllegalValueError from cylc.flow.parsec.validate import ( BroadcastConfigValidator, CylcConfigValidator as VDR, DurationFloat, ListValueError, IllegalItemError, ParsecValidator, parsec_validate ) @pytest.fixture def sample_spec(): with Conf('myconf') as myconf: with Conf('section1'): Conf('value1', default='') Conf('value2', default='what?') with Conf('section2'): Conf('enabled', VDR.V_BOOLEAN) with Conf('section3'): Conf('title', default='default', options=['1', '2']) Conf( 'amounts', VDR.V_INTEGER_LIST, default=[1, 2, 3], # options=[[1, 2, 3]] ) with Conf('entries'): Conf('key') Conf('value') with Conf(''): Conf('section300000', default='') Conf('ids', VDR.V_INTEGER_LIST) return myconf def validator_invalid_values(): """ Data provider or invalid values for parsec validator. All values must not be null (covered elsewhere), and not dict's. Possible invalid scenarios must include: - cfg[key] is a list AND a value is not in list of the possible values - OR - cfg[key] is not a list AND cfg[key] not in the list of possible values :return: a list with sets of tuples for the test parameters :rtype: list """ values = [] # variables reused throughout spec = None msg = None # set 1 (t, f, f, t) with Conf('base') as spec: Conf('value', VDR.V_INTEGER_LIST, default=1, options=[1, 2, 3, 4]) cfg = OrderedDictWithDefaults() cfg['value'] = "1, 2, 3" msg = None values.append((spec, cfg, msg)) # set 2 (t, t, f, t) with Conf('base') as spec: Conf('value', VDR.V_INTEGER_LIST, default=1, options=[1, 2, 3, 4]) cfg = OrderedDictWithDefaults() cfg['value'] = "1, 2, 5" msg = '(type=option) value = 5' values.append((spec, cfg, msg)) # set 3 (f, f, t, f) with Conf('base') as spec: Conf('value', VDR.V_INTEGER, default=1, options=[2, 3, 4]) cfg = OrderedDictWithDefaults() cfg['value'] = "2" msg = None values.append((spec, cfg, msg)) # set 4 (f, f, t, t) with Conf('base') as spec: Conf('value', VDR.V_INTEGER, default=1, options=[1, 2, 3, 4]) cfg = OrderedDictWithDefaults() cfg['value'] = "5" msg = '(type=option) value = 5' values.append((spec, cfg, msg)) return values @pytest.fixture def strip_and_unquote_list(): return [ [ '"a,b", c, "d e"', # input ["a,b", "c", "d e"] # expected ], [ 'foo bar baz', # input ["foo bar baz"] # expected ], [ '"a", \'b\', c', # input ["a", "b", "c"] # expected ], [ 'a b c, d e f', # input ["a b c", "d e f"] # expected ], ] def test_list_value_error(): keys = ['a,', 'b', 'c'] value = 'a sample value' error = ListValueError(keys, value, "who cares") output = str(error) expected = '(type=list) [a,][b]c = a sample value - (who cares)' assert expected == output def test_list_value_error_with_exception(): keys = ['a,', 'b', 'c'] value = 'a sample value' exc = Exception('test') error = ListValueError(keys, value, "who cares", exc) output = str(error) expected = '(type=list) [a,][b]c = a sample value - (test: who cares)' assert expected == output def test_illegal_value_error(): value_type = 'ClassA' keys = ['a,', 'b', 'c'] value = 'a sample value' error = IllegalValueError(value_type, keys, value) output = str(error) expected = "(type=ClassA) [a,][b]c = a sample value" assert expected == output def test_illegal_value_error_with_exception(): value_type = 'ClassA' keys = ['a,', 'b', 'c'] value = 'a sample value' exc = Exception('test') error = IllegalValueError(value_type, keys, value, exc) output = str(error) expected = "(type=ClassA) [a,][b]c = a sample value - (test)" assert expected == output def test_illegal_item_error(): keys = ['a,', 'b', 'c'] key = 'a sample value' error = IllegalItemError(keys, key) output = str(error) expected = "[a,][b][c]a sample value" assert expected == output def test_illegal_item_error_message(): keys = ['a,', 'b', 'c'] key = 'a sample value' message = "invalid" error = IllegalItemError(keys, key, message) output = str(error) expected = "[a,][b][c]a sample value - (invalid)" assert expected == output def test_parsec_validator_invalid_key(sample_spec): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section22'] = 'abc' with pytest.raises(IllegalItemError): parsec_validator.validate(cfg, sample_spec) def test_parsec_validator_invalid_key_no_spec(sample_spec): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section22'] = 'abc' # remove the user-defined section from the spec sample_spec._children = { key: value for key, value in sample_spec._children.items() if key != '__MANY__' } with pytest.raises(IllegalItemError): parsec_validator.validate(cfg, sample_spec) def test_parsec_validator_invalid_key_with_many_spaces(sample_spec): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section 3000000'] = 'test' with pytest.raises(IllegalItemError) as cm: parsec_validator.validate(cfg, sample_spec) assert str(cm.exception) == "section 3000000 - (consecutive spaces)" @pytest.mark.parametrize('spec, cfg, msg', validator_invalid_values()) def test_parsec_validator_invalid_key_with_many_invalid_values( spec, cfg, msg ): parsec_validator = ParsecValidator() if msg is not None: with pytest.raises(IllegalValueError) as cm: parsec_validator.validate(cfg, spec) assert msg == str(cm.value) else: # cylc.flow.parsec_validator.validate(cfg, spec) # let's use the alias `parsec_validate` here parsec_validate(cfg, spec) # TBD assertIsNotNone when 2.6+ assert parsec_validator is not None def test_parsec_validator_warn_options(caplog): """Test the "warn_options" option. This should turn invalid option errors into warnings. """ with Conf('base') as spec: Conf( 'foo', VDR.V_STRING_LIST, default=1, options=['a', 'b'], warn_options=True, ) parsec_validator = ParsecValidator() caplog.set_level(logging.WARNING, CYLC_LOG) parsec_validator.validate({'foo': 'b, c'}, spec) # there should be one (and only one) warning assert caplog.messages == [ '(type=option) foo = c\nInvalid items have been removed' ] def test_parsec_validator_invalid_key_with_many_1(sample_spec): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section3000000'] = OrderedDictWithDefaults() parsec_validator.validate(cfg, sample_spec) # TBD assertIsNotNone when 2.6+ assert parsec_validator is not None def test_parsec_validator_invalid_key_with_many_2(sample_spec): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section3'] = OrderedDictWithDefaults() cfg['section3']['title'] = '1' cfg['section3']['entries'] = OrderedDictWithDefaults() cfg['section3']['entries']['key'] = 'name' cfg['section3']['entries']['value'] = "1, 2, 3, 4" parsec_validator.validate(cfg, sample_spec) # TBD assertIsNotNone when 2.6+ assert parsec_validator is not None def test_parsec_validator(sample_spec): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section3'] = OrderedDictWithDefaults() cfg['section3']['title'] = None parsec_validator.validate(cfg, sample_spec) # TBD assertIsNotNone when 2.6+ assert parsec_validator is not None # --- static methods def test_coerce_none_fails(): with pytest.raises(AttributeError): ParsecValidator.coerce_boolean(None, []) with pytest.raises(AttributeError): ParsecValidator.coerce_float(None, []) with pytest.raises(AttributeError): ParsecValidator.coerce_int(None, []) def test_coerce_boolean(): """Test coerce_boolean.""" validator = ParsecValidator() # The good for value, result in [ ('True', True), (' True ', True), ('"True"', True), ("'True'", True), ('true', True), (' true ', True), ('"true"', True), ("'true'", True), ('False', False), (' False ', False), ('"False"', False), ("'False'", False), ('false', False), (' false ', False), ('"false"', False), ("'false'", False), ('', None), (' ', None) ]: assert validator.coerce_boolean(value, ['whatever']) == result # The bad for value in [ 'None', ' Who cares? ', '3.14', '[]', '[True]', 'True, False' ]: with pytest.raises(IllegalValueError): validator.coerce_boolean(value, ['whatever']) @pytest.mark.parametrize( 'value, expected', [ ('3', 3.0), ('9.80', 9.80), ('3.141592654', 3.141592654), ('"3.141592654"', 3.141592654), ("'3.141592654'", 3.141592654), ('-3', -3.0), ('-3.1', -3.1), ('0', 0.0), ('-0', -0.0), ('0.0', 0.0), ('1e20', 1.0e20), ('6.02e23', 6.02e23), ('-1.6021765e-19', -1.6021765e-19), ('6.62607004e-34', 6.62607004e-34), ] ) def test_coerce_float(value: str, expected: float): """Test coerce_float.""" assert ( ParsecValidator.coerce_float(value, ['whatever']) == approx(expected) ) def test_coerce_float__empty(): # not a number assert ParsecValidator.coerce_float('', ['whatever']) is None @pytest.mark.parametrize( 'value', ['None', ' Who cares? ', 'True', '[]', '[3.14]', '3.14, 2.72'] ) def test_coerce_float__bad(value: str): with pytest.raises(IllegalValueError): ParsecValidator.coerce_float(value, ['whatever']) @pytest.mark.parametrize( 'value, expected', [ ('', []), ('3', [3.0]), ('2*3.141592654', [3.141592654, 3.141592654]), ('12*8, 8*12.0', [8.0] * 12 + [12.0] * 8), ('-3, -2, -1, -0.0, 1.0', [-3.0, -2.0, -1.0, -0.0, 1.0]), ('6.02e23, -1.6021765e-19, 6.62607004e-34', [6.02e23, -1.6021765e-19, 6.62607004e-34]), ] ) def test_coerce_float_list(value: str, expected: List[float]): """Test coerce_float_list.""" items = ParsecValidator.coerce_float_list(value, ['whatever']) assert items == approx(expected) @pytest.mark.parametrize( 'value', ['None', 'e, i, e, i, o', '[]', '[3.14]', 'pi, 2.72', '2*True'] ) def test_coerce_float_list__bad(value: str): with pytest.raises(IllegalValueError): ParsecValidator.coerce_float_list(value, ['whatever']) @pytest.mark.parametrize( 'value, expected', [ ('0', 0), ('3', 3), ('-3', -3), ('-0', -0), ('653456', 653456), ('-8362583645365', -8362583645365) ] ) def test_coerce_int(value: str, expected: int): """Test coerce_int.""" assert ParsecValidator.coerce_int(value, ['whatever']) == expected def test_coerce_int__empty(): assert ParsecValidator.coerce_int('', ['whatever']) is None # not a number @pytest.mark.parametrize( 'value', ['None', ' Who cares? ', 'True', '4.8', '[]', '[3]', '60*60'] ) def test_coerce_int__bad(value: str): with pytest.raises(IllegalValueError): ParsecValidator.coerce_int(value, ['whatever']) def test_coerce_int_list(): """Test coerce_int_list.""" validator = ParsecValidator() # The good for value, results in [ ('', []), ('3', [3]), ('1..10, 11..20..2', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19]), ('18 .. 24', [18, 19, 20, 21, 22, 23, 24]), ('18 .. 24 .. 3', [18, 21, 24]), ('-10..10..3', [-10, -7, -4, -1, 2, 5, 8]), ('10*3, 4*-6', [3] * 10 + [-6] * 4), ('10*128, -78..-72, 2048', [128] * 10 + [-78, -77, -76, -75, -74, -73, -72, 2048]) ]: assert validator.coerce_int_list(value, ['whatever']) == results # The bad for value in [ 'None', 'e, i, e, i, o', '[]', '1..3, x', 'one..ten' ]: with pytest.raises(IllegalValueError): validator.coerce_int_list(value, ['whatever']) @pytest.mark.parametrize( 'value, expected', [ ('', ''), ('Hello World!', 'Hello World!'), ('"Hello World!"', 'Hello World!'), ('"Hello Cylc\'s World!"', 'Hello Cylc\'s World!'), ("'Hello World!'", 'Hello World!'), ('0', '0'), ('My list is:\nfoo, bar, baz\n', 'My list is:\nfoo, bar, baz'), (' Hello:\n foo\n bar\n baz\n', 'Hello:\nfoo\nbar\nbaz'), (' Hello:\n foo\n Greet\n baz\n', 'Hello:\n foo\nGreet\n baz'), ('False', 'False'), ('None', 'None'), (['a', 'b'], 'a\nb'), ('abc#def', 'abc'), ] ) def test_coerce_str(value: str, expected: str): """Test coerce_str.""" validator = ParsecValidator() # The good assert validator.coerce_str(value, ['whatever']) == expected def test_coerce_str_list(): """Test coerce_str_list.""" validator = ParsecValidator() # The good for value, results in [ ('', []), ('Hello', ['Hello']), ('"Hello"', ['Hello']), ('1', ['1']), ('Mercury, Venus, Earth, Mars', ['Mercury', 'Venus', 'Earth', 'Mars']), ('Mercury, Venus, Earth, Mars,\n"Jupiter",\n"Saturn"\n', ['Mercury', 'Venus', 'Earth', 'Mars', 'Jupiter', 'Saturn']), ('New Zealand, United Kingdom', ['New Zealand', 'United Kingdom']) ]: assert validator.coerce_str_list(value, ['whatever']) == results @pytest.mark.parametrize('value, expected', [ param( "'a'", 'a', id="single quotes" ), param( '"\'a\'"', "'a'", id="single quotes inside double quotes" ), param( '" a b" # comment', ' a b', id="comment outside" ), param( '"""bene\ngesserit"""', 'bene\ngesserit', id="multiline double quotes" ), param( "'''kwisatz\n haderach'''", 'kwisatz\n haderach', id="multiline single quotes" ), param( '"""a\nb""" # comment', 'a\nb', id="multiline with comment outside" ), ]) def test_unquote(value: str, expected: str): """Test strip_and_unquote.""" assert ParsecValidator._unquote(['a'], value) == expected @pytest.mark.parametrize('value', [ '"""', "'''", "'don't do this'", ]) def test_strip_and_unquote__bad(value: str): with pytest.raises(IllegalValueError): ParsecValidator.strip_and_unquote(['a'], value) def test_strip_and_unquote_list_parsec(): """Test strip_and_unquote_list using ParsecValidator.""" for value, results in [ ('"a"\n"b"', ['a', 'b']), ('"a", "b"', ['a', 'b']), ('"a", "b"', ['a', 'b']), ('"c" # d', ['c']), ('"a", "b", "c" # d', ['a', 'b', 'c']), ('"a"\n"b"\n"c" # d', ['a', 'b', 'c']), ("'a', 'b'", ['a', 'b']), ("'c' #d", ['c']), ("'a', 'b', 'c' # d", ['a', 'b', 'c']), ("'a'\n'b'\n'c' # d", ['a', 'b', 'c']), ('a, b, c,', ['a', 'b', 'c']), ('a, b, c # d', ['a', 'b', 'c']), ('a, b, c\n"d"', ['a', 'b', 'd']), ('a, b, c\n"d" # e', ['a', 'b', '"d"']) ]: assert results == ParsecValidator.strip_and_unquote_list( ['a'], value) def test_strip_and_unquote_list_cylc(strip_and_unquote_list): """Test strip_and_unquote_list using CylcConfigValidator.""" validator = VDR() for values in strip_and_unquote_list: value = values[0] expected = values[1] output = validator.strip_and_unquote_list(keys=[], value=value) assert expected == output def test_strip_and_unquote_list_multiparam(): with pytest.raises(ListValueError): ParsecValidator.strip_and_unquote_list( ['a'], 'a, b, c' ) def test_coerce_cycle_point(): """Test coerce_cycle_point.""" validator = VDR() # The good for value, result in [ ('', None), ('3', '3'), ('2018', '2018'), ('20181225T12Z', '20181225T12Z'), ('2018-12-25T12:00+11:00', '2018-12-25T12:00+11:00')]: assert validator.coerce_cycle_point(value, ['whatever']) == result # The bad for value in [ 'None', ' Who cares? ', 'True', '1, 2', '20781340E10']: with pytest.raises(IllegalValueError): validator.coerce_cycle_point(value, ['whatever']) def test_coerce_cycle_point_format(): """Test coerce_cycle_point_format.""" validator = VDR() # The good for value, result in [ ('', None), ('%Y%m%dT%H%M%z', '%Y%m%dT%H%M%z'), ('CCYYMMDDThhmmZ', 'CCYYMMDDThhmmZ'), ('XCCYYMMDDThhmmZ', 'XCCYYMMDDThhmmZ')]: assert ( validator.coerce_cycle_point_format(value, ['whatever']) == result ) # The bad # '/' and ':' not allowed in cylc cycle points (they are used in paths). for value in ['%i%j', 'Y/M/D', '%Y-%m-%dT%H:%MZ']: with pytest.raises(IllegalValueError): validator.coerce_cycle_point_format(value, ['whatever']) def test_coerce_cycle_point_time_zone(): """Test coerce_cycle_point_time_zone.""" validator = VDR() # The good for value, result in [ ('', None), ('Z', 'Z'), ('+0000', '+0000'), ('+0100', '+0100'), ('+1300', '+1300'), ('-0630', '-0630')]: assert ( validator.coerce_cycle_point_time_zone(value, ['whatever']) == result ) # The bad for value in ['None', 'Big Bang Time', 'Standard Galaxy Time']: with pytest.raises(IllegalValueError): validator.coerce_cycle_point_time_zone(value, ['whatever']) def test_coerce_interval(): """Test coerce_interval.""" validator = VDR() # The good for value, result in [ ('', None), ('P3D', DurationFloat(259200)), ('PT10M10S', DurationFloat(610))]: assert validator.coerce_interval(value, ['whatever']) == result # The bad for value in ['None', '5 days', '20', '-12']: with pytest.raises(IllegalValueError): validator.coerce_interval(value, ['whatever']) @pytest.mark.parametrize( 'value, expected', [ ('', []), ('P3D', [DurationFloat(259200)]), ('P3D, PT10M10S', [DurationFloat(259200), DurationFloat(610)]), ('25*PT30M,10*PT1H', [DurationFloat(1800)] * 25 + [DurationFloat(3600)] * 10) ] ) def test_coerce_interval_list(value: str, expected: List[DurationFloat]): """Test coerce_interval_list.""" assert VDR.coerce_interval_list(value, ['whatever']) == approx(expected) @pytest.mark.parametrize( 'value', ['None', '5 days', '20', 'PT10S, -12'] ) def test_coerce_interval_list__bad(value: str): with pytest.raises(IllegalValueError): VDR.coerce_interval_list(value, ['whatever']) def test_coerce_parameter_list(): """Test coerce_parameter_list.""" validator = VDR() # The good for value, result in [ ('', []), ('planet', ['planet']), ('planet, star, galaxy', ['planet', 'star', 'galaxy']), ('1..5, 21..25', [1, 2, 3, 4, 5, 21, 22, 23, 24, 25]), ('-15, -10, -5, -1..1', [-15, -10, -5, -1, 0, 1])]: assert validator.coerce_parameter_list(value, ['whatever']) == result # The bad for value in ['foo/bar', 'p1, 1..10', '2..3, 4, p', 'x:,']: with pytest.raises(IllegalValueError): validator.coerce_parameter_list(value, ['whatever']) def test_coerce_xtrigger(): """Test coerce_xtrigger.""" validator = VDR() # The good for value, result in [ ('foo(x="bar")', 'foo(x=bar)'), ('foo(x, y, z="zebra")', 'foo(x, y, z=zebra)')]: assert ( validator.coerce_xtrigger(value, ['whatever']).get_signature() == result ) # The bad for value in [ '', 'foo(', 'foo)', 'foo,bar']: with pytest.raises(IllegalValueError): validator.coerce_xtrigger(value, ['whatever']) def test_type_help_examples(): types = { **ParsecValidator.V_TYPE_HELP, **VDR.V_TYPE_HELP } validator = VDR() for vdr, info in types.items(): coercer = validator.coercers[vdr] if len(info) > 2: for example in info[2]: try: coercer(example, [None]) except Exception: raise Exception( f'Example "{example}" failed for type "{vdr}"' ) @pytest.mark.parametrize( 'value, expected', [ param( """ a="don't have a cow" a=${a#*have} echo "$a" # let's see what happens """, "a=\"don't have a cow\"\na=${a#*have}\necho \"$a\"" " # let's see what happens", id="multiline", ), param('"sleep 30 # ja!" ', 'sleep 30 # ja!', id="quoted"), ], ) def test_broadcast_coerce_str(value: str, expected: str): assert BroadcastConfigValidator.coerce_str(value, ['whatever']) == expected cylc-flow-8.7.0/tests/unit/parsec/test_parsec.py0000664000175000017500000000234715264420307022106 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import unittest from cylc.flow.parsec.exceptions import ParsecError class TestParsec(unittest.TestCase): def test_parsec_error_msg(self): parsec_error = ParsecError() self.assertEqual('', str(parsec_error)) parsec_error = ParsecError('foo') self.assertEqual('foo', str(parsec_error)) def test_parsec_error_str(self): msg = 'Turbulence!' parsec_error = ParsecError(msg) self.assertEqual(msg, str(parsec_error)) cylc-flow-8.7.0/tests/unit/parsec/__init__.py0000664000175000017500000000140715264420307021325 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . cylc-flow-8.7.0/tests/unit/parsec/test_config.py0000664000175000017500000002353015264420307022073 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from pathlib import Path import tempfile import pytest from cylc.flow.parsec.config import ( ConfigNode as Conf, ParsecConfig ) from cylc.flow.parsec.exceptions import ( IllegalItemError, InvalidConfigError, ItemNotFoundError, ) from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults from cylc.flow.parsec.upgrade import upgrader from cylc.flow.parsec.validate import ( cylc_config_validate, CylcConfigValidator as VDR ) def test_loadcfg(sample_spec): with tempfile.NamedTemporaryFile() as output_file_name: with tempfile.NamedTemporaryFile() as rcfile: parsec_config = ParsecConfig( spec=sample_spec, upgrader=None, # new spec output_fname=output_file_name.name, tvars=None, validator=None # use default ) rcfile.write(""" [section1] value1 = 'test' value2 = 'test' [section2] enabled = True [section3] title = 'Ohm' [[entries]] key = 'product' value = 1, 2, 3, 4 """.encode()) rcfile.seek(0) parsec_config.loadcfg(rcfile.name, "File test_loadcfg") sparse = parsec_config.sparse value = sparse['section3']['entries']['value'] assert [1, 2, 3, 4] == value # calling it multiple times should still work parsec_config.loadcfg(rcfile.name, "File test_loadcfg") parsec_config.loadcfg(rcfile.name, "File test_loadcfg") sparse = parsec_config.sparse value = sparse['section3']['entries']['value'] assert [1, 2, 3, 4] == value def test_loadcfg_override(sample_spec): """Test that loading a second config file overrides common settings but leaves in settings only present in the first""" with tempfile.NamedTemporaryFile() as output_file_name: parsec_config = ParsecConfig( spec=sample_spec, upgrader=None, output_fname=output_file_name.name, tvars=None, validator=None ) with tempfile.NamedTemporaryFile() as conf_file1: conf_file1.write(""" [section1] value1 = 'frodo' value2 = 'sam' """.encode()) conf_file1.seek(0) parsec_config.loadcfg(conf_file1.name, "File test_loadcfg") sparse = parsec_config.sparse assert sparse['section1']['value1'] == 'frodo' assert sparse['section1']['value2'] == 'sam' with tempfile.NamedTemporaryFile() as conf_file2: conf_file2.write(""" [section1] value2 = 'pippin' """.encode()) conf_file2.seek(0) parsec_config.loadcfg(conf_file2.name, "File test_loadcfg") sparse = parsec_config.sparse assert sparse['section1']['value1'] == 'frodo' assert sparse['section1']['value2'] == 'pippin' def test_loadcfg_with_upgrade(sample_spec): def upg(cfg, description): u = upgrader(cfg, description) u.obsolete('1.0', ['section3', 'entries']) u.upgrade() with tempfile.NamedTemporaryFile() as output_file_name: with tempfile.NamedTemporaryFile() as rcfile: parsec_config = ParsecConfig( spec=sample_spec, upgrader=upg, output_fname=output_file_name.name, tvars=None, validator=None # use default ) rcfile.write(""" [section1] value1 = 'test' value2 = 'test' [section2] enabled = True [section3] title = 'Ohm' [[entries]] key = 'product' value = 1, 2, 3, 4 """.encode()) rcfile.seek(0) parsec_config.loadcfg(rcfile.name, "1.1") sparse = parsec_config.sparse # removed by the obsolete upgrade assert 'entries' not in sparse['section3'] def test_validate(): """ An interesting aspect of the ParsecConfig.validate, is that if you have a sparse dict produced by this class, and you call the validate on that dict again, you may have TypeErrors. That occurs because the values like 'True' are validated against the spec and converted from Strings with quotes, to bool types. So the next type you run the validation if expects Strings... :return: """ with Conf('myconf') as spec: with Conf('section'): Conf('name', VDR.V_STRING) Conf('address', VDR.V_STRING) parsec_config = ParsecConfig( spec=spec, upgrader=None, # new spec output_fname=None, # not going to call the loadcfg tvars=None, validator=None # use default ) sparse = OrderedDictWithDefaults() parsec_config.validate(sparse) # empty dict is OK with pytest.raises(IllegalItemError): sparse = OrderedDictWithDefaults() sparse['name'] = 'True' parsec_config.validate(sparse) # name is not valid sparse = OrderedDictWithDefaults() sparse['section'] = OrderedDictWithDefaults() sparse['section']['name'] = 'Wind' sparse['section']['address'] = 'Corner' parsec_config.validate(sparse) @pytest.fixture def parsec_config_2(tmp_path: Path): with Conf('myconf') as spec: with Conf('section'): Conf('name', VDR.V_STRING) Conf('address', VDR.V_INTEGER_LIST) with Conf('allow_many'): Conf('', VDR.V_STRING, '') with Conf('so_many'): with Conf(''): Conf('color', VDR.V_STRING) Conf('horsepower', VDR.V_INTEGER) parsec_config = ParsecConfig(spec, validator=cylc_config_validate) conf_file = tmp_path / 'myconf' conf_file.write_text(""" [section] name = test [allow_many] anything = yup [so_many] [[legs]] horsepower = 123 """) parsec_config.loadcfg(conf_file, "1.0") return parsec_config def test_expand(parsec_config_2: ParsecConfig): parsec_config_2.expand() sparse = parsec_config_2.sparse assert sparse['allow_many']['anything'] == 'yup' assert '__MANY__' not in sparse['allow_many'] def test_get(parsec_config_2: ParsecConfig): cfg = parsec_config_2.get(keys=None, sparse=False) assert cfg == parsec_config_2.dense cfg = parsec_config_2.get(keys=None, sparse=True) assert cfg == parsec_config_2.sparse cfg = parsec_config_2.get(keys=['section'], sparse=True) assert cfg == parsec_config_2.sparse['section'] @pytest.mark.parametrize('keys, expected', [ (['section', 'name'], 'test'), (['section', 'a'], InvalidConfigError), (['alloy_many', 'anything'], InvalidConfigError), (['allow_many', 'anything'], 'yup'), (['allow_many', 'a'], ItemNotFoundError), (['so_many', 'legs', 'horsepower'], 123), (['so_many', 'legs', 'color'], ItemNotFoundError), (['so_many', 'legs', 'a'], InvalidConfigError), (['so_many', 'teeth', 'horsepower'], ItemNotFoundError), ]) def test_get__sparse(parsec_config_2: ParsecConfig, keys, expected): if isinstance(expected, type) and issubclass(expected, Exception): with pytest.raises(expected): parsec_config_2.get(keys, sparse=True) else: assert parsec_config_2.get(keys, sparse=True) == expected def test_mdump_none(config, sample_spec, capsys): cfg = config(sample_spec, ''' [section1] value1 = abc value2 = def ''') cfg.mdump() std = capsys.readouterr() assert std.out == '' assert std.err == '' def test_mdump_some(config, sample_spec, capsys): cfg = config(sample_spec, ''' [section1] value1 = abc value2 = def ''') cfg.mdump( [ ['section1', 'value1'], ['section1', 'value2'], ] ) std = capsys.readouterr() assert std.out == 'abc\ndef\n' assert std.err == '' def test_mdump_oneline(config, sample_spec, capsys): cfg = config(sample_spec, ''' [section1] value1 = abc value2 = def ''') cfg.mdump( [ ['section1', 'value1'], ['section1', 'value2'], ], oneline=True ) std = capsys.readouterr() assert std.out == 'abc def\n' assert std.err == '' def test_get_none(config, sample_spec): cfg = config(sample_spec, '') # blank config assert cfg.get(sparse=True) == {} def test__get_namespace_parents(): """It returns a list of parents and nothing else""" with Conf('myconfig.cylc') as myconf: with Conf('a'): with Conf('b'): with Conf(''): with Conf('d'): Conf('') with Conf('x'): Conf('y') cfg = ParsecConfig(myconf) assert cfg.manyparents == [ ['a', 'b'], ['a', 'b', '__MANY__', 'd'], ] cylc-flow-8.7.0/tests/unit/parsec/conftest.py0000664000175000017500000000422715264420307021416 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.parsec.config import ( ParsecConfig, ConfigNode as Conf ) from cylc.flow.parsec.validate import ( CylcConfigValidator as VDR ) @pytest.fixture def config(tmp_path): """Returns a function for parsing Parsec configurations.""" def _inner(spec, conf): """Parse conf against spec and return the result. Arguments: spec (cylc.flow.parsec.config.ConfigNode): The spec to parse the config against. conf (str): Multiline string containing the configuration. Returns: cylc.flow.parsec.ParsecConfig """ filepath = tmp_path / 'cfg.cylc' with open(filepath, 'w+') as filehandle: filehandle.write(conf) cfg = ParsecConfig(spec) cfg.loadcfg(filepath) return cfg return _inner @pytest.fixture def sample_spec(): """An example cylc.flow.parsec.config.ConfigNode.""" with Conf('myconf') as myconf: with Conf('section1'): Conf('value1', VDR.V_STRING, '') Conf('value2', VDR.V_STRING, 'what?') with Conf('section2'): Conf('enabled', VDR.V_BOOLEAN, False) with Conf('section3'): Conf('title', VDR.V_STRING) with Conf('entries'): Conf('key', VDR.V_STRING) Conf('value', VDR.V_INTEGER_LIST) return myconf cylc-flow-8.7.0/tests/unit/parsec/test_include.py0000664000175000017500000000752415264420307022256 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Basic tests for the include module and its functions. Some functions rely on file system operations, and are probably better tested by functional tests. So this suite of unit tests should not cover all the module features. """ import os import tempfile import unittest from cylc.flow.parsec.exceptions import ParsecError from cylc.flow.parsec.include import inline, IncludeFileNotFoundError class TestInclude(unittest.TestCase): def test_include_file_not_found_error(self): dir_temp = tempfile.mkdtemp() with tempfile.NamedTemporaryFile(dir=dir_temp) as tf: file_list = [tf.name, tf.name, tf.name] error = IncludeFileNotFoundError(file_list) self.assertTrue(" via " in str(error)) def test_inline_error_mismatched_quotes(self): """The inline function throws an error when you have the %include statement with a value without the correct balance for quotes, e.g. %include "abc.txt """ with self.assertRaises(ParsecError): inline( lines=["%include 'abc.txt"], dir_=None, filename=None, level=None) def test_inline(self): with tempfile.NamedTemporaryFile() as tf: filename = tf.name file_lines = [ "#!jinja2", "[section]", "value 1" ] file_lines_with_include = file_lines + [ "%include '{0}'".format(filename) ] # same as before as there was no include lines r = inline(lines=file_lines, dir_=os.path.dirname(tf.name), filename=filename) self.assertEqual(file_lines, r) # here the include line is removed, so the value returned # is still file_lines, not file_lines_with_include r = inline(lines=file_lines_with_include, dir_=os.path.dirname(tf.name), filename=filename) self.assertEqual(file_lines, r) # the for_grep adds some marks helpful for when grep'ing the file r = inline(lines=file_lines_with_include, dir_=os.path.dirname(tf.name), filename=filename, for_grep=True) expected = file_lines + [ "#++++ START INLINED INCLUDE FILE {0}".format(filename), "#++++ END INLINED INCLUDE FILE {0}".format(filename) ] self.assertEqual(expected, r) # for_edit would call the backup function, which triggers file # system operations. So we avoid testing that option here. # test that whatever is in the included file appears in the output tf.write("[section2]".encode()) tf.flush() r = inline(lines=file_lines_with_include, dir_=os.path.dirname(tf.name), filename=filename) expected = file_lines + [ "[section2]" ] self.assertEqual(expected, r) cylc-flow-8.7.0/tests/unit/parsec/test_fileparse.py0000664000175000017500000006154115264420307022604 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from contextlib import suppress import os from pathlib import Path import re import sqlite3 from tempfile import NamedTemporaryFile from textwrap import dedent from types import SimpleNamespace import warnings import pytest from pytest import param from cylc.flow import __version__ as cylc_version from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults from cylc.flow.parsec.exceptions import ( FileParseError, IncludeFileNotFoundError, Jinja2Error, ParsecError, ) from cylc.flow.parsec.fileparse import ( EXTRA_VARS_TEMPLATE, _get_fpath_for_source, _prepend_old_templatevars, addict, addsect, get_cylc_env_vars, merge_template_vars, multiline, parse, process_plugins, read_and_proc, ) def test_file_parse_error(): error = FileParseError(reason="No reason") assert str(error) == "No reason" error = FileParseError("", index=2) assert str(error) == ( " (line 3)\n" "(line numbers match 'cylc view -p')" ) error = FileParseError("", line="test") assert str(error) == ":\n test" error = FileParseError("ERROR", lines={"a": ["1", "2"], "b": ["3"]}) assert str(error) == '\n'.join([ 'ERROR', 'File a', ' 1', ' 2\t<--', 'File b', ' 3\t<--' ]) def test_addsect(): cfg = OrderedDictWithDefaults() cfg["section1"] = OrderedDictWithDefaults() cfg["section1"]["subsection"] = OrderedDictWithDefaults() new_section_name = "test" existing_section_name = "section1" parents = ["section1"] empty_cfg = OrderedDictWithDefaults() addsect(empty_cfg, "", []) assert len(empty_cfg) == 1 assert "" in empty_cfg assert len(cfg) == 1 addsect(cfg, existing_section_name, []) assert len(cfg) == 1 assert "test" not in cfg["section1"] addsect(cfg, new_section_name, parents) assert len(cfg) == 1 assert "test" in cfg["section1"] def test_addict_error_line_already_encountered(): with pytest.raises(FileParseError): addict({"title": "a"}, "title", None, ["title"], 1) def test_addict_new_value_added(): for key, val in [ ('a', '1'), ('b', '2'), ('c', '3') ]: cfg = OrderedDictWithDefaults() addict(cfg, key, val, [], 0) assert key in cfg def test_addict_replace_value(): cfg = OrderedDictWithDefaults() cfg['country'] = 'ABC' addict(cfg, 'country', 'test', [], 0) assert cfg['country'] == 'test' def test_addict_replace_value_1(): """Special case depending on key and parents, for - key is 'graph' AND parents['scheduling']['graph'] OR - len(parents)==3 AND parents['scheduling']['graph'][?] """ # set 1 key is graph, parents is wrong (T, F) cfg = OrderedDictWithDefaults() cfg['graph'] = 'ABC' addict(cfg, 'graph', 'test', [], 0) assert cfg['graph'] == 'test' # set 2 key is graph, parents right (T, T) cfg = { 'scheduling': { 'graph': { 'graph': 'ABC' } } } addict(cfg, 'graph', 'test', ['scheduling', 'graph'], 0) assert ( cfg['scheduling']['graph']['graph'] ) == ['ABC', 'test'] # other side of boolean expression # set 3 len(parents) is 3, parents is wrong (T, F) cfg = { 'scheduling': { 'graph': { 'team': { 'graph': 'ABC' } }, 'notme': { 'team': { 'graph': '1' } } } } addict(cfg, 'graph', 'test', ['scheduling', 'notme', 'team'], 0) assert ( cfg['scheduling']['notme']['team']['graph'] ) == 'test' # set 3 len(parents) is 3, parents is right (T, T) cfg = { 'scheduling': { 'graph': { 'team': { 'graph': 'ABC' } } } } addict(cfg, 'graph', 'test', ['scheduling', 'graph', 'team'], 0) assert cfg['scheduling']['graph']['team']['graph'] == ['ABC', 'test'] @pytest.mark.parametrize( 'flines, value, index, maxline, exc, expected', ( param( [], "'''single line'''", 0, 0, None, ("'''single line'''", 0), id='single-line', ), param( ["'''single line"], "'''single line", # missing closing quote 0, 0, FileParseError, { 'reason': 'Multiline string not closed', 'line': "'''single line", }, id='missing-closing-quote', ), param( ["'''", "single line"], "'''\n '''single line", # missing closing quote 0, 0, FileParseError, {'reason': 'Invalid line', 'line': "'''"}, id='missing-closing-quote2', ), param( ["", "another value"], # multiline, but we forgot to close quotes "'''a\n#b", 0, 1, FileParseError, {'reason': "Multiline string not closed"}, id='multiline-missing-closing-quote', ), param( ["", "c'''"], "'''a\n#b", 0, 1, None, ("'''a\n#b\nc'''", 1), id='good-path', ), param( ["", "c'''"], # multiline, but we forgot to close quotes "'''a\n#b", 0, 10000, # no error. The function will stop before on the quotes None, ("'''a\n#b\nc'''", 1), id='multiline-missing-closing-quote2', ), param( ["", "c", "hello", ""], # quotes out of balance "'''a\n#b", 0, 3, FileParseError, {'reason': "Multiline string not closed"}, id='unbalanced-quotes', ), param( ["", "c", "hello", ""], "'''a\n#b", 0, 4, # one too many IndexError, None, id='one-too-many', ), param( ["", "a'''c", "hello", ""], "'''a\n#b", 0, 3, FileParseError, {'reason': 'Invalid line', 'line': "a'''c"}, id='invalid-quotes', ), ), ) def test_multiline(flines, value, index, maxline, exc, expected): if exc is not None: with pytest.raises(exc) as cm: multiline(flines, value, index, maxline) if isinstance(cm.value, FileParseError): exc = cm.value for key, attr in expected.items(): assert getattr(exc, key) == attr else: r = multiline(flines, value, index, maxline) assert r == expected def test_read_and_proc_no_template_engine(): with NamedTemporaryFile() as tf: fpath = tf.name template_vars = None viewcfg = { 'jinja2': False, 'contin': False, 'inline': False, } tf.write("a=b\n".encode()) tf.flush() r = read_and_proc(fpath=fpath, template_vars=template_vars, viewcfg=viewcfg) assert r == ['a=b'] # last \\ is ignored, becoming just '' tf.write("c=\\\nd\n\\".encode()) tf.flush() viewcfg = { 'jinja2': False, 'contin': True, 'inline': False, } r = read_and_proc(fpath=fpath, template_vars=template_vars, viewcfg=viewcfg) assert r == ['a=b', 'c=d', ''] def test_inline(): with NamedTemporaryFile() as tf: fpath = tf.name template_vars = None viewcfg = { 'jinja2': False, 'contin': False, 'inline': True, 'mark': None, 'single': None, 'label': None, } with NamedTemporaryFile() as include_file: include_file.write("c=d".encode()) include_file.flush() tf.write(("a=b\n%include \"{0}\"" .format(include_file.name)).encode()) tf.flush() r = read_and_proc(fpath=fpath, template_vars=template_vars, viewcfg=viewcfg) assert r == ['a=b', 'c=d'] def test_inline_error(): with NamedTemporaryFile() as tf: fpath = tf.name template_vars = None viewcfg = { 'jinja2': False, 'contin': False, 'inline': True, 'mark': None, 'single': None, 'label': None, } tf.write("a=b\n%include \"404.txt\"".encode()) tf.flush() with pytest.raises(IncludeFileNotFoundError) as cm: read_and_proc(fpath=fpath, template_vars=template_vars, viewcfg=viewcfg) assert "404.txt" in str(cm.value) def test_read_and_proc_jinja2(): with NamedTemporaryFile() as tf: fpath = tf.name template_vars = { 'name': 'Cylc' } viewcfg = { 'jinja2': True, 'contin': False, 'inline': False, } tf.write("#!jinja2\na={{ name }}\n".encode()) tf.flush() r = read_and_proc(fpath=fpath, template_vars=template_vars, viewcfg=viewcfg) assert r == ['a=Cylc'] def test_read_and_proc_cwd(tmp_path): """The template processor should be able to read workflow files. This relies on moving to the config dir during file parsing. """ sdir = tmp_path / "sub" sdir.mkdir() for sub in ["a", "b", "c"]: (sdir / sub).touch() viewcfg = { 'jinja2': True, 'contin': False, 'inline': False } tmpf = tmp_path / "a.conf" with open(tmpf, 'w') as tf: tf.write( '#!Jinja2' '\n{% from "os" import listdir %}' '\n{% for f in listdir("sub") %}' '\n{{f}}' '\n{% endfor %}' ) with open(tmpf, 'r') as tf: r = read_and_proc(fpath=tf.name, viewcfg=viewcfg) assert sorted(r) == ['a', 'b', 'c'] def test_read_and_proc_jinja2_error(): with NamedTemporaryFile() as tf: fpath = tf.name template_vars = { 'name': 'Cylc' } viewcfg = { 'jinja2': True, 'contin': False, 'inline': False, } tf.write("#!jinja2\na={{ name \n".encode()) tf.flush() with pytest.raises(Jinja2Error) as cm: read_and_proc(fpath=fpath, template_vars=template_vars, viewcfg=viewcfg) assert ( "unexpected end of template, expected " "'end of print statement'." ) in str(cm.value) def test_read_and_proc_jinja2_error_missing_shebang(): with NamedTemporaryFile() as tf: fpath = tf.name template_vars = { 'name': 'Cylc' } viewcfg = { 'jinja2': True, 'contin': False, 'inline': False, } # first line is missing shebang! tf.write("a={{ name }}\n".encode()) tf.flush() r = read_and_proc(fpath=fpath, template_vars=template_vars, viewcfg=viewcfg) assert r == ['a={{ name }}'] def test_read_and_proc_jinja2_warnings( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture, ): """Warnings from users' custom jinja2 filters etc should be logged.""" (fpath := tmp_path / 'flow.cylc').write_text(r"#!jinja2\n") msg = "Mock deprecation warning" def mock_jinja2process(fpath, *a, **k): if fpath.endswith('flow.cylc'): warnings.warn(msg, category=DeprecationWarning) return [] monkeypatch.setattr( 'cylc.flow.parsec.jinja2support.jinja2process', mock_jinja2process ) read_and_proc( fpath=str(fpath), viewcfg={'jinja2': True, 'contin': False, 'inline': False}, ) assert len(caplog.records) == 1 rec = caplog.records[0] assert rec.levelname == 'WARNING' assert re.match( ( rf"The following warnings .* during Jinja2 preprocessing.*\s+" rf"{__file__}:\d+: DeprecationWarning: {msg}" ), rec.message, ) def test_read_and_proc_jinja2_warnings_invalid_escape( tmp_path: Path, caplog: pytest.LogCaptureFixture ): """When a warning comes from a j2 template expression, it should be logged with the cylc config filepath.""" (fpath := tmp_path / 'flow.cylc').write_text( dedent(r""" #!jinja2 {% from 'warnings' import warn %} {% do warn('Mock warning') %} """).lstrip() ) read_and_proc( fpath=str(fpath), viewcfg={'jinja2': True, 'contin': False, 'inline': False}, ) assert ( f"{fpath}:: UserWarning: Mock warning" in caplog.text ) def test_parse_keys_only_singleline(): with NamedTemporaryFile() as of, NamedTemporaryFile() as tf: fpath = tf.name template_vars = { 'name': 'Cylc' } tf.write("#!jinja2\na={{ name }}\n".encode()) tf.flush() r = parse(fpath=fpath, output_fname=of.name, template_vars=template_vars) expected = OrderedDictWithDefaults() expected['a'] = 'Cylc' assert r == expected of.flush() output_file_contents = of.read().decode() assert output_file_contents == 'a=Cylc\n' def test_parse_keys_only_multiline(): with NamedTemporaryFile() as of, NamedTemporaryFile() as tf: fpath = tf.name template_vars = { 'name': 'Cylc' } tf.write( "#!jinja2\na='''value is \\\n{{ name }}'''\n".encode()) tf.flush() r = parse(fpath=fpath, output_fname=of.name, template_vars=template_vars) expected = OrderedDictWithDefaults() expected['a'] = "'''value is Cylc'''" assert r == expected def test_parse_invalid_line(): with NamedTemporaryFile() as of, NamedTemporaryFile() as tf: fpath = tf.name template_vars = { 'name': 'Cylc' } tf.write("#!jinja2\n{{ name }}\n".encode()) tf.flush() with pytest.raises(FileParseError) as cm: parse(fpath=fpath, output_fname=of.name, template_vars=template_vars) exc = cm.value assert exc.reason == 'Invalid line' assert exc.line_num == 1 assert exc.line == 'Cylc' def test_parse_comments(): with NamedTemporaryFile() as of, NamedTemporaryFile() as tf: fpath = tf.name template_vars = { 'name': 'Cylc' } tf.write("#!jinja2\na={{ name }}\n# comment!".encode()) tf.flush() r = parse(fpath=fpath, output_fname=of.name, template_vars=template_vars) expected = OrderedDictWithDefaults() expected['a'] = 'Cylc' assert r == expected of.flush() output_file_contents = of.read().decode() assert output_file_contents == 'a=Cylc\n# comment!\n' def test_parse_with_sections(): with NamedTemporaryFile() as of, NamedTemporaryFile() as tf: fpath = tf.name template_vars = { 'name': 'Cylc' } tf.write(("#!jinja2\n[section1]\n" "a={{ name }}\n# comment!\n" "[[subsection1]]\n" "[[subsection2]]\n" "[section2]").encode()) tf.flush() r = parse(fpath=fpath, output_fname=of.name, template_vars=template_vars) expected = OrderedDictWithDefaults() expected['section1'] = OrderedDictWithDefaults() expected['section1']['a'] = 'Cylc' expected['section1']['subsection1'] = OrderedDictWithDefaults() expected['section1']['subsection2'] = OrderedDictWithDefaults() expected['section2'] = OrderedDictWithDefaults() assert r == expected of.flush() output_file_contents = of.read().decode() assert output_file_contents == ( '[section1]\na=Cylc\n# comment!\n' '[[subsection1]]\n' '[[subsection2]]\n' '[section2]\n' ) def test_parse_with_sections_missing_bracket(): with NamedTemporaryFile() as tf: fpath = tf.name template_vars = { 'name': 'Cylc' } tf.write( "#!jinja2\n[[section1]\na={{ name }}\n# comment!".encode()) tf.flush() with pytest.raises(FileParseError) as cm: parse(fpath=fpath, output_fname="", template_vars=template_vars) exc = cm.value assert exc.reason == 'bracket mismatch' assert exc.line == '[[section1]' def test_parse_with_sections_error_wrong_level(): with NamedTemporaryFile() as of, NamedTemporaryFile() as tf: fpath = tf.name template_vars = { 'name': 'Cylc' } tf.write(("#!jinja2\n[section1]\n" "a={{ name }}\n# comment!\n" "[[[subsection1]]]\n") # expected [[]] instead! .encode()) tf.flush() with pytest.raises(FileParseError) as cm: parse(fpath=fpath, output_fname=of.name, template_vars=template_vars) exc = cm.value assert exc.line_num == 4 assert exc.line == '[[[subsection1]]]' def test_unclosed_multiline(): with NamedTemporaryFile() as tf: fpath = tf.name template_vars = { 'name': 'Cylc' } tf.write((''' [scheduling] [[graph]] R1 = """ foo [runtime] [[foo]] script = """ echo hello world """ ''').encode()) tf.flush() with pytest.raises(FileParseError) as cm: parse(fpath=fpath, output_fname="", template_vars=template_vars) exc = cm.value assert exc.reason == 'Invalid line' assert 'echo hello world' in exc.line assert 'Did you forget to close [scheduling][graph]R1?' in str(exc) @pytest.mark.parametrize( 'expect, native_tvars, plugin_result, log', [ pytest.param( {'FOO': 123}, {'FOO': 123}, { 'templating_detected': None, 'template_variables': {'FOO': 122} }, ['Overriding FOO: 122 -> 123'], id='no templating engine set' ), pytest.param( {'FOO': 125}, {'FOO': 125}, { 'templating_detected': 'qux', 'template_variables': {'FOO': 124} }, ['Overriding FOO: 124 -> 125'], id='Variable overridden' ), pytest.param( {'FOO': 126}, {'FOO': 126}, { 'templating_detected': 'qux', 'template_variables': {'FOO': 126} }, [], id='Variable overridden quietly' ) ] ) def test_merge_template_vars(caplog, expect, native_tvars, plugin_result, log): assert merge_template_vars(native_tvars, plugin_result) == expect assert [r.msg for r in caplog.records] == log @pytest.fixture def _mock_old_template_vars_db(tmp_path): def _inner(create_srclink=True): # Create a fake workflow dir: (tmp_path / 'flow.cylc').touch() (tmp_path / 'log').mkdir() db_path = tmp_path / 'log/db' db_path.touch() # Set up a fake workflow database: conn = sqlite3.connect(str(db_path)) conn.execute( "CREATE TABLE workflow_template_vars" "(key TEXT, value TEXT, PRIMARY KEY(key)) ;" ) conn.execute( "INSERT INTO workflow_template_vars VALUES" " ('Marius', '\"Consul\"')" ) conn.execute( "CREATE TABLE workflow_params" "(key TEXT, value TEXT, PRIMARY KEY(key)) ;" ) conn.execute( "INSERT INTO workflow_params VALUES" f" ('cylc_version', '{cylc_version}')" ) conn.commit() conn.close() # Simulate being an installed rundir by creating a sourcelink: if create_srclink: src = (tmp_path / 'src') src.mkdir(exist_ok=True) link = tmp_path.parent / '_cylc-install/source' link.parent.mkdir(exist_ok=True) with suppress(FileExistsError): # We don't mind the link persisting. os.symlink(src, link) return tmp_path / 'flow.cylc' yield _inner @pytest.mark.parametrize( 'expect, tvars', [ param( {'Marius': 'Consul', 'Julius': 'Pontifex'}, {'Julius': 'Pontifex'}, id='It adds a new key at the end' ), param( {'Marius': 'Tribune'}, {'Marius': 'Tribune'}, id='It overrides an existing key' ), ] ) def test__prepend_old_templatevars(_mock_old_template_vars_db, expect, tvars): # Create a target for a source symlink result = _prepend_old_templatevars( _mock_old_template_vars_db(), tvars) assert result == expect def test_get_fpath_for_source(tmp_path): # Create rundir and srcdir: srcdir = tmp_path / 'source' rundir = tmp_path / 'run' srcdir.mkdir() rundir.mkdir() (rundir / 'flow.cylc').touch() (srcdir / 'flow.cylc').touch() # Mock Options object: opts = SimpleNamespace() # It raises an error if source is not linked: with pytest.raises( ParsecError, match=f'Cannot validate {rundir} against source:' ): opts.against_source = True _get_fpath_for_source(rundir / 'flow.cylc', opts) # Create symlinks: link = rundir / '_cylc-install/source' link.parent.mkdir(exist_ok=True) os.symlink(srcdir, link) # It does nothing if opts.against_source is False: opts.against_source = False assert _get_fpath_for_source( rundir / 'flow.cylc', opts) == rundir / 'flow.cylc' # It gets source dir if opts.against_source is True: opts.against_source = True assert _get_fpath_for_source( rundir / 'flow.cylc', opts) == str(srcdir / 'flow.cylc') def test_user_has_no_cwd(tmp_path): """Test we can parse a config file even if cwd does not exist.""" cwd = tmp_path / "cwd" os.mkdir(cwd) os.chdir(cwd) os.rmdir(cwd) # (I am now located in a non-existent directory. Outrageous!) with NamedTemporaryFile() as tf: fpath = tf.name tf.write((''' [scheduling] [[graph]] R1 = "foo" ''').encode()) tf.flush() # Should not raise FileNotFoundError from os.getcwd(): parse(fpath=fpath, output_fname="") def test_get_cylc_env_vars(monkeypatch): """It should return CYLC env vars but not CYLC_VERSION or CYLC_ENV_NAME.""" monkeypatch.setattr( 'os.environ', { "CYLC_VERSION": "betwixt", "CYLC_ENV_NAME": "between", "CYLC_QUESTION": "que?", "CYLC_ANSWER": "42", "FOO": "foo" } ) assert ( get_cylc_env_vars() == { "CYLC_QUESTION": "que?", "CYLC_ANSWER": "42", } ) class EntryPointWrapper: """Wraps a method to make it look like an entry point.""" def __init__(self, fcn): self.name = fcn.__name__ self.fcn = fcn def load(self): return self.fcn @EntryPointWrapper def pre_configure_basic(*_, **__): """Simple plugin that returns one env var and one template var.""" return {'env': {'foo': 44}, 'template_variables': {}} def test_plugins_not_called_on_global_config(monkeypatch): monkeypatch.setattr( 'cylc.flow.plugins.iter_entry_points', lambda x: [pre_configure_basic] ) result = process_plugins('/pennine/way/flow.cylc', {}) assert result != EXTRA_VARS_TEMPLATE result = process_plugins('/appalachian/trail/global.cylc', {}) assert result == EXTRA_VARS_TEMPLATE cylc-flow-8.7.0/tests/unit/parsec/test_fileparse_templating_clash.py0000664000175000017500000000337215264420307026200 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.parsec import fileparse from cylc.flow.parsec.fileparse import read_and_proc from cylc.flow.parsec.exceptions import TemplateVarLanguageClash @pytest.mark.parametrize( 'templating, hashbang', [ ['other', 'jinja2'], ['jinja2', 'other'] ] ) def test_read_and_proc_raises_TemplateVarLanguageClash( monkeypatch, tmp_path, templating, hashbang ): """func fails when diffn't templating engines set in hashbang and plugin. """ def fake_process_plugins(_, __): extra_vars = { 'env': {}, 'template_variables': {'foo': 52}, 'templating_detected': templating } return extra_vars monkeypatch.setattr(fileparse, 'process_plugins', fake_process_plugins) file_ = tmp_path / 'flow.cylc' file_.write_text( f'#!{hashbang}\nfoo' ) with pytest.raises(TemplateVarLanguageClash) as exc: read_and_proc(file_) assert exc.type == TemplateVarLanguageClash cylc-flow-8.7.0/tests/unit/parsec/test_util.py0000664000175000017500000004062515264420307021607 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from io import StringIO import pytest from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults from cylc.flow.parsec.util import ( SECTION_EXPAND_PATTERN, expand_many_section, fast_listjoin, itemstr, listjoin, m_override, pdeepcopy, poverride, printcfg, replicate, un_many, ) from cylc.flow.parsec.validate import DurationFloat @pytest.mark.parametrize('method', (listjoin, fast_listjoin)) def test_listjoin(method): assert method(None) == '' assert method(None, 'test') == 'test' assert method([]) == '' assert method([], 'test') == 'test' assert method([None], 'test') == 'test' assert method(['test', 'test']) == 'test, test' if method == listjoin: # intelligent quoting assert method(['a#', 'b,', "c'"]) == "'a#', 'b,', \"c'\"" assert method(['test,', 'test']) == '\'test,\', test' # integer list rationalisation assert method([10, 11, 12]) == '10..12' assert method([10, 10, 10]) == '3*10' assert method([10, 10, 10, 42]) == '3*10, 42' def test_duration_float(): """It should prevent the creation of duplicate instances.""" # these calls should only create two discrete instances of DurationFloat a = DurationFloat(1.0) b = DurationFloat(1.0) c = DurationFloat(42.0) # a and b should be no only equal but also the same instance assert a == b assert id(a) == id(b) assert str(a) == 'PT1S' # a and c should be unequal and different instances assert a != c assert id(a) != id(c) assert str(c) == 'PT42S' # --- printcfg def test_printcfg(): cfg = OrderedDictWithDefaults() cfg['root'] = OrderedDictWithDefaults() cfg['root']['special'] = 1 cfg['root']['normal'] = 0 cfg['root'][None] = None cfg[None] = None myhandle = StringIO() printcfg(cfg, handle=myhandle) expected = "\n[root]\n special = 1\n normal = 0\n \n" actual = myhandle.getvalue() assert actual == expected def test_printcfg_none_str_is_none(): cfg = OrderedDictWithDefaults() cfg['root'] = OrderedDictWithDefaults() cfg['root']['special'] = 1 cfg['root']['normal'] = 0 cfg['root'][None] = None cfg[None] = None myhandle = StringIO() printcfg(cfg, handle=myhandle, none_str=None) expected = "[root]\n special = 1\n normal = 0\n" actual = myhandle.getvalue() assert actual == expected def test_printcfg_list_values(): cfg = OrderedDictWithDefaults() cfg['root'] = OrderedDictWithDefaults() cfg['root']['special'] = ['a', 'b', 'c', None] cfg['root']['normal'] = 0 myhandle = StringIO() printcfg(cfg, handle=myhandle, none_str='d') expected = "[root]\n special = a, b, c, d\n normal = 0\n" actual = myhandle.getvalue() assert actual == expected def test_printcfg_break_lines(): cfg = OrderedDictWithDefaults() cfg['root'] = OrderedDictWithDefaults() cfg['root']['special'] = "\nthis is\nvalid" cfg['root']['normal'] = 0 myhandle = StringIO() printcfg(cfg, handle=myhandle) expected = ( "[root]\n special = \"\"\"\n \n " " this is\n valid\n \"\"\"\n normal = 0\n" ) actual = myhandle.getvalue() assert actual == expected # --- replicate def test_replicate(): replicate('Name', None) # does nothing, no exception/error source_1 = OrderedDictWithDefaults() source_1["name"] = "sea" source_1["origin"] = "kitchen" source_1.defaults_ = {"brewery": False} source_2 = OrderedDictWithDefaults() source_2["name"] = ["sea", "legume"] source_2["origin"] = "fridge" source_3 = OrderedDictWithDefaults() source_3["name"] = OrderedDictWithDefaults() source_3["name"]["value"] = "oil" source_3["name"]["key"] = 1 source_3["name"].defaults_ = {"value": 1} target_1 = OrderedDictWithDefaults() target_2 = OrderedDictWithDefaults() target_3 = OrderedDictWithDefaults() replicate(target_1, source_1) replicate(target_2, source_2) replicate(target_3, source_3) # Note: assertDictEqual not available for Python 2.6 assert str(target_1) == str(source_1) assert str(target_2) == str(source_2) assert str(target_3) == str(source_3) # --- pdeepcopy def test_pdeepcopy(): """This is tested entirely by the tests in replicate as well""" source = OrderedDictWithDefaults() source["name"] = OrderedDictWithDefaults() source["name"]["value"] = "oil" source["name"]["key"] = 1 source["name"].defaults_ = {"value": 1} target = pdeepcopy(source) assert target == source # --- poverride def test_poverride_append(): source = OrderedDictWithDefaults() source["name"] = OrderedDictWithDefaults() source["name"]["value"] = "oil" source["name"]["key"] = [1, 2, 3, 4] target = OrderedDictWithDefaults() target["name"] = OrderedDictWithDefaults() target["name"]["index"] = 0 poverride(target, None) # harmless, and no error/exception poverride(target, source) expected = OrderedDictWithDefaults() expected["index"] = 0 expected["value"] = "oil" expected["key"] = [1, 2, 3, 4] assert target["name"] == expected def test_poverride_prepend(): source = OrderedDictWithDefaults() source["name"] = OrderedDictWithDefaults() source["name"]["value"] = "oil" source["name"]["key"] = [1, 2, 3, 4] target = OrderedDictWithDefaults() target["name"] = OrderedDictWithDefaults() target["name"]["index"] = 0 poverride(target, None) # harmless, and no error/exception poverride(target, source, prepend=True) expected = OrderedDictWithDefaults() expected["key"] = [1, 2, 3, 4] expected["value"] = "oil" expected["index"] = 0 assert target["name"] == expected # -- m_override def test_m_override(): source = OrderedDictWithDefaults() source["name"] = OrderedDictWithDefaults() source["name"]["index"] = "oil" source["name2"] = OrderedDictWithDefaults() source["name2"]["key"] = [1, 2, 3, 4] source["name2"]["text"] = "" source["name2"]["subdict"] = OrderedDictWithDefaults() source["name2"]["subdict"]['family'] = 'ALL' target = OrderedDictWithDefaults() target["name"] = OrderedDictWithDefaults() target["name"]["index"] = 0 target['__MANY__'] = OrderedDictWithDefaults() target['__MANY__']['name2'] = OrderedDictWithDefaults() target['__MANY__']['subdict'] = OrderedDictWithDefaults() target['__MANY__']['subdict']['__MANY__'] = OrderedDictWithDefaults() target['__MANY__']['subdict']['__MANY__']['family'] = 'LATIN' target['__MANY__']['key'] = [] target['__MANY__']['text'] = "Ad infinitum" assert target["name"]["index"] == 0 m_override(target, source) assert target["name"]["index"] == "oil" def test_m_override_many_with_many(): source = OrderedDictWithDefaults() source["name"] = OrderedDictWithDefaults() source["name"]["index"] = "oil" source["name2"] = OrderedDictWithDefaults() source["name2"]["key"] = [1, 2, 3, 4] source["name2"]["text"] = "" source["name2"]["subdict"] = OrderedDictWithDefaults() source["name2"]["subdict"]['family'] = 'ALL' target = OrderedDictWithDefaults() target["name"] = OrderedDictWithDefaults() target["name"]["index"] = 0 target['__MANY__'] = OrderedDictWithDefaults() target['__MANY__']['name2'] = OrderedDictWithDefaults() target['__MANY__']['subdict'] = OrderedDictWithDefaults() target['__MANY__']['subdict']['__MANY__'] = OrderedDictWithDefaults() target['__MANY__']['subdict']['__MANY__']['family'] = 'LATIN' target['__MANY__']['key'] = [] target['__MANY__']['text'] = "Ad infinitum" # code is OK until here # It appears this is valid for now, but may change later target['__MANY__']['__MANY__'] = OrderedDictWithDefaults() with pytest.raises(Exception): m_override(target, source) def test_m_override_without_many_1(): source = OrderedDictWithDefaults() source["name"] = OrderedDictWithDefaults() source["name"]["value"] = "oil" source["name"]["key"] = [1, 2, 3, 4] target = OrderedDictWithDefaults() target["name"] = OrderedDictWithDefaults() target["name"]["index"] = 0 with pytest.raises(Exception): m_override(target, source) def test_m_override_without_many_2(): source = OrderedDictWithDefaults() source["name"] = OrderedDictWithDefaults() source["name"]["index"] = "oil" source["name2"] = OrderedDictWithDefaults() source["name2"]["key"] = [1, 2, 3, 4] target = OrderedDictWithDefaults() target["name"] = OrderedDictWithDefaults() target["name"]["index"] = 0 target['__MANY__'] = OrderedDictWithDefaults() target['__MANY__']['name2'] = OrderedDictWithDefaults() # target['__MANY__']['key'] = [] with pytest.raises(Exception): m_override(target, source) def test_m_override_without_many_3(): source = OrderedDictWithDefaults() source["name"] = OrderedDictWithDefaults() source["name"]["index"] = "oil" source["name2"] = OrderedDictWithDefaults() source["name2"]["key"] = [1, 2, 3, 4] source["name2"]["text"] = "" source["name2"]["subdict"] = OrderedDictWithDefaults() source["name2"]["subdict"]['family'] = 'ALL' target = OrderedDictWithDefaults() target["name"] = OrderedDictWithDefaults() target["name"]["index"] = 0 target['__MANY__'] = OrderedDictWithDefaults() target['__MANY__']['name2'] = OrderedDictWithDefaults() target['__MANY__']['subdict'] = OrderedDictWithDefaults() target['__MANY__']['key'] = [] target['__MANY__']['text'] = "Ad infinitum" with pytest.raises(Exception): m_override(target, source) def test_m_override_without_many_4(): source = OrderedDictWithDefaults() source["name"] = OrderedDictWithDefaults() source["name"]["index"] = "oil" source["name2"] = OrderedDictWithDefaults() source["name2"]["key"] = [1, 2, 3, 4] source["name2"]["text"] = "" source["name2"]["subdict"] = OrderedDictWithDefaults() source["name2"]["subdict"]['family'] = 'ALL' target = OrderedDictWithDefaults() target["name"] = OrderedDictWithDefaults() target["name"]["index"] = 0 target['__MANY__'] = OrderedDictWithDefaults() target['__MANY__']['name2'] = OrderedDictWithDefaults() target['__MANY__']['key'] = [] target['__MANY__']['text'] = "Ad infinitum" with pytest.raises(Exception): m_override(target, source) # --- un_many def test_un_many(): target = OrderedDictWithDefaults() target["name"] = OrderedDictWithDefaults() target["name"]["index"] = 0 target['__MANY__'] = OrderedDictWithDefaults() target['__MANY__']['name2'] = OrderedDictWithDefaults() target['__MANY__']['subdict'] = OrderedDictWithDefaults() target['__MANY__']['key'] = [] target['__MANY__']['text'] = "Ad infinitum" un_many(None) # harmless, no error/exception un_many(target) assert '__MANY__' not in list(target) def test_un_many_keyerror(): """ Only way that this may happen is if dict is updated elsewhere. """ class MyODWD(OrderedDictWithDefaults): def __delitem__(self, _): raise KeyError() target = MyODWD() target["name"] = "Anything" target.defaults_ = {} target.defaults_["name"] = True target.defaults_["__MANY__"] = True target['__MANY__'] = MyODWD() target['__MANY__']['name2'] = MyODWD() target['__MANY__']['subdict'] = MyODWD() target['__MANY__']['key'] = [] target['__MANY__']['text'] = "Ad infinitum" un_many(None) # harmless, no error/exception un_many(target) assert target assert '__MANY__' in target def test_un_many_keyerror_no_default(): """ Only way that this may happen is if dict is updated elsewhere. And in this case, when there is no defaults_, the API raises the current KeyError. """ class MyODWD(OrderedDictWithDefaults): def __delitem__(self, _): raise KeyError() target = MyODWD() target["name"] = "Anything" target['__MANY__'] = MyODWD() target['__MANY__']['name2'] = MyODWD() target['__MANY__']['subdict'] = MyODWD() target['__MANY__']['key'] = [] target['__MANY__']['text'] = "Ad infinitum" un_many(None) # harmless, no error/exception with pytest.raises(KeyError): un_many(target) # --- itemstr def test_itemstr(): parents = ["parent1", "parent2"] text = itemstr(parents=parents, item="Value", value="Anything") assert text == '[parent1][parent2]Value = Anything' def test_itemstr_no_item(): parents = ["parent1", "parent2", "Value"] text = itemstr(parents=parents, item=None, value="Anything") assert text == '[parent1][parent2]Value = Anything' def test_itemstr_no_parents_no_item(): text = itemstr(parents=None, item=None, value='Anything') assert text == 'Anything' def test_itemstr_no_parents(): text = itemstr(parents=None, item="Value", value='Anything') assert text == 'Value = Anything' def test_itemstr_no_parents_no_value(): text = itemstr(parents=None, item="Value", value=None) assert text == 'Value' # --- expand_many_section @pytest.mark.parametrize( 'in_,out', [ # basically a fancy version of string.split(',') ('foo', ['foo']), ('foo,bar', ['foo', 'bar']), ('foo, bar', ['foo', ' bar']), # doesn't remove whitespace # except that it doesn't split quoted things ('"foo", "bar"', ['"foo"', ' "bar"']), ('"foo,", "b,ar"', ['"foo,"', ' "b,ar"']), # doesn't split in " quotes ("'foo', 'bar'", ["'foo'", " 'bar'"]), ("'foo,', 'b,ar'", ["'foo,'", " 'b,ar'"]), # doesn"t split in ' quotes ] ) def test_SECTION_EXPAND_PATTERN(in_, out): """It should split sections which contain commas. This is used in order to expand [foo, bar] into [foo] and [bar]. """ assert SECTION_EXPAND_PATTERN.findall(in_) == out @pytest.mark.parametrize( 'in_,out', [ ('foo,bar', ['foo', 'bar']), ('foo , bar', ['foo', 'bar']), ('"foo", "bar"', ['foo', 'bar']), ('"foo,", "b,ar"', ['foo,', 'b,ar']), ] ) def test_expand_many_section_expand(in_, out): """It should expand sections which contain commas. E.G. it should expand [foo, bar] into [foo] and [bar]. """ config = {in_: {'whatever': True}} assert list(expand_many_section(config)) == out def test_expand_many_section_order(): """It should maintain order when expanding sections.""" assert list(expand_many_section({ 'a': {}, 'b, a': {}, 'c, b, a, d': {}, 'e': {}, 'a, e': {}, 'f, e': {}, 'g, h': {}, })) == ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'] def test_expand_many_section_merge(): """It should merge sections together in definition order.""" config = expand_many_section({ 'b': {'x': 1}, 'b, a, c, d': {'x': 2}, 'c': {'x': 3}, }) assert config == { 'b': {'x': 2}, 'a': {'x': 2}, 'c': {'x': 3}, 'd': {'x': 2}, } # bonus marks: ensure all values copied rather than referenced config['a']['x'] = 4 assert config['b']['x'] == 2 def test_expand_many_section_merge_deep(): """It should deep-merge nested sections - see replicate().""" config = expand_many_section({ 'b': {'x': {'y': 1}}, 'b, a, c, d': {'x': {'y': 2}}, 'c': {'x': {'y': 3}}, }) assert config == { 'b': {'x': {'y': 2}}, 'a': {'x': {'y': 2}}, 'c': {'x': {'y': 3}}, 'd': {'x': {'y': 2}}, } # bonus marks: ensure all values are unique objects # (i.e. they have been copied rather than referenced) config['a']['x']['y'] = 4 assert config['b']['x']['y'] == 2 cylc-flow-8.7.0/tests/unit/parsec/test_jinja2support.py0000664000175000017500000001262215264420307023440 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from pathlib import Path import sys from textwrap import dedent import jinja2 import pytest from cylc.flow.parsec.jinja2support import ( Jinja2AssertionError, Jinja2Error, PyModuleLoader, assert_helper, jinja2environment, jinja2process, raise_helper, ) from cylc.flow.util import sstrip def test_raise_helper(): message = 'Ops' with pytest.raises(Jinja2AssertionError) as cm: raise_helper(message=message) assert str(cm.value) == "Ops" def test_assert_helper(): assert_helper(logical=True, message="Doesn't matter") # harmless with pytest.raises(Exception): assert_helper(logical=False, message="Doesn't matter") @pytest.fixture def custom_filter_global_and_test(tmp_path: Path): """Provide custom Jinja2 filter, global and test.""" (filters_dir := tmp_path / 'Jinja2Filters').mkdir() with open(filters_dir / "exclaim.py", "w") as tf: tf.write(dedent(r""" def exclaim(val): return f"{val}!!!" """)) (globals_dir := tmp_path / 'Jinja2Globals').mkdir() with open(globals_dir / "assassins.py", "w") as tf: tf.write(r"assassins = 'creed'") (tests_dir := tmp_path / 'Jinja2Tests').mkdir() with open(tests_dir / "big.py", "w") as tf: tf.write(dedent(r""" def big(n: int) -> bool: return n > 9000 """)) def test_jinja2environment(tmp_path: Path, custom_filter_global_and_test): env = jinja2environment(tmp_path) # our jinja env contains the following keys in the global namespace assert 'environ' in env.globals assert 'raise' in env.globals assert 'assert' in env.globals assert env.filters['exclaim']('UP') == 'UP!!!' assert env.globals['assassins'] == 'creed' assert env.tests['big'](5) is False def test_jinja2process(tmp_path, custom_filter_global_and_test): lines = [ r"skipped", r"My name is {{ name }}", r"", r"Custom global: {{ assassins }}", r"Custom filter: {{ 'UP' | exclaim }}", r"Custom test: {% if 5 is big %}big{% else %}small{% endif %}", ] variables = {'name': 'Cylc'} assert jinja2process(None, lines, tmp_path, variables) == [ "My name is Cylc", "Custom global: creed", "Custom filter: UP!!!", "Custom test: small", ] def test_jinja2process_missing_variables(tmp_path): lines = ["skipped", "My name is {{ name }}", ""] with pytest.raises(Jinja2Error) as exc: jinja2process(None, lines, tmp_path, template_vars=None) assert 'jinja2.UndefinedError' in str(exc) def test_pymoduleloader(tmp_path): filters_dir = tmp_path / 'Jinja2filters' filters_dir.mkdir() with open(filters_dir / 'jinja2jinja.py', 'bw+') as tf: tf.write( "def jinja2jinja():\n raise Exception('It works!')".encode()) tf.seek(0) env = jinja2environment(tmp_path) module_loader = PyModuleLoader() template = module_loader.load(environment=env, name='sys') assert sys.path == template.module.path template2 = module_loader.load( environment=env, name='__python__.sys') assert template.module.path == template2.module.path def test_pymoduleloader_invalid_module(tmp_path): filters_dir = tmp_path / 'Jinja2filters' filters_dir.mkdir() with open(filters_dir / 'jinja2jinja.py', 'bw+') as tf: tf.write( "def jinja2jinja():\n raise Exception('It works!')".encode()) tf.seek(0) env = jinja2environment(tmp_path) module_loader = PyModuleLoader() with pytest.raises(jinja2.TemplateNotFound): module_loader.load(environment=env, name='no way jose') def test_restore_deprecated_interfaces(tmp_path, log_filter): """It should log warnings when back-supported Jinja2 interfaces are used. See https://github.com/cylc/cylc-flow/pull/7325 """ # flow.cylc flow_cylc = tmp_path / 'flow.cylc' flow_cylc.write_text(sstrip(''' #!Jinja2 # {{ 1 | foo }} ''')) # Jinja2Filters/foo.py (tmp_path / 'Jinja2Filters').mkdir() ((tmp_path / 'Jinja2Filters') / 'foo.py').write_text(sstrip(''' from jinja2 import contextfilter @contextfilter # NOTE: contextfilter is deprecated! def foo(x, y): return y * 2 ''' + '\n')) # process Jinja2 template assert jinja2process( flow_cylc, flow_cylc.read_text().splitlines(), str(tmp_path), ) == ['# 2'] # warning should be raised assert log_filter( contains=( 'The Jinja2 function contextfilter was renamed to pass_context' ) ) cylc-flow-8.7.0/tests/unit/parsec/test_upgrade.py0000664000175000017500000002050215264420307022251 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import unittest import pytest from cylc.flow.parsec.upgrade import upgrader, converter from cylc.flow.parsec.exceptions import UpgradeError from cylc.flow.parsec.OrderedDict import OrderedDict def test_simple(): """A quick test of overall functionality.""" cfg = { 'item one': 1, 'item two': 'move me up', 'section A': { 'abc': 5, 'cde': 'foo', 'gah': 'bar' }, 'hostnames': { 'host 1': { 'work dir': '/a/b/c', 'running dir': '/a/b/c/d' }, 'host 2': { 'work dir': '/x/b/c', 'running dir': '/x/b/c/d' }, } } x2 = converter(lambda x: 2 * x, 'value x 2') upg = upgrader(cfg, 'test file') # successive upgrades are incremental - at least until I think of a # good way to remember what items have already been translated... upg.deprecate('1.3', ['item one'], ['item ONE'], x2) upg.deprecate('1.3', ['section A'], ['Heading A']) # NOTE change to new item keys here! upg.deprecate('1.3', ['Heading A', 'cde'], ['Heading A', 'CDE']) upg.deprecate( '1.4', ['Heading A', 'abc'], ['Heading A', 'abc'], cvtr=x2 ) upg.deprecate( '1.4.1', ['item two'], ['Heading A', 'item two'] ) upg.deprecate('1.5', ['hostnames'], ['hosts']) upg.deprecate( '1.5', ['hosts', '__MANY__', 'running dir'], ['hosts', '__MANY__', 'run dir']) # obsolete() but with a custom message - `[Heading A]gah` will be deleted: upg.deprecate( '1.3', ['Heading A', 'gah'], None, cvtr=converter(lambda x: x, 'Yaba daba do')) upg.upgrade() assert cfg == { 'item ONE': 2, 'Heading A': { 'CDE': 'foo', 'abc': 10, 'item two': 'move me up' }, 'hosts': { 'host 1': { 'work dir': '/a/b/c', 'run dir': '/a/b/c/d' }, 'host 2': { 'work dir': '/x/b/c', 'run dir': '/x/b/c/d' } } } def test_conflicting_items(): cfg = { 'item one': 1, 'item two': 2, } def get_upgrader(): upg = upgrader(cfg, 'test file') upg.deprecate('1.3', ['item one'], ['item two']) return upg # specifying both the old and new variants of a config should result in an # error upg = get_upgrader() with pytest.raises(UpgradeError): upg.upgrade() # unless the new config is unset cfg['item two'] = None upg = get_upgrader() upg.upgrade() class TestUpgrade(unittest.TestCase): def setUp(self): self.cfg = OrderedDict() self.cfg['section'] = OrderedDict() self.cfg['section']['a'] = '1' self.cfg['section']['b'] = '2' self.u = upgrader(self.cfg, "1.0 to 2.0") def test_converter(self): def callback(i): return 1 + i c = converter(callback=callback, descr="My callback") self.assertEqual("My callback", c.describe()) self.assertEqual(2, c.convert(1)) def test_constructor(self): self.assertEqual(self.cfg, self.u.cfg) self.assertEqual("1.0 to 2.0", self.u.descr) def test_deprecate(self): # b is being deprecated; use c instead self.u.deprecate('entry', ['section', 'b'], ['section', 'c']) self.assertTrue('entry' in self.u.upgrades) def callback(i): return 10 * i c = converter(callback=callback, descr="My callback") self.u.deprecate(vn='entry', oldkeys=[1, 2, 3], newkeys=[10, 20, 30], cvtr=c) # assert the key exists before deprecation self.assertTrue('b' in self.cfg['section']) self.assertFalse('c' in self.cfg['section']) self.u.upgrade() # assert the key exists before deprecation self.assertFalse('b' in self.cfg['section']) self.assertTrue('c' in self.cfg['section']) def test_obsolete(self): # b is obsolete, so the value is omitted self.u.obsolete(vn='entry', oldkeys=['section', 'b']) self.assertTrue('entry' in self.u.upgrades) self.assertTrue('b' in self.cfg['section']) self.u.upgrade() self.assertFalse('b' in self.cfg['section']) self.u.obsolete( vn='entry', oldkeys=['section', 'b'], ) def test_get_item(self): for keys, results in [ (['section', 'a'], '1'), (['section', 'b'], '2'), (['section'], {'a': '1', 'b': '2'}) ]: item = self.u.get_item(keys) self.assertEqual(results, item) def test_put_item(self): for keys, value in [ (['section', 'a'], '100'), (['section', 'b'], '200'), (['special', 'c'], '3'), ]: self.u.put_item(keys, value) self.assertEqual(self.u.get_item(keys), value) def test_expand_not_many(self): upg = { 'new': None, 'cvt': None, 'is_section': False, 'old': [ ] } self.assertEqual([upg], self.u.expand(upg)) def test_expand_too_many(self): upg = { 'new': None, 'cvt': None, 'is_section': True, 'old': [ 'section', '__MANY__', '__MANY__' ] } with self.assertRaises(UpgradeError) as cm: self.u.expand(upg) self.assertTrue('Multiple simultaneous __MANY__ not supported' in str(cm.exception)) def test_expand_deprecate_many_mismatch(self): upg = { 'new': [ 'section', '__MANY__' ], 'cvt': None, 'is_section': False, 'old': [ 'section', '__MANY__', 'b' ] } with self.assertRaises(UpgradeError) as cm: self.u.expand(upg) self.assertEqual('__MANY__ mismatch', str(cm.exception)) def test_expand_deprecate(self): def callback(i): return i c = converter(callback=callback, descr="My callback") upg = { 'new': [ 'section', '__MANY__', 'e' ], 'cvt': c, 'is_section': False, 'old': [ 'section', '__MANY__', 'c' ] } self.u.upgrade() expanded = self.u.expand(upg) self.assertEqual(2, len(expanded)) self.assertEqual(['section', 'a', 'e'], expanded[0]['new']) def test_expand_obsolete(self): upg = { 'new': None, 'cvt': None, 'is_section': False, 'old': [ 'section', '__MANY__', 'a' ] } self.cfg['__MANY__'] = OrderedDict() self.cfg['__MANY__']['name'] = 'Arthur' self.u.obsolete('entry', ['section', '__MANY__']) self.u.upgrade() expanded = self.u.expand(upg) self.assertEqual(1, len(expanded)) self.assertTrue(expanded[0]['new'] is None) def test_template_in_converter_description(caplog, capsys): """Before and after values are available to the conversion descriptor""" cfg = {'old': 42} u = upgrader(cfg, 'Whateva') u.deprecate( '2.0.0', ['old'], ['new'], cvtr=converter(lambda x: x + 20, '{old} -> {new}'), ) u.upgrade() assert cfg == {'new': 62} assert '42 -> 62' in caplog.records[1].message cylc-flow-8.7.0/tests/unit/parsec/test_dict_tree.py0000664000175000017500000001104615264420307022567 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.parsec.OrderedDict import ( DictTree, OrderedDictWithDefaults as ODD ) import pytest def test_eq(): """It compares to identical instances.""" assert DictTree({'a': 1}) == DictTree({'a': 1}) assert DictTree({'a': 1}) != DictTree({'a': 2}) assert DictTree({'a': 1}) != {'a': 1} def test_nesting(): """It slices without duplicating data.""" this_a = dict(b=1) this = dict(a=this_a) that_a = dict(b=2) that = dict(a=that_a) both = DictTree(this, that) sub = both['a'] assert sub._tree == (this_a, that_a) r_this_a, r_that_a = sub._tree assert id(r_this_a) == id(this_a) assert id(r_that_a) == id(that_a) def test_iter(): """It yields keys from all dictionaries.""" a = DictTree({'a': 1}, {'a': 2, 'b': 3}) assert list(sorted(a)) == ['a', 'b'] def test_iter_defaults(): """It doesn't yield keys from defaults.""" this = ODD(a=1) that = ODD(b=2) this.defaults = dict(c=3) that.defaults = dict(d=4) a = DictTree(this, that) assert list(sorted(a)) == ['a', 'b'] def test_getitem(): """It returns values prioritised correctly via the `getitem` interface.""" this = { 'a': 1, 'b': 2, 'f': { 'x': 5, 'y': 6 } } that = { 'b': 3, 'd': 4, 'f': { 'y': 7, 'z': 8 } } a = DictTree(this, that) # key from this assert a['a'] == 1 # key from that assert a['d'] == 4 # key from this overrides key from that assert a['b'] == 2 # dict from both f = a['f'] assert f._tree == (this['f'], that['f']) assert f['x'] == 5 # key from this assert f['y'] == 6 # key from this which is also in that assert f['z'] == 8 # key from that # mutate definition object this['f']['y'] = 42 assert a['f']['y'] == 42 def test_get(): """It returns values correctly via the `get` interface.""" a = DictTree({'a': 1}, {'b': 2}) # key exists in this assert a.get('a') == 1 assert a.get('a', 42) == 1 # key exists in that assert a.get('b') == 2 assert a.get('b', 42) == 2 # key does not exist assert a.get('e') is None assert a.get('e', 42) == 42 def test_get_dict(): """It slices correctly via the `get` interface.""" a = DictTree({'a': {'b': 1}}, {'a': {'b': 2}}) assert a.get('a') == DictTree({'b': 1}, {'b': 2}) def test_odd_getitem(): """It returns default values if none of the dicts contains the key.""" this = ODD(a=1) this.defaults_ = dict(c=3) that = ODD(b=2) that.defaults_ = dict(d=4) both = DictTree(this, that) # key from this assert both['a'] == 1 # key from that assert both['b'] == 2 # default from this assert both['c'] == 3 # default from that assert both['d'] == 4 # missing value with pytest.raises(KeyError): both['e'] def test_defaults_getitem_dict(): """It returns defaults if a value isn't set.""" this = ODD() that = ODD(a=2) both = DictTree(this, that) # default in this gets overridden by value in that this.defaults_ = dict(a=1) that.defaults_ = dict() assert both['a'] == 2 # default in first dict gets priority this.defaults_ = dict(b=1) that.defaults_ = dict(b=2) assert both['b'] == 1 def test_defaults_getitem_nested(): """It preserves the defaults_ behaviour of ODD in nested dicts.""" this = ODD() that = ODD(a=2) both = DictTree(dict(a=this), dict(a=that)) # default in this gets overridden by value in that this.defaults_ = dict(a=1) that.defaults_ = dict() assert both['a']['a'] == 2 # default in first dict gets priority this.defaults_ = dict(b=1) that.defaults_ = dict(b=2) assert both['a']['b'] == 1 cylc-flow-8.7.0/tests/unit/test_templatevars.py0000664000175000017500000001550115264420307022057 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from pathlib import Path import pytest from pytest import param import tempfile import unittest from cylc.flow import __version__ as cylc_version from cylc.flow.exceptions import ServiceFileError, InputError from cylc.flow.rundb import CylcWorkflowDAO from cylc.flow.templatevars import ( get_template_vars_from_db, load_template_vars ) from cylc.flow.workflow_files import WorkflowFiles class TestTemplatevars(unittest.TestCase): def test_load_template_vars_no_params(self): self.assertFalse(load_template_vars()) def test_load_template_vars_from_string(self): pairs = [ "name='John'", "type='Human'", "age='12'" ] expected = { "name": "John", "type": "Human", "age": "12" } self.assertEqual(expected, load_template_vars(template_vars=pairs)) def test_load_template_vars_from_file(self): with tempfile.NamedTemporaryFile() as tf: tf.write(""" name='John' type='Human' # a comment # type=Test age='12' """.encode()) tf.flush() expected = { "name": "John", "type": "Human", "age": "12" } self.assertEqual( expected, load_template_vars(template_vars=None, template_vars_file=tf.name)) def test_load_template_vars_from_string_and_file_1(self): """Text pair variables take precedence over file.""" pairs = [ "name='John'", "age='12'" ] with tempfile.NamedTemporaryFile() as tf: tf.write(""" name='Mariah' type='Human' # a comment # type=Test age='70' """.encode()) tf.flush() expected = { "name": "John", "type": "Human", "age": "12" } self.assertEqual( expected, load_template_vars(template_vars=pairs, template_vars_file=tf.name)) def test_load_template_vars_from_string_and_file_2(self): """Text pair variables take precedence over file.""" pairs = [ "str='str'", "int=12", "float=12.3", "bool=True", "none=None" ] expected = { 'str': 'str', 'int': 12, 'float': 12.3, 'bool': True, 'none': None } self.assertEqual(expected, load_template_vars(template_vars=pairs)) @pytest.fixture(scope='module') def _setup_db(tmp_path_factory): tmp_path: Path = tmp_path_factory.mktemp('test_get_old_tvars') logfolder = tmp_path / WorkflowFiles.LogDir.DIRNAME logfolder.mkdir() db_path = logfolder / WorkflowFiles.LogDir.DB with CylcWorkflowDAO(db_path, create_tables=True) as dao: dao.connect().execute( r''' INSERT INTO workflow_params VALUES (?, ?) ''', ("cylc_version", cylc_version), ) dao.connect().executemany( r''' INSERT INTO workflow_template_vars VALUES (?, ?) ''', [ ("FOO", "42"), ("BAR", "'hello world'"), ("BAZ", "'foo', 'bar', 48"), ("QUX", "['foo', 'bar', 21]"), ], ) dao.connect().commit() yield get_template_vars_from_db(tmp_path) @pytest.mark.parametrize( 'key, expect', ( ('FOO', 42), ('BAR', 'hello world'), ('BAZ', ('foo', 'bar', 48)), ('QUX', ['foo', 'bar', 21]) ) ) def test_get_old_tvars(key, expect, _setup_db): """It can extract a variety of items from a workflow database. """ assert _setup_db[key] == expect def test_get_old_tvars_fails_if_cylc_7_db(tmp_path): """get_template_vars_from_db fails with error if db file is not a valid Cylc 8 DB. """ dbfile = tmp_path / WorkflowFiles.LogDir.DIRNAME / WorkflowFiles.LogDir.DB dbfile.parent.mkdir() dbfile.touch() with pytest.raises(ServiceFileError, match='database is incompatible'): get_template_vars_from_db(tmp_path) @pytest.mark.parametrize( 'tvars_lists, expect', ( param( ['FOO=a,b,c'], {'FOO': ['a', 'b', 'c']}, id='basic' ), param( ['FOO=a,"b,b",c'], {'FOO': ['a', 'b,b', 'c']}, id='contains-quotes' ), ) ) def test_load_var_lists(tvars_lists, expect): """ load_template_vars should parse template vars lists and dump these into template vars. """ result = load_template_vars(templatevars_lists=tvars_lists) assert result == expect def test_load_var_lists_warns_overwriting(caplog): """It wars user if same tvar name used twice.""" load_template_vars(template_vars=["BAZ='HEY'", "BAZ='THERE'"]) assert 'BAZ' in caplog.records[-1].message load_template_vars(templatevars_lists=['FOO=a,b', 'FOO=b,c']) assert 'FOO' in caplog.records[-1].message def test_load_var_lists_fails(): """ load_template_vars should fail if the same var is set using -s and -z """ errmsg = r"FOO=a and FOO=\['a', 'b'\]" with pytest.raises(InputError, match=errmsg): load_template_vars( template_vars=['FOO="a"'], templatevars_lists=['FOO=a,b'] ) def test_load_vars_invalid_value_pairs(): with pytest.raises(InputError, match="FOO\n.*qux\n.*BAZ\n.*QUX"): load_template_vars( template_vars=['FOO', 'qux'], templatevars_lists=['BAZ', 'QUX'] ) def test_load_template_vars_ValueError_fm_file(monkeypatch): from io import StringIO handle = StringIO('Hello') monkeypatch.setattr('builtins.open', lambda *_: handle) with pytest.raises(InputError, match='Hello'): load_template_vars(template_vars_file='foo') cylc-flow-8.7.0/tests/unit/test_pbs_multi_cluster.py0000664000175000017500000000432615264420307023112 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import unittest from cylc.flow.job_runner_handlers.pbs_multi_cluster import ( PBSMulticlusterHandler, ) def get_test_filter_poll_many_output(): return [ ("header1\nheader2\n123.localhost", ["123.localhost@localhost"]), ( """header1 header2 12.localhost job123 09.jd-01.foo.bar""", [ "12.localhost@localhost", "job123", # unchanged "09.jd-01.foo.bar@jd-01.foo.bar", ] ), ] def get_test_manip_job_id(): return [ ("1.localhost", "1.localhost@localhost"), ("10000.jd-01", "10000.jd-01@jd-01"), (" 1077 ", "1077") # unchanged ] class TestPBSMultiCluster(unittest.TestCase): def test_filter_poll_many_output(self): """Basic tests for filter_poll_many_output.""" for out, expected in get_test_filter_poll_many_output(): job_ids = PBSMulticlusterHandler.filter_poll_many_output(out) self.assertEqual(expected, job_ids) def test_manip_job_id(self): """Basic tests for manip_job_id.""" for job_id, expected in get_test_manip_job_id(): mod_job_id = PBSMulticlusterHandler.manip_job_id(job_id) self.assertEqual(expected, mod_job_id) def test_export_handler(self): import cylc.flow.job_runner_handlers.pbs_multi_cluster as m self.assertTrue(hasattr(m, 'JOB_RUNNER_HANDLER')) cylc-flow-8.7.0/tests/unit/test_prerequisite.py0000664000175000017500000002222015264420307022065 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from functools import partial from typing import Optional import pytest from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.cycling.loader import ISO8601_CYCLING_TYPE, get_point from cylc.flow.exceptions import TriggerExpressionError from cylc.flow.id import Tokens, detokenise from cylc.flow.prerequisite import Prerequisite, SatisfiedState from cylc.flow.run_modes import RunMode detok = partial(detokenise, selectors=True, relative=True) @pytest.fixture def prereq(set_cycling_type): set_cycling_type(ISO8601_CYCLING_TYPE, "Z") prereq = Prerequisite(get_point('2000')) prereq[(1999, 'a', 'succeeded')] = True prereq[(2000, 'b', 'succeeded')] = False prereq[(2000, 'c', 'succeeded')] = False prereq[(2001, 'd', 'custom')] = False return prereq def test_satisfied(prereq: Prerequisite): assert prereq._satisfied == { # the pre-initial dependency should be marked as satisfied ('1999', 'a', 'succeeded'): 'satisfied naturally', # all others should not ('2000', 'b', 'succeeded'): False, ('2000', 'c', 'succeeded'): False, ('2001', 'd', 'custom'): False, } # No cached satisfaction state yet: assert prereq._cached_satisfied is None # Calling self.is_satisfied() should cache the result: assert not prereq.is_satisfied() assert prereq._cached_satisfied is False # mark two prerequisites as satisfied prereq.satisfy_me([ Tokens('2000/b:succeeded', relative=True), Tokens('2000/c:succeeded', relative=True), ]) assert prereq._satisfied == { # the pre-initial dependency should be marked as satisfied ('1999', 'a', 'succeeded'): 'satisfied naturally', # the two newly-satisfied dependency should be satisfied ('2000', 'b', 'succeeded'): 'satisfied naturally', ('2000', 'c', 'succeeded'): 'satisfied naturally', # the remaining dependency should not ('2001', 'd', 'custom'): False, } # Should have reset cached satisfaction state: assert prereq._cached_satisfied is None assert not prereq.is_satisfied() # mark all prereqs as satisfied prereq.set_satisfied() assert prereq._satisfied == { # the pre-initial dependency should be marked as satisfied ('1999', 'a', 'succeeded'): 'satisfied naturally', # the two newly-satisfied dependency should be satisfied ('2000', 'b', 'succeeded'): 'satisfied naturally', ('2000', 'c', 'succeeded'): 'satisfied naturally', # the remaining dependency should be marked as forse-satisfied ('2001', 'd', 'custom'): 'force satisfied', } # Should have set cached satisfaction state as must be true now: assert prereq._cached_satisfied is True assert prereq.is_satisfied() def test_getitem_setitem(prereq: Prerequisite): msg = ('2000', 'b', 'succeeded') # __getitem__: assert prereq[msg] is False # __setitem__: prereq[msg] = True assert prereq[msg] == 'satisfied naturally' prereq[msg] = 'force satisfied' assert prereq[msg] == 'force satisfied' # coercion of cycle point assert prereq[(2000, 'b', 'succeeded')] == 'force satisfied' assert prereq[(get_point('2000'), 'b', 'succeeded')] == 'force satisfied' def test_iter(prereq: Prerequisite): assert list(prereq) == [ ('1999', 'a', 'succeeded'), ('2000', 'b', 'succeeded'), ('2000', 'c', 'succeeded'), ('2001', 'd', 'custom'), ] assert [p.task for p in prereq] == ['a', 'b', 'c', 'd'] def test_items(prereq: Prerequisite): assert list(prereq.items()) == [ (('1999', 'a', 'succeeded'), 'satisfied naturally'), (('2000', 'b', 'succeeded'), False), (('2000', 'c', 'succeeded'), False), (('2001', 'd', 'custom'), False), ] def test_set_conditional_expr(prereq: Prerequisite): assert not prereq.is_satisfied() prereq.set_conditional_expr('1999/a succeeded | 2000/b succeeded') assert prereq.is_satisfied() def test_iter_target_point_strings(prereq): assert set(prereq.iter_target_point_strings()) == { '1999', '2000', '2001', } def test_get_target_points(prereq): assert set(prereq.get_target_points()) == { get_point('1999'), get_point('2000'), get_point('2001'), } @pytest.fixture def satisfied_states_prereq(): """Fixture for testing the full range of possible satisfied states.""" prereq = Prerequisite(IntegerPoint('2')) prereq[('1', 'a', 'x')] = True prereq[('1', 'b', 'x')] = False prereq[('1', 'c', 'x')] = 'satisfied from database' prereq[('1', 'd', 'x')] = 'force satisfied' prereq[('1', 'e', 'x')] = 'satisfied by skip mode' return prereq def test_unset_naturally_satisfied(satisfied_states_prereq: Prerequisite): satisfied_states_prereq[('1', 'a', 'y')] = True satisfied_states_prereq[('1', 'a', 'z')] = 'force satisfied' for id_, expected in [ ('1/a', True), ('1/b', False), ('1/c', True), ('1/d', False), ('1/e', True), ]: assert ( satisfied_states_prereq.unset_naturally_satisfied(id_) == expected ) assert satisfied_states_prereq._satisfied == { ('1', 'a', 'x'): False, ('1', 'a', 'y'): False, ('1', 'a', 'z'): 'force satisfied', ('1', 'b', 'x'): False, ('1', 'c', 'x'): False, ('1', 'd', 'x'): 'force satisfied', ('1', 'e', 'x'): False, } def test_set_satisfied(satisfied_states_prereq: Prerequisite): satisfied_states_prereq.set_satisfied() assert satisfied_states_prereq._satisfied == { ('1', 'a', 'x'): 'satisfied naturally', ('1', 'b', 'x'): 'force satisfied', ('1', 'c', 'x'): 'satisfied from database', ('1', 'd', 'x'): 'force satisfied', ('1', 'e', 'x'): 'satisfied by skip mode', } def test_satisfy_me(): prereq = Prerequisite(IntegerPoint('2')) for task_name in ('a', 'b', 'c'): prereq[('1', task_name, 'x')] = False assert not prereq.is_satisfied() assert prereq._cached_satisfied is False prereq.satisfy_me( [Tokens('//1/a:x'), Tokens('//1/d:x'), Tokens('//1/c:y')], ) assert prereq._satisfied == { ('1', 'a', 'x'): 'satisfied naturally', ('1', 'b', 'x'): False, ('1', 'c', 'x'): False, } # should have reset cached satisfaction state assert prereq._cached_satisfied is None prereq.satisfy_me( [Tokens('//1/a:x'), Tokens('//1/b:x')], forced=True, ) assert prereq._satisfied == { # 1/a:x unaffected as already satisfied ('1', 'a', 'x'): 'satisfied naturally', ('1', 'b', 'x'): 'force satisfied', ('1', 'c', 'x'): False, } @pytest.mark.parametrize('forced, mode, expected', [ (False, None, 'satisfied naturally'), (True, None, 'force satisfied'), (True, RunMode.SKIP, 'force satisfied'), (False, RunMode.SKIP, 'satisfied by skip mode'), ]) def test_satisfy_me__override_false( forced: bool, mode: Optional[RunMode], expected: SatisfiedState, ): """Test satisfying an unsatisfied prereq with different states.""" prereq = Prerequisite(IntegerPoint('2')) prereq[('1', 'a', 'x')] = False prereq.satisfy_me([Tokens('//1/a:x')], forced=forced, mode=mode) assert prereq[('1', 'a', 'x')] == expected @pytest.mark.parametrize('mode', [None, RunMode.SKIP]) @pytest.mark.parametrize('forced', [True, False]) @pytest.mark.parametrize('existing', [ 'satisfied from database', 'force satisfied', 'satisfied naturally', ]) def test_satisfy_me__override_truthy( existing: SatisfiedState, forced: bool, mode: Optional[RunMode], ): """Test that satisfying an already-satisfied prereq doesn't change it.""" prereq = Prerequisite(IntegerPoint('2')) prereq[('1', 'a', 'x')] = existing prereq.satisfy_me([Tokens('//1/a:x')], forced=forced, mode=mode) assert prereq[('1', 'a', 'x')] == existing @pytest.mark.parametrize( 'expr, err', ( ('int("df")', 'invalid literal'), ('7 +', 'invalid syntax'), )) def test__eval_satisfied_raises(expr, err, monkeypatch): monkeypatch.setattr( 'cylc.flow.prerequisite.Prerequisite.get_raw_conditional_expression', lambda _: expr ) prereq = Prerequisite(IntegerPoint('1')) prereq.conditional_expression = expr with pytest.raises(TriggerExpressionError, match=err): prereq._eval_satisfied() cylc-flow-8.7.0/tests/unit/test_c3mro.py0000664000175000017500000000522215264420307020372 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import unittest from cylc.flow.c3mro import C3 class TestC3mro(unittest.TestCase): def test_tree_is_empty_by_default(self): c3 = C3() self.assertFalse(c3.tree) def test_tree_parameter(self): c3 = C3({'a': 1}) self.assertTrue(c3.tree) def test_simple_inheritance(self): parents = {} parents['object'] = [] parents['string'] = ['object'] c3 = C3(parents) string_hierarchy = c3.mro('string') self.assertEqual(['string', 'object'], string_hierarchy) def test_simple_inheritance_extra_nodes(self): parents = {} parents['object'] = [] parents['string'] = ['object'] # nodes not related to string parents['root'] = [] parents['diamond'] = ['root'] c3 = C3(parents) self.assertEqual(['string', 'object'], c3.mro('string')) self.assertEqual(['diamond', 'root'], c3.mro('diamond')) def test_empty_tree_key_error(self): parents = {} c3 = C3(parents) with self.assertRaises(KeyError): c3.mro('test') def test_multiple_inheritance_error_py23(self): parents = {} parents['object'] = [] parents['x'] = ['object'] parents['y'] = ['object'] parents['a'] = ['x', 'y'] parents['b'] = ['y', 'x'] parents['z'] = ['a', 'b'] c3 = C3(parents) # see class docstring, this is the case #2 with self.assertRaises(Exception) as cm: c3.mro('z') self.assertTrue("ERROR: z: bad runtime namespace inheritance hierarchy" in str(cm.exception)) def test_mro_of_none(self): with self.assertRaises(Exception) as cm: C3.merge([[], ['x', 'y', 'o'], ['y', 'x', 'o'], []], None) self.assertTrue("ERROR: bad runtime namespace inheritance hierarchy" in str(cm.exception)) cylc-flow-8.7.0/tests/unit/test_task_proxy.py0000664000175000017500000001702715264420307021560 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from typing import Callable, Optional from unittest.mock import Mock import pytest from pytest import param from cylc.flow.cycling import PointBase from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.cycling.iso8601 import ISO8601Point from cylc.flow.flow_mgr import FlowNums from cylc.flow.id import Tokens from cylc.flow.task_proxy import TaskProxy from cylc.flow.taskdef import TaskDef def test_get_point_as_seconds_cached(): """Already-computed value is returned immediately.""" mock_itask = Mock(point_as_seconds=42) assert TaskProxy.get_point_as_seconds(mock_itask) == 42 def test_get_point_as_seconds_known_tz(set_cycling_type): """Cycle point with explicit UTC timezone.""" set_cycling_type(ISO8601Point.TYPE) mock_itask = Mock( point=ISO8601Point('19700101T00Z').standardise(), point_as_seconds=None, ) assert TaskProxy.get_point_as_seconds(mock_itask) == 0 def test_get_point_as_seconds_unknown_tz(monkeypatch): """Cycle point without timezone triggers local-tz offset adjustment.""" # Mock point_parse to return a timepoint with unknown timezone mock_timepoint = Mock( seconds_since_unix_epoch=3600, time_zone=Mock(unknown=True), ) monkeypatch.setattr( 'cylc.flow.task_proxy.point_parse', lambda _: mock_timepoint, ) # Pretend local timezone is UTC+5:30 monkeypatch.setattr( 'cylc.flow.task_proxy.get_local_time_zone', lambda: (5, 30), ) mock_itask = Mock( point='19700101T01', point_as_seconds=None, ) result = TaskProxy.get_point_as_seconds(mock_itask) assert result == 3600 + (5 * 3600 + 30 * 60) @pytest.mark.parametrize( 'itask_point, offset_str, expected', [ param( # date -u -d 19700101 "+%s" ISO8601Point('19700101T00Z'), 'PT0M', 0, id="zero epoch" ), param( # 2025 is not a leap year: Jan 1 + P2M = P59D ISO8601Point('20250101T00Z'), 'PT0M', 1735689600, id="nonleap base" ), param( ISO8601Point('20250101T00Z'), 'P59D', 1740787200, id="nonleap off1" ), param( ISO8601Point('20250101T00Z'), 'P2M', 1740787200, id="nonleap off2" ), param( # 2024 is a leap year: Jan 1 + P2M = P60D ISO8601Point('20240101T00Z'), 'PT0M', 1704067200, id="leap base" ), param( ISO8601Point('20240101T00Z'), 'P60D', 1709251200, id="leap off1" ), param( ISO8601Point('20240101T00Z'), 'P2M', 1709251200, id="leap off2" ), ] ) def test_get_clock_trigger_time( itask_point: PointBase, offset_str: str, expected: int, set_cycling_type: Callable ) -> None: """Test get_clock_trigger_time() for exact and inexact offsets.""" set_cycling_type(itask_point.TYPE) mock_itask = Mock( point=itask_point.standardise(), clock_trigger_times={} ) assert TaskProxy.get_clock_trigger_time( mock_itask, mock_itask.point, offset_str) == expected def test_is_ready_to_run_held(): """A held task is not ready to run.""" mock_itask = Mock(state=Mock(is_held=True)) assert TaskProxy.is_ready_to_run(mock_itask) is False def test_is_ready_to_run_try_timer(): """A task with an active try timer delegates to is_delay_done().""" mock_timer = Mock() mock_timer.is_delay_done.return_value = True mock_itask = Mock( state=Mock(is_held=False, status='submission-failed'), try_timers={'submission-failed': mock_timer}, ) assert TaskProxy.is_ready_to_run(mock_itask) is True assert mock_timer.is_delay_done.call_count == 1 @pytest.mark.parametrize( 'is_waiting, prereqs, ext_trigs, xtrigs, expected', [ param(True, True, True, True, True, id="all-satisfied"), param(False, True, True, True, False, id="not-waiting"), param(True, False, True, True, False, id="prereqs-unsatisfied"), param(True, True, False, True, False, id="ext-trigs-unsatisfied"), param(True, True, True, False, False, id="xtrigs-unsatisfied"), ] ) def test_is_ready_to_run_conditions( is_waiting, prereqs, ext_trigs, xtrigs, expected ): """Test the final return with various combinations of conditions.""" mock_itask = Mock( state=Mock(is_held=False, status='waiting'), try_timers={}, ) mock_itask.state.return_value = is_waiting mock_itask.prereqs_are_satisfied = Mock(return_value=prereqs) mock_itask.state.external_triggers_all_satisfied.return_value = ext_trigs mock_itask.state.xtriggers_all_satisfied.return_value = xtrigs assert TaskProxy.is_ready_to_run(mock_itask) is expected @pytest.mark.parametrize( 'name_str, expected', [('beer', True), ('FAM', True), ('root', True), ('horse', False), ('F*', True), ('*', True)] ) def test_name_match(name_str: str, expected: bool): """Test TaskProxy.name_match(). For a task named "beer" in family "FAM". """ mock_tdef = Mock(namespace_hierarchy=['root', 'FAM', 'beer']) mock_tdef.name = 'beer' mock_itask = Mock(tdef=mock_tdef) assert TaskProxy.name_match(mock_itask, name_str) is expected @pytest.mark.parametrize( 'status_str, expected', [param('waiting', True, id="Basic"), param('w*', False, id="Globs don't work"), param(None, True, id="None always matches")] ) def test_status_match(status_str: Optional[str], expected: bool): """Test TaskProxy.status_match(). For a task with status "waiting". """ mock_itask = Mock(state=Mock(status='waiting')) assert TaskProxy.status_match(mock_itask, status_str) is expected @pytest.mark.parametrize('itask_flow_nums, flow_nums, expected', [ param({1, 2}, {2}, {2}, id="subset"), param({2}, {1, 2}, {2}, id="superset"), param({1, 2}, {3, 4}, set(), id="disjoint"), param({1, 2}, set(), {1, 2}, id="all-matches-num"), param(set(), {1, 2}, set(), id="num-doesnt-match-none"), param(set(), set(), set(), id="all-doesnt-match-none"), ]) def test_match_flows( itask_flow_nums: FlowNums, flow_nums: FlowNums, expected: FlowNums ): mock_itask = Mock(flow_nums=itask_flow_nums) assert TaskProxy.match_flows(mock_itask, flow_nums) == expected def test_match_flows_copy(): """Test that this method does not return the same reference as itask.flow_nums, otherwise you could end up unexpectedly mutating itask.flow_nums.""" mock_itask = Mock(flow_nums={1, 2}) result = TaskProxy.match_flows(mock_itask, set()) assert result == mock_itask.flow_nums assert result is not mock_itask.flow_nums def test_job_tokens(): itask = TaskProxy( Tokens('wflow'), TaskDef('foo', {}, None, None), IntegerPoint('10'), submit_num=3, ) assert str(itask.job_tokens) == 'wflow//10/foo/03' cylc-flow-8.7.0/tests/unit/test_async_util.py0000664000175000017500000001515515264420307021527 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio from inspect import signature import logging from pathlib import Path from random import random import pytest from cylc.flow.async_util import ( pipe, asyncqgen, scandir, ) LOG = logging.getLogger('test') @pipe() async def a_range(n): for num in range(n): LOG.info(f'a_range({n})') yield num @pipe async def even(x): LOG.info(f'even({x})') return x % 2 == 0 @pipe async def mult(x, y, kwarg='useless kwarg'): LOG.info(f'mult{x, y}') return x * y @pipe async def sleepy(x): """A filter which waits a while then passes.""" LOG.info(f'sleepy({x})') await asyncio.sleep(0.1) return True @pytest.mark.parametrize('preserve_order', (True, False)) async def test_pipe(preserve_order): """It passes values through the pipe.""" pipe = a_range(5) | even | mult(2) pipe.preserve_order = preserve_order result = [] async for num in pipe: result.append(num) assert result == [ 0, 4, 8, ] @pytest.mark.parametrize('preserve_order', (True, False)) async def test_pipe_single(preserve_order): """It allow single-step pipes.""" pipe = a_range(5) pipe.preserve_order = preserve_order result = [] async for num in pipe: result.append(num) assert result == [ 0, 1, 2, 3, 4 ] @pytest.mark.parametrize('preserve_order', (True, False)) async def test_pipe_reusable(preserve_order): """It can be re-used once depleted.""" pipe = a_range(5) | even | mult(2) pipe.preserve_order = preserve_order for _ in range(5): result = [] async for num in pipe: result.append(num) assert result == [ 0, 4, 8, ] @pytest.mark.parametrize('preserve_order', (True, False)) async def test_pipe_filter_stop(preserve_order): """It yields values early with the filter_stop argument.""" pipe = a_range(5) | even(filter_stop=False) pipe |= mult(10) pipe.preserve_order = preserve_order result = [] async for num in pipe: result.append(num) # the even numbers should be multiplied by 10 # the odd numbers should be yielded early (so don't get multiplied) assert result == [ 0, 1, 20, 3, 40, ] @pipe async def one(x): await asyncio.sleep(random() / 5) return x @pytest.mark.parametrize('preserve_order', (True, False)) async def test_pipe_preserve_order(preserve_order): """It should control result order according to pipe configuration.""" n = 50 pipe = a_range(n) | one | one | one pipe.preserve_order = preserve_order result = [] async for item in pipe: result.append(item) # the odds of getting 50 items in order by chance are pretty slim assert (result == list(range(n))) is preserve_order @pytest.mark.parametrize('preserve_order', (True, False)) async def test_pipe_concurrent(caplog, preserve_order): """It runs pipes concurrently. It is easy to make something which appears to be concurrent, this test is intended to ensure that it actually IS concurrent. """ pipe = a_range(5) | even | sleepy | mult(2) pipe.preserve_order = preserve_order caplog.set_level(logging.INFO, 'test') async for num in pipe: pass order = [ # a list of the log messages generated by each step of the pipe # as it processes an item x[2].split('(')[0] for x in caplog.record_tuples ] assert 'mult' in order assert len(order) == 4 * 4 # 4 steps * 4 items yielded by a_range # ensure that the steps aren't completed in order (as sync code would) # the sleep should ensure this # NOTE: not the best test but better than nothing assert order != [ 'a_range', 'even', 'sleepy', 'mult' ] * 4 def test_pipe_str(): """It has helpful textual representations.""" pipe = a_range(5) | even(filter_stop=False) | mult(10, kwarg=42) assert str(pipe) == 'a_range(5)' assert repr(pipe) == 'a_range(5) | even() | mult(10, kwarg=42)' @pipe() # NOTE: these brackets are what the next function is testing async def div(x, y): return x / y def test_pipe_brackets(): """Ensure that pipe functions can be declared with or without brackets.""" pipe = a_range(5) | div assert repr(pipe) == 'a_range(5) | div()' @pipe async def documented(x: str, y: int = 0): """The docstring for the pipe function.""" pass def test_documentation(): """It should preserve the docstring, signature & annotations of the wrapped function.""" assert documented.__doc__ == 'The docstring for the pipe function.' assert documented.__annotations__ == {'x': str, 'y': int} assert str(signature(documented)) == '(x: str, y: int = 0)' def test_rewind(): """It should be possible to move through the pipe stages.""" pipe = a_range | mult | even assert pipe.fastforward().rewind() == pipe async def test_asyncqgen(): """It should provide an async gen interface to an async queue.""" queue = asyncio.Queue() gen = asyncqgen(queue) await queue.put(1) await queue.put(2) await queue.put(3) ret = [] async for item in gen: ret.append(item) assert ret == [1, 2, 3] async def test_scandir(tmp_path: Path): """It should list directory contents (including symlinks).""" (tmp_path / 'a').touch() (tmp_path / 'b').touch() (tmp_path / 'c').symlink_to(tmp_path / 'b') assert sorted(await scandir(tmp_path)) == [ Path(tmp_path, 'a'), Path(tmp_path, 'b'), Path(tmp_path, 'c') ] async def test_scandir_non_exist(tmp_path: Path): """scandir() should raise FileNotFoundError if called on a path that doesn't exist.""" with pytest.raises(FileNotFoundError): await scandir(tmp_path / 'HORSE') cylc-flow-8.7.0/tests/unit/test_job_file.py0000664000175000017500000004670315264420307021131 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # Tests for functions contained in cylc.flow.job_file. # TODO remove the unittest dependency - it should not be necessary. from contextlib import suppress import io import os from pathlib import Path import pytest from tempfile import NamedTemporaryFile from textwrap import dedent from cylc.flow import ( __version__, __file__ as cylc_flow_file, ) from cylc.flow.job_file import ( JobFileWriter, ) from cylc.flow.platforms import platform_from_name @pytest.mark.parametrize( 'in_value, out_value', [('~foo/bar bar', '~foo/"bar bar"'), ('~/bar bar', '~/"bar bar"'), ('~/a', '~/"a"'), ('test', '"test"'), ('~', '~'), ('~a', '~a'), ('foo%s', '"foo%s"'), ('foo%(i)d', '"foo3"')] ) def test_get_variable_value_definition(in_value, out_value): """Test the value for single/tilde variables are correctly quoted, and parameter environment templates are handled""" param_dict = {'i': 3} res = JobFileWriter._get_variable_value_definition(in_value, param_dict) assert out_value == res @pytest.fixture def fixture_get_platform(): """ Allows pytest to cache default platform dictionary. Args: custom_settings (dict): settings that you wish to override. Returns: platforms dictionary. """ def inner_func(custom_settings=None): platform = platform_from_name() if custom_settings is not None: platform.update(custom_settings) return platform yield inner_func def test_write(fixture_get_platform): """Test write function outputs jobscript file correctly.""" with NamedTemporaryFile() as local_job_file_path: local_job_file_path = local_job_file_path.name platform = fixture_get_platform( { "job runner command template": "woof", } ) job_conf = { "platform": platform, "task_id": "1/baa", "workflow_name": "farm_noises", "work_d": "farm_noises/work_d", "uuid_str": "neigh", 'environment': {'cow': '~/moo', 'sheep': '~baa/baa', 'duck': '~quack'}, "job_d": "1/baa/01", "try_num": 1, "flow_nums": {1}, # "job_runner_name": "background", "param_var": {"duck": "quack", "mouse": "squeak"}, "execution_time_limit": "moo", "namespace_hierarchy": ["root", "baa", "moo"], "dependencies": ['moo', 'neigh', 'quack'], "init-script": "This is the init script", "env-script": "This is the env script", "err-script": "This is the err script", "pre-script": "This is the pre script", "script": "This is the script", "post-script": "This is the post script", "exit-script": "This is the exit script", } JobFileWriter().write(local_job_file_path, job_conf) assert (os.path.exists(local_job_file_path)) size_of_file = os.stat(local_job_file_path).st_size # This test only needs to check that the file is created and is # non-empty as each section is covered by individual unit tests. assert size_of_file > 10 """Test the header is correctly written""" expected = ('#!/bin/bash -l\n#\n# ++++ THIS IS A CYLC JOB SCRIPT ' '++++\n# Workflow: farm_noises\n# Task: 1/baa\n# Job ' 'log directory: 1/baa/01\n# Job runner: ' 'background\n# Job runner command template: woof\n#' ' Execution time limit: moo') platform = fixture_get_platform( {"job runner command template": "woof"} ) job_conf = { "platform": platform, "job runner": "background", "execution_time_limit": "moo", "workflow_name": "farm_noises", "task_id": "1/baa", "job_d": "1/baa/01" } with io.StringIO() as fake_file: JobFileWriter()._write_header(fake_file, job_conf) assert fake_file.getvalue() == expected @pytest.mark.parametrize( 'job_conf,expected', [ ( # basic { "platform": { "job runner": "loadleveler", "job runner command template": "test_workflow", }, "directives": {"moo": "foo", "cluck": "bar"}, "workflow_name": "farm_noises", "task_id": "1/baa", "job_d": "1/test_task_id/01", "job_file_path": "directory/job", "execution_time_limit": 60 }, ('\n\n# DIRECTIVES:\n# @ job_name = farm_noises.baa.1' '\n# @ output = directory/job.out\n# @ error = directory/' 'job.err\n# @ wall_clock_limit = 120,60\n# @ moo = foo' '\n# @ cluck = bar\n# @ queue') ), ( # Check no directives is correctly written { "platform": { "job runner": "slurm", "job runner command template": "test_workflow" }, "directives": {}, "workflow_name": "farm_noises", "task_id": "1/baa", "job_d": "1/test_task_id/01", "job_file_path": "directory/job", "execution_time_limit": 60 }, ( '\n\n# DIRECTIVES:\n#SBATCH ' '--job-name=baa.1.farm_noises\n#SBATCH ' '--output=directory/job.out\n#SBATCH --error=directory/' 'job.err\n#SBATCH --time=1:00' ) ), ( # Check pbs max job name length { "platform": { "job runner": "pbs", "job runner command template": "test_workflow", "job name length maximum": 15 }, "directives": {}, "workflow_name": "farm_noises", "task_id": "1/baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "job_d": "1/test_task_id/01", "job_file_path": "directory/job", "execution_time_limit": 60 }, ('\n\n# DIRECTIVES:\n#PBS -N baaaaaaaaaaaaaa\n#PBS -o ' 'directory/job.out\n#PBS -e directory/job.err\n#PBS -l ' 'walltime=60') ), ( # Check sge directives are correctly written { "platform": { "job runner": "sge", "job runner command template": "test_workflow", }, "directives": {"-V": "", "-q": "queuename", "-l": "s_vmem=1G,s_cpu=60"}, "workflow_name": "farm_noises", "task_id": "1/baa", "job_d": "1/test_task_id/01", "job_file_path": "$HOME/directory/job", "execution_time_limit": 1000 }, ('\n\n# DIRECTIVES:\n#$ -N farm_noises.baa.1\n#$ -o directory/' 'job.out\n#$ -e directory/job.err\n#$ -l h_rt=0:16:40\n#$ -V\n#' '$ -q queuename\n#$ -l s_vmem=1G,s_cpu=60' ) ) ], ids=["1", "2", "3", "4"]) def test_write_directives(fixture_get_platform, job_conf: dict, expected: str): """Test the directives section of job script file is correctly written""" with io.StringIO() as fake_file: JobFileWriter()._write_directives(fake_file, job_conf) assert fake_file.getvalue() == expected @pytest.mark.parametrize( "job_runner", ["at", "background", "loadleveler", "pbs", "sge", "slurm"]) def test_traps_for_each_job_runner(job_runner: str): """Test traps for each job runner""" platform = platform_from_name() platform.update({ "job runner": f"{job_runner}", }) job_conf = { "platform": platform, "directives": {}, "workflow_name": 'test_traps_for_each_job_runner', } with io.StringIO() as fake_file: JobFileWriter()._write_prelude(fake_file, job_conf) output = fake_file.getvalue() assert "CYLC_FAIL_SIGNALS='EXIT ERR TERM XCPU" in output @pytest.mark.parametrize('CYLC_ENV_NAME', ['myenv', None]) def test_write_prelude( monkeypatch: pytest.MonkeyPatch, fixture_get_platform, CYLC_ENV_NAME ): """Test the prelude section of job script file is correctly written. """ if CYLC_ENV_NAME: monkeypatch.setenv('CYLC_ENV_NAME', CYLC_ENV_NAME) else: with suppress(KeyError): monkeypatch.delenv('CYLC_ENV_NAME') monkeypatch.setattr('cylc.flow.flags.verbosity', 2) expected = [ "CYLC_FAIL_SIGNALS='EXIT ERR TERM XCPU'", "CYLC_VACATION_SIGNALS='USR1'", "export PATH=moo/baa:$PATH", "export CYLC_VERBOSE=true", "export CYLC_DEBUG=true", f"export CYLC_VERSION='{__version__}'", ] if CYLC_ENV_NAME: expected.append(f"export CYLC_ENV_NAME='{CYLC_ENV_NAME}'") expected.extend([ 'export CYLC_WORKFLOW_ID="test_write_prelude"', "export CYLC_WORKFLOW_INITIAL_CYCLE_POINT='20200101T0000Z'", "export CYLC_COVERAGE='1'", ]) job_conf = { "workflow_name": "test_write_prelude", "platform": fixture_get_platform({ "job runner": "loadleveler", "job runner command template": "test_workflow", "host": "localhost", "copyable environment variables": [ "CYLC_WORKFLOW_INITIAL_CYCLE_POINT" ], "cylc path": "moo/baa" }), "directives": {"restart": "yes"}, } for k, v in { "CYLC_WORKFLOW_INITIAL_CYCLE_POINT": "20200101T0000Z", "CYLC_WORKFLOW_NAME": "test_write_prelude", "CYLC_WORKFLOW_NAME_BASE": "test_write_prelude", 'CYLC_COVERAGE': '1' }.items(): monkeypatch.setenv(k, v) with io.StringIO() as fake_file: # copyable environment variables JobFileWriter()._write_prelude(fake_file, job_conf) assert fake_file.getvalue().strip().splitlines() == expected def test_write_workflow_environment(fixture_get_platform, monkeypatch): """Test workflow environment is correctly written in jobscript""" # set some workflow environment conditions monkeypatch.setattr('cylc.flow.flags.verbosity', 2) workflow_env = {'CYLC_UTC': 'True', 'CYLC_CYCLING_MODE': 'integer', 'CYLC_WORKFLOW_NAME': 'blargh/quack', 'CYLC_WORKFLOW_NAME_BASE': 'quack'} job_file_writer = JobFileWriter() job_file_writer.set_workflow_env(workflow_env) # workflow env not correctly setting...check this expected = ('\n\ncylc__job__inst__cylc_env() {\n # CYLC WORKFLOW ' 'ENVIRONMENT:\n export CYLC_CYCLING_MODE="integer"\n ' ' export CYLC_UTC="True"' '\n export CYLC_WORKFLOW_NAME="blargh/quack"' '\n export CYLC_WORKFLOW_NAME_BASE="quack"' '\n export TZ="UTC"' '\n export CYLC_WORKFLOW_UUID="neigh"') job_conf = { "platform": fixture_get_platform({ "host": "localhost", }), "workflow_name": "farm_noises", "uuid_str": "neigh" } with io.StringIO() as fake_file: job_file_writer._write_workflow_environment(fake_file, job_conf) result = fake_file.getvalue() assert result == expected def test_write_script(): """Test script is correctly written in jobscript""" expected = ( "\n\ncylc__job__inst__init_script() {\n# INIT-SCRIPT:\n" "This is the init script\n}\n\ncylc__job__inst__env_script()" " {\n# ENV-SCRIPT:\nThis is the env script\n}\n\n" "cylc__job__inst__err_script() {\n# ERR-SCRIPT:\nThis is the err " "script\n}\n\ncylc__job__inst__pre_script() {\n# PRE-SCRIPT:\n" "This is the pre script\n}\n\ncylc__job__inst__script() {\n" "# SCRIPT:\nThis is the script\n}\n\ncylc__job__inst__post_script" "() {\n# POST-SCRIPT:\nThis is the post script\n}\n\n" "cylc__job__inst__exit_script() {\n# EXIT-SCRIPT:\n" "This is the exit script\n}") job_conf = { "init-script": "This is the init script", "env-script": "This is the env script", "err-script": "This is the err script", "pre-script": "This is the pre script", "script": "This is the script", "post-script": "This is the post script", "exit-script": "This is the exit script", "workflow_name": "test_write_script", } with io.StringIO() as fake_file: JobFileWriter()._write_script(fake_file, job_conf) assert fake_file.getvalue() == expected def test_no_script_section_with_comment_only_script(): """Test jobfilewriter does not generate script section when script is comment only""" expected = ("") job_conf = { "init-script": "", "env-script": "", "err-script": "", "pre-script": "#This is the pre script/n #moo /n#baa", "script": "", "post-script": "", "exit-script": "", "workflow_name": "test_no_script_section_with_comment_only_script" } with io.StringIO() as fake_file: JobFileWriter()._write_script(fake_file, job_conf) blah = fake_file.getvalue() print(blah) assert fake_file.getvalue() == expected def test_write_task_environment(): """Test task environment is correctly written in jobscript""" # set some task environment conditions expected = ('\n\n # CYLC TASK ENVIRONMENT:\n ' 'export CYLC_TASK_COMMS_METHOD=ssh\n ' 'export CYLC_TASK_JOB="1/moo/01"\n export ' 'CYLC_TASK_NAMESPACE_HIERARCHY="baa moo"\n export ' 'CYLC_TASK_TRY_NUMBER=1\n export ' 'CYLC_TASK_FLOW_NUMBERS=1\n export ' 'CYLC_PROFILER=true\n export ' 'CYLC_PROFILER_CGROUPS=exit_light\n export ' 'CYLC_PROFILER_POLL_INTERVAL=1\n export ' 'CYLC_TASK_PARAM_duck="quack"\n export ' 'CYLC_TASK_PARAM_mouse="squeak"\n ' 'CYLC_TASK_WORK_DIR_BASE=\'farm_noises/work_d\'\n}') job_conf = { "platform": { 'communication method': 'ssh', 'profiler': { "activate": "true", "cgroups path": 'exit_light', "polling interval": 1 } }, "job_d": "1/moo/01", "namespace_hierarchy": ["baa", "moo"], "dependencies": ['moo', 'neigh', 'quack'], "try_num": 1, "flow_nums": {1}, "param_var": {"duck": "quack", "mouse": "squeak"}, "work_d": "farm_noises/work_d", "workflow_name": "test_write_task_environment", } with io.StringIO() as fake_file: JobFileWriter()._write_task_environment(fake_file, job_conf) assert fake_file.getvalue() == expected def test_write_runtime_environment(): """Test runtime environment is correctly written in jobscript""" expected = ( '\n\ncylc__job__inst__user_env() {\n # TASK RUNTIME ' 'ENVIRONMENT:\n export cow sheep duck\n' ' cow=~/"moo"\n sheep=~baa/"baa"\n ' 'duck=~quack\n}') job_conf = { 'environment': { 'cow': '~/moo', 'sheep': '~baa/baa', 'duck': '~quack' }, "workflow_name": "test_write_runtime_environment", } with io.StringIO() as fake_file: JobFileWriter()._write_runtime_environment(fake_file, job_conf) assert fake_file.getvalue() == expected def test_write_epilogue(): """Test epilogue is correctly written in jobscript""" expected = '\n' + dedent(''' CYLC_RUN_DIR="${CYLC_RUN_DIR:-$HOME/cylc-run}" . "${CYLC_RUN_DIR}/${CYLC_WORKFLOW_ID}/.service/etc/job.sh" cylc__job__main #EOF: 1/moo/01 ''') job_conf = {'job_d': "1/moo/01"} with io.StringIO() as fake_file: JobFileWriter()._write_epilogue(fake_file, job_conf) assert fake_file.getvalue() == expected def test_write_global_init_scripts(fixture_get_platform): """Test global init script is correctly written in jobscript""" job_conf = { "platform": fixture_get_platform({ "global init-script": ( 'export COW=moo\n' 'export PIG=oink\n' 'export DONKEY=HEEHAW\n' ) }) } expected = '\n' + dedent(''' # GLOBAL INIT-SCRIPT: export COW=moo export PIG=oink export DONKEY=HEEHAW ''') with io.StringIO() as fake_file: JobFileWriter()._write_global_init_script(fake_file, job_conf) assert fake_file.getvalue() == expected def test_homeless_platform(fixture_get_platform): """Ensure there are no uses of $HOME before the global init-script. This is to allow users to configure a $HOME on machines with no $HOME directory. """ job_conf = { "platform": fixture_get_platform({ 'global init-script': 'some-script' }), "task_id": "1/a", "workflow_name": "b", "work_d": "c/d", "uuid_str": "e", 'environment': {}, 'cow': '~/moo', "job_d": "1/a/01", "try_num": 1, "flow_nums": {1}, # "job_runner_name": "background", "param_var": {}, "execution_time_limit": None, "namespace_hierarchy": [], "dependencies": [], "init-script": "", "env-script": "", "err-script": "", "pre-script": "", "script": "", "post-script": "", "exit-script": "", } with NamedTemporaryFile() as local_job_file_path: local_job_file_path = local_job_file_path.name JobFileWriter().write(local_job_file_path, job_conf) with open(local_job_file_path, 'r') as local_job_file: job_script = local_job_file.read() # ensure that $HOME is not used before the global init-script for line in job_script.splitlines(): if line.startswith(' '): # ignore env/script functions which aren't run until later continue if line == '# GLOBAL INIT-SCRIPT:': # quit once we've hit the global init-script break if 'HOME' in line: # bail if $HOME is used raise Exception(f'$HOME found in {line}\n{job_script}') # also ensure there is no use of $HOME in the job.sh script with open(Path(cylc_flow_file).parent / 'etc/job.sh', 'r') as job_sh: job_sh_txt = job_sh.read() if 'HOME' in job_sh_txt: raise Exception('$HOME found in job.sh\n{job_sh_txt}') cylc-flow-8.7.0/tests/unit/__init__.py0000664000175000017500000000140715264420307020050 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . cylc-flow-8.7.0/tests/unit/test_config.py0000664000175000017500000015723515264420307020630 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from contextlib import suppress import logging from optparse import Values import os from pathlib import Path from textwrap import dedent from types import SimpleNamespace from typing import ( Any, Callable, Dict, List, Optional, Tuple, Type, ) from unittest.mock import Mock import pytest from cylc.flow import ( CYLC_LOG, ) from cylc.flow.config import WorkflowConfig from cylc.flow.cycling import loader from cylc.flow.cycling.iso8601 import ISO8601Point from cylc.flow.cycling.loader import ( INTEGER_CYCLING_TYPE, ISO8601_CYCLING_TYPE, ) from cylc.flow.exceptions import ( InputError, PointParsingError, WorkflowConfigError, XtriggerConfigError, ) from cylc.flow.parsec.exceptions import ( Jinja2Error, ) from cylc.flow.scheduler_cli import RunOptions from cylc.flow.scripts.validate import ValidateOptions from cylc.flow.task_outputs import ( TASK_OUTPUT_SUCCEEDED, ) from cylc.flow.wallclock import ( get_utc_mode, set_utc_mode, ) from cylc.flow.workflow_files import WorkflowFiles Fixture = Any param = pytest.param class TestWorkflowConfig: """Test class for the Cylc WorkflowConfig object.""" def test_xfunction_imports( self, mock_glbl_cfg: 'Fixture', tmp_path: 'Path'): """Test for a workflow configuration with valid xtriggers""" mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[localhost]] hosts = localhost ''' ) python_dir = tmp_path / "lib" / "python" python_dir.mkdir(parents=True) name_a_tree_file = python_dir / "name_a_tree.py" # NB: we are not returning a lambda, instead we have a scalar name_a_tree_file.write_text("""name_a_tree = lambda: 'jacaranda'""") flow_file = tmp_path / WorkflowFiles.FLOW_FILE flow_config = """ [scheduler] allow implicit tasks = True [scheduling] initial cycle point = 2018-01-01 [[xtriggers]] tree = name_a_tree() [[graph]] R1 = '@tree => qux' """ flow_file.write_text(flow_config) workflow_config = WorkflowConfig( workflow="name_a_tree", fpath=flow_file, options=SimpleNamespace() ) assert 'tree' in workflow_config.xtrigger_collator.functx_map def test_xfunction_import_error(self, mock_glbl_cfg, tmp_path): """Test for error when a xtrigger function cannot be imported.""" mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[localhost]] hosts = localhost ''' ) python_dir = tmp_path / "lib" / "python" python_dir.mkdir(parents=True) caiman_file = python_dir / "caiman.py" # NB: we are not returning a lambda, instead we have a scalar caiman_file.write_text("""caiman = lambda: True""") flow_file = tmp_path / WorkflowFiles.FLOW_FILE flow_config = """ [scheduling] initial cycle point = 2018-01-01 [[xtriggers]] oopsie = piranha() [[graph]] R1 = '@oopsie => qux' """ flow_file.write_text(flow_config) with pytest.raises(XtriggerConfigError) as excinfo: WorkflowConfig( workflow="caiman_workflow", fpath=flow_file, options=SimpleNamespace() ) assert "No module named 'piranha'" in str(excinfo.value) def test_xfunction_attribute_error(self, mock_glbl_cfg, tmp_path): """Test for error when a xtrigger function cannot be imported.""" mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[localhost]] hosts = localhost ''' ) python_dir = tmp_path / "lib" / "python" python_dir.mkdir(parents=True) capybara_file = python_dir / "capybara.py" # NB: we are not returning a lambda, instead we have a scalar capybara_file.write_text("""toucan = lambda: True""") flow_file = tmp_path / WorkflowFiles.FLOW_FILE flow_config = """ [scheduling] initial cycle point = 2018-01-01 [[xtriggers]] oopsie = capybara() [[graph]] R1 = '@oopsie => qux' """ flow_file.write_text(flow_config) with pytest.raises(XtriggerConfigError) as excinfo: WorkflowConfig(workflow="capybara_workflow", fpath=flow_file, options=SimpleNamespace()) assert "module 'capybara' has no attribute 'capybara'" in str( excinfo.value) def test_xfunction_not_callable(self, mock_glbl_cfg, tmp_path): """Test for error when a xtrigger function is not callable.""" mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[localhost]] hosts = localhost ''' ) python_dir = tmp_path / "lib" / "python" python_dir.mkdir(parents=True) not_callable_file = python_dir / "not_callable.py" # NB: we are not returning a lambda, instead we have a scalar not_callable_file.write_text("""not_callable = 42""") flow_file = tmp_path / WorkflowFiles.FLOW_FILE flow_config = """ [scheduling] initial cycle point = 2018-01-01 [[xtriggers]] oopsie = not_callable() [[graph]] R1 = '@oopsie => qux' """ flow_file.write_text(flow_config) with pytest.raises(XtriggerConfigError) as excinfo: WorkflowConfig( workflow="workflow_with_not_callable", fpath=flow_file, options=SimpleNamespace() ) assert "callable" in str(excinfo.value) @pytest.mark.parametrize( 'fam_txt', [pytest.param('"SOMEFAM"', id="double quoted"), pytest.param('\'SOMEFAM\'', id="single quoted"), pytest.param('SOMEFAM', id="unquoted")] ) def test_family_inheritance_and_quotes( fam_txt: str, mock_glbl_cfg: Callable, tmp_flow_config: Callable ) -> None: """Test that inheritance does not ignore items, if not all quoted. For example: inherit = 'MAINFAM', SOMEFAM inherit = 'BIGFAM', SOMEFAM See bug #2700 for more/ """ mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[localhost]] hosts = localhost ''' ) id_ = 'test' file_path = tmp_flow_config(id_, f''' [scheduler] allow implicit tasks = True [task parameters] major = 1..5 minor = 10..20 [scheduling] [[graph]] R1 = """hello => MAINFAM hello => SOMEFAM""" [runtime] [[root]] script = true [[MAINFAM]] [[SOMEFAM]] [[ goodbye_0 ]] inherit = 'MAINFAM', {fam_txt} ''') config = WorkflowConfig( id_, file_path, template_vars={}, options=Values() ) assert ('goodbye_0_major1_minor10' in config.runtime['descendants']['MAINFAM_major1_minor10']) assert ('goodbye_0_major1_minor10' in config.runtime['descendants']['SOMEFAM']) @pytest.mark.parametrize( ('cycling_type', 'scheduling_cfg', 'expected_icp', 'expected_err'), [ pytest.param( ISO8601_CYCLING_TYPE, {'initial cycle point': None}, None, (WorkflowConfigError, "requires an initial cycle point"), id="Lack of icp" ), pytest.param( INTEGER_CYCLING_TYPE, {'initial cycle point': None}, '1', None, id="Default icp for integer cycling type" ), pytest.param( INTEGER_CYCLING_TYPE, {'initial cycle point': "now"}, None, (PointParsingError, "invalid literal for int()"), id="Non-integer ICP for integer cycling type" ), pytest.param( INTEGER_CYCLING_TYPE, {'initial cycle point': "20500808T0000Z"}, None, (PointParsingError, "invalid literal for int()"), id="More non-integer ICP for integer cycling type" ), pytest.param( ISO8601_CYCLING_TYPE, {'initial cycle point': "1"}, None, (PointParsingError, "Invalid ISO 8601 date representation"), id="Non-ISO8601 ICP for ISO8601 cycling type" ), pytest.param( ISO8601_CYCLING_TYPE, {'initial cycle point': 'now'}, '20050102T0615+0530', None, id="ICP = now" ), pytest.param( ISO8601_CYCLING_TYPE, {'initial cycle point': 'previous(T00)'}, '20050102T0000+0530', None, id="ICP = prev" ), pytest.param( ISO8601_CYCLING_TYPE, { 'initial cycle point': '2013', 'initial cycle point constraints': ['T00', 'T12'] }, '20130101T0000+0530', None, id="Constraints" ), pytest.param( ISO8601_CYCLING_TYPE, { 'initial cycle point': '2021-01-20', 'initial cycle point constraints': ['--01-19', '--01-21'] }, None, (WorkflowConfigError, "does not meet the constraints"), id="Violated constraints" ), pytest.param( ISO8601_CYCLING_TYPE, { 'initial cycle point': 'a', }, None, (WorkflowConfigError, 'Invalid ISO 8601 date representation: a'), id="invalid" ), ] ) def test_process_icp( cycling_type: str, scheduling_cfg: Dict[str, Any], expected_icp: Optional[str], expected_err: Optional[Tuple[Type[Exception], str]], monkeypatch: pytest.MonkeyPatch, set_cycling_type: 'Fixture' ) -> None: """Test WorkflowConfig.process_initial_cycle_point(). "now" is assumed to be 2005-01-02T06:15+0530 Params: cycling_type: Workflow cycling type. scheduling_cfg: 'scheduling' section of workflow config. expected_icp: The expected icp value that gets set. expected_err: Exception class expected to be raised plus the message. """ set_cycling_type(cycling_type, time_zone="+0530") mocked_config = SimpleNamespace( cycling_type=cycling_type, options=SimpleNamespace(icp=None), cfg={ 'scheduling': { 'initial cycle point constraints': [], **scheduling_cfg }, }, ) monkeypatch.setattr('cylc.flow.config.get_current_time_string', lambda: '20050102T0615+0530') if expected_err: err, msg = expected_err with pytest.raises(err) as exc: WorkflowConfig.process_initial_cycle_point(mocked_config) assert msg in str(exc.value) else: WorkflowConfig.process_initial_cycle_point(mocked_config) assert mocked_config.cfg[ 'scheduling']['initial cycle point'] == expected_icp assert str(mocked_config.initial_point) == expected_icp @pytest.mark.parametrize( 'cycling_type, options, expected, expected_err', [ ( ISO8601_CYCLING_TYPE, SimpleNamespace(startcp='20210120T1700+0530'), '20210120T1700+0530', None, ), ( ISO8601_CYCLING_TYPE, SimpleNamespace(startcp='now'), '20050102T0615+0530', None, ), ( ISO8601_CYCLING_TYPE, SimpleNamespace(startcp='previous(T00)'), '20050102T0000+0530', None, ), ( ISO8601_CYCLING_TYPE, SimpleNamespace(startcp=None), '18990501T0000+0530', None, ), ( ISO8601_CYCLING_TYPE, SimpleNamespace( starttask=['20090802T0615+0530/foo', '20090802T0515+0530/bar'] ), '20090802T0515+0530', None, ), ( ISO8601_CYCLING_TYPE, SimpleNamespace( startcp='20210120T1700+0530', starttask=['20090802T0615+0530/foo'], ), None, ( InputError, "--start-cycle-point and --start-task are mutually exclusive", ), ), ( INTEGER_CYCLING_TYPE, SimpleNamespace(startcp='10'), '10', None, ) ], ) def test_process_startcp( cycling_type: str, options: SimpleNamespace, expected: str, expected_err: Optional[Tuple[Type[Exception], str]], monkeypatch: pytest.MonkeyPatch, set_cycling_type: 'Fixture' ) -> None: """Test WorkflowConfig.process_start_cycle_point(). An icp of 1899-05-01T00+0530 is assumed, and "now" is assumed to be 2005-01-02T06:15+0530 Params: options: SimpleNamespace with startcp and starttask attributes. expected: The expected startcp value that gets set. expected_err: Expected exception. """ set_cycling_type(cycling_type, time_zone='+0530') mocked_config = Mock( spec=WorkflowConfig, cycling_type=cycling_type, initial_point='18990501T0000+0530', options=options, ) monkeypatch.setattr('cylc.flow.config.get_current_time_string', lambda: '20050102T0615+0530') if expected_err is not None: err, msg = expected_err with pytest.raises(err) as exc: WorkflowConfig.process_start_cycle_point(mocked_config) assert msg in str(exc.value) else: WorkflowConfig.process_start_cycle_point(mocked_config) assert str(mocked_config.start_point) == expected @pytest.mark.parametrize( 'cycling_type, scheduling_cfg, options_fcp, expected_fcp, expected_err', [ pytest.param( ISO8601_CYCLING_TYPE, { 'initial cycle point': '2021', 'final cycle point': None, }, None, None, None, id="No fcp" ), pytest.param( ISO8601_CYCLING_TYPE, { 'initial cycle point': '2021', 'final cycle point': '', }, None, None, None, id="Empty fcp in cfg" # This test is needed because fcp is treated as string by parsec, # unlike other cycle point settings (allows for e.g. '+P1Y') ), pytest.param( ISO8601_CYCLING_TYPE, { 'initial cycle point': '2016', 'final cycle point': '2021', }, None, '20210101T0000+0530', None, id="fcp in cfg" ), pytest.param( ISO8601_CYCLING_TYPE, { 'initial cycle point': '2016', 'final cycle point': '2021', }, '2019', '20190101T0000+0530', None, id="Overridden by cli option" ), pytest.param( ISO8601_CYCLING_TYPE, { 'initial cycle point': '2017-02-11', 'final cycle point': '+P4D', }, None, '20170215T0000+0530', None, id="Relative fcp" ), pytest.param( ISO8601_CYCLING_TYPE, { 'initial cycle point': '2017-02-11', 'final cycle point': '+P4D+PT3H-PT2H', }, None, '20170215T0100+0530', None, id="Relative fcp chained" ), pytest.param( ISO8601_CYCLING_TYPE, { 'initial cycle point': '2017-02-11', 'final cycle point': '---04', }, None, '20170215T0000+0530', None, id="Relative truncated fcp", marks=pytest.mark.xfail # https://github.com/metomi/isodatetime/issues/80 ), pytest.param( INTEGER_CYCLING_TYPE, { 'initial cycle point': '1', 'final cycle point': '4', }, None, '4', None, id="Integer cycling" ), pytest.param( INTEGER_CYCLING_TYPE, { 'initial cycle point': '1', 'final cycle point': '+P2', }, None, '3', None, id="Relative fcp, integer cycling" ), pytest.param( ISO8601_CYCLING_TYPE, { 'initial cycle point': '2013', 'final cycle point': '2009', }, None, None, (WorkflowConfigError, "initial cycle point:20130101T0000+0530 is after the " "final cycle point"), id="fcp before icp" ), pytest.param( ISO8601_CYCLING_TYPE, { 'initial cycle point': '2013', 'final cycle point': '-PT1S', }, None, None, (WorkflowConfigError, "initial cycle point:20130101T0000+0530 is after the " "final cycle point"), id="Negative relative fcp" ), pytest.param( ISO8601_CYCLING_TYPE, { 'initial cycle point': '2013', 'final cycle point': '2021', 'final cycle point constraints': ['T00', 'T12'] }, None, '20210101T0000+0530', None, id="Constraints" ), pytest.param( ISO8601_CYCLING_TYPE, { 'initial cycle point': '2013', 'final cycle point': '2021-01-19', 'final cycle point constraints': ['--01-19', '--01-21'] }, '2021-01-20', None, (WorkflowConfigError, "does not meet the constraints"), id="Violated constraints" ), pytest.param( ISO8601_CYCLING_TYPE, { 'initial cycle point': '2013', 'final cycle point': '2021', }, 'reload', '20210101T0000+0530', None, id="--fcp=reload" ), ] ) def test_process_fcp( cycling_type: str, scheduling_cfg: dict, options_fcp: Optional[str], expected_fcp: Optional[str], expected_err: Optional[Tuple[Type[Exception], str]], set_cycling_type: 'Fixture' ) -> None: """Test WorkflowConfig.process_final_cycle_point(). Params: cycling_type: Workflow cycling type. scheduling_cfg: 'scheduling' section of workflow config. options_fcp: The fcp set by cli option. expected_fcp: The expected fcp value that gets set. expected_err: Exception class expected to be raised plus the message. """ set_cycling_type(cycling_type, time_zone='+0530') mocked_config = SimpleNamespace( cycling_type=cycling_type, cfg={ 'scheduling': { 'final cycle point constraints': [], **scheduling_cfg, }, }, initial_point=loader.get_point( scheduling_cfg['initial cycle point'] ).standardise(), final_point=None, options=SimpleNamespace(fcp=options_fcp), ) if expected_err: err, msg = expected_err with pytest.raises(err) as exc: WorkflowConfig.process_final_cycle_point(mocked_config) assert msg in str(exc.value) else: WorkflowConfig.process_final_cycle_point(mocked_config) assert mocked_config.cfg[ 'scheduling']['final cycle point'] == expected_fcp assert str(mocked_config.final_point) == str(expected_fcp) @pytest.mark.parametrize( ('cfg_stopcp', 'options_stopcp', 'expected_value', 'expected_options_value', 'expected_warning'), [ pytest.param( None, None, None, None, None, id="no-stopcp" ), pytest.param( '1993', None, '1993', None, None, id="stopcp" ), pytest.param( '1993', '1066', '1066', '1066', None, id="stop-cp-and-cli-option" ), pytest.param( '1993', 'reload', '1993', None, None, id="stop-cp-and-cli-reload-option" ), pytest.param( '3000', None, None, None, "will have no effect as it is after the final cycle point", id="stopcp-beyond-fcp" ), pytest.param( '+P12Y -P2Y', None, '2000', None, None, id="stopcp-relative-to-icp" ), ] ) def test_process_stop_cycle_point( cfg_stopcp: Optional[str], options_stopcp: Optional[str], expected_value: Optional[str], expected_options_value: Optional[str], expected_warning: Optional[str], set_cycling_type: Callable, caplog: pytest.LogCaptureFixture ): """Test WorkflowConfig.process_stop_cycle_point(). Params: cfg_stopcp: [scheduling]stop after cycle point options_stopcp: The stopcp from cli option / database. expected_value: The expected stopcp value that gets set. expected_options_value: The expected options.stopcp that gets set. expected_warning: Expected warning message, if any. """ set_cycling_type(ISO8601_CYCLING_TYPE, dump_format='CCYY') caplog.set_level(logging.WARNING, CYLC_LOG) fcp = loader.get_point('2012').standardise() mock_config = SimpleNamespace( cfg={ 'scheduling': { 'stop after cycle point': cfg_stopcp } }, initial_point=ISO8601Point('1990'), final_point=fcp, stop_point=None, options=RunOptions(stopcp=options_stopcp), ) WorkflowConfig.process_stop_cycle_point(mock_config) assert str(mock_config.stop_point) == str(expected_value) assert mock_config.cfg['scheduling']['stop after cycle point'] == ( expected_value ) assert mock_config.options.stopcp == expected_options_value if expected_warning: assert expected_warning in caplog.text else: assert not caplog.record_tuples @pytest.mark.parametrize( 'cfg_fcp, cfg_stopcp, opts, warning_expected', [ pytest.param( '2005', '2017', {}, True, id="cfg stopcp > fcp bad" ), pytest.param( '2017', '2017', {}, False, id="cfg stopcp == fcp ok" ), pytest.param( '', '', {'fcp': '2005', 'stopcp': '2017'}, True, id="options stopcp > fcp bad" ), pytest.param( '', '', {'fcp': '2017', 'stopcp': '2017'}, False, id="options stopcp == fcp ok" ), pytest.param( '2017', '2005', {'stopcp': '2022'}, True, id="options stopcp > cfg fcp bad" ), pytest.param( '2017', '2005', {'stopcp': '2022'}, True, id="options stopcp > cfg fcp bad" ), pytest.param( '2022', '2017', {'fcp': '2005'}, True, id="cfg stopcp > options fcp bad" ), pytest.param( '', '2022', {}, False, id="no fcp" ), ] ) def test_stopcp_after_fcp( cfg_fcp: str, cfg_stopcp: str, opts: Dict[str, str], warning_expected: bool, tmp_flow_config: Callable, caplog: pytest.LogCaptureFixture, ): """Test that setting a stop after cycle point that is beyond the final cycle point is handled correctly.""" caplog.set_level(logging.WARNING, CYLC_LOG) id_ = 'cassini' flow_file: 'Path' = tmp_flow_config(id_, f""" [scheduler] allow implicit tasks = True [scheduling] initial cycle point = 1997 final cycle point = {cfg_fcp} stop after cycle point = {cfg_stopcp} [[graph]] P1Y = huygens """) cfg = WorkflowConfig(id_, flow_file, options=RunOptions(**opts)) msg = "will have no effect as it is after the final cycle point" if warning_expected: assert msg in caplog.text assert cfg.stop_point is None else: assert msg not in caplog.text if cfg_stopcp or opts.get('stopcp'): assert cfg.stop_point @pytest.mark.parametrize( 'scheduling_cfg, scheduling_expected, expected_err', [ pytest.param( { 'graph': {} }, None, (WorkflowConfigError, "No workflow dependency graph defined"), id="Empty graph" ), pytest.param( { 'graph': {'R1': 'foo'} }, { 'cycling mode': 'integer', 'initial cycle point': '1', 'final cycle point': '1', 'graph': {'R1': 'foo'} }, None, id="Pure acyclic graph" ), pytest.param( { 'cycling mode': "", 'graph': {'R1': 'foo'} }, { 'cycling mode': "", 'graph': {'R1': 'foo'} }, None, id="Pure acyclic graph but datetime cycling" ), pytest.param( { 'graph': {'R1': 'foo', 'R2': 'bar'} }, { 'graph': {'R1': 'foo', 'R2': 'bar'} }, None, id="Acyclic graph with >1 recurrence" ), ] ) def test_prelim_process_graph( scheduling_cfg: Dict[str, Any], scheduling_expected: Optional[Dict[str, Any]], expected_err: Optional[Tuple[Type[Exception], str]]): """Test WorkflowConfig.prelim_process_graph(). Params: scheduling_cfg: 'scheduling' section of workflow config. scheduling_expected: The expected scheduling section after preliminary processing. expected_err: Exception class expected to be raised plus the message. """ mock_config = SimpleNamespace(cfg={ 'scheduling': scheduling_cfg }) if expected_err: err, msg = expected_err with pytest.raises(err) as exc: WorkflowConfig.prelim_process_graph(mock_config) assert msg in str(exc.value) else: WorkflowConfig.prelim_process_graph(mock_config) assert mock_config.cfg['scheduling'] == scheduling_expected def test_utc_mode(caplog, mock_glbl_cfg): """Test that UTC mode is handled correctly.""" caplog.set_level(logging.WARNING, CYLC_LOG) def _test(utc_mode, expected, expected_warnings=0): mock_glbl_cfg( 'cylc.flow.config.glbl_cfg', f''' [scheduler] UTC mode = {utc_mode['glbl']} ''' ) mock_config = SimpleNamespace( cfg={ 'scheduler': { 'UTC mode': utc_mode['workflow'] } }, options=SimpleNamespace(utc_mode=utc_mode['stored']), ) WorkflowConfig.process_utc_mode(mock_config) assert mock_config.cfg['scheduler']['UTC mode'] is expected assert get_utc_mode() is expected assert len(caplog.record_tuples) == expected_warnings caplog.clear() tests = [ { 'utc_mode': {'glbl': True, 'workflow': None, 'stored': None}, 'expected': True }, { 'utc_mode': {'glbl': True, 'workflow': False, 'stored': None}, 'expected': False }, { # On restart 'utc_mode': {'glbl': False, 'workflow': None, 'stored': True}, 'expected': True }, { # Changed config value between restarts 'utc_mode': {'glbl': False, 'workflow': False, 'stored': True}, 'expected': True, 'expected_warnings': 1 } ] for case in tests: _test(**case) def test_cycle_point_tz(caplog, monkeypatch): """Test that `[scheduler]cycle point time zone` is handled correctly.""" caplog.set_level(logging.WARNING, CYLC_LOG) local_tz = '-0230' monkeypatch.setattr( 'cylc.flow.config.get_local_time_zone_format', lambda: local_tz ) def _test(cp_tz, utc_mode, expected, expected_warnings=0): set_utc_mode(utc_mode) mock_config = SimpleNamespace( cfg={ 'scheduler': { 'cycle point time zone': cp_tz['workflow'], }, }, options=SimpleNamespace(cycle_point_tz=cp_tz['stored']), ) WorkflowConfig.process_cycle_point_tz(mock_config) assert mock_config.cfg['scheduler'][ 'cycle point time zone'] == expected assert len(caplog.record_tuples) == expected_warnings caplog.clear() tests = [ { 'cp_tz': {'workflow': None, 'stored': None}, 'utc_mode': True, 'expected': 'Z' }, { 'cp_tz': {'workflow': None, 'stored': None}, 'utc_mode': False, 'expected': 'Z' }, { 'cp_tz': {'workflow': '+0530', 'stored': None}, 'utc_mode': True, 'expected': '+0530' }, { # On restart 'cp_tz': {'workflow': None, 'stored': '+0530'}, 'utc_mode': True, 'expected': '+0530', 'expected_warnings': 1 }, { # Changed config value between restarts 'cp_tz': {'workflow': '+0530', 'stored': '-0030'}, 'utc_mode': True, 'expected': '-0030', 'expected_warnings': 1 }, { 'cp_tz': {'workflow': 'Z', 'stored': 'Z'}, 'utc_mode': False, 'expected': 'Z' } ] for case in tests: _test(**case) def test_rsync_includes_will_not_accept_sub_directories(tmp_flow_config): id_ = 'rsynctest' flow_file = tmp_flow_config(id_, """ [scheduling] initial cycle point = 2020-01-01 [[dependencies]] graph = "blah => deeblah" [scheduler] install = /, /foo, /foo/, foo/bar, foo/bar/ """) with pytest.raises(WorkflowConfigError) as exc: WorkflowConfig( workflow=id_, fpath=flow_file, options=Values() ) assert '"/" - Paths cannot be absolute.' in str(exc.value) assert '"/foo" - Paths cannot be absolute.' in str(exc.value) assert ( '"foo/bar" - Only top-level files/directories can be configured.' in str(exc.value) ) assert ( '"foo/bar/" - Only top-level files/directories can be configured.' in str(exc.value) ) @pytest.mark.parametrize( 'cylc_var, expected_err', [ ["CYLC_WORKFLOW_NAME", None], ["CYLC_BEEF_WELLINGTON", (Jinja2Error, "is undefined")], ] ) def test_jinja2_cylc_vars(tmp_flow_config, cylc_var, expected_err): """Defined CYLC_ variables should be available to Jinja2 during parsing. This test is not located in the jinja2_support unit test module because CYLC_ variables are only defined during workflow config parsing. """ reg = 'nodule' flow_file = tmp_flow_config(reg, """#!Jinja2 # {{""" + cylc_var + """}} [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = foo """) if expected_err is None: WorkflowConfig(workflow=reg, fpath=flow_file, options=Values()) else: with pytest.raises(expected_err[0]) as exc: WorkflowConfig(workflow=reg, fpath=flow_file, options=Values()) assert expected_err[1] in str(exc) def test_valid_rsync_includes_returns_correct_list(tmp_flow_config): """Test that the rsync includes in the correct """ id_ = 'rsynctest' flow_file = tmp_flow_config(id_, """ [scheduling] initial cycle point = 2020-01-01 [[dependencies]] graph = "blah => deeblah" [scheduler] install = dir/, dir2/, file1, file2 allow implicit tasks = True """) config = WorkflowConfig( workflow=id_, fpath=flow_file, options=Values() ) rsync_includes = WorkflowConfig.get_validated_rsync_includes(config) assert rsync_includes == ['dir/', 'dir2/', 'file1', 'file2'] @pytest.mark.parametrize( 'cycling_type, runahead_limit, valid', [ (INTEGER_CYCLING_TYPE, 'P14', True), (ISO8601_CYCLING_TYPE, 'P14', True), (ISO8601_CYCLING_TYPE, 'PT12H', True), (ISO8601_CYCLING_TYPE, 'P7D', True), (ISO8601_CYCLING_TYPE, 'P2W', True), (ISO8601_CYCLING_TYPE, '4', True), (INTEGER_CYCLING_TYPE, 'PT12H', False), (INTEGER_CYCLING_TYPE, 'P7D', False), (INTEGER_CYCLING_TYPE, '4', False), (ISO8601_CYCLING_TYPE, '', False), (ISO8601_CYCLING_TYPE, 'asdf', False) ] ) def test_process_runahead_limit( cycling_type: str, runahead_limit: str, valid: bool, set_cycling_type: Callable ) -> None: set_cycling_type(cycling_type) mock_config = SimpleNamespace(cycling_type=cycling_type) mock_config.cfg = { 'scheduling': { 'runahead limit': runahead_limit } } if valid: WorkflowConfig.process_runahead_limit(mock_config) else: with pytest.raises(WorkflowConfigError) as exc: WorkflowConfig.process_runahead_limit(mock_config) assert "bad runahead limit" in str(exc.value).lower() @pytest.mark.parametrize( 'opt', [None, 'check_circular'] ) def test_check_circular(opt, monkeypatch, caplog, tmp_flow_config): """Test WorkflowConfig._check_circular().""" # ----- Setup ----- caplog.set_level(logging.INFO, CYLC_LOG) options = SimpleNamespace(is_validate=True) if opt: setattr(options, opt, True) id_ = 'circular' flow_file = tmp_flow_config(id_, """ [scheduling] cycling mode = integer [[graph]] R1 = "a => b => c => d => e => a" [runtime] [[a, b, c, d, e]] script = True """) def WorkflowConfig__assert_err_raised(): with pytest.raises(WorkflowConfigError) as exc: WorkflowConfig(workflow=id_, fpath=flow_file, options=options) assert "circular edges detected" in str(exc.value) # ----- The actual test ----- WorkflowConfig__assert_err_raised() # Now artificially lower the limit and re-test: monkeypatch.setattr( 'cylc.flow.config.WorkflowConfig.CHECK_CIRCULAR_LIMIT', 4) if opt != 'check_circular': # Will no longer raise WorkflowConfig(workflow='circular', fpath=flow_file, options=options) msg = "will not check graph for circular dependencies" assert msg in caplog.text else: WorkflowConfig__assert_err_raised() @pytest.mark.parametrize( 'graph', (('foo:x => bar'), ('foo:x')) ) def test_undefined_custom_output(graph: str, tmp_flow_config: Callable): """Test error on undefined custom output referenced in graph.""" id_ = 'custom_out1' flow_file = tmp_flow_config(id_, f""" [scheduling] [[graph]] R1 = "{graph}" [runtime] [[foo, bar]] """) with pytest.raises(WorkflowConfigError) as cm: WorkflowConfig(workflow=id_, fpath=flow_file, options=Values()) assert "Undefined custom output" in str(cm.value) def test_invalid_custom_output_msg(tmp_flow_config: Callable): """Test invalid output message (colon not allowed).""" id_ = 'invalid_output' flow_file = tmp_flow_config(id_, """ [scheduling] [[graph]] R1 = "foo:x => bar" [runtime] [[bar]] [[foo]] [[[outputs]]] x = "the quick: brown fox" """) with pytest.raises(WorkflowConfigError) as cm: WorkflowConfig( workflow=id_, fpath=flow_file, options=Values()) assert ( 'Invalid task message "[runtime][foo][outputs]x = ' 'the quick: brown fox"' ) in str(cm.value) @pytest.mark.parametrize( 'graph', [ "foo:x => bar", "foo:start => bar", "foo:submit => bar", ] ) def test_implicit_success_required(tmp_flow_config, graph): """Check foo:succeed is required if success/fail not used in the graph.""" id_ = 'blargh' flow_file = tmp_flow_config(id_, f""" [scheduling] [[graph]] R1 = {graph} [runtime] [[bar]] [[foo]] [[[outputs]]] x = "the quick brown fox" """) cfg = WorkflowConfig(workflow=id_, fpath=flow_file, options=None) assert cfg.taskdefs['foo'].outputs[TASK_OUTPUT_SUCCEEDED][1] @pytest.mark.parametrize('allow_implicit_tasks', [True, False, None]) def test_implicit_tasks( allow_implicit_tasks: Optional[bool], caplog: pytest.LogCaptureFixture, tmp_flow_config: Callable ): """Test that the prescence of implicit tasks in the config is handled correctly. Params: allow_implicit_tasks: Value of "[scheduler]allow implicit tasks". extra_msg_expected: If True, there should be the note on how to allow implicit tasks in the err msg. """ # Setup id_ = 'rincewind' allow_implicit_tasks_text = ( f'allow implicit tasks = {allow_implicit_tasks}' if allow_implicit_tasks is not None else '' ) flow_file: 'Path' = tmp_flow_config( id_, dedent(f""" [scheduler] {allow_implicit_tasks_text} [scheduling] [[graph]] R1 = foo """) ) caplog.set_level(logging.DEBUG, CYLC_LOG) if allow_implicit_tasks is True: expected_exc = None else: expected_exc = WorkflowConfigError # Test args: dict = {'workflow': id_, 'fpath': flow_file, 'options': None} if expected_exc: with pytest.raises( expected_exc, match=r"implicit tasks detected.*" ) as excinfo: WorkflowConfig(**args) if allow_implicit_tasks is None: assert "To allow implicit tasks" in str(excinfo.value) else: WorkflowConfig(**args) @pytest.mark.parametrize('workflow_meta', [True, False]) @pytest.mark.parametrize('url_type', ['good', 'bad', 'ugly', 'broken']) def test_process_urls(log_filter, workflow_meta, url_type): if url_type == 'good': # valid cylc 8 syntax url = '%(workflow)s' elif url_type == 'bad': # no variable called "foo" url = '%(foo)s' elif url_type == 'broken': # invalid syntax (missing the trailing "s") url = '%(suite_name)' elif url_type == 'ugly': # valid cylc 7 syntax url = '%(suite_name)s' config = SimpleNamespace() config.workflow = 'my-workflow' if workflow_meta: config.cfg = { 'meta': {'URL': url}, 'runtime': {} } else: config.cfg = { 'meta': {'URL': ''}, 'runtime': {'foo': {'meta': {'URL': url}}}, } if url_type == 'good': WorkflowConfig.process_metadata_urls(config) elif url_type in {'bad', 'broken'}: with pytest.raises(InputError): WorkflowConfig.process_metadata_urls(config) elif url_type == 'ugly': WorkflowConfig.process_metadata_urls(config) assert log_filter( contains='Detected deprecated template variables', ) @pytest.mark.parametrize('opts', [ValidateOptions(), RunOptions()]) @pytest.mark.parametrize( 'recurrence, should_warn', [ # Format 3: ('P0Y', True), ('R//P0Y', True), ('R2//P0Y', True), ('R1//P0Y', False), # Format 4: ('R/P0M', True), ('R1/P0M', False), # Format 1: ('R/2002-09-01/2002-09-01', True), ('R1/2002-09-01/2002-09-01', False), ('R/2002-08-31/2002-09-02', False), ] ) def test_zero_interval( recurrence: str, should_warn: bool, opts: Values, tmp_flow_config: Callable, log_filter: Callable, ): """Test that a zero-duration recurrence with >1 repetition gets an appropriate warning.""" id_ = 'ordinary' flow_file: 'Path' = tmp_flow_config(id_, f""" [scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 2002-08-30 final cycle point = 2002-09-14 [[graph]] {recurrence} = slidescape36 """) WorkflowConfig(id_, flow_file, options=opts) logged = log_filter( level=logging.WARNING, contains="Cannot have more than 1 repetition for zero-duration" ) if should_warn: assert logged else: assert not logged @pytest.mark.parametrize( 'icp, fcp_expr, expected_fcp', [ ('2021-02-28', '+P1M+P1D', '2021-03-29'), ('2019-02-28', '+P1D+P1M', '2019-04-01'), ('2008-07-01', '+P1M-P1D', '2008-07-31'), ('2004-07-01', '-P1D+P1M', '2004-07-30'), ('1992-02-29', '+P1Y+P1M', '1993-03-28'), ('1988-02-29', '+P1M+P1Y', '1989-03-29'), ('1910-08-14', '+P2D-PT6H', '1910-08-15T18:00'), ('1850-04-10', '+P1M-P1D+PT1H', '1850-05-09T01:00'), ('1066-10-14', '+PT1H+PT1M', '1066-10-14T01:01'), ] ) def test_chain_expr( icp: str, fcp_expr: str, expected_fcp: str, tmp_flow_config: Callable, ): """Test a "chain expression" final cycle point offset. Note the order matters when "nominal" units (years, months) are used. """ id_ = 'osgiliath' flow_file: 'Path' = tmp_flow_config(id_, f""" [scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = {icp} final cycle point = {fcp_expr} [[graph]] P1D = faramir """) cfg = WorkflowConfig(id_, flow_file, options=ValidateOptions()) assert cfg.final_point == ISO8601Point(expected_fcp).standardise() @pytest.mark.parametrize( 'runtime_cfg', ( pytest.param( {'foo': {'remote': {'host': 'bar'}}}, id='no-owners' ), pytest.param( {'foo': {'remote': {'owner': 'tim'}}}, id='one-owner' ), pytest.param( { 'foo': {'remote': {'owner': 'tim'}}, 'bar': {'remote': {'owner': 'oliver'}}, 'baz': {'remote': {'owner': 'ronnie'}}, }, id='3-owners' ), pytest.param( { 'foo': {'remote': {'owner': 'tim'}}, 'bar': {'remote': {'owner': 'oliver'}}, 'baz': {'remote': {'owner': 'ronnie'}}, 'qux': {'remote': {'owner': 'tim'}}, 'aleph': {'remote': {'owner': 'oliver'}}, 'bet': {'remote': {'owner': 'ronnie'}}, }, id='6-owners' ), ) ) def test_check_for_owner(runtime_cfg): """check_for_owner raises a list of [runtime][task][remote]owner set.""" if 'owner' in str(runtime_cfg): with pytest.raises(WorkflowConfigError) as exc: WorkflowConfig.check_for_owner(runtime_cfg) # Assert is the correct error message: assert exc.match('owner\" is obsolete') # Assert error message has right number of lines: else: # Assert function doesn't raise if no owner set: assert WorkflowConfig.check_for_owner(runtime_cfg) is None @pytest.fixture(scope='module') def awe_config(mod_tmp_flow_config: Callable) -> WorkflowConfig: """Return a workflow config object.""" id_ = 'awe' flow_file = mod_tmp_flow_config(id_, ''' [scheduling] cycling mode = integer [[graph]] P1 = ordinary & sterling R1/2 = fra_mauro [runtime] [[USA, MOON]] [[ordinary, sterling]] inherit = USA [[fra_mauro]] inherit = MOON ''') return WorkflowConfig( workflow=id_, fpath=flow_file, options=ValidateOptions() ) @pytest.mark.parametrize( 'name, expected', [ pytest.param( 'ordinary', ['ordinary'], id="task name" ), pytest.param( 'USA', ['ordinary', 'sterling'], id="family name" ), pytest.param( 'fra*', ['fra_mauro'], id="glob task name" ), pytest.param( 'U*', ['ordinary', 'sterling'], id="glob family name" ), pytest.param( '*', ['ordinary', 'sterling', 'fra_mauro'], id="glob everything" ), pytest.param( 'butte', [], id="no match" ), ] ) def test_find_taskdefs( name: str, expected: List[str], awe_config: WorkflowConfig ): assert sorted( t.name for t in awe_config.find_taskdefs(name) ) == sorted(expected) def test__warn_if_queues_have_implicit_tasks(caplog): """It Warns that queues imply tasks undefined in runtime. """ config = { 'scheduling': {'queues': { 'q1': {'members': ['foo']}, 'q2': {'members': ['bar', 'baz']} }}, 'runtime': {} } taskdefs = {} max_warning_lines = 2 WorkflowConfig._warn_if_queues_have_implicit_tasks( config, taskdefs, max_warning_lines) result = caplog.records[0].message assert "'foo' in queue 'q1'" in result assert "'bar' in queue 'q2'" in result assert "'baz'" not in result assert f"showing first {max_warning_lines}" in result @pytest.mark.parametrize( 'installed, run_dir, cylc_vars', [ pytest.param( False, # not installed (parsing a source dir) None, # no run directory passed to config object by scheduler { 'CYLC_WORKFLOW_NAME': True, # expected environment variables 'CYLC_WORKFLOW_ID': False, 'CYLC_WORKFLOW_RUN_DIR': False, 'CYLC_WORKFLOW_WORK_DIR': False, 'CYLC_WORKFLOW_SHARE_DIR': False, 'CYLC_WORKFLOW_LOG_DIR': False, }, id="source-dir" ), pytest.param( True, None, { 'CYLC_WORKFLOW_NAME': True, 'CYLC_WORKFLOW_ID': True, 'CYLC_WORKFLOW_RUN_DIR': True, 'CYLC_WORKFLOW_WORK_DIR': False, 'CYLC_WORKFLOW_SHARE_DIR': False, 'CYLC_WORKFLOW_LOG_DIR': False, }, id="run-dir" ), pytest.param( True, "/some/path", { 'CYLC_WORKFLOW_NAME': True, 'CYLC_WORKFLOW_ID': True, 'CYLC_WORKFLOW_RUN_DIR': True, 'CYLC_WORKFLOW_WORK_DIR': True, 'CYLC_WORKFLOW_SHARE_DIR': True, 'CYLC_WORKFLOW_LOG_DIR': True, }, id="run-dir-from-scheduler" ), ] ) def test_cylc_env_at_parsing( tmp_path: 'Path', monkeypatch: pytest.MonkeyPatch, installed, run_dir, cylc_vars ): """Check that CYLC_ environment vars exported during config file parsing are appropriate to the workflow context (source, installed, or running). """ # Purge environment from previous tests. for key in cylc_vars.keys(): with suppress(KeyError): del os.environ[key] flow_file = tmp_path / WorkflowFiles.FLOW_FILE flow_config = """ [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = 'foo' """ flow_file.write_text(flow_config) # Make it look as if path is relative to cylc-run (i.e. installed). monkeypatch.setattr( 'cylc.flow.config.is_relative_to', lambda _a, _b: installed ) # Parse the workflow config then check the environment. WorkflowConfig( workflow="name", fpath=flow_file, options=SimpleNamespace(), run_dir=run_dir ) cylc_env = [k for k in os.environ.keys() if k.startswith('CYLC_')] for var, expected in cylc_vars.items(): if expected: assert var in cylc_env else: assert var not in cylc_env @pytest.mark.parametrize( 'registered_outputs, tasks_and_outputs, fails', ( param([], ['foo:x'], True, id='output-unregistered'), param([], ['foo:x?'], True, id='optional-output-unregistered'), param([], ['foo'], False, id='no-modifier-unregistered'), param(['x'], ['foo:x'], False, id='output-registered'), param([], ['foo:succeed'], False, id='alt-default-ok'), param([], ['foo:failed'], False, id='default-ok'), ) ) def test_check_outputs( tmp_flow_config, registered_outputs, tasks_and_outputs, fails ): flow_file = tmp_flow_config('wflow', """ [scheduler] allow implicit tasks = true [scheduling] [[graph]] R1 = foo """) cfg = WorkflowConfig('', flow_file, ValidateOptions()) cfg.cfg['runtime']['foo']['outputs'] = registered_outputs if fails: with pytest.raises( WorkflowConfigError, match='Undefined custom output' ): cfg.check_terminal_outputs(tasks_and_outputs) else: assert cfg.check_terminal_outputs(tasks_and_outputs) is None def test_upg_wflow_event_names(tmp_flow_config, log_filter): """Cylc 7 workflow handler/mail event names are upgraded.""" events = 'inactivity, abort, stalled' expected = ['inactivity timeout', 'abort', 'stall'] flow_file = tmp_flow_config('foo', f""" [scheduler] allow implicit tasks = true [[events]] handler events = {events} mail events = {events} [scheduling] [[graph]] R1 = foo """) cfg = WorkflowConfig('foo', str(flow_file), ValidateOptions()) for item in ('handler events', 'mail events'): assert cfg.cfg['scheduler']['events'][item] == expected assert log_filter( logging.WARNING, regex=( r'Deprecated config items were automatically upgraded[\s\S]*' r'support for the old item will be removed at Cylc 8\.9' ), ) @pytest.mark.parametrize('item', ['handler events', 'mail events']) def test_val_wflow_event_names(item, tmp_flow_config, log_filter): """Any invalid workflow handler/mail events raise a warning.""" flow_file = tmp_flow_config('foo', f""" [scheduler] allow implicit tasks = true [[events]] {item} = abort, badger, stall, alpaca [scheduling] [[graph]] R1 = foo """) WorkflowConfig('foo', str(flow_file), ValidateOptions()) assert log_filter(contains=f"{item} = badger, alpaca") @pytest.mark.parametrize('item', ['handler events', 'mail events']) def test_check_task_event_names(item, tmp_flow_config, log_filter): """Any invalid task handler events are warned about.""" flow_file = tmp_flow_config('foo', f""" [scheduling] [[graph]] R1 = foo [runtime] [[foo]] [[[events]]] {item} = submitted, late, warning, custom,\ execution timeout, badger, owl, retry, horse [[[outputs]]] owl = who """) WorkflowConfig('foo', str(flow_file), ValidateOptions()) assert log_filter(contains=( f"Invalid event name(s) for [runtime][foo][events]{item}: " "badger, horse" )) def test_jinja2_lib_python(tmp_flow_config): """Modules in lib/python are available in custom Jinja2.""" flow_file: Path = tmp_flow_config((id_ := 'wflow'), r""" #!jinja2 {% from 'my_module' import my_const %} [meta] x = {{ my_const }} [scheduler] allow implicit tasks = true [scheduling] [[graph]] R1 = {{ 'foo' | my_filter }} """) (lib_dir := flow_file.parent / 'lib' / 'python').mkdir(parents=True) (lib_dir / 'my_module.py').write_text( r"my_const = 'bar'" ) (filters_dir := flow_file.parent / 'Jinja2Filters').mkdir() (filters_dir / 'my_filter.py').write_text( dedent(r""" from my_module import my_const def my_filter(val): return f"{val}_{my_const}" """) ) conf = WorkflowConfig(id_, str(flow_file), ValidateOptions()) assert conf.cfg['meta']['x'] == 'bar' assert conf.cfg['scheduling']['graph']['R1'] == 'foo_bar' @pytest.mark.parametrize('old, new', [ ('suite', 'workflow'), ('suite_uuid', 'uuid'), ('suite_url', 'workflow_url') ]) @pytest.mark.parametrize('handlers', [ 'handlers', 'startup handlers', 'shutdown handlers', 'stall timeout handlers', ]) def test_depr_wflow_event_template_vars( handlers, old, new, tmp_flow_config, log_filter ): """Deprecated workflow event template variables are logged.""" flow_file = tmp_flow_config('foo', f""" [scheduler] [[events]] {handlers} = echo %({old})s [scheduling] [[graph]] R1 = foo [runtime] [[foo]] """) WorkflowConfig('foo', str(flow_file), ValidateOptions()) assert log_filter( logging.WARNING, contains=( f'The workflow event handler template variable "%({old})s" is ' f'deprecated and will be removed in Cylc 8.9 - ' f'use "%({new})s" instead.' ) ) @pytest.mark.parametrize('old, new', [ ('suite', 'workflow'), ('suite_uuid', 'uuid'), ('batch_sys_name', 'job_runner_name'), ('batch_sys_job_id', 'job_id'), ('user@host', 'platform_name'), ]) @pytest.mark.parametrize('handlers', [ 'handlers', 'started handlers', 'submission failed handlers', 'execution timeout handlers', 'critical handlers', ]) def test_depr_task_event_template_vars( handlers, old, new, tmp_flow_config, log_filter ): """Deprecated task event template variables are logged. See also tests/flakyfunctional/events/41-task-event-template-deprecated.t """ flow_file = tmp_flow_config('foo', f""" [scheduling] [[graph]] R1 = foo [runtime] [[foo]] [[[events]]] {handlers} = echo %({old})s """) WorkflowConfig('foo', str(flow_file), ValidateOptions()) assert log_filter( logging.WARNING, contains=( f'The task event handler template variable "%({old})s" is ' f'deprecated and will be removed in Cylc 8.9 - ' f'use "%({new})s" instead.' ) ) cylc-flow-8.7.0/tests/unit/option_parsers.py0000664000175000017500000000330615264420307021360 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import optparse import pytest from cylc.flow.option_parsers import Options @pytest.fixture def simple_parser(): """Simple option parser.""" parser = optparse.OptionParser() parser.add_option('-a', action='store') parser.add_option('-b', action='store_true') parser.add_option('-c', default='C') return parser def test_options(simple_parser): """It is a substitute for an optparse options object.""" options = Options(parser=simple_parser) opts = options(a=1, b=True) # we can access options as attributes assert opts.a == 1 assert opts.b is True # defaults are automatically substituted assert opts.c == 'C' # get-like syntax should work assert opts.get('d', 42) == 42 # invalid keys result in KeyErrors with pytest.raises(KeyError): opts.d with pytest.raises(KeyError): opts(d=1) # just for fun we can still use dict syntax assert opts['a'] == 1 cylc-flow-8.7.0/tests/unit/conftest.py0000664000175000017500000001724015264420307020140 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Standard pytest fixtures for unit tests.""" from pathlib import Path from textwrap import dedent from typing import ( Any, Callable, Optional, Union, ) from unittest.mock import ( Mock, create_autospec, ) import pytest from cylc.flow.cycling.iso8601 import init as iso8601_init from cylc.flow.cycling.loader import ( INTEGER_CYCLING_TYPE, ISO8601_CYCLING_TYPE, ) from cylc.flow.data_store_mgr import DataStoreMgr from cylc.flow.install import ( link_runN, unlink_runN, ) from cylc.flow.scheduler import Scheduler from cylc.flow.workflow_files import WorkflowFiles from cylc.flow.xtrigger_mgr import XtriggerManager # Type alias for monkeymock() MonkeyMock = Callable[..., Mock] @pytest.fixture def monkeymock(monkeypatch: pytest.MonkeyPatch): """Fixture that patches a function/attr with a Mock and returns that Mock. Args: pypath: The Python-style import path to be patched. **kwargs: Any kwargs to set on the Mock. Example: mock_clean = monkeymock('cylc.flow.workflow_files.clean') something() # calls workflow_files.clean assert mock_clean.called is True """ def _monkeymock(pypath: str, **kwargs: Any) -> Mock: _mock = Mock(**kwargs) monkeypatch.setattr(pypath, _mock) return _mock return _monkeymock def _tmp_run_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): """Fixture that patches the cylc-run dir to the tests's {tmp_path}/cylc-run, and optionally creates a workflow run dir inside. Adds the runN symlink automatically if the workflow ID ends with /run__. Args: id_: Workflow name. installed: If True, make it look like the workflow was installed using cylc install (creates _cylc-install dir). named: If True and installed is True, the _cylc-install dir will be created in the parent to make it look like this is a named run. Example: run_dir = tmp_run_dir('foo') # Or: cylc_run_dir = tmp_run_dir() """ cylc_run_dir = tmp_path / 'cylc-run' cylc_run_dir.mkdir(exist_ok=True) monkeypatch.setattr('cylc.flow.pathutil._CYLC_RUN_DIR', cylc_run_dir) def __tmp_run_dir( id_: Optional[str] = None, installed: bool = False, named: bool = False ) -> Path: if not id_: return cylc_run_dir run_dir = cylc_run_dir.joinpath(id_) run_dir.mkdir(parents=True, exist_ok=True) (run_dir / WorkflowFiles.FLOW_FILE).touch(exist_ok=True) (run_dir / WorkflowFiles.Service.DIRNAME).mkdir(exist_ok=True) if run_dir.name.startswith('run'): unlink_runN(run_dir.parent) link_runN(run_dir) if installed: if named: if len(Path(id_).parts) < 2: raise ValueError("Named run requires two-level id_") (run_dir.parent / WorkflowFiles.Install.DIRNAME).mkdir( exist_ok=True) else: (run_dir / WorkflowFiles.Install.DIRNAME).mkdir(exist_ok=True) return run_dir return __tmp_run_dir @pytest.fixture def tmp_run_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): return _tmp_run_dir(tmp_path, monkeypatch) @pytest.fixture(scope='module') def mod_tmp_run_dir(tmp_path_factory: pytest.TempPathFactory): """Module-scoped version of tmp_run_dir()""" tmp_path = tmp_path_factory.getbasetemp() with pytest.MonkeyPatch.context() as mp: yield _tmp_run_dir(tmp_path, mp) def _tmp_src_dir(tmp_path: Path): """Fixture that creates a temporary workflow source dir. (Actually the fixture is below, this is the re-usable meat of it.) Args: path: Path of source dir relative to cylc-src/. Example: src_dir = tmp_src_dir('foo') """ def __tmp_src_dir(path: Union[Path, str]) -> Path: cylc_src_dir = tmp_path / 'cylc-src' cylc_src_dir.mkdir(exist_ok=True) src_dir = cylc_src_dir / path src_dir.mkdir(parents=True) (src_dir / WorkflowFiles.FLOW_FILE).touch() return src_dir return __tmp_src_dir @pytest.fixture def tmp_src_dir(tmp_path: Path): # This is the actual tmp_src_dir fixture return _tmp_src_dir(tmp_path) @pytest.fixture(scope='module') def mod_tmp_src_dir(tmp_path_factory: pytest.TempPathFactory): """Module-scoped version of tmp_src_dir()""" tmp_path = tmp_path_factory.getbasetemp() return _tmp_src_dir(tmp_path) @pytest.fixture def set_cycling_type(monkeypatch: pytest.MonkeyPatch): """Initialize the Cylc cycling type. Args: ctype: The cycling type (integer or iso8601). time_zone: If using ISO8601/datetime cycling type, you can specify a custom time zone to use. dump_format: If using ISO8601, specify custom dump format. """ def _set_cycling_type( ctype: str = INTEGER_CYCLING_TYPE, time_zone: Optional[str] = 'Z', dump_format: Optional[str] = None, ) -> None: class _DefaultCycler: TYPE = ctype monkeypatch.setattr( 'cylc.flow.cycling.loader.DefaultCycler', _DefaultCycler ) if ctype == ISO8601_CYCLING_TYPE: monkeypatch.setattr( 'cylc.flow.cycling.iso8601.WorkflowSpecifics', iso8601_init( time_zone=time_zone, custom_dump_format=dump_format ), ) return _set_cycling_type @pytest.fixture def xtrigger_mgr() -> XtriggerManager: """A fixture to build an XtriggerManager which uses a mocked proc_pool, and uses a mocked broadcast_mgr.""" workflow_name = "sample_workflow" user = "john-foo" schd = create_autospec(Scheduler, workflow=workflow_name, owner=user) schd.proc_pool = Mock(put_command=lambda *a, **k: True) schd.workflow_db_mgr = Mock(housekeep=lambda *a, **k: True) schd.broadcast_mgr = Mock(put_broadcast=lambda *a, **k: True) schd.data_store_mgr = DataStoreMgr(schd) return XtriggerManager(schd) @pytest.fixture() def prevent_symlinking(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr( 'cylc.flow.pathutil.make_symlink_dir', lambda *_, **__: {} ) def _tmp_flow_config(tmp_run_dir: Callable): """Create a temporary flow config file for use in init'ing WorkflowConfig. Args: id_: Workflow name. config: The flow file content. Returns the path to the flow file. """ def __tmp_flow_config(id_: str, config: str) -> 'Path': run_dir: 'Path' = tmp_run_dir(id_) flow_file = run_dir / WorkflowFiles.FLOW_FILE flow_file.write_text(dedent(config).lstrip()) return flow_file return __tmp_flow_config @pytest.fixture def tmp_flow_config(tmp_run_dir: Callable): return _tmp_flow_config(tmp_run_dir) @pytest.fixture(scope='module') def mod_tmp_flow_config(mod_tmp_run_dir: Callable): return _tmp_flow_config(mod_tmp_run_dir) cylc-flow-8.7.0/tests/unit/xtriggers/0000775000175000017500000000000015264420307017753 5ustar alastairalastaircylc-flow-8.7.0/tests/unit/xtriggers/test_echo.py0000664000175000017500000000260615264420307022306 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.exceptions import WorkflowConfigError from cylc.flow.xtriggers.echo import validate import pytest from pytest import param def test_validate_good(): validate({ 'args': (), 'kwargs': {'succeed': False, 'egg': 'fried', 'potato': 'baked'} }) @pytest.mark.parametrize( 'all_args', ( param({'args': (False,), 'kwargs': {}}, id='no-kwarg'), param( {'args': (), 'kwargs': {'spud': 'mashed'}}, id='no-succeed-kwarg' ), ), ) def test_validate_exceptions(all_args): with pytest.raises(WorkflowConfigError, match='^Requires'): validate(all_args) cylc-flow-8.7.0/tests/unit/xtriggers/test_xrandom.py0000664000175000017500000000277315264420307023045 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from pytest import param from cylc.flow.xtriggers.xrandom import validate from cylc.flow.exceptions import WorkflowConfigError def test_validate_good(): validate({'percent': 1, 'secs': 0, '_': 'HelloWorld'}) @pytest.mark.parametrize( 'args, err', ( param({'percent': 'foo'}, r"'percent", id='percent-not-numeric'), param({'percent': 101}, r"'percent", id='percent>100'), param({'percent': -1}, r"'percent", id='percent<0'), param({'percent': 100, 'secs': 1.1}, r"'secs'", id='secs-not-int'), ) ) def test_validate_exceptions(args, err): """Illegal args and kwargs cause a WorkflowConfigError raised.""" with pytest.raises(WorkflowConfigError, match=f'^{err}'): validate(args) cylc-flow-8.7.0/tests/unit/xtriggers/__init__.py0000664000175000017500000000000015264420307022052 0ustar alastairalastaircylc-flow-8.7.0/tests/unit/xtriggers/test_wall_clock.py0000664000175000017500000000370515264420307023503 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.xtriggers.wall_clock import _wall_clock, validate from metomi.isodatetime.parsers import DurationParser import pytest from pytest import param @pytest.mark.parametrize('trigger_time, expected', [ (499, True), (500, False), ]) def test_wall_clock( monkeypatch: pytest.MonkeyPatch, trigger_time: int, expected: bool ): monkeypatch.setattr( 'cylc.flow.xtriggers.wall_clock.time', lambda: 500 ) assert _wall_clock(trigger_time) == expected @pytest.fixture def monkeypatch_interval_parser(monkeypatch): """Interval parse only works normally if a WorkflowSpecifics object identify the parser to be used. """ monkeypatch.setattr( 'cylc.flow.xtriggers.wall_clock.interval_parse', DurationParser().parse ) def test_validate_good(monkeypatch_interval_parser): validate({'offset': 'PT1H'}) @pytest.mark.parametrize( 'args, err', ( param({'offset': 1}, "^Invalid", id='invalid-offset-int'), param({'offset': 'Zaphod'}, "^Invalid", id='invalid-offset-str'), ) ) def test_validate_exceptions( monkeypatch_interval_parser, args, err ): with pytest.raises(Exception, match=err): validate(args) cylc-flow-8.7.0/tests/unit/xtriggers/test_workflow_state.py0000664000175000017500000002040515264420307024437 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from pathlib import Path from shutil import copytree, rmtree import sqlite3 from typing import Any, Callable import pytest from cylc.flow.dbstatecheck import output_fallback_msg from cylc.flow.exceptions import WorkflowConfigError from cylc.flow.rundb import CylcWorkflowDAO from cylc.flow.xtriggers.suite_state import suite_state from cylc.flow.xtriggers.workflow_state import ( _workflow_state_backcompat, validate, workflow_state, ) def test_inferred_run(tmp_run_dir: 'Callable', capsys: pytest.CaptureFixture): """Test that the workflow_state xtrigger infers the run number. Method: the faked run-dir has no DB to connect to, but the WorkflowPoller prints inferred ID to stderr if the run-dir exists. """ id_ = 'isildur' expected_workflow_id = f'{id_}/run1' cylc_run_dir = str(tmp_run_dir()) tmp_run_dir(expected_workflow_id, installed=True, named=True) workflow_state(id_ + '//3000/precious') assert expected_workflow_id in capsys.readouterr().err # Now test we can see workflows in alternate cylc-run directories # e.g. for `cylc workflow-state` or xtriggers targeting another user. alt_cylc_run_dir = cylc_run_dir + "_alt" # copy the cylc-run dir to alt location and delete the original. copytree(cylc_run_dir, alt_cylc_run_dir, symlinks=True) rmtree(cylc_run_dir) # It can no longer parse IDs in the original cylc-run location. workflow_state(id_) assert expected_workflow_id not in capsys.readouterr().err # But it can via an explicit alternate run directory. workflow_state(id_, alt_cylc_run_dir=alt_cylc_run_dir) assert expected_workflow_id in capsys.readouterr().err def test_c8_db_back_compat( tmp_run_dir: 'Callable', capsys: pytest.CaptureFixture, ): """Test workflow_state xtrigger backwards compatibility with Cylc < 8.3.0 database.""" id_ = 'nazgul' run_dir: Path = tmp_run_dir(id_) db_file = run_dir / 'log' / 'db' db_file.parent.mkdir(exist_ok=True) # Note: don't use CylcWorkflowDAO here as DB should be frozen conn = sqlite3.connect(str(db_file)) try: conn.execute(r""" CREATE TABLE workflow_params( key TEXT, value TEXT, PRIMARY KEY(key) ); """) conn.execute(r""" CREATE TABLE task_states( name TEXT, cycle TEXT, flow_nums TEXT, time_created TEXT, time_updated TEXT, submit_num INTEGER, status TEXT, flow_wait INTEGER, is_manual_submit INTEGER, PRIMARY KEY(name, cycle, flow_nums) ); """) conn.execute(r""" CREATE TABLE task_outputs( cycle TEXT, name TEXT, flow_nums TEXT, outputs TEXT, PRIMARY KEY(cycle, name, flow_nums) ); """) conn.executemany( r'INSERT INTO "workflow_params" VALUES(?,?);', [('cylc_version', '8.2.7'), ('cycle_point_format', '%Y'), ('cycle_point_tz', 'Z')] ) conn.execute(r""" INSERT INTO "task_states" VALUES( 'gimli','2012','[1]','2023-01-30T18:19:15Z', '2023-01-30T18:19:15Z',1,'succeeded',0,0 ); """) conn.execute(r""" INSERT INTO "task_outputs" VALUES( '2012','gimli','[1]', '["submitted", "started", "succeeded", "axe"]' ); """) conn.commit() finally: conn.close() gimli = f'{id_}//2012/gimli' satisfied, _ = workflow_state(gimli) assert satisfied satisfied, _ = workflow_state(f'{gimli}:succeeded') assert satisfied satisfied, _ = workflow_state(f'{gimli}:axe', is_message=True) assert satisfied _, err = capsys.readouterr() assert not err # Output label selector falls back to message # (won't work if message != output label) satisfied, _ = workflow_state(f'{gimli}:axe', is_trigger=True) assert satisfied _, err = capsys.readouterr() assert output_fallback_msg in err def test__workflow_state_backcompat(tmp_run_dir: 'Callable'): """Test the _workflow_state_backcompat & suite_state functions on a *current* Cylc database.""" id_ = 'dune' run_dir: Path = tmp_run_dir(id_) db_file = run_dir / 'log' / 'db' db_file.parent.mkdir(exist_ok=True) with CylcWorkflowDAO(db_file, create_tables=True) as dao: conn = dao.connect() conn.executemany( r'INSERT INTO "workflow_params" VALUES(?,?);', [('cylc_version', '8.3.0'), ('cycle_point_format', '%Y'), ('cycle_point_tz', 'Z')] ) conn.execute(r""" INSERT INTO "task_states" VALUES( 'arrakis','2012','[1]','2023-01-30T18:19:15Z', '2023-01-30T18:19:15Z',1,'succeeded',0,0 ); """) conn.execute(r""" INSERT INTO "task_outputs" VALUES( '2012','arrakis','[1]', '{ "submitted": "submitted", "started": "started", "succeeded": "succeeded", "paul": "lisan al-gaib" }' ); """) conn.commit() func: Any for func in (_workflow_state_backcompat, suite_state): satisfied, _ = func(id_, 'arrakis', '2012') assert satisfied satisfied, _ = func(id_, 'arrakis', '2012', status='succeeded') assert satisfied # Both output label and message work satisfied, _ = func(id_, 'arrakis', '2012', message='paul') assert satisfied satisfied, _ = func(id_, 'arrakis', '2012', message='lisan al-gaib') assert satisfied def test_validate_ok(): """Validate returns ok with valid args.""" validate({ 'workflow_task_id': 'foo//1/bar', 'is_trigger': False, 'is_message': False, 'offset': 'PT1H', 'flow_num': 44, }) @pytest.mark.parametrize( 'id_', (('foo//1'),) ) def test_validate_fail_bad_id(id_): """Validation failure for bad id""" with pytest.raises(WorkflowConfigError, match='Full ID needed'): validate({ 'workflow_task_id': id_, 'offset': 'PT1H', 'flow_num': 44, }) @pytest.mark.parametrize( 'flow_num', ((4.25260), ('Belguim')) ) def test_validate_fail_non_int_flow(flow_num): """Validate failure for non integer flow numbers.""" with pytest.raises(WorkflowConfigError, match='must be an integer'): validate({ 'workflow_task_id': 'foo//1/bar', 'offset': 'PT1H', 'flow_num': flow_num, }) def test_validate_polling_config(): """It should reject invalid or unreliable polling configurations. See https://github.com/cylc/cylc-flow/issues/6157 """ with pytest.raises(WorkflowConfigError, match='No such task state'): validate({ 'workflow_task_id': 'foo//1/bar:elephant', 'is_trigger': False, 'is_message': False, 'flow_num': 44, }) with pytest.raises(WorkflowConfigError, match='Cannot poll for'): validate({ 'workflow_task_id': 'foo//1/bar:waiting', 'is_trigger': False, 'is_message': False, 'flow_num': 44, }) with pytest.raises(WorkflowConfigError, match='is not reliable'): validate({ 'workflow_task_id': 'foo//1/bar:submitted', 'is_trigger': False, 'is_message': False, 'flow_num': 44, }) cylc-flow-8.7.0/tests/unit/test_task_remote_cmd.py0000664000175000017500000000527615264420307022520 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests for remote initialisation.""" from pathlib import Path from unittest.mock import patch from pytest import CaptureFixture from cylc.flow.task_remote_cmd import remote_init from cylc.flow.workflow_files import WorkflowFiles def test_existing_key_raises_error(tmp_path: Path, capsys: CaptureFixture): """Test .service directory that contains existing incorrect key, results in REMOTE INIT FAILED""" rundir = tmp_path / 'some_rund' srvdir = rundir / WorkflowFiles.Service.DIRNAME srvdir.mkdir(parents=True) (srvdir / 'client_wrong.key').touch() assert remote_init('test_install_target', str(rundir)) assert capsys.readouterr().out == ( "REMOTE INIT FAILED\nUnexpected authentication key" " 'client_wrong.key' exists. Check global.cylc install target is" " configured correctly for this platform.\n") @patch('os.path.expandvars') def test_unexpandable_symlink_env_var_returns_failed( mocked_expandvars, capsys): """Test unexpandable symlinks return REMOTE INIT FAILED""" mocked_expandvars.side_effect = ['some/rund/path', '$blah'] assert remote_init('test_install_target', 'some_rund', 'run=$blah') assert capsys.readouterr().out == ( "REMOTE INIT FAILED\nError occurred when symlinking." " $blah contains an invalid environment variable.\n") def test_existing_client_key_dir_raises_error( tmp_path: Path, capsys: CaptureFixture): """Test .service directory that contains existing incorrect key, results in REMOTE INIT FAILED """ rundir = tmp_path / 'some_rund' keydir = rundir / WorkflowFiles.Service.DIRNAME / "client_public_keys" keydir.mkdir(parents=True) assert remote_init('test_install_target', rundir) assert capsys.readouterr().out == ( f"REMOTE INIT FAILED\nUnexpected key directory exists: {keydir}" " Check global.cylc install target is configured correctly for this" " platform.\n") cylc-flow-8.7.0/tests/unit/test_platforms_get_platform.py0000664000175000017500000002106315264420307024122 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # Tests for the platform lookup module's get_platform method. from typing import Callable, Dict, Optional import pytest from cylc.flow.platforms import ( get_localhost_install_target, get_platform ) from cylc.flow.exceptions import PlatformLookupError def test_get_platform_no_args(): # If no task conf is given, we get localhost args. assert get_platform()['hosts'] == ['localhost'] @pytest.mark.parametrize( 'platform_re', [ None, 'localhost', 'localhost, otherplatform', 'otherplatform, localhost', 'localhost, xylophone\\d{1,5}' ] ) def test_get_localhost_platform(mock_glbl_cfg, platform_re): # Check that an arbitrary string name returns a sensible platform mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', f''' [platforms] [[localhost]] hosts = localhost ssh command = ssh -oConnectTimeout=42 [[{platform_re}]] hosts = localhost ssh command = ssh -oConnectTimeout=24 ''' ) platform = get_platform('localhost') if platform_re: assert platform['ssh command'] == 'ssh -oConnectTimeout=24' else: assert platform['ssh command'] == 'ssh -oConnectTimeout=42' @pytest.mark.parametrize( 'platform_re', [ 'saffron', 'sumac|saffron', 'sumac, saffron', 'sumac|asafoetida, saffron', ] ) def test_get_platform_from_platform_name_str(mock_glbl_cfg, platform_re): # Check that an arbitrary string name returns a sensible platform mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', f''' [platforms] [[{platform_re}]] hosts = saff01 job runner = slurm ''' ) platform = get_platform('saffron') assert platform['hosts'] == ['saff01'] assert platform['job runner'] == 'slurm' @pytest.mark.parametrize( 'task_conf, err_expected', [ ( { 'platform': 'localhost', 'remote': { 'host': 'localhost' } }, True ), ( { 'platform': 'gondor', 'remote': { 'retrieve job logs': False } }, True ), ( { 'platform': 'gondor', 'remote': { 'host': None } }, False ), ] ) def test_get_platform_cylc7_8_syntax_mix_fails( task_conf: dict, err_expected: bool, mock_glbl_cfg: Callable ): """If a task with a mix of Cylc7 and 8 syntax is passed to get_platform this should return an error. """ mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[gondor]] hosts = denethor ''' ) if err_expected: with pytest.raises( PlatformLookupError, match=( r"Task .* has the following deprecated '\[runtime\]' " r"setting\(s\) which cannot be used with 'platform.*" ) ): get_platform(task_conf) else: get_platform(task_conf) def test_get_platform_from_config_with_platform_name(mock_glbl_cfg): # A platform name is present, and no clashing cylc7 configs are: mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[mace]] hosts = mace001, mace002 job runner = slurm ''' ) task_conf = {'platform': 'mace'} platform = get_platform(task_conf) assert platform['hosts'] == ['mace001', 'mace002'] assert platform['job runner'] == 'slurm' @pytest.mark.parametrize( 'task_conf, expected_platform_name', [ ( { 'remote': {'host': 'cumin'}, 'job': {'batch system': 'slurm'} }, 'ras_el_hanout' ), ( {'remote': {'host': 'cumin'}}, 'spice_bg' ), ( {'job': {'batch system': 'batchyMcBatchFace'}}, 'local_job_runner' ), ( {'script': 'true'}, 'localhost' ), ( { 'remote': {'host': 'localhost'}, 'job': { 'batch system': None, 'batch submit command template': None, 'execution polling intervals': None } }, 'localhost' ), ( { 'remote': {'host': 'cylcdevbox'}, 'job': { 'batch system': None, 'batch submit command template': None, 'execution polling intervals': None } }, 'cylcdevbox' ) ] ) def test_get_platform_using_platform_name_from_job_info( mock_glbl_cfg, task_conf, expected_platform_name ): """Calculate platform from Cylc 7 config: n.b. If this fails we don't warn because this might lead to many thousands of warnings This should not contain a comprehensive set of use-cases - these should be coverend by the unit tests for `platform_from_host_items` """ mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[ras_el_hanout]] hosts = rose, chilli, cumin, paprika job runner = slurm [[spice_bg]] hosts = rose, chilli, cumin, paprika [[local_job_runner]] hosts = localhost job runner = batchyMcBatchFace [[cylcdevbox]] hosts = cylcdevbox ''' ) assert get_platform(task_conf)['name'] == expected_platform_name def test_get_platform_groups_basic(mock_glbl_cfg): """get platform from group works. Additionally, ensure that we stop after selecting the first appropriate platform. """ mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[aleph, bet, alpha, beta]] [platform groups] [[hebrew_letters]] platforms = alpha, beta [[[selection]]] method = definition order [[aleph]] # Group with same name as platform to try and # trip up the platform selection logic after it # has processed [[.*_letters]] below platforms = alpha [[.*_letters]] platforms = aleph, bet [[[selection]]] method = definition order ''' ) output = get_platform('hebrew_letters') assert output['name'] == 'aleph' @pytest.mark.parametrize( 'task_conf, expected_err_msg', [ ({'platform': '$(host)'}, None), ({'platform': '$(host)-suffix'}, None), ({'platform': '`echo ${chamber}`'}, "backticks are not supported") ] ) def test_get_platform_subshell( task_conf: Dict[str, str], expected_err_msg: Optional[str]): """Test get_platform() for subshell platform definition.""" if expected_err_msg: with pytest.raises(PlatformLookupError) as err: get_platform(task_conf) assert expected_err_msg in str(err.value) else: assert get_platform(task_conf) is None def test_get_localhost_install_target(): assert get_localhost_install_target() == 'localhost' def test_localhost_different_install_target(mock_glbl_cfg): mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[localhost]] install target = file_system_1 ''' ) assert get_localhost_install_target() == 'file_system_1' cylc-flow-8.7.0/tests/unit/job_runner_handlers/0000775000175000017500000000000015264420307021760 5ustar alastairalastaircylc-flow-8.7.0/tests/unit/job_runner_handlers/test_lsf.py0000664000175000017500000000463715264420307024167 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.job_runner_handlers.lsf import JOB_RUNNER_HANDLER @pytest.mark.parametrize( 'job_conf,lines', [ ( # basic { 'directives': {}, 'execution_time_limit': 180, 'job_file_path': 'cylc-run/chop/log/job/1/axe/01/job', 'workflow_name': 'chop', 'task_id': '1/axe', }, [ '#BSUB -J axe.1.chop', '#BSUB -o cylc-run/chop/log/job/1/axe/01/job.out', '#BSUB -e cylc-run/chop/log/job/1/axe/01/job.err', '#BSUB -W 3', ], ), ( # some useful directives { 'directives': { '-q': 'forever', '-B': '', '-ar': '', }, 'execution_time_limit': 200, 'job_file_path': 'cylc-run/chop/log/job/1/axe/01/job', 'workflow_name': 'chop', 'task_id': '1/axe', }, [ '#BSUB -J axe.1.chop', '#BSUB -o cylc-run/chop/log/job/1/axe/01/job.out', '#BSUB -e cylc-run/chop/log/job/1/axe/01/job.err', '#BSUB -W 4', '#BSUB -q forever', '#BSUB -B', '#BSUB -ar', ], ), ], ) def test_format_directives(job_conf: dict, lines: list): assert JOB_RUNNER_HANDLER.format_directives(job_conf) == lines def test_get_submit_stdin(): outs = JOB_RUNNER_HANDLER.get_submit_stdin(__file__, None) assert outs[0].name == __file__ assert outs[1] is None cylc-flow-8.7.0/tests/unit/job_runner_handlers/test_moab.py0000664000175000017500000000443715264420307024317 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.job_runner_handlers.moab import JOB_RUNNER_HANDLER @pytest.mark.parametrize( 'job_conf,lines', [ ( # basic { 'directives': {}, 'execution_time_limit': 180, 'job_file_path': 'cylc-run/chop/log/job/1/axe/01/job', 'workflow_name': 'chop', 'task_id': '1/axe', }, [ '#PBS -N axe.1.chop', '#PBS -o cylc-run/chop/log/job/1/axe/01/job.out', '#PBS -e cylc-run/chop/log/job/1/axe/01/job.err', '#PBS -l walltime=180', ], ), ( # some useful directives { 'directives': { '-q': 'forever', '-V': '', '-l mem': '256gb', }, 'execution_time_limit': 180, 'job_file_path': 'cylc-run/chop/log/job/1/axe/01/job', 'workflow_name': 'chop', 'task_id': '1/axe', }, [ '#PBS -N axe.1.chop', '#PBS -o cylc-run/chop/log/job/1/axe/01/job.out', '#PBS -e cylc-run/chop/log/job/1/axe/01/job.err', '#PBS -l walltime=180', '#PBS -q forever', '#PBS -V', '#PBS -l mem=256gb', ], ), ], ) def test_format_directives(job_conf: dict, lines: list): assert JOB_RUNNER_HANDLER.format_directives(job_conf) == lines cylc-flow-8.7.0/tests/unit/job_runner_handlers/test_slurm_packjob.py0000664000175000017500000000566015264420307026233 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.job_runner_handlers.slurm_packjob import JOB_RUNNER_HANDLER @pytest.mark.parametrize( 'job_conf,lines', [ ( # heterogeneous job, old-style (packjob) { 'directives': { '-p': 'middle', 'packjob_0_--mem': '128gb', 'packjob_1_--mem': '256gb', }, 'execution_time_limit': 200, 'job_file_path': 'cylc-run/chop/log/job/1/axe/01/job', 'workflow_name': 'chop', 'task_id': '1/axe', }, [ '#SBATCH --job-name=axe.1.chop', ( '#SBATCH --output=' 'cylc-run/chop/log/job/1/axe/01/job.out' ), ( '#SBATCH --error=' 'cylc-run/chop/log/job/1/axe/01/job.err' ), '#SBATCH --time=3:20', '#SBATCH -p=middle', '#SBATCH --mem=128gb', '#SBATCH packjob', '#SBATCH --mem=256gb', ], ), ], ) def test_format_directives(job_conf: dict, lines: list): assert JOB_RUNNER_HANDLER.format_directives(job_conf) == lines @pytest.mark.parametrize( 'job_ids,cmd', [ [['1234567'], ['squeue', '-h', '-j', '1234567']], [ ['1234567', '709394', '30624700'], ['squeue', '-h', '-j', '1234567,709394,30624700'], ], ], ) def test_get_poll_many_cmd(job_ids: list, cmd: list): assert JOB_RUNNER_HANDLER.get_poll_many_cmd(job_ids) == cmd @pytest.mark.parametrize( 'out,job_ids', [ [ """HEADING 1234567 JOB PROPERTIES 709394 JOB PROPERTIES 30624700 JOB PROPERTIES """, ['1234567', '30624700', '709394'], ], [ """HEADING 1234567+0 JOB PROPERTIES (HETERO) 1234567+1 JOB PROPERTIES (HETERO) 709394 JOB PROPERTIES 30624700 JOB PROPERTIES """, ['1234567', '30624700', '709394'], ], ], ) def test_filter_poll_many_output(job_ids: list, out: str): assert sorted(JOB_RUNNER_HANDLER.filter_poll_many_output(out)) == job_ids cylc-flow-8.7.0/tests/unit/job_runner_handlers/test_loadleveler.py0000664000175000017500000000577115264420307025701 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.job_runner_handlers.loadleveler import JOB_RUNNER_HANDLER from cylc.flow.job_runner_handlers.loadleveler import LoadlevelerHandler @pytest.mark.parametrize( 'job_conf,lines', [ ( # basic { 'directives': {}, 'execution_time_limit': 180, 'job_file_path': 'cylc-run/chop/log/job/1/axe/01/job', 'workflow_name': 'chop', 'task_id': '1/axe', }, [ '# @ job_name = chop.axe.1', '# @ output = cylc-run/chop/log/job/1/axe/01/job.out', '# @ error = cylc-run/chop/log/job/1/axe/01/job.err', '# @ wall_clock_limit = 240,180', '# @ queue' ], ), ( # some useful directives { 'directives': { '-q': 'forever', '-V': '', '-l mem': '256gb', }, 'execution_time_limit': 180, 'job_file_path': 'cylc-run/chop/log/job/1/axe/01/job', 'workflow_name': 'chop', 'task_id': '1/axe', }, [ '# @ job_name = chop.axe.1', '# @ output = cylc-run/chop/log/job/1/axe/01/job.out', '# @ error = cylc-run/chop/log/job/1/axe/01/job.err', '# @ wall_clock_limit = 240,180', '# @ -q = forever', '# @ -V', '# @ -l mem = 256gb', '# @ queue' ], ), ], ) def test_format_directives(job_conf: dict, lines: list): assert JOB_RUNNER_HANDLER.format_directives(job_conf) == lines def test_filter_poll_many_output(): configuration = ''' Id Owner Submitted ST PRI Class Running On ---------------- ---------- ----------- -- --- -------- ---------- mars.498.0 brownap 5/20 11:31 R 100 silver mars mars.499.0 brownap 5/20 11:31 R 50 No_Class mars mars.501.0 brownap 5/20 11:31 I 50 silver ''' out = ['Id', '----------------', 'mars.498', 'mars.499', 'mars.501'] assert LoadlevelerHandler.filter_poll_many_output(configuration) == out cylc-flow-8.7.0/tests/unit/job_runner_handlers/test_slurm.py0000664000175000017500000001235415264420307024540 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.job_runner_handlers.slurm import JOB_RUNNER_HANDLER @pytest.mark.parametrize( 'job_conf,lines', [ ( # basic { 'directives': {}, 'execution_time_limit': 180, 'job_file_path': 'cylc-run/chop/log/job/1/axe/01/job', 'workflow_name': 'chop', 'task_id': '1/axe', }, [ '#SBATCH --job-name=axe.1.chop', ( '#SBATCH --output=' 'cylc-run/chop/log/job/1/axe/01/job.out' ), ( '#SBATCH --error=' 'cylc-run/chop/log/job/1/axe/01/job.err' ), '#SBATCH --time=3:00', ], ), ( # task name with % character { 'directives': {}, 'execution_time_limit': 180, 'job_file_path': ( 'cylc-run/chop/log/job/1/axe%40HEAD/01/job' ), 'workflow_name': 'chop', 'task_id': '1/axe%40HEAD', }, [ '#SBATCH --job-name=axe%40HEAD.1.chop', ( '#SBATCH --output' '=cylc-run/chop/log/job/1/axe%%40HEAD/01/job.out' ), ( '#SBATCH --error' '=cylc-run/chop/log/job/1/axe%%40HEAD/01/job.err' ), '#SBATCH --time=3:00', ], ), ( # some useful directives { 'directives': { '-p': 'middle', '--no-requeue': '', '--mem': '256gb', }, 'execution_time_limit': 200, 'job_file_path': 'cylc-run/chop/log/job/1/axe/01/job', 'workflow_name': 'chop', 'task_id': '1/axe', }, [ '#SBATCH --job-name=axe.1.chop', ( '#SBATCH --output=' 'cylc-run/chop/log/job/1/axe/01/job.out' ), ( '#SBATCH --error=' 'cylc-run/chop/log/job/1/axe/01/job.err' ), '#SBATCH --time=3:20', '#SBATCH -p=middle', '#SBATCH --no-requeue', '#SBATCH --mem=256gb', ], ), ( # heterogeneous job { 'directives': { '-p': 'middle', 'hetjob_0_--mem': '128gb', 'hetjob_1_--mem': '256gb', }, 'execution_time_limit': 200, 'job_file_path': 'cylc-run/chop/log/job/1/axe/01/job', 'workflow_name': 'chop', 'task_id': '1/axe', }, [ '#SBATCH --job-name=axe.1.chop', ( '#SBATCH --output=' 'cylc-run/chop/log/job/1/axe/01/job.out' ), ( '#SBATCH --error=' 'cylc-run/chop/log/job/1/axe/01/job.err' ), '#SBATCH --time=3:20', '#SBATCH -p=middle', '#SBATCH --mem=128gb', '#SBATCH hetjob', '#SBATCH --mem=256gb', ], ), ], ) def test_format_directives(job_conf: dict, lines: list): assert JOB_RUNNER_HANDLER.format_directives(job_conf) == lines @pytest.mark.parametrize( 'job_ids,cmd', [ [['1234567'], ['squeue', '-h', '-j', '1234567']], [ ['1234567', '709394', '30624700'], ['squeue', '-h', '-j', '1234567,709394,30624700'], ], ], ) def test_get_poll_many_cmd(job_ids: list, cmd: list): assert JOB_RUNNER_HANDLER.get_poll_many_cmd(job_ids) == cmd @pytest.mark.parametrize( 'out,job_ids', [ [ """HEADING 1234567 JOB PROPERTIES 709394 JOB PROPERTIES 30624700 JOB PROPERTIES """, ['1234567', '30624700', '709394'], ], [ """HEADING 1234567+0 JOB PROPERTIES (HETERO) 1234567+1 JOB PROPERTIES (HETERO) 709394 JOB PROPERTIES 30624700 JOB PROPERTIES """, ['1234567', '30624700', '709394'], ], ], ) def test_filter_poll_many_output(job_ids: list, out: str): assert sorted(JOB_RUNNER_HANDLER.filter_poll_many_output(out)) == job_ids cylc-flow-8.7.0/tests/unit/job_runner_handlers/test_pbs.py0000664000175000017500000001276515264420307024170 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.job_runner_handlers.pbs import ( JOB_RUNNER_HANDLER, PBSHandler ) from cylc.flow.job_runner_mgr import JobRunnerManager VERY_LONG_STR = 'x' * 240 @pytest.mark.parametrize( 'job_conf,lines', [ pytest.param( { 'directives': {}, 'execution_time_limit': 180, 'job_file_path': 'cylc-run/chop/log/job/1/axe/01/job', 'workflow_name': 'chop', 'task_id': '1/axe', 'platform': { 'job runner': 'pbs', 'job name length maximum': 100, }, }, [ '#PBS -N axe.1.chop', '#PBS -o cylc-run/chop/log/job/1/axe/01/job.out', '#PBS -e cylc-run/chop/log/job/1/axe/01/job.err', '#PBS -l walltime=180', ], id='basic', ), pytest.param( { 'directives': {}, 'execution_time_limit': 180, 'job_file_path': 'cylc-run/chop/log/job/1/axe/01/job', 'workflow_name': 'chop', 'task_id': VERY_LONG_STR, 'platform': { 'job runner': 'pbs', }, }, [ '#PBS -N ' f'None.{VERY_LONG_STR[: PBSHandler.JOB_NAME_LEN_MAX - 5]}', '#PBS -o cylc-run/chop/log/job/1/axe/01/job.out', '#PBS -e cylc-run/chop/log/job/1/axe/01/job.err', '#PBS -l walltime=180', ], id='long-job-name', ), pytest.param( { 'directives': {}, 'execution_time_limit': 180, 'job_file_path': 'cylc-run/chop/log/job/1/axe/01/job', 'workflow_name': 'chop', 'task_id': '1/axe', 'platform': { 'job runner': 'pbs', 'job name length maximum': 6, }, }, [ '#PBS -N axe.1.', '#PBS -o cylc-run/chop/log/job/1/axe/01/job.out', '#PBS -e cylc-run/chop/log/job/1/axe/01/job.err', '#PBS -l walltime=180', ], id='truncate-job-name', ), pytest.param( { 'directives': { '-q': 'forever', '-V': '', '-l mem': '256gb', }, 'execution_time_limit': 180, 'job_file_path': 'cylc-run/chop/log/job/1/axe/01/job', 'workflow_name': 'chop', 'task_id': '1/axe', 'platform': { 'job runner': 'pbs', 'job name length maximum': 100, }, }, [ '#PBS -N axe.1.chop', '#PBS -o cylc-run/chop/log/job/1/axe/01/job.out', '#PBS -e cylc-run/chop/log/job/1/axe/01/job.err', '#PBS -l walltime=180', '#PBS -q forever', '#PBS -V', '#PBS -l mem=256gb', ], id='custom-directives', ), ], ) def test_format_directives(job_conf: dict, lines: list): assert JOB_RUNNER_HANDLER.format_directives(job_conf) == lines def test_filter_poll_many_output(): """It should strip trailing junk from job IDs. Job IDs are assumed to be a series of numbers, optionally followed by a full-stop and some other letters and numbers which are not needed for job tracking purposes. Job IDs are not expected to start with letters e.g. `abc.456` is not supported. """ assert JOB_RUNNER_HANDLER.filter_poll_many_output(''' Job id Name User Time Use S Queue ---------------- ---------------- ---------------- -------- - ----- 12345.foo.bar.baz test-pbs xxxxxxx 0 Q reomq 23456.foo test-pbs xxxxxxx 0 Q romeq 34567 test-pbs xxxxxxx 1 Q romeq abc.456 test-pbs xxxxxxx 2 Q romeq abcdef test-pbs xxxxxxx 2 Q romeq ''') == ['12345', '23456', '34567'] def test_filter_submit_output(tmp_path): """See notes for test_filter_poll_many_output.""" status_file = tmp_path / 'submit_out' status_file.touch() def test(out): return JobRunnerManager._filter_submit_output( status_file, JOB_RUNNER_HANDLER, out, '', )[2] assert test(' 12345.foo.bar.baz') == '12345' assert test(' 12345.foo') == '12345' assert test(' 12345') == '12345' cylc-flow-8.7.0/tests/unit/test_timer.py0000664000175000017500000000336315264420307020473 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging from time import sleep import pytest from cylc.flow.timer import Timer def test_Timer(caplog: pytest.LogCaptureFixture): """Test the Timer class.""" caplog.set_level(logging.INFO) timer = Timer("bob timeout", 1.0) # timer attributes assert timer.name == "bob timer" assert timer.interval == "PT1S" # start timer timer.reset() assert caplog.records[-1].message == "PT1S bob timer starts NOW" # check timeout sleep(2) assert timer.timed_out() assert caplog.records[-1].message == "bob timer timed out after PT1S" # stop should do nothing after timeout caplog.clear() timer.stop() assert not caplog.records # start timer again, then stop it timer.reset() assert caplog.records[-1].message == "PT1S bob timer starts NOW" timer.stop() assert caplog.records[-1].message == "bob timer stopped" # another stop should do nothing caplog.clear() timer.stop() assert not caplog.records cylc-flow-8.7.0/tests/unit/test_pathutil.py0000664000175000017500000004510215264420307021202 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests for "cylc.flow.pathutil".""" import logging import os from pathlib import Path import pytest from pytest import param from typing import Callable, Dict, Iterable, List, Set from unittest.mock import Mock, patch, call from cylc.flow.exceptions import InputError, WorkflowFilesError from cylc.flow.pathutil import ( EXPLICIT_RELATIVE_PATH_REGEX, expand_path, get_dirs_to_symlink, get_next_rundir_number, get_remote_workflow_run_dir, get_remote_workflow_run_job_dir, get_workflow_run_dir, get_workflow_run_job_dir, get_workflow_run_scheduler_log_dir, get_workflow_run_scheduler_log_path, get_workflow_run_pub_db_path, get_workflow_run_config_log_dir, get_workflow_run_share_dir, get_workflow_run_work_dir, get_workflow_test_log_path, is_relative_to, make_localhost_symlinks, make_workflow_run_tree, parse_rm_dirs, remove_dir_and_target, remove_dir_or_file, remove_empty_parents, get_workflow_name_from_id ) from .conftest import MonkeyMock HOME = Path.home() @pytest.mark.parametrize( 'string, match_expected', [ ('./foo/bar', True), ('../foo', True), ('./', True), ('../', True), ('.', True), ('..', True), ('foo/bar', False), ('.foo/bar', False), ('foo/..', False), ] ) def test_explicit_relative_path_regex(string: str, match_expected: bool): assert bool(EXPLICIT_RELATIVE_PATH_REGEX.match(string)) is match_expected @pytest.mark.parametrize( 'path, expected', [('~/moo', os.path.join(HOME, 'moo')), ('$HOME/moo', os.path.join(HOME, 'moo')), ('~/$FOO/moo', os.path.join(HOME, 'foo', 'bar', 'moo')), ('$NON_EXIST/moo', '$NON_EXIST/moo')] ) def test_expand_path( path: str, expected: str, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv('FOO', 'foo/bar') monkeypatch.delenv('NON_EXIST', raising=False) assert expand_path(path) == expected @pytest.mark.parametrize( 'func, extra_args, expected', [ (get_remote_workflow_run_dir, (), "$HOME/cylc-run/foo"), ( get_remote_workflow_run_dir, ("comes", "true"), "$HOME/cylc-run/foo/comes/true", ), ( get_remote_workflow_run_job_dir, (), "$HOME/cylc-run/foo/log/job"), ( get_remote_workflow_run_job_dir, ("comes", "true"), "$HOME/cylc-run/foo/log/job/comes/true", ) ] ) def test_get_remote_workflow_run_dirs( func: Callable, extra_args: Iterable[str], expected: str, ) -> None: """Tests for get_remote_workflow_run_[|job|work]_dir""" if extra_args: result = func('foo', *extra_args) else: result = func('foo') assert result == expected @pytest.mark.parametrize( 'func, tail1', [(get_workflow_run_dir, ''), (get_workflow_run_job_dir, '/log/job'), (get_workflow_run_scheduler_log_dir, '/log/scheduler'), (get_workflow_run_config_log_dir, '/log/config'), (get_workflow_run_share_dir, '/share'), (get_workflow_run_work_dir, '/work')] ) @pytest.mark.parametrize( 'args, tail2', [([], ''), (['comes', 'true'], '/comes/true')] ) def test_get_workflow_run_dirs( func: Callable, tail1: str, args: List[str], tail2: str ) -> None: """Usage of get_workflow_run_*dir. Params: func: get_remote_* function to test tail1: expected tail of return value from configuration args: extra *args tail2: expected tail of return value from extra args """ homedir = os.getenv("HOME") expected_result = f'{homedir}/cylc-run/my-workflow/dream{tail1}{tail2}' assert func('my-workflow/dream', *args) == expected_result @pytest.mark.parametrize( 'func, tail', [(get_workflow_run_scheduler_log_path, '/log/scheduler/log'), (get_workflow_run_pub_db_path, '/log/db'), (get_workflow_test_log_path, '/log/scheduler/reftest.log')] ) def test_get_workflow_run_names(func: Callable, tail: str) -> None: """Usage of get_workflow_run_*name. Params: func: get_remote_* function to test cfg: configuration used in mocked global configuration tail: expected tail of return value from configuration """ homedir = os.getenv("HOME") assert ( func('my-workflow/dream') == f'{homedir}/cylc-run/my-workflow/dream{tail}' ) def test_make_workflow_run_tree( tmp_run_dir: Callable, caplog: pytest.LogCaptureFixture ) -> None: run_dir: Path = tmp_run_dir('my-workflow') caplog.set_level(logging.DEBUG) # Only used for debugging test make_workflow_run_tree('my-workflow') # Check that directories have been created for subdir in [ '', 'log/scheduler', 'log/job', 'log/config', 'share', 'work' ]: assert (run_dir / subdir).is_dir() is True @pytest.mark.parametrize( 'mocked_glbl_cfg, output', [ pytest.param( # basic ''' [install] [[symlink dirs]] [[[the_matrix]]] run = $DEE work = $DAH log = $DUH log/job =$RAY share = $DOH share/cycle = $DAH ''', { 'run': '$DEE/cylc-run/morpheus', 'work': '$DAH/cylc-run/morpheus/work', 'log': '$DUH/cylc-run/morpheus/log', 'log/job': '$RAY/cylc-run/morpheus/log/job', 'share': '$DOH/cylc-run/morpheus/share', 'share/cycle': '$DAH/cylc-run/morpheus/share/cycle' }, id="basic" ), pytest.param( # remove nested run symlinks ''' [install] [[symlink dirs]] [[[the_matrix]]] run = $DEE work = $DAH log = $DEE share = $DOH share/cycle = $DAH ''', { 'run': '$DEE/cylc-run/morpheus', 'work': '$DAH/cylc-run/morpheus/work', 'share': '$DOH/cylc-run/morpheus/share', 'share/cycle': '$DAH/cylc-run/morpheus/share/cycle' }, id="remove nested run symlinks" ), pytest.param( # remove only nested run symlinks ''' [install] [[symlink dirs]] [[[the_matrix]]] run = $DOH log = $DEE share = $DEE ''', { 'run': '$DOH/cylc-run/morpheus', 'log': '$DEE/cylc-run/morpheus/log', 'share': '$DEE/cylc-run/morpheus/share' }, id="remove only nested run symlinks" ), pytest.param( # blank entries ''' [install] [[symlink dirs]] [[[the_matrix]]] run = log = "" share = work = " " ''', {}, id="blank entries" ) ] ) def test_get_dirs_to_symlink( mocked_glbl_cfg: str, output: Dict[str, str], mock_glbl_cfg: Callable, monkeypatch: pytest.MonkeyPatch ) -> None: # Set env var 'DEE', but we expect it to be unexpanded monkeypatch.setenv('DEE', 'poiuytrewq') mock_glbl_cfg('cylc.flow.pathutil.glbl_cfg', mocked_glbl_cfg) dirs = get_dirs_to_symlink('the_matrix', 'morpheus') assert dirs == output @patch('cylc.flow.pathutil.get_workflow_run_dir') @patch('cylc.flow.pathutil.get_dirs_to_symlink') def test_make_localhost_symlinks_calls_make_symlink_for_each_key_value_dir( mocked_dirs_to_symlink: Mock, mocked_get_workflow_run_dir: Mock, monkeypatch: pytest.MonkeyPatch, monkeymock: MonkeyMock ) -> None: mocked_dirs_to_symlink.return_value = { 'run': '$DOH/trinity', 'log': '$DEE/trinity/log', 'share': '$DEE/trinity/share' } mocked_get_workflow_run_dir.return_value = "rund" for v in ('DOH', 'DEE'): monkeypatch.setenv(v, 'expanded') mocked_make_symlink = monkeymock('cylc.flow.pathutil.make_symlink_dir') make_localhost_symlinks('rund', 'workflow') mocked_make_symlink.assert_has_calls([ call('rund', 'expanded/trinity'), call('rund/log', 'expanded/trinity/log'), call('rund/share', 'expanded/trinity/share') ]) @patch('cylc.flow.pathutil.get_workflow_run_dir') @patch('cylc.flow.pathutil.make_symlink_dir') @patch('cylc.flow.pathutil.get_dirs_to_symlink') def test_incorrect_environment_variables_raise_error( mocked_dirs_to_symlink, mocked_make_symlink, mocked_get_workflow_run_dir, monkeypatch: pytest.MonkeyPatch ): monkeypatch.delenv('doh', raising=False) mocked_dirs_to_symlink.return_value = { 'run': '$doh/cylc-run/test_workflow'} mocked_get_workflow_run_dir.return_value = "rund" with pytest.raises(WorkflowFilesError) as excinfo: make_localhost_symlinks('rund', 'test_workflow') assert ( "Can't symlink to $doh/cylc-run/test_workflow\n" "Undefined variables, check global config: $doh" ) in str(excinfo.value) @pytest.mark.parametrize( 'filetype, expected_err', [('dir', None), ('file', NotADirectoryError), (None, FileNotFoundError)] ) def test_remove_dir_and_target(filetype, expected_err, tmp_path): """Test that remove_dir_and_target() can delete nested dirs and handle bad paths.""" test_path = tmp_path.joinpath('foo/bar') if filetype == 'dir': # Test removal of sub directories too sub_dir = test_path.joinpath('baz') sub_dir.mkdir(parents=True) sub_dir_file = sub_dir.joinpath('meow') sub_dir_file.touch() elif filetype == 'file': test_path = tmp_path.joinpath('meow') test_path.touch() if expected_err: with pytest.raises(expected_err): remove_dir_and_target(test_path) else: remove_dir_and_target(test_path) assert test_path.exists() is False assert test_path.is_symlink() is False @pytest.mark.parametrize( 'target, expected_err', [('dir', None), ('file', NotADirectoryError), (None, None)] ) def test_remove_dir_and_target_symlinks(target, expected_err, tmp_path): """Test that remove_dir_and_target() can delete symlinks, including the target.""" target_path = tmp_path.joinpath('x/y') target_path.mkdir(parents=True) tmp_path.joinpath('a').mkdir() symlink_path = tmp_path.joinpath('a/b') if target == 'dir': # Add a file into the the target dir to check it removes that ok target_path.joinpath('meow').touch() symlink_path.symlink_to(target_path) elif target == 'file': target_path = target_path.joinpath('meow') target_path.touch() symlink_path.symlink_to(target_path) elif target is None: symlink_path.symlink_to(target_path) # Break symlink target_path.rmdir() if expected_err: with pytest.raises(expected_err): remove_dir_and_target(symlink_path) else: remove_dir_and_target(symlink_path) for path in [symlink_path, target_path]: assert path.exists() is False assert path.is_symlink() is False @pytest.mark.parametrize( 'func', [remove_dir_and_target, remove_dir_or_file] ) def test_remove_relative(func: Callable, tmp_path: Path): """Test that you cannot use remove_dir_and_target() or remove_dir_or_file() on relative paths. When removing a path, we want to be absolute-ly sure where it is! """ # cd to temp dir in case we accidentally succeed in deleting the path os.chdir(tmp_path) with pytest.raises(ValueError) as cm: func('foo/bar') assert 'Path must be absolute' in str(cm.value) def test_remove_dir_or_file(tmp_path: Path): """Test remove_dir_or_file()""" a_file = tmp_path.joinpath('fyle') a_file.touch() assert a_file.exists() remove_dir_or_file(a_file) assert a_file.exists() is False a_symlink = tmp_path.joinpath('simlynk') a_file.touch() a_symlink.symlink_to(a_file) assert a_symlink.is_symlink() remove_dir_or_file(a_symlink) assert a_symlink.is_symlink() is False assert a_file.exists() a_dir = tmp_path.joinpath('der') # Add contents to check whole tree is removed sub_dir = a_dir.joinpath('sub_der') sub_dir.mkdir(parents=True) sub_dir.joinpath('fyle').touch() assert a_dir.exists() remove_dir_or_file(a_dir) assert a_dir.exists() is False def test_remove_empty_parents(tmp_path: Path): """Test that _remove_empty_parents() doesn't remove parents containing a sibling.""" # -- Setup -- id_ = 'foo/bar/baz/qux' path = tmp_path.joinpath(id_) tmp_path.joinpath('foo/bar/baz').mkdir(parents=True) # Note qux does not exist, but that shouldn't matter sibling_reg = 'foo/darmok' sibling_path = tmp_path.joinpath(sibling_reg) sibling_path.mkdir() # -- Test -- remove_empty_parents(path, id_) assert tmp_path.joinpath('foo/bar').exists() is False assert tmp_path.joinpath('foo').exists() is True # Check it skips non-existent dirs, and stops at the right place too tmp_path.joinpath('foo/bar').mkdir() sibling_path.rmdir() remove_empty_parents(path, id_) assert tmp_path.joinpath('foo').exists() is False assert tmp_path.exists() is True @pytest.mark.parametrize( 'path, tail, exc_msg', [ pytest.param( 'meow/foo/darmok', 'foo/darmok', "path must be absolute", id="relative path" ), pytest.param( '/meow/foo/darmok', '/foo/darmok', "tail must not be an absolute path", id="absolute tail" ), pytest.param( '/meow/foo/darmok', 'foo/jalad', "path '/meow/foo/darmok' does not end with 'foo/jalad'", id="tail not in path" ) ] ) def test_remove_empty_parents_bad(path: str, tail: str, exc_msg: str): """Test that _remove_empty_parents() fails appropriately with bad args.""" with pytest.raises(ValueError) as exc: remove_empty_parents(path, tail) assert exc_msg in str(exc.value) @pytest.mark.parametrize( 'dirs, expected', [ ([" "], set()), (["foo", "bar"], {"foo", "bar"}), (["foo:bar", "baz/*"], {"foo", "bar", "baz/*"}), ([" :foo :bar:"], {"foo", "bar"}), (["foo/:bar//baz "], {"foo/", "bar/baz"}), ([".foo", "..bar", " ./gah"], {".foo", "..bar", "gah"}) # Note '..bar' is a valid filename (doesn't point to parent dir) ] ) def test_parse_rm_dirs(dirs: List[str], expected: Set[str]): """Test parse_dirs()""" assert parse_rm_dirs(dirs) == expected @pytest.mark.parametrize( 'dirs, err_msg', [ (["foo:/bar"], "--rm option cannot take absolute paths"), ([".."], "cannot take paths that point to the run directory or above"), (["foo:../bar"], "cannot take paths that point to the run directory or above"), (["foo:bar/../../gah"], "cannot take paths that point to the run directory or above"), ] ) def test_parse_rm_dirs__bad(dirs: List[str], err_msg: str): """Test parse_dirs() with bad inputs""" with pytest.raises(InputError) as exc: parse_rm_dirs(dirs) assert err_msg in str(exc.value) @pytest.mark.parametrize( 'expect, files, runN', [ param(1, [], False, id='1st run (from filenames)'), param(2, ['run1'], False, id='2nd run (from filenames)'), param( 1000, ['run20', 'run400', 'run999'], False, id='1000th run (from filenames)' ), param( 6, ['run1', 'run5'], False, id='Non-sequential (from filenames)'), param(2, ['run1'], True, id='2nd run (from symlink)'), param(100, ['run1', 'run99'], True, id='100th run (from symlink)'), param(42, ['foo', 'foo12', 'run41'], False, id='with dirs not runX') ] ) def test_get_next_rundir_number(tmp_path, expect, files, runN): for file_ in files: (tmp_path / file_).mkdir() if runN: (tmp_path / 'runN').symlink_to(tmp_path / files[-1]) assert get_next_rundir_number(tmp_path) == expect @pytest.mark.parametrize( 'name, id_, src', ( param('my_workflow1', 'my_workflow1', False, id='--no-run-name'), param('my_workflow2', 'my_workflow2/run22', False, id='installed'), param( 'my_workflow3', 'my_workflow3/foo', False, id='--run-name="foo"'), param('my_workflow4', 'my_workflow4', True, id='not installed'), ) ) def test_get_workflow_name_from_id( tmp_path, monkeypatch, name: str, id_: str, src: bool ) -> None: """It gets the correct name. args: name: Workflow name id: Workflow id src: Is this workflow a source or installed workflow. """ monkeypatch.setattr( 'cylc.flow.pathutil.get_cylc_run_dir', lambda: tmp_path) (tmp_path / name).mkdir(exist_ok=True) if not src: (tmp_path / name / '_cylc-install').mkdir(exist_ok=True) (tmp_path / id_).mkdir(exist_ok=True) result = get_workflow_name_from_id(id_) assert result == name @pytest.mark.parametrize('abs_path', [True, False]) @pytest.mark.parametrize('path1, path2, expected', [ param('/foo/bar/baz', '/foo/bar', True, id="child"), param('/foo/bar', '/foo/bar/baz', False, id="parent"), param('/foo/bar', '/foo/bar', True, id="same-path"), param('/cat/dog', '/hat/bog', False, id="different"), param('/a/b/c/../x/y', '/a/b/x', True, id="trickery"), ]) def test_is_relative_to(abs_path, path1, path2, expected): if not abs_path: # absolute & relative versions of same test path1.lstrip('/') path2.lstrip('/') assert is_relative_to(path1, path2) == expected cylc-flow-8.7.0/tests/unit/test_command_validation.py0000664000175000017500000000665515264420307023212 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import re import pytest from cylc.flow.command_validation import ( ERR_OPT_FLOW_COMBINE, ERR_OPT_FLOW_VAL_INT_NEW_NONE, flow_opts, is_tasks, ) from cylc.flow.cycling.loader import ISO8601_CYCLING_TYPE from cylc.flow.exceptions import InputError from cylc.flow.flow_mgr import FLOW_NEW, FLOW_NONE from cylc.flow.id import TaskTokens @pytest.mark.parametrize('flow_strs, expected_msg', [ ([FLOW_NEW, '1'], ERR_OPT_FLOW_COMBINE.format(FLOW_NEW)), ([FLOW_NONE, '1'], ERR_OPT_FLOW_COMBINE.format(FLOW_NONE)), ([FLOW_NONE, FLOW_NEW], ERR_OPT_FLOW_COMBINE.format(FLOW_NONE)), (['a'], ERR_OPT_FLOW_VAL_INT_NEW_NONE), (['1', 'a'], ERR_OPT_FLOW_VAL_INT_NEW_NONE), ]) async def test_trigger_invalid(flow_strs, expected_msg): """Ensure invalid flow values are rejected during command validation.""" with pytest.raises(InputError) as exc_info: flow_opts(flow_strs, False) assert str(exc_info.value) == expected_msg async def test_is_tasks(set_cycling_type): set_cycling_type(ISO8601_CYCLING_TYPE) # tokens should be parsed assert is_tasks({'20000101T0000Z/a', '20010101T0000Z/b'}) == { TaskTokens('20000101T0000Z', 'a'), TaskTokens('20010101T0000Z', 'b'), } # cycle points should be standardised unless they are globs assert is_tasks({'2000/a', '20010101/b', '*/c', '[23]000/d'}) == { TaskTokens('20000101T0000Z', 'a'), TaskTokens('20010101T0000Z', 'b'), TaskTokens('*', 'c'), TaskTokens('[23]000', 'd'), } # the namespace should default to "root" unless provided assert is_tasks({'*', '2000'}) == { TaskTokens('*', 'root'), TaskTokens('20000101T0000Z', 'root'), } # invalid IDs result in errors with pytest.raises(InputError, match='Invalid ID: //, ///, ////'): is_tasks({'//', '///', '////', '2000/a'}) # last ID is valid # invalid cycle points result in errors with pytest.raises( InputError, match=re.escape('Invalid cycle point: (42)/answer, 2000Z, abc'), ): is_tasks({'(42)/answer', '2000Z', 'abc', '2000/a'}) # last ID is valid # job IDs result in errors with pytest.raises( InputError, match=re.escape('This command does not take job IDs: */b/02, 1/a/01'), ): is_tasks({'1/a/01', '*/b/02'}) # combinations of errors are reported with pytest.raises( InputError, match=( re.escape('Invalid ID: ///, ////') + re.escape('\nInvalid cycle point: 200Z, abc') + re.escape('\nThis command does not take job IDs: */b/02, 1/a/01') ), ): is_tasks({'///', '////', '200Z', 'abc', '1/a/01', '*/b/02'}) cylc-flow-8.7.0/tests/unit/test_workflow_events.py0000664000175000017500000000657615264420307022622 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ tests for functions in cylc.flow.workflow_events.py """ import pytest from types import SimpleNamespace from cylc.flow.workflow_events import ( WorkflowEventHandler, get_template_variables, process_mail_footer, ) @pytest.mark.parametrize( 'key, workflow_cfg, glbl_cfg, expected', [ ('handlers', True, True, ['stall']), ('handlers', False, True, None), ('handlers', False, False, None), ('mail events', True, True, []), ('mail events', False, True, ['abort']), ('mail events', False, False, None), ('from', True, True, 'docklands@railway'), ('from', False, True, 'highway@mixture'), ('from', False, False, None), ('abort on workflow timeout', True, True, True), ('abort on workflow timeout', False, True, True), ('abort on workflow timeout', False, False, False), ] ) def test_get_events_handler( mock_glbl_cfg, key, workflow_cfg, glbl_cfg, expected ): """Test order of precedence for getting event handler configuration.""" mock_glbl_cfg( 'cylc.flow.workflow_events.glbl_cfg', ( ''' [scheduler] [[mail]] from = highway@mixture [[events]] abort on workflow timeout = True mail events = abort ''' if glbl_cfg else '' ), ) config = SimpleNamespace() config.cfg = { 'scheduler': { 'events': {'handlers': ['stall'], 'mail events': []}, 'mail': {'from': 'docklands@railway'}, } if workflow_cfg else {'events': {}} } assert WorkflowEventHandler.get_events_conf(config, key) == expected def test_process_mail_footer(caplog, log_filter): schd = SimpleNamespace( config=SimpleNamespace(cfg={'meta': {}}), host='myhost', owner='me', server=SimpleNamespace(port=42), uuid_str=None, workflow='my_workflow', ) template_vars = get_template_variables(schd, '', '') # test all variables assert process_mail_footer( '%(host)s|%(port)s|%(owner)s|%(suite)s|%(workflow)s', template_vars ) == 'myhost|42|me|my_workflow|my_workflow\n' assert not log_filter(contains='Ignoring bad mail footer template') # test invalid variable assert process_mail_footer('%(invalid)s', template_vars) == '' assert log_filter(contains='Ignoring bad mail footer template') # test broken template caplog.clear() assert process_mail_footer('%(invalid)s', template_vars) == '' assert log_filter(contains='Ignoring bad mail footer template') cylc-flow-8.7.0/tests/unit/tui/0000775000175000017500000000000015264420307016536 5ustar alastairalastaircylc-flow-8.7.0/tests/unit/tui/test_overlay.py0000664000175000017500000000536515264420307021641 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import ast from unittest.mock import Mock import pytest import urwid from cylc.flow.tui.app import BINDINGS import cylc.flow.tui.overlay from cylc.flow.workflow_status import WorkflowStatus @pytest.fixture def overlay_functions(): """List overlay all generator functions in cylc.flow.tui.overlay Uses ast to parse functions out of the module. """ filepath = cylc.flow.tui.overlay.__file__ with open(filepath, 'r') as source_file: tree = ast.parse(source_file.read(), filename=filepath) return [ getattr(cylc.flow.tui.overlay, obj.name) for obj in tree.body if isinstance(obj, ast.FunctionDef) and not obj.name.startswith('_') ] def test_interface(overlay_functions): """Ensure all overlay functions have the correct signature.""" for function in overlay_functions: # mock up an app object to keep things working app = Mock( filters={'tasks': {}, 'workflows': {'id': '.*'}}, bindings=BINDINGS, tree_walker=Mock( get_focus=Mock( return_value=[ Mock( get_node=Mock( return_value=Mock( get_value=lambda: { 'id_': '~u/a', 'type_': 'workflow', 'data': { 'status': WorkflowStatus.RUNNING.value, }, } ) ) ) ] ) ) ) widget, options = function(app) assert isinstance(widget, urwid.Widget) assert isinstance(options, dict) assert 'width' in options assert 'height' in options cylc-flow-8.7.0/tests/unit/tui/__init__.py0000664000175000017500000000000015264420307020635 0ustar alastairalastaircylc-flow-8.7.0/tests/unit/tui/test_data.py0000664000175000017500000000346615264420307021071 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow import __version__ import cylc.flow.tui.data from cylc.flow.tui.data import ( _QUERY, VersionIncompat, generate_mutation, get_query, ) def test_generate_mutation(monkeypatch): """It should produce a GraphQL mutation with the args filled in.""" arg_types = { 'foo': 'String!', 'bar': '[Int]' } monkeypatch.setattr(cylc.flow.tui.data, 'ARGUMENT_TYPES', arg_types) assert generate_mutation( 'my_mutation', {'foo': 'foo', 'bar': 'bar', 'user': 'user'} ) == ''' mutation($foo: String!, $bar: [Int]) { my_mutation (foos: $foo, bars: $bar) { result } } ''' def test_query_compat(): """It should return a query or raise an exception.""" # old version - unsupported with pytest.raises(VersionIncompat, match='6.11.4'): get_query('6.11.4') # current version - supported assert get_query(__version__) == _QUERY # future version - supported assert get_query('9.0.0') == _QUERY cylc-flow-8.7.0/tests/unit/tui/test_util.py0000664000175000017500000002374515264420307021137 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from datetime import ( datetime, timedelta, timezone ) from unittest.mock import Mock import pytest from cylc.flow.tui.util import ( JOB_ICON, TASK_ICONS, render_node, compute_tree, get_task_icon ) from cylc.flow.wallclock import ( get_time_string, get_current_time_string ) def test_render_node__job_info(): """It renders job information nodes.""" assert render_node( None, {'a': 1, 'b': 2}, 'job_info' ) == [ 'a 1\n', 'b 2' ] def test_render_node__job(): """It renders job nodes.""" assert render_node( None, {'state': 'succeeded', 'submitNum': 1}, 'job' ) == [ '#01 ', [('job_succeeded', JOB_ICON)] ] def test_render_node__task__succeeded(): """It renders tasks.""" node = Mock() node.get_child_node = lambda _: None assert render_node( node, { 'name': 'foo', 'state': 'succeeded', 'isHeld': False, 'isQueued': False, 'isRunahead': False, 'flowNums': '[1]', }, 'task' ) == [ ('body', TASK_ICONS['succeeded']), ' ', ('body', 'foo'), ] def test_render_node__task__running(): """It renders running tasks.""" child = Mock() child.get_value = lambda: {'data': { 'startedTime': get_current_time_string(), 'state': 'running' }} node = Mock() node.get_child_node = lambda _: child assert render_node( node, { 'name': 'foo', 'state': 'running', 'isHeld': False, 'isQueued': False, 'isRunahead': False, 'flowNums': '[1]', 'task': {'meanElapsedTime': 100} }, 'task' ) == [ ('body', TASK_ICONS['running']), ' ', ('job_running', JOB_ICON), ' ', ('body', 'foo'), ] def test_render_node__family(): """It renders families.""" assert render_node( None, { 'state': 'succeeded', 'isHeld': False, 'isQueued': False, 'isRunahead': False, 'id': 'myid' }, 'family' ) == [ [('body', TASK_ICONS['succeeded'])], ' ', 'myid' ] def test_render_node__cycle_point(): """It renders cycle points.""" assert render_node( None, {'id': 'myid'}, 'cycle_point' ) == 'myid' @pytest.mark.parametrize( 'status,is_held,is_queued,is_runahead,start_offset,mean_time,expected', [ # task states ('waiting', False, False, False, None, None, ['○']), ('submitted', False, False, False, None, None, ['⊙']), ('running', False, False, False, None, None, ['⊙']), ('succeeded', False, False, False, None, None, ['●']), ('submit-failed', False, False, False, None, None, ['⊘']), ('failed', False, False, False, None, None, ['⊗']), # progress indicator ('running', False, False, False, 0, 100, ['⊙']), ('running', False, False, False, 25, 100, ['◔']), ('running', False, False, False, 50, 100, ['◑']), ('running', False, False, False, 75, 100, ['◕']), ('running', False, False, False, 100, 100, ['◕']), # is-held modifier ('waiting', True, False, False, None, None, ['\u030E', '○']), # is-queued modifier ('waiting', False, True, False, None, None, ['\u033F', '○']), # is-runahead modifier ('waiting', False, False, True, None, None, ['\u0340', '○']), ] ) def test_get_task_icon( status, is_held, is_queued, is_runahead, start_offset, mean_time, expected): """It renders task icons.""" start_time = None if start_offset is not None: start_time = get_time_string( datetime.now(timezone.utc) - timedelta(seconds=start_offset) ) assert ( ( get_task_icon( status, is_held=is_held, is_queued=is_queued, is_runahead=is_runahead, colour='custom', start_time=start_time, mean_time=mean_time, ) ) ) == [('custom', char) for char in expected] def test_compute_tree(): """It computes a tree in the right structure for urwid. Note this test doesn't use full data or proper ids because it's purpose is not to test the GraphQL interface but to ensure the assumptions made by compute_tree check out. """ tree = compute_tree({ 'workflows': [{ 'id': 'workflow id', 'port': 1234, 'cyclePoints': [ { 'id': '1/family-suffix', 'cyclePoint': '1' } ], 'familyProxies': [ { # top level family 'name': 'FOO', 'id': '1/FOO', 'cyclePoint': '1', 'firstParent': {'name': 'root', 'id': '1/root'} }, { # nested family 'name': 'FOOT', 'id': '1/FOOT', 'cyclePoint': '1', 'firstParent': {'name': 'FOO', 'id': '1/FOO'} }, ], 'taskProxies': [ { # top level task 'name': 'pub', 'id': '1/pub', 'firstParent': {'name': 'root', 'id': '1/root'}, 'cyclePoint': '1', 'jobs': [] }, { # child task (belongs to family) 'name': 'fan', 'id': '1/fan', 'firstParent': {'name': 'fan', 'id': '1/fan'}, 'cyclePoint': '1', 'jobs': [] }, { # nested child task (belongs to incestuous family) 'name': 'fool', 'id': '1/fool', 'firstParent': {'name': 'FOOT', 'id': '1/FOOT'}, 'cyclePoint': '1', 'jobs': [] }, { # a task which has jobs 'name': 'worker', 'id': '1/worker', 'firstParent': {'name': 'root', 'id': '1/root'}, 'cyclePoint': '1', 'jobs': [ {'id': '1/worker/03', 'submitNum': '3'}, {'id': '1/worker/02', 'submitNum': '2'}, {'id': '1/worker/01', 'submitNum': '1'} ] } ] }] }) # the root node assert tree['type_'] == 'root' assert tree['id_'] == 'root' assert len(tree['children']) == 1 # the workflow node workflow = tree['children'][0] assert workflow['type_'] == 'workflow' assert workflow['id_'] == 'workflow id' assert set(workflow['data']) == { # whatever if present on the node should end up in data 'cyclePoints', 'familyProxies', 'id', 'port', 'taskProxies' } assert len(workflow['children']) == 1 # the cycle point node cycle = workflow['children'][0] assert cycle['type_'] == 'cycle' assert cycle['id_'] == '//1' assert list(cycle['data']) == [ 'id', 'cyclePoint' ] assert len(cycle['children']) == 3 assert [ node['id_'] for node in cycle['children'] ] == [ # test alphabetical sorting '1/FOO', '1/pub', '1/worker' ] # test family node family = cycle['children'][0] assert family['type_'] == 'family' assert family['id_'] == '1/FOO' assert list(family['data']) == [ 'name', 'id', 'cyclePoint', 'firstParent' ] assert len(family['children']) == 1 # test nested family nested_family = family['children'][0] assert nested_family['type_'] == 'family' assert nested_family['id_'] == '1/FOOT' assert list(nested_family['data']) == [ 'name', 'id', 'cyclePoint', 'firstParent' ] assert len(nested_family['children']) == 1 # test task task = nested_family['children'][0] assert task['type_'] == 'task' assert task['id_'] == '1/fool' assert list(task['data']) == [ 'name', 'id', 'firstParent', 'cyclePoint', 'jobs' ] assert len(task['children']) == 0 # test task with jobs task = cycle['children'][-1] assert [ # test sorting job['id_'] for job in task['children'] ] == [ '1/worker/03', '1/worker/02', '1/worker/01' ] # test job job = task['children'][0] assert job['type_'] == 'job' assert job['id_'] == '1/worker/03' assert list(job['data']) == [ 'id', 'submitNum' ] assert len(job['children']) == 1 # test job info job_info = job['children'][0] assert job_info['type_'] == 'job_info' assert job_info['id_'] == '1/worker/03_info' assert list(job_info['data']) == [ 'id', 'submitNum' ] assert len(job_info['children']) == 0 cylc-flow-8.7.0/tests/unit/test_loggingutil.py0000664000175000017500000002235015264420307021674 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging import re import sys from io import TextIOWrapper from pathlib import Path from time import sleep from typing import Callable, cast from unittest import mock import pytest from pytest import param from cylc.flow import LOG from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.cfgspec.globalcfg import GlobalConfig from cylc.flow.loggingutil import ( CylcLogFormatter, RotatingLogFileHandler, get_reload_start_number, get_sorted_logs_by_time, patch_log_level, set_timestamps, ) @pytest.fixture def rotating_log_file_handler(tmp_path: Path): """Fixture to create a RotatingLogFileHandler for testing.""" log_file = tmp_path / "log" log_file.touch() handler = cast('RotatingLogFileHandler', None) orig_stream = cast('TextIOWrapper', None) def inner( *args, level: int = logging.INFO, **kwargs ) -> RotatingLogFileHandler: nonlocal handler, orig_stream handler = RotatingLogFileHandler(log_file, *args, **kwargs) orig_stream = handler.stream # next line is important as pytest can have a "Bad file descriptor" # due to a FileHandler with default "a" (pytest tries to r/w). handler.mode = "a+" # enable the logger LOG.setLevel(level) LOG.addHandler(handler) return handler yield inner # clean up LOG.setLevel(logging.INFO) LOG.removeHandler(handler) @mock.patch("cylc.flow.loggingutil.glbl_cfg") def test_value_error_raises_system_exit( mocked_glbl_cfg, rotating_log_file_handler ): """Test that a ValueError when writing to a log stream won't result in multiple exceptions (what could lead to infinite loop in some occasions. Instead, it **must** raise a SystemExit""" # mock objects used when creating the file handler mocked = mock.MagicMock() mocked_glbl_cfg.return_value = mocked mocked.get.return_value = 100 file_handler = rotating_log_file_handler(level=logging.INFO) # Disable raising uncaught exceptions in logging, due to file # handler using stdin.fileno. See the following links for more. # https://github.com/pytest-dev/pytest/issues/2276 & # https://github.com/pytest-dev/pytest/issues/1585 logging.raiseExceptions = False # first message will initialize the stream and the handler LOG.info("What could go") # here we change the stream of the handler file_handler.stream = mock.MagicMock() file_handler.stream.seek = mock.MagicMock() # in case where file_handler.stream.seek.side_effect = ValueError with pytest.raises(SystemExit): # next call will call the emit method and use the mocked stream LOG.info("wrong?!") # clean up logging.raiseExceptions = True @pytest.mark.parametrize( 'dev_info, expect', [ param( True, ( '%(asctime)s %(levelname)-2s - [%(module)s:%(lineno)d] - ' '%(message)s' ), id='dev_info=True' ), param( False, '%(asctime)s %(levelname)-2s - %(message)s', id='dev_info=False' ) ] ) def test_CylcLogFormatter__init__dev_info(dev_info, expect): """dev_info switch changes the logging format string.""" formatter = CylcLogFormatter(dev_info=dev_info) assert formatter._fmt == expect def test_update_log_archive(tmp_run_dir: Callable): """Test log archive performs as expected""" run_dir = tmp_run_dir('some_workflow') log_dir = Path(run_dir / 'log' / 'scheduler') log_dir.mkdir(exist_ok=True, parents=True) log_file = log_dir.joinpath('log') log_file.touch() log_object = RotatingLogFileHandler( log_file, no_detach=False, restart_num=0 ) for i in range(1, 5): (log_dir / f'{i:02d}-start-{i:02d}.log').touch() log_object.update_log_archive(2) assert list((log_dir.iterdir())).sort() == [ Path(log_dir / 'log'), Path(log_dir / '03-start-03.log'), Path(log_dir / '04-start-04.log')].sort() def test_get_sorted_logs_by_time(tmp_run_dir: Callable): run_dir = tmp_run_dir('some_workflow') config_log_dir = Path(run_dir / 'log' / 'config') config_log_dir.mkdir(exist_ok=True, parents=True) for file in ['01-start-01.cylc', '02-start-01.cylc', '03-restart-02.cylc', '04-reload-02.cylc']: (config_log_dir / file).touch() # Sleep required to ensure modification times are sufficiently # different for sort sleep(0.1) loggies = get_sorted_logs_by_time(config_log_dir, "*.cylc") assert loggies == [f'{config_log_dir}/01-start-01.cylc', f'{config_log_dir}/02-start-01.cylc', f'{config_log_dir}/03-restart-02.cylc', f'{config_log_dir}/04-reload-02.cylc'] def test_get_reload_number(tmp_run_dir: Callable): run_dir = tmp_run_dir('some_reloaded_workflow') config_log_dir = Path(run_dir / 'log' / 'config') config_log_dir.mkdir(exist_ok=True, parents=True) for file in [ '01-start-01.cylc', '02-reload-01.cylc', '03-restart-02.cylc', '04-restart-02.cylc' ]: (config_log_dir / file).touch() # Sleeps required to ensure modification times are sufficiently # different for sort sleep(0.1) config_logs = get_sorted_logs_by_time(config_log_dir, "*.cylc") assert get_reload_start_number(config_logs) == '02' def test_get_reload_number_no_logs(tmp_run_dir: Callable): run_dir = tmp_run_dir('another_reloaded_workflow') config_log_dir = Path(run_dir / 'log' / 'config') config_log_dir.mkdir(exist_ok=True, parents=True) config_logs = get_sorted_logs_by_time(config_log_dir, "*.cylc") assert get_reload_start_number(config_logs) == '01' def test_set_timestamps(capsys): """The enable and disable timestamp methods do what they say""" # Setup log handler log_handler = logging.StreamHandler(sys.stderr) log_handler.setFormatter(CylcLogFormatter()) LOG.addHandler(log_handler) # Log some messages with timestamps on or off: LOG.warning('foo') set_timestamps(LOG, False) LOG.warning('bar') set_timestamps(LOG, True) LOG.warning('baz') # Check 1st and 3rd error have something timestamp-like: errors = capsys.readouterr().err.split('\n') assert re.match('^[0-9]{4}', errors[0]) assert re.match('^WARNING - bar', errors[1]) assert re.match('^[0-9]{4}', errors[2]) LOG.removeHandler(log_handler) def test_log_emit_and_glbl_cfg( monkeypatch: pytest.MonkeyPatch, rotating_log_file_handler ): """Test that log calls do not access the global config object. Doing so can have the side effect of expanding the global config object so should be avoided - see https://github.com/cylc/cylc-flow/issues/6244 """ rotating_log_file_handler(level=logging.DEBUG) mock_cfg = mock.Mock(spec=GlobalConfig) monkeypatch.setattr( 'cylc.flow.cfgspec.globalcfg.GlobalConfig', mock.Mock( spec=GlobalConfig, get_inst=lambda *a, **k: mock_cfg ) ) # Check mocking is correct: glbl_cfg().get(['kinesis']) assert mock_cfg.get.call_args_list == [mock.call(['kinesis'])] mock_cfg.reset_mock() # Check log emit does not access global config object: LOG.debug("Entering zero gravity") assert mock_cfg.get.call_args_list == [] def test_patch_log_level(caplog: pytest.LogCaptureFixture): """Test patch_log_level temporarily changes the log level.""" caplog.set_level(logging.DEBUG) logger = logging.getLogger("forest") logger.setLevel(logging.ERROR) logger.info("nope") assert not caplog.records with patch_log_level(logger, logging.INFO): LOG.info("yep") assert len(caplog.records) == 1 logger.info("nope") assert len(caplog.records) == 1 def test_patch_log_level__reset(caplog: pytest.LogCaptureFixture): """Test patch_log_level resets the log level correctly after use, not affected by the parent logger level - see https://github.com/cylc/cylc-flow/pull/6327 """ caplog.set_level(logging.ERROR) logger = logging.getLogger("woods") assert logger.level == logging.NOTSET with patch_log_level(logger, logging.INFO): logger.info("emitted but not captured, as caplog is at ERROR level") assert not caplog.records caplog.set_level(logging.INFO) logger.info("yep") assert len(caplog.records) == 1 assert logger.level == logging.NOTSET cylc-flow-8.7.0/tests/unit/post_install/0000775000175000017500000000000015264420307020450 5ustar alastairalastaircylc-flow-8.7.0/tests/unit/post_install/test_log_vc_info.py0000664000175000017500000002635615264420307024361 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import json import logging from pathlib import Path import pytest from pytest import MonkeyPatch, TempPathFactory from secrets import token_hex import shutil import subprocess from typing import Any, Callable, Tuple from unittest.mock import Mock from cylc.flow.plugins.log_vc_info import ( INFO_FILENAME, VCSNotInstalledError, _get_git_commit, _run_cmd, get_status, get_vc_info, main, write_diff, ) from cylc.flow.workflow_files import WorkflowFiles Fixture = Any BASIC_FLOW_1 = """ [scheduling] [[graph]] R1 = foo """ BASIC_FLOW_2 = """ [scheduling] [[graph]] R1 = bar """ BASIC_FLOW_3 = """ [scheduling] [[graph]] R1 = foobar """ require_git = pytest.mark.skipif( shutil.which('git') is None, reason="git is not installed" ) require_svn = pytest.mark.skipif( shutil.which('svn') is None, reason="svn is not installed" ) @pytest.fixture(scope='module') def git_source_repo(tmp_path_factory: TempPathFactory) -> Tuple[str, str]: """Init a git repo for a workflow source dir. The repo has uncommitted changes. This dir is reused by all tests requesting it in this module. Returns (source_dir_path, commit_hash) """ source_dir: Path = tmp_path_factory.getbasetemp() / 'git_repo' source_dir.mkdir() subprocess.run(['git', 'init'], cwd=source_dir, check=True) flow_file = source_dir / 'flow.cylc' flow_file.write_text(BASIC_FLOW_1) subprocess.run(['git', 'add', '-A'], cwd=source_dir, check=True) subprocess.run( ['git', 'commit', '-am', '"Initial commit"'], cwd=source_dir, check=True, capture_output=True) # Overwrite file to introduce uncommitted changes: flow_file.write_text(BASIC_FLOW_2) # Also add new file: (source_dir / 'gandalf.md').touch() commit_sha = subprocess.run( ['git', 'rev-parse', 'HEAD'], cwd=source_dir, check=True, capture_output=True, text=True ).stdout.splitlines()[0] return (str(source_dir), commit_sha) @pytest.fixture(scope='module') def svn_source_repo(tmp_path_factory: TempPathFactory) -> Tuple[str, str, str]: """Init an svn repo & working copy for a workflow source dir. The working copy has a flow.cylc file with uncommitted changes. This dir is reused by all tests requesting it in this module. Returns (source_dir_path, repository_UUID, repository_path) """ tmp_path: Path = tmp_path_factory.getbasetemp() repo = tmp_path.joinpath('svn_repo') subprocess.run( ['svnadmin', 'create', 'svn_repo'], cwd=tmp_path, check=True) uuid = subprocess.run( ['svnlook', 'uuid', repo], check=True, capture_output=True, text=True ).stdout.splitlines()[0] project_dir = tmp_path.joinpath('project') project_dir.mkdir() project_dir.joinpath('flow.cylc').write_text(BASIC_FLOW_1) subprocess.run( ['svn', 'import', project_dir, f'file://{repo}/project/trunk', '-m', '"Initial import"'], check=True) source_dir = tmp_path.joinpath('svn_working_copy') subprocess.run( ['svn', 'checkout', f'file://{repo}/project/trunk', source_dir], check=True) flow_file = source_dir.joinpath('flow.cylc') # Overwrite file to introduce uncommitted changes: flow_file.write_text(BASIC_FLOW_2) return (str(source_dir), uuid, str(repo)) @require_git def test_get_git_commit(git_source_repo: Tuple[str, str]): """Test get_git_commit()""" source_dir, commit_sha = git_source_repo assert _get_git_commit(source_dir) == commit_sha @require_git def test_get_status_git(git_source_repo: Tuple[str, str]): """Test get_status() for a git repo""" source_dir, commit_sha = git_source_repo assert get_status('git', source_dir) == [ " M flow.cylc", "?? gandalf.md" ] @require_git def test_get_vc_info_git(git_source_repo: Tuple[str, str]): """Test get_vc_info() for a git repo""" source_dir, commit_sha = git_source_repo vc_info = get_vc_info(source_dir) assert vc_info is not None expected = [ ('version control system', "git"), ('repository version', f"{commit_sha[:7]}-dirty"), ('commit', commit_sha), ('working copy root path', source_dir), ('status', [ " M flow.cylc", "?? gandalf.md" ]) ] assert list(vc_info.items()) == expected @require_git def test_write_diff_git(git_source_repo: Tuple[str, str], tmp_path: Path): """Test write_diff() for a git repo""" source_dir, _ = git_source_repo run_dir = tmp_path / 'run_dir' (run_dir / WorkflowFiles.LogDir.DIRNAME).mkdir(parents=True) (run_dir / WorkflowFiles.LogDir.DIRNAME / WorkflowFiles.LogDir.INSTALL).mkdir(exist_ok=True) with open(run_dir / WorkflowFiles.LogDir.DIRNAME / WorkflowFiles.LogDir.INSTALL / "01-install.log", "w") as _: pass diff_file = write_diff('git', source_dir, run_dir) assert diff_file.parts[-1] == "01-uncommitted.diff" diff_lines = diff_file.read_text().splitlines() assert diff_lines[0].startswith("# Auto-generated diff") assert ( "diff --git a/flow.cylc b/flow.cylc" in diff_lines or "diff --git c/flow.cylc w/flow.cylc" in diff_lines ) for line in ( "- R1 = foo", "+ R1 = bar", ): assert line in diff_lines flow_file = Path(source_dir) / 'flow.cylc' flow_file.write_text(BASIC_FLOW_3) with open(run_dir / WorkflowFiles.LogDir.DIRNAME / WorkflowFiles.LogDir.INSTALL / "02-reinstall.log", "w") as _: pass diff_file = write_diff('git', source_dir, run_dir) assert diff_file.parts[-1] == "02-uncommitted.diff" diff_lines = diff_file.read_text().splitlines() assert diff_lines[0].startswith("# Auto-generated diff") assert ( "diff --git a/flow.cylc b/flow.cylc" in diff_lines or "diff --git c/flow.cylc w/flow.cylc" in diff_lines ) for line in ( "- R1 = foo", "+ R1 = foobar", ): assert line in diff_lines @require_git def test_main_git(git_source_repo: Tuple[str, str], tmp_run_dir: Callable): """Test the written JSON info file.""" source_dir, _ = git_source_repo run_dir: Path = tmp_run_dir('frodo') main(source_dir, None, run_dir) with open( run_dir / WorkflowFiles.LogDir.DIRNAME / WorkflowFiles.LogDir.VERSION / INFO_FILENAME, 'r' ) as f: loaded = json.loads(f.read()) assert isinstance(loaded, dict) assert loaded['version control system'] == 'git' assert isinstance(loaded['status'], list) assert len(loaded['status']) == 2 @require_svn def test_get_vc_info_svn(svn_source_repo: Tuple[str, str, str]): """Test get_vc_info() for an svn working copy""" source_dir, uuid, repo_path = svn_source_repo vc_info = get_vc_info(source_dir) assert vc_info is not None expected = [ ('version control system', "svn"), ('working copy root path', str(source_dir)), ('url', f"file://{repo_path}/project/trunk"), ('repository uuid', uuid), ('revision', "1"), ('status', ["M flow.cylc"]) ] assert list(vc_info.items()) == expected @require_svn def test_write_diff_svn(svn_source_repo: Tuple[str, str, str], tmp_path: Path): """Test write_diff() for an svn working copy""" source_dir, _, _ = svn_source_repo run_dir = tmp_path / 'run_dir' (run_dir / WorkflowFiles.LogDir.DIRNAME).mkdir(parents=True) diff_file = write_diff('svn', source_dir, run_dir) diff_lines = diff_file.read_text().splitlines() assert diff_lines[0].startswith("# Auto-generated diff") for line in (f"--- {source_dir}/flow.cylc (revision 1)", f"+++ {source_dir}/flow.cylc (working copy)", "- R1 = foo", "+ R1 = bar"): assert line in diff_lines def test_not_repo(tmp_path: Path, monkeypatch: MonkeyPatch): """Test get_vc_info() and main() for a dir that is not a supported repo""" source_dir = Path(tmp_path, 'git_repo') source_dir.mkdir() flow_file = source_dir.joinpath('flow.cylc') flow_file.write_text(BASIC_FLOW_1) mock_write_vc_info = Mock() monkeypatch.setattr('cylc.flow.plugins.log_vc_info.write_vc_info', mock_write_vc_info) mock_write_diff = Mock() monkeypatch.setattr('cylc.flow.plugins.log_vc_info.write_diff', mock_write_diff) assert get_vc_info(source_dir) is None assert main(source_dir, None, None) is False # type: ignore assert mock_write_vc_info.called is False assert mock_write_diff.called is False @require_git def test_no_base_commit_git(tmp_path: Path): """Test get_vc_info() and write_diff() for a recently init'd git source dir that does not have a base commit yet.""" source_dir = Path(tmp_path, 'new_git_repo') source_dir.mkdir() subprocess.run(['git', 'init'], cwd=source_dir, check=True) flow_file = source_dir.joinpath('flow.cylc') flow_file.write_text(BASIC_FLOW_1) run_dir = tmp_path / 'run_dir' (run_dir / WorkflowFiles.LogDir.DIRNAME).mkdir(parents=True) vc_info = get_vc_info(source_dir) assert vc_info is not None assert list(vc_info.items()) == [ ('version control system', "git"), ('working copy root path', str(source_dir)), ('status', ["?? flow.cylc"]) ] # Diff file expected to be empty (only containing comment lines), # but should work without raising diff_file = write_diff('git', source_dir, run_dir) for line in diff_file.read_text().splitlines(): assert line.startswith('#') @require_svn def test_untracked_svn_subdir( svn_source_repo: Tuple[str, str, str], log_filter ): repo_dir, *_ = svn_source_repo source_dir = Path(repo_dir, 'jar_jar_binks') source_dir.mkdir() assert get_vc_info(source_dir) is None assert log_filter(logging.WARNING, contains="$ svn info") def test_not_installed( tmp_path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, log_filter, ): """Test what happens if version control software is not installed.""" fake_vcs = token_hex(8) with pytest.raises(VCSNotInstalledError): _run_cmd(fake_vcs, [], cwd=tmp_path) monkeypatch.setattr( 'cylc.flow.plugins.log_vc_info.INFO_COMMANDS', {fake_vcs: []} ) caplog.set_level(logging.DEBUG) assert get_vc_info(tmp_path) is None assert log_filter( logging.DEBUG, contains=f"{fake_vcs} does not appear to be installed", ) cylc-flow-8.7.0/tests/unit/test_id_cli.py0000664000175000017500000004775515264420307020613 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging import os from pathlib import Path import pytest from shutil import copytree, rmtree from cylc.flow import CYLC_LOG from cylc.flow.async_util import pipe from cylc.flow.exceptions import InputError, WorkflowFilesError from cylc.flow.id import detokenise, tokenise, Tokens from cylc.flow.id_cli import ( _expand_workflow_tokens, _parse_src_path, _validate_constraint, _validate_workflow_ids, _validate_number, cli_tokenise, parse_ids_async, ) from cylc.flow.pathutil import get_cylc_run_dir from cylc.flow.workflow_files import WorkflowFiles @pytest.fixture def mock_exists(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr( 'pathlib.Path.exists', lambda path: path.name != WorkflowFiles.SUITE_RC, ) @pytest.fixture(scope='module') def abc_src_dir(tmp_path_factory): """Src dir containing three workflows, a, b & c.""" cwd_before = Path.cwd() tmp_path = tmp_path_factory.getbasetemp() os.chdir(tmp_path) for name in ('a', 'b', 'c'): Path(tmp_path, name).mkdir() Path(tmp_path, name, WorkflowFiles.FLOW_FILE).touch() yield tmp_path os.chdir(cwd_before) @pytest.mark.parametrize( 'ids_in,ids_out', [ (('a//',), ['a']), (('a//', 'a//'), ['a']), (('a//', 'b//'), ['a', 'b']), ] ) async def test_parse_ids_workflows(ids_in, ids_out, mock_exists): """It should parse workflows & tasks.""" workflows, _ = await parse_ids_async(*ids_in, constraint='workflows') assert list(workflows) == ids_out assert list(workflows.values()) == [[] for _ in workflows] @pytest.mark.parametrize( 'ids_in,ids_out', [ (('./a',), ['a']), ] ) async def test_parse_ids_workflows_src(ids_in, ids_out, abc_src_dir): """It should parse src workflows.""" workflows, _ = await parse_ids_async( *ids_in, src=True, constraint='workflows', ) assert list(workflows) == ids_out assert list(workflows.values()) == [[] for _ in workflows] @pytest.mark.parametrize( 'ids_in,ids_out', [ ( ('a//i',), {'a': ['//i']}, ), ( ('a//i', 'a//j'), {'a': ['//i', '//j']}, ), ( ('a//i', 'b//i'), {'a': ['//i'], 'b': ['//i']}, ), ( ('a//', '//i', 'b//', '//i'), {'a': ['//i'], 'b': ['//i']}, ), ] ) async def test_parse_ids_tasks(mock_exists, ids_in, ids_out): """It should parse workflow tasks in two formats.""" workflows, _ = await parse_ids_async(*ids_in, constraint='tasks') assert { workflow_id: [detokenise(tokens) for tokens in tokens_list] for workflow_id, tokens_list in workflows.items() } == ids_out @pytest.mark.parametrize( 'ids_in,ids_out', [ ( ('./a', '//i'), {'a': ['//i']} ), ( ('./a', '//i', '//j', '//k'), {'a': ['//i', '//j', '//k']} ), ] ) async def test_parse_ids_tasks_src(mock_exists, ids_in, ids_out, abc_src_dir): """It should parse workflow tasks for src workflows.""" workflows, _ = await parse_ids_async( *ids_in, constraint='tasks', src=True) assert { workflow_id: [detokenise(tokens) for tokens in tokens_list] for workflow_id, tokens_list in workflows.items() } == ids_out @pytest.mark.parametrize( 'ids_in,ids_out', [ (('a//',), {'a': []}), ( ('a//', 'b//', 'c//'), {'a': [], 'b': [], 'c': []} ), (('a//i',), {'a': ['//i']}), (('a//', '//i'), {'a': ['//i']}), ( ('a//', '//i', '//j', '//k'), {'a': ['//i', '//j', '//k']}, ), (('a//', '//i', 'b//'), {'a': ['//i'], 'b': []}), ] ) async def test_parse_ids_mixed(ids_in, ids_out, mock_exists): """It should parse mixed workflows & tasks.""" workflows, _ = await parse_ids_async(*ids_in, constraint='mixed') assert { workflow_id: [detokenise(tokens) for tokens in tokens_list] for workflow_id, tokens_list in workflows.items() } == ids_out @pytest.mark.parametrize( 'ids_in,ids_out', [ (('./a',), {'a': []}), (('./a', '//i'), {'a': ['//i']}), (('./a', '//i', '//j', '//k'), {'a': ['//i', '//j', '//k']}), ] ) async def test_parse_ids_mixed_src(ids_in, ids_out, abc_src_dir, mock_exists): """It should parse mixed workflows & tasks from src workflows.""" workflows, _ = await parse_ids_async( *ids_in, constraint='mixed', src=True ) assert { workflow_id: [detokenise(tokens) for tokens in tokens_list] for workflow_id, tokens_list in workflows.items() } == ids_out @pytest.mark.parametrize( 'ids_in,errors', [ (('a//',), False), (('a//', 'b//'), False), (('a//', 'b//', 'c//'), True), ] ) async def test_parse_ids_max_workflows(ids_in, errors, mock_exists): """It should validate input against the max_workflows constraint.""" try: await parse_ids_async( *ids_in, constraint='workflows', max_workflows=2) except InputError: if not errors: raise else: if errors: raise Exception('Should have raised InputError') @pytest.mark.parametrize( 'ids_in,errors', [ (('a//', '//i'), False), (('a//', '//i', '//j'), False), (('a//', '//i', '//j', '//k'), True), ] ) async def test_parse_ids_max_tasks(ids_in, errors, mock_exists): """It should validate input against the max_tasks constraint.""" try: await parse_ids_async(*ids_in, constraint='tasks', max_tasks=2) except InputError: if not errors: raise else: if errors: raise Exception('Should have raised InputError') async def test_parse_ids_infer_run_name(tmp_run_dir): """It should infer the run name for auto-numbered installations.""" # it doesn't do anything for a named run tmp_run_dir('foo/bar', named=True, installed=True) workflows, *_ = await parse_ids_async('foo//', constraint='workflows') assert list(workflows) == ['foo'] # it correctly identifies the latest run tmp_run_dir('bar/run1') workflows, *_ = await parse_ids_async('bar//', constraint='workflows') assert list(workflows) == ['bar/run1'] tmp_run_dir('bar/run2') workflows, *_ = await parse_ids_async('bar//', constraint='workflows') assert list(workflows) == ['bar/run2'] # it leaves the ID alone if infer_latest_runs = False workflows, *_ = await parse_ids_async( 'bar//', constraint='workflows', infer_latest_runs=False, ) assert list(workflows) == ['bar'] # Now test we can see workflows in alternate cylc-run directories # e.g. for `cylc workflow-state` or xtriggers targeting another user. cylc_run_dir = get_cylc_run_dir() alt_cylc_run_dir = cylc_run_dir + "_alt" # copy the cylc-run dir to alt location and delete the original. copytree(cylc_run_dir, alt_cylc_run_dir, symlinks=True) rmtree(cylc_run_dir) # It can no longer parse IDs in the original cylc-run location. with pytest.raises(InputError): workflows, *_ = await parse_ids_async( 'bar//', constraint='workflows', infer_latest_runs=True, ) # But it can if we specify the alternate location. workflows, *_ = await parse_ids_async( 'bar//', constraint='workflows', infer_latest_runs=True, alt_run_dir=alt_cylc_run_dir ) assert list(workflows) == ['bar/run2'] @pytest.fixture def patch_expand_workflow_tokens(monkeypatch): """Define the output of scan events.""" def _patch_expand_workflow_tokens(_ids): async def _expand_workflow_tokens_impl(tokens, match_active=True): for id_ in _ids: yield tokens.duplicate(workflow=id_) monkeypatch.setattr( 'cylc.flow.id_cli._expand_workflow_tokens_impl', _expand_workflow_tokens_impl, ) _patch_expand_workflow_tokens(['xxx']) return _patch_expand_workflow_tokens @pytest.mark.parametrize( 'ids_in,ids_out,multi_mode', [ # multi mode should be True if multiple workflows are defined (['a//'], ['a'], False), (['a//', 'b//'], ['a', 'b'], True), # or if pattern matching is used, irrespective of the number of matches (['*//'], ['xxx'], True), ] ) async def test_parse_ids_multi_mode( patch_expand_workflow_tokens, ids_in, ids_out, multi_mode, mock_exists ): """It should glob for workflows. Note: More advanced tests for this in the integration tests. """ workflows, _multi_mode = await parse_ids_async( *ids_in, constraint='workflows', match_workflows=True, ) assert list(workflows) == ids_out assert _multi_mode == multi_mode @pytest.fixture def src_dir(tmp_path): """A src dir containing a workflow called "a".""" cwd_before = Path.cwd() src_dir = (tmp_path / 'a') src_dir.mkdir() src_file = src_dir / 'flow.cylc' src_file.touch() other_dir = (tmp_path / 'blargh') other_dir.mkdir() other_file = other_dir / 'nugget' other_file.touch() os.chdir(tmp_path) yield src_dir os.chdir(cwd_before) def test_parse_src_path(src_dir, monkeypatch): """It should locate src dirs.""" # valid absolute path workflow_id, src_path, src_file_path = _parse_src_path( str(src_dir.resolve()) ) assert workflow_id == 'a' assert src_path == src_dir assert src_file_path == src_dir / 'flow.cylc' # broken absolute path with pytest.raises(InputError): workflow_id, src_path, src_file_path = _parse_src_path( str(src_dir.resolve()) + 'xyz' ) # valid ./relative path workflow_id, src_path, src_file_path = _parse_src_path('./a') assert workflow_id == 'a' assert src_path == src_dir assert src_file_path == src_dir / 'flow.cylc' # broken relative path with pytest.raises(InputError): _parse_src_path('./xxx') # relative '.' dir (invalid) with pytest.raises(WorkflowFilesError) as exc_ctx: workflow_id, src_path, src_file_path = _parse_src_path('.') assert 'No flow.cylc or suite.rc file in' in str(exc_ctx.value) # relative 'invalid/' (invalid) with pytest.raises(InputError) as exc_ctx: _parse_src_path('xxx/flow.cylc') assert 'Not a valid workflow ID or source directory' in str(exc_ctx.value) # Might be a workflow ID res = _parse_src_path('the/quick/brown/fox') assert res is None # Might be a workflow ID, even though there's a matching relative path res = _parse_src_path('a') assert res is None # Not a src directory (dir) with pytest.raises(WorkflowFilesError) as exc_ctx: _parse_src_path('./blargh') assert 'No flow.cylc or suite.rc file in' in str(exc_ctx.value) # Not a src directory (file) with pytest.raises(InputError) as exc_ctx: _parse_src_path('./blargh/nugget') assert 'Path is not a source directory' in str(exc_ctx.value) # move into the src dir monkeypatch.chdir(src_dir) # relative '.' dir (valid) workflow_id, src_path, src_file_path = _parse_src_path('.') assert workflow_id == 'a' assert src_path == src_dir assert src_file_path == src_dir / 'flow.cylc' # relative './' (invalid) with pytest.raises(InputError) as exc_ctx: _parse_src_path('./flow.cylc') assert 'Not a valid workflow ID or source directory' in str(exc_ctx.value) # suite.rc & flow.cylc both present: (src_dir / 'suite.rc').touch() with pytest.raises(WorkflowFilesError) as exc_ctx: _parse_src_path(str(src_dir)) assert 'Both flow.cylc and suite.rc' in str(exc_ctx.value) async def test_parse_ids_src_path(src_dir): workflows, src_path = await parse_ids_async( './a', src=True, constraint='workflows', ) assert workflows == {'a': []} @pytest.mark.parametrize( 'ids_in,error_msg', [ ( ['/home/me/whatever'], 'Invalid ID: /home/me/whatever', ), ( ['foo/..'], 'cannot be a path that points to the cylc-run directory or above', ), ( ['~alice/foo'], "Operating on other users' workflows is not supported", ), ] ) async def test_parse_ids_invalid_ids( ids_in, error_msg, monkeypatch: pytest.MonkeyPatch ): """It should error for invalid IDs.""" monkeypatch.setattr('cylc.flow.id_cli.get_user', lambda: 'rincewind') with pytest.raises(Exception) as exc_ctx: await parse_ids_async( *ids_in, constraint='workflows', ) assert error_msg in str(exc_ctx.value) async def test_parse_ids_current_user( monkeypatch: pytest.MonkeyPatch, mock_exists ): """It should work if the user in the ID is the current user.""" monkeypatch.setattr('cylc.flow.id_cli.get_user', lambda: 'rincewind') await parse_ids_async('~rincewind/luggage', constraint='workflows') async def test_parse_ids_file(tmp_run_dir): """It should reject IDs that are paths to files.""" tmp_path = tmp_run_dir('x') tmp_file = tmp_path / 'tmpfile' tmp_file.touch() (tmp_path / WorkflowFiles.FLOW_FILE).touch() # using a directory should work await parse_ids_async( str(tmp_path.relative_to(get_cylc_run_dir())), constraint='workflows', ) with pytest.raises(Exception) as exc_ctx: # using a file should not await parse_ids_async( str(tmp_file.relative_to(get_cylc_run_dir())), constraint='workflows', ) assert 'Workflow ID cannot be a file' in str(exc_ctx.value) async def test_parse_ids_constraint(mock_exists): """It should validate input against the constraint.""" # constraint: workflows await parse_ids_async('a//', constraint='workflows') with pytest.raises(InputError): await parse_ids_async('a//b', constraint='workflows') # constraint: tasks await parse_ids_async('a//b', constraint='tasks') with pytest.raises(InputError): await parse_ids_async('a//', constraint='tasks') # constraint: mixed await parse_ids_async('a//', constraint='mixed') await parse_ids_async('a//b', constraint='mixed') # constraint: invalid with pytest.raises(ValueError): await parse_ids_async('foo', constraint='bar') async def test_parse_ids_src_run(abc_src_dir, tmp_run_dir): """It should locate the flow file when src=True.""" # locate flow file for a src workflow workflows, flow_file_path = await parse_ids_async( './a', src=True, constraint='workflows', ) assert list(workflows) == ['a'] assert flow_file_path == abc_src_dir / 'a' / WorkflowFiles.FLOW_FILE # locate flow file for a run workflow run_dir = tmp_run_dir('b') workflows, flow_file_path = await parse_ids_async( 'b', src=True, constraint='workflows', ) assert list(workflows) == ['b'] assert flow_file_path == run_dir / WorkflowFiles.FLOW_FILE def test_validate_constraint(): """It should validate tokens against the constraint.""" # constraint=workflows _validate_constraint(Tokens(workflow='a'), constraint='workflows') with pytest.raises(InputError): _validate_constraint(Tokens(cycle='a'), constraint='workflows') with pytest.raises(InputError): _validate_constraint(Tokens(), constraint='workflows') # constraint=tasks _validate_constraint(Tokens(cycle='a'), constraint='tasks') with pytest.raises(InputError): _validate_constraint(Tokens(workflow='a'), constraint='tasks') with pytest.raises(InputError): _validate_constraint(Tokens(), constraint='tasks') # constraint=mixed _validate_constraint(Tokens(workflow='a'), constraint='mixed') _validate_constraint(Tokens(cycle='a'), constraint='mixed') with pytest.raises(InputError): _validate_constraint(Tokens(), constraint='mixed') def test_validate_workflow_ids_basic(tmp_run_dir): _validate_workflow_ids(Tokens('workflow'), src_path='') with pytest.raises(InputError): _validate_workflow_ids(Tokens('~alice/workflow'), src_path='') run_dir = tmp_run_dir('b') with pytest.raises(InputError): _validate_workflow_ids( Tokens('workflow'), src_path=run_dir / 'flow.cylc', ) def test_validate_workflow_ids_warning(caplog): """It should warn when the run number is provided as a cycle point.""" caplog.set_level(logging.WARN, CYLC_LOG) _validate_workflow_ids(Tokens('workflow/run1//cycle/task'), src_path='') assert caplog.messages == [] _validate_workflow_ids(Tokens('workflow//run1'), src_path='') assert caplog.messages == ['Did you mean: workflow/run1'] caplog.clear() _validate_workflow_ids(Tokens('workflow//run1/cycle/task'), src_path='') assert caplog.messages == ['Did you mean: workflow/run1//cycle/task'] def test_validate_number(): _validate_number(Tokens('a'), max_workflows=1) with pytest.raises(InputError): _validate_number(Tokens('a'), Tokens('b'), max_workflows=1) t1 = Tokens(cycle='1') t2 = Tokens(cycle='2') _validate_number(t1, max_tasks=1) with pytest.raises(InputError): _validate_number(t1, t2, max_tasks=1) _validate_number(t1, max_tasks=1) _validate_number(Tokens('a//1'), Tokens('a//2'), max_workflows=1) _validate_number( Tokens('a'), Tokens('//2'), Tokens('//3'), max_workflows=1 ) with pytest.raises(InputError): _validate_number(Tokens('a//1'), Tokens('b//1'), max_workflows=1) _validate_number(Tokens('a//1'), Tokens('b//1'), max_workflows=2) @pytest.fixture def no_scan(monkeypatch): """Disable the filesystem part of scan.""" @pipe async def _scan(): # something that looks like scan but doesn't do anything yield monkeypatch.setattr('cylc.flow.network.scan.scan', _scan) async def test_expand_workflow_tokens_impl_selector(no_scan): """It should reject filters it can't handle.""" tokens = tokenise('~user/*') await _expand_workflow_tokens([tokens]) tokens = tokens.duplicate(workflow_sel='stopped') with pytest.raises(InputError): await _expand_workflow_tokens([tokens]) @pytest.mark.parametrize('identifier, expected', [ ( '//2024-01-01T00:fail/a', {'cycle': '2024-01-01T00', 'cycle_sel': 'fail', 'task': 'a'} ), ( '//2024-01-01T00:00Z/a', {'cycle': '2024-01-01T00:00Z', 'task': 'a'} ), ( '//2024-01-01T00:00Z:fail/a', {'cycle': '2024-01-01T00:00Z', 'cycle_sel': 'fail', 'task': 'a'} ), ( '//2024-01-01T00:00:00+05:30/a', {'cycle': '2024-01-01T00:00:00+05:30', 'task': 'a'} ), ( '//2024-01-01T00:00:00+05:30:f/a', {'cycle': '2024-01-01T00:00:00+05:30', 'cycle_sel': 'f', 'task': 'a'} ), ( # Nonsensical example, but whatever... '//2024-01-01T00:00Z:00Z/a', {'cycle': '2024-01-01T00:00Z', 'cycle_sel': '00Z', 'task': 'a'} ) ]) def test_iso_long_fmt(identifier, expected): assert { k: v for k, v in cli_tokenise(identifier).items() if v is not None } == expected cylc-flow-8.7.0/tests/unit/plugins/0000775000175000017500000000000015264420307017416 5ustar alastairalastaircylc-flow-8.7.0/tests/unit/plugins/test_pre_configure.py0000664000175000017500000001264215264420307023663 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test the pre_configure entry point.""" from functools import partial from itertools import permutations from random import random import pytest from cylc.flow.exceptions import PluginError from cylc.flow.parsec.exceptions import ParsecError from cylc.flow.parsec.fileparse import process_plugins class EntryPointWrapper: """Wraps a method to make it look like an entry point.""" def __init__(self, fcn): self.name = getattr(fcn, '__name__', str(fcn)) self.fcn = fcn def load(self): return self.fcn @EntryPointWrapper def pre_configure_basic(*_, **__): """Simple plugin that returns one env var and one template var.""" return { 'env': { 'ANSWER': '42' }, 'template_variables': { 'QUESTION': 'What do you get if you multiply 7 by 6?' } } @EntryPointWrapper def pre_configure_templating_detected(*_, **__): """Plugin that detects a random templating engine.""" return { 'templating_detected': str(random()) } @EntryPointWrapper def pre_configure_error(*_, **__): """Plugin that raises an exception.""" raise Exception('foo') def test_pre_configure(monkeypatch): """It should call the plugin.""" monkeypatch.setattr( 'cylc.flow.plugins.iter_entry_points', lambda namespace: ( [pre_configure_basic] if namespace == 'cylc.pre_configure' else [] ) ) extra_vars = process_plugins('/', None) assert extra_vars == { 'env': { 'ANSWER': '42' }, 'template_variables': { 'QUESTION': 'What do you get if you multiply 7 by 6?' }, 'templating_detected': None } def test_pre_configure_duplicate(monkeypatch): """It should error when plugins clash.""" monkeypatch.setattr( 'cylc.flow.plugins.iter_entry_points', lambda namespace: ( [ pre_configure_basic, pre_configure_basic, ] if namespace == 'cylc.pre_configure' else [] ) ) with pytest.raises(ParsecError): process_plugins('/', None) def test_pre_configure_templating_detected(monkeypatch): """It should error when plugins clash (for templating).""" monkeypatch.setattr( 'cylc.flow.plugins.iter_entry_points', lambda namespace: ( [ pre_configure_templating_detected, pre_configure_templating_detected, ] if namespace == 'cylc.pre_configure' else [] ) ) with pytest.raises(ParsecError): process_plugins('/', None) def test_pre_configure_exception(monkeypatch): """It should wrap plugin errors.""" monkeypatch.setattr( 'cylc.flow.plugins.iter_entry_points', lambda namespace: ( [ pre_configure_error, ] if namespace == 'cylc.pre_configure' else [] ) ) with pytest.raises(PluginError) as exc_ctx: process_plugins('/', None) # the context of the original error should be preserved in the raised # exception assert exc_ctx.value.entry_point == 'cylc.pre_configure' assert exc_ctx.value.plugin_name == 'pre_configure_error' assert str(exc_ctx.value.exc) == 'foo' assert str(exc_ctx.value) == ( "Error in plugin cylc.pre_configure.pre_configure_error\n" "Exception: foo" ) def _pre_configure(*a, templating_detected=None, **k): """A mock pre-configure plugin where templating_detected can be overridden. """ return { 'env': {}, 'template_variables': {}, 'templating_detected': templating_detected, } @pytest.mark.parametrize( 'plugins', permutations( [ # four plugins which return differing, but compatible results EntryPointWrapper( partial(_pre_configure, templating_detected='a') ), EntryPointWrapper( partial(_pre_configure, templating_detected='a') ), EntryPointWrapper( partial(_pre_configure, templating_detected=None) ), EntryPointWrapper( partial(_pre_configure, templating_detected=None) ), ], ), ) def test_pre_configure_multiple_compatible(monkeypatch, plugins): """The order in which plugins are run shouldn't affect templating_detected. See https://github.com/cylc/cylc-rose/issues/440 """ monkeypatch.setattr( 'cylc.flow.plugins.iter_entry_points', lambda namespace: plugins if namespace == 'cylc.pre_configure' else [], ) extra_vars = process_plugins('/', None) assert extra_vars['templating_detected'] == 'a' cylc-flow-8.7.0/tests/unit/test_workflow_files.py0000664000175000017500000004351615264420307022413 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging from pathlib import Path from typing import ( Any, Callable, Dict, Optional, Type, Union, ) import pytest from cylc.flow import workflow_files from cylc.flow.exceptions import ( InputError, WorkflowFilesError, ) from cylc.flow.util import sstrip from cylc.flow.workflow_files import ( WorkflowFiles, abort_if_flow_file_in_path, check_flow_file, check_reserved_dir_names, get_symlink_dirs, get_workflow_title, infer_latest_run, is_installed, validate_workflow_name, ) from .filetree import ( FILETREE_1, FILETREE_2, FILETREE_3, FILETREE_4, create_filetree, ) NonCallableFixture = Any @pytest.mark.parametrize( 'path, expected', [('a/b/c', '/mock_cylc_dir/a/b/c'), ('/a/b/c', '/a/b/c')] ) def test_get_cylc_run_abs_path( path: str, expected: str, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr('cylc.flow.pathutil._CYLC_RUN_DIR', '/mock_cylc_dir') assert workflow_files.get_cylc_run_abs_path(path) == expected @pytest.mark.parametrize('is_abs_path', [False, True]) def test_is_valid_run_dir(is_abs_path: bool, tmp_run_dir: Callable): """Test that a directory is correctly identified as a valid run dir when it contains a service dir. """ cylc_run_dir: Path = tmp_run_dir() prefix = str(cylc_run_dir) if is_abs_path else '' # What if no dir there? assert workflow_files.is_valid_run_dir( Path(prefix, 'nothing/here')) is False # What if only flow.cylc exists but no service dir? # (Non-run dirs can still contain flow.cylc) run_dir = cylc_run_dir.joinpath('foo/bar') run_dir.mkdir(parents=True) flow_file = run_dir.joinpath(WorkflowFiles.FLOW_FILE) flow_file.touch() assert workflow_files.is_valid_run_dir(Path(prefix, 'foo/bar')) is True # What if service dir exists? flow_file.unlink() run_dir.joinpath(WorkflowFiles.Service.DIRNAME).mkdir() assert workflow_files.is_valid_run_dir(Path(prefix, 'foo/bar')) is True @pytest.mark.parametrize( 'id_, expected_err, expected_msg', [('foo/bar/', None, None), ('/foo/bar', WorkflowFilesError, "cannot be an absolute path"), ('$HOME/alone', WorkflowFilesError, "invalid workflow name"), ('./foo', WorkflowFilesError, "invalid workflow name"), ('meow/..', WorkflowFilesError, "cannot be a path that points to the cylc-run directory or above")] ) def test_validate_workflow_name(id_, expected_err, expected_msg): if expected_err: with pytest.raises(expected_err) as exc: validate_workflow_name(id_) if expected_msg: assert expected_msg in str(exc.value) else: validate_workflow_name(id_) @pytest.mark.parametrize( 'name, err_expected', [ # Basic ok: ('foo/bar/baz', False), # Reserved dir names: ('foo/log/baz', True), ('foo/runN/baz', True), ('foo/run9000/baz', True), ('work', True), # If not exact match, but substring, that's fine: ('foo/underrunN/baz', False), ('foo/overrun2', False), ('slog', False) ] ) def test_check_reserved_dir_names(name: str, err_expected: bool): if err_expected: with pytest.raises(WorkflowFilesError) as exc_inf: check_reserved_dir_names(name) assert "cannot contain a directory named" in str(exc_inf.value) else: check_reserved_dir_names(name) def test_validate_workflow_name__reserved_name(): """Check that validate_workflow_name() doesn't check for reserved dir names unless we tell it to with the arg.""" name = 'foo/runN' validate_workflow_name(name) with pytest.raises(WorkflowFilesError): validate_workflow_name(name, check_reserved_names=True) @pytest.mark.parametrize( 'path, implicit_runN, expected_reg', [ ('{cylc_run}/numbered/workflow', True, 'numbered/workflow/run2'), ('{cylc_run}/numbered/workflow', False, 'numbered/workflow'), ('{cylc_run}/numbered/workflow/runN', True, 'numbered/workflow/run2'), ('{cylc_run}/numbered/workflow/runN', False, 'numbered/workflow/run2'), ('{cylc_run}/numbered/workflow/run1', True, 'numbered/workflow/run1'), ('{cylc_run}/numbered/workflow/run1', False, 'numbered/workflow/run1'), ('{cylc_run}/non_numbered/workflow', True, 'non_numbered/workflow'), ('{cylc_run}/non_numbered/workflow', False, 'non_numbered/workflow'), ] ) def test_infer_latest_run( path: str, implicit_runN: bool, expected_reg: str, tmp_run_dir: Callable, ) -> None: """Test infer_latest_run(). Params: path: Input arg. implicit_runN: Input arg. expected_reg: The id_ part of the expected returned tuple. """ # Setup cylc_run_dir: Path = tmp_run_dir() run_dir = cylc_run_dir / 'numbered' / 'workflow' run_dir.mkdir(parents=True) (run_dir / 'run1').mkdir() (run_dir / 'run2').mkdir() (run_dir / 'runN').symlink_to('run2') run_dir = cylc_run_dir / 'non_numbered' / 'workflow' / 'named_run' run_dir.mkdir(parents=True) path: Path = Path(path.format(cylc_run=cylc_run_dir)) expected = (cylc_run_dir / expected_reg, expected_reg) # Test assert infer_latest_run(path, implicit_runN) == expected # Check implicit_runN=True is the default: if implicit_runN: assert infer_latest_run(path) == expected @pytest.mark.parametrize('warn_arg', [True, False]) def test_infer_latest_run_warns_for_runN( warn_arg: bool, log_filter: Callable, tmp_run_dir: Callable, monkeypatch: pytest.MonkeyPatch ): """Tests warning is produced to discourage use of /runN in workflow_id""" (tmp_run_dir() / 'run1').mkdir() runN_path = tmp_run_dir() / 'runN' runN_path.symlink_to('run1') monkeypatch.setattr('cylc.flow.LOG.level', logging.INFO) infer_latest_run(runN_path, warn_runN=warn_arg) filtered_log = log_filter( logging.WARNING, contains="You do not need to include" ) assert filtered_log if warn_arg else not filtered_log @pytest.mark.parametrize( ('reason', 'error_type'), [ ('not dir', WorkflowFilesError), ('not symlink', WorkflowFilesError), ('broken symlink', WorkflowFilesError), ('invalid target', WorkflowFilesError), ('not exist', InputError) ] ) def test_infer_latest_run__bad( reason: str, error_type: Type[Exception], tmp_run_dir: Callable, ) -> None: # -- Setup -- cylc_run_dir: Path = tmp_run_dir() run_dir = cylc_run_dir / 'sulu' run_dir.mkdir() runN_path = run_dir / 'runN' err_msg = f"{runN_path} symlink not valid" if reason == 'not dir': (run_dir / 'run1').touch() runN_path.symlink_to('run1') elif reason == 'not symlink': runN_path.mkdir() elif reason == 'broken symlink': runN_path.symlink_to('run1') elif reason == 'invalid target': # noqa: SIM106 (run_dir / 'palpatine').mkdir() runN_path.symlink_to('palpatine') err_msg = ( f"{runN_path} symlink target not valid: palpatine" ) elif reason == 'not exist': run_dir = run_dir / 'not-exist' err_msg = ( f"Workflow ID not found: sulu/not-exist\n" f"(Directory not found: {run_dir})" ) else: raise ValueError(reason) # -- Test -- with pytest.raises(error_type) as excinfo: infer_latest_run(run_dir) assert str(excinfo.value) == err_msg @pytest.mark.parametrize( 'filetree, expected', [ pytest.param( FILETREE_1, {'log': 'sym/cylc-run/foo/bar/log'}, id="filetree1" ), pytest.param( FILETREE_2, { 'share/cycle': 'sym-cycle/cylc-run/foo/bar/share/cycle', 'share': 'sym-share/cylc-run/foo/bar/share', '': 'sym-run/cylc-run/foo/bar/' }, id="filetree2" ), pytest.param( FILETREE_3, { 'share/cycle': 'sym-cycle/cylc-run/foo/bar/share/cycle', '': 'sym-run/cylc-run/foo/bar/' }, id="filetree3" ), pytest.param( FILETREE_4, {'share/cycle': 'sym-cycle/cylc-run/foo/bar/share/cycle'}, id="filetree4" ), ] ) def test_get_symlink_dirs( filetree: Dict[str, Any], expected: Dict[str, Union[Path, str]], tmp_run_dir: Callable, tmp_path: Path ): """Test get_symlink_dirs(). Params: filetree: The directory structure to test against. expected: The expected return dictionary, except with the values being relative to tmp_path instead of absolute paths. """ # Setup cylc_run_dir = tmp_run_dir() create_filetree(filetree, tmp_path, tmp_path) id_ = 'foo/bar' for k, v in expected.items(): expected[k] = Path(tmp_path / v) # Test assert get_symlink_dirs(id_, cylc_run_dir / id_) == expected @pytest.mark.parametrize( 'flow_file_target, suiterc_exists, err, expected_file', [ pytest.param( WorkflowFiles.SUITE_RC, True, None, WorkflowFiles.FLOW_FILE, id="flow.cylc symlinked to suite.rc" ), pytest.param( WorkflowFiles.SUITE_RC, False, WorkflowFilesError, None, id="flow.cylc symlinked to non-existent suite.rc" ), pytest.param( 'inside.cylc', True, WorkflowFilesError, None, id="flow.cylc symlinked to file in run dir, suite.rc exists" ), pytest.param( 'inside.cylc', False, None, WorkflowFiles.FLOW_FILE, id="flow.cylc symlinked to file in run dir, no suite.rc" ), pytest.param( '../outside.cylc', True, WorkflowFilesError, None, id="flow.cylc symlinked to file outside, suite.rc exists" ), pytest.param( '../outside.cylc', False, None, WorkflowFiles.FLOW_FILE, id="flow.cylc symlinked to file outside, no suite.rc" ), pytest.param( None, True, None, WorkflowFiles.SUITE_RC, id="No flow.cylc, suite.rc exists" ), pytest.param( None, False, WorkflowFilesError, None, id="No flow.cylc, no suite.rc" ), ] ) def test_check_flow_file_symlinks( flow_file_target: Optional[str], suiterc_exists: bool, err: Optional[Type[Exception]], expected_file: Optional[str], tmp_path: Path ) -> None: """Test check_flow_file() when flow.cylc is a symlink or doesn't exist. Params: flow_file_target: Relative path of the flow.cylc symlink's target, or None if the symlink doesn't exist. suiterc_exists: Whether there is a suite.rc file in the dir. err: Type of exception if expected to get raised. expected_file: Which file's path should get returned, when symlink_suiterc_arg is FALSE (otherwise it will always be flow.cylc, assuming no exception occurred). """ run_dir = tmp_path / 'espresso' flow_file = run_dir / WorkflowFiles.FLOW_FILE suiterc = run_dir / WorkflowFiles.SUITE_RC run_dir.mkdir() (run_dir / '../outside.cylc').touch() (run_dir / 'inside.cylc').touch() if suiterc_exists: suiterc.touch() if flow_file_target: flow_file.symlink_to(flow_file_target) if err: with pytest.raises(err): check_flow_file(run_dir) else: assert expected_file is not None # otherwise test is wrong result = check_flow_file(run_dir) assert result == run_dir / expected_file @pytest.fixture() def flow_cylc(tmp_path): return tmp_path / WorkflowFiles.FLOW_FILE @pytest.fixture() def suite_rc(tmp_path): return tmp_path / WorkflowFiles.SUITE_RC def test_check_flow_file__flow_cylc( flow_cylc, tmp_path, ): """flow.cylc""" flow_cylc.touch() assert check_flow_file(tmp_path, allow_suite_rc=False) == flow_cylc assert check_flow_file(tmp_path, allow_suite_rc=True) == flow_cylc def test_check_flow_file__flow_cylc_and_suite_rc( tmp_path, suite_rc, flow_cylc, ): """flow.cylc, suite.rc""" flow_cylc.touch() suite_rc.touch() with pytest.raises( WorkflowFilesError, match=rf'Both {WorkflowFiles.FLOW_FILE} and {WorkflowFiles.SUITE_RC}', ): check_flow_file(tmp_path, allow_suite_rc=False) with pytest.raises( WorkflowFilesError, match=rf'Both {WorkflowFiles.FLOW_FILE} and {WorkflowFiles.SUITE_RC}', ): check_flow_file(tmp_path, allow_suite_rc=True) def test_check_flow_file__flow_cylc_symlink( flow_cylc, suite_rc, caplog, tmp_path, log_filter, ): """flow.cylc -> suite.rc""" suite_rc.touch() flow_cylc.symlink_to(suite_rc) with pytest.raises( WorkflowFilesError, match=rf'{WorkflowFiles.SUITE_RC} found', ): check_flow_file(tmp_path, allow_suite_rc=False) caplog.clear() assert check_flow_file(tmp_path, allow_suite_rc=True) == flow_cylc assert log_filter( contains=( f'{WorkflowFiles.FLOW_FILE} is a symlink to' f' {WorkflowFiles.SUITE_RC}' ) ) def test_check_flow__suite_rc( suite_rc, tmp_path, ): """suite.rc""" suite_rc.touch() with pytest.raises( WorkflowFilesError, match=rf'{WorkflowFiles.SUITE_RC} found', ): check_flow_file(tmp_path, allow_suite_rc=False) assert check_flow_file(tmp_path, allow_suite_rc=True) == suite_rc def test_check_flow_file__invalid_symlink( flow_cylc, suite_rc, tmp_path, ): """suite.rc -> flow.cylc""" flow_cylc.touch() suite_rc.symlink_to(flow_cylc) with pytest.raises( WorkflowFilesError, match=rf'Both {WorkflowFiles.FLOW_FILE} and {WorkflowFiles.SUITE_RC}', ): check_flow_file(tmp_path, allow_suite_rc=False) with pytest.raises( WorkflowFilesError, match=rf'Both {WorkflowFiles.FLOW_FILE} and {WorkflowFiles.SUITE_RC}', ): check_flow_file(tmp_path, allow_suite_rc=True) def test_check_flow_file__valid_symlink( flow_cylc, tmp_path, ): """flow.cylc -> a.cylc""" a_cylc = tmp_path / 'a.cylc' a_cylc.touch() flow_cylc.symlink_to(a_cylc) assert check_flow_file(tmp_path, allow_suite_rc=False) == flow_cylc assert check_flow_file(tmp_path, allow_suite_rc=True) == flow_cylc def test_check_flow_file__flow_cylc_dir( flow_cylc, tmp_path, ): """flow.cylc/""" flow_cylc.mkdir() with pytest.raises( WorkflowFilesError, match=rf'No {WorkflowFiles.FLOW_FILE} file', ): check_flow_file(tmp_path, allow_suite_rc=False) with pytest.raises( WorkflowFilesError, match=( rf'No {WorkflowFiles.FLOW_FILE}' rf' or {WorkflowFiles.SUITE_RC} file' ), ): check_flow_file(tmp_path, allow_suite_rc=True) def test_check_flow_file__suite_rc_dir( suite_rc, tmp_path, ): """suite.rc/""" suite_rc.mkdir() with pytest.raises( WorkflowFilesError, match=( rf'(No {WorkflowFiles.FLOW_FILE}' rf'|{WorkflowFiles.SUITE_RC} found)' ), # NOTE: either "No flow.cylc" or "suite.rc found" would be acceptable ): check_flow_file(tmp_path, allow_suite_rc=False) with pytest.raises( WorkflowFilesError, match=( rf'No {WorkflowFiles.FLOW_FILE}' rf' or {WorkflowFiles.SUITE_RC} file' ), ): check_flow_file(tmp_path, allow_suite_rc=True) @pytest.mark.parametrize( 'id_, installed, named, expected', [('reg1/run1', True, True, True), ('reg2', True, False, True), ('reg3', False, False, False)] ) def test_is_installed(tmp_run_dir: Callable, id_, installed, named, expected): """Test is_installed correctly identifies presence of _cylc-install dir""" cylc_run_dir: Path = tmp_run_dir(id_, installed=installed, named=named) actual = is_installed(cylc_run_dir) assert actual == expected def test_validate_abort_if_flow_file_in_path(): assert abort_if_flow_file_in_path(Path("path/to/wflow")) is None with pytest.raises(InputError) as exc_info: abort_if_flow_file_in_path(Path("path/to/wflow/flow.cylc")) assert "Not a valid workflow ID or source directory" in str(exc_info.value) def test_get_workflow_title(tmp_run_dir): cylc_run_dir: Path = tmp_run_dir('x', installed=True, named=False) (cylc_run_dir / 'flow.cylc').write_text(sstrip(''' [meta] title = foo [some junk] whatevs = True ''')) # it should extract the title from a flow.cylc file assert get_workflow_title('x') == 'foo' # it should extract the title from a suite.rc file (cylc_run_dir / 'flow.cylc').rename(cylc_run_dir / 'suite.rc') assert get_workflow_title('x') == 'foo' # but it will only work if the [meta] section comes first (cylc_run_dir / 'suite.rc').write_text( '[some other junk]\n whatevs = False\n' + (cylc_run_dir / 'suite.rc').read_text() ) assert get_workflow_title('x') == 'No title provided' cylc-flow-8.7.0/tests/unit/test_db_compat.py0000664000175000017500000001504215264420307021300 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Compatibility tests for handling old workflow databases.""" from functools import partial import sqlite3 from unittest.mock import Mock import pytest from cylc.flow.exceptions import CylcError, ServiceFileError from cylc.flow.task_pool import TaskPool from cylc.flow.workflow_db_mgr import ( CylcWorkflowDAO, WorkflowDatabaseManager, ) @pytest.fixture def _setup_db(tmp_path): """Fixture to create old DB.""" def _inner(stmts, db_file_name='sql.db'): db_file = tmp_path / db_file_name db_file.parent.mkdir(parents=True, exist_ok=True) # Note: cannot use CylcWorkflowDAO here as creating outdated DB conn = sqlite3.connect(str(db_file)) conn.execute(( r'CREATE TABLE task_states(name TEXT, cycle TEXT, flow_nums TEXT,' r' time_created TEXT, time_updated TEXT, submit_num INTEGER,' r' status TEXT, flow_wait INTEGER, is_manual_submit INTEGER,' r' PRIMARY KEY(name, cycle, flow_nums));') ) conn.execute(( r'CREATE TABLE task_jobs(cycle TEXT, name TEXT,' r' submit_num INTEGER, is_manual_submit INTEGER,' r' try_num INTEGER, time_submit TEXT, time_submit_exit TEXT,' r' submit_status INTEGER, time_run TEXT, time_run_exit TEXT,' r' run_signal TEXT, run_status INTEGER, platform_name TEXT,' r' job_runner_name TEXT, job_id TEXT,' r' PRIMARY KEY(cycle, name, submit_num));' )) for stmt, values in stmts: conn.execute(stmt, values) conn.execute(( r"INSERT INTO task_jobs VALUES" r" ('10090101T0000Z', 'foo', 1, 0, 1, '2022-12-05T14:46:06Z'," r" '2022-12-05T14:46:07Z', 0, '2022-12-05T14:46:10Z'," r" '2022-12-05T14:46:39Z', '', 0, 'localhost', 'background'," r" 4377)" )) conn.commit() conn.close() return db_file return _inner def test_upgrade_pre_810_fails_on_multiple_flows(_setup_db): stmts = [( r'INSERT INTO task_states VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', ( 'foo', '10050101T0000Z', '[1, 3]', '2022-12-05T14:46:33Z', '2022-12-05T14:46:40Z', 1, 'succeeded', 0, 0, ), )] db_file_name = _setup_db(stmts) with CylcWorkflowDAO(db_file_name) as dao, pytest.raises( CylcError, match='^Cannot .* 8.0.x to 8.1.0 .* used.$' ): WorkflowDatabaseManager.upgrade_pre_810(dao) def test_upgrade_pre_810_pass_on_single_flow(_setup_db): stmts = [( r'INSERT INTO task_states VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', ( 'foo', '10050101T0000Z', '[1]', '2022-12-05T14:46:33Z', '2022-12-05T14:46:40Z', 1, 'succeeded', 0, 0, ), )] db_file_name = _setup_db(stmts) with CylcWorkflowDAO(db_file_name) as dao: WorkflowDatabaseManager.upgrade_pre_810(dao) result = dao.connect().execute( 'SELECT DISTINCT flow_nums FROM task_jobs;' ).fetchall()[0][0] assert result == '[1]' def test_check_workflow_db_compat(_setup_db, capsys): """method can pick private or public db to check. """ # Create public and private databases with different cylc versions: create = r'CREATE TABLE workflow_params(key TEXT, value TEXT)' insert = r'INSERT INTO workflow_params VALUES (?, ?)' pri_path = _setup_db( [ (create, tuple()), (insert, ('cylc_version', '7.99.99')), ], db_file_name='private/db', ) pub_path = _setup_db( [ (create, tuple()), (insert, ('cylc_version', '7.99.98')), ], db_file_name='public/db' ) with pytest.raises(ServiceFileError, match='99.98'): WorkflowDatabaseManager.check_db_compatibility(pub_path) with pytest.raises(ServiceFileError, match='99.99'): WorkflowDatabaseManager.check_db_compatibility(pri_path) def test_pre_830_task_action_timers(_setup_db): """Test back compat for task_action_timers table. Before 8.3.0, TaskEventMailContext had an extra field "ctx_type" at index 1. TaskPool.load_db_task_action_timers() should be able to discard this field. """ stmts = [ ( r''' CREATE TABLE task_action_timers( cycle TEXT, name TEXT, ctx_key TEXT, ctx TEXT, delays TEXT, num INTEGER, delay TEXT, timeout TEXT, PRIMARY KEY(cycle, name, ctx_key) ); ''', tuple(), ), ( r'INSERT INTO task_action_timers VALUES (?, ?, ?, ?, ?, ?, ?, ?)', ( '1', 'foo', '[["event-mail", "failed"], 9]', ( '["TaskEventMailContext", ["event-mail", "event-mail",' ' "notifications@fbc.gov", "jfaden"]]' ), '[0.0]', 1, '0.0', '1709229449.61275', ) ), ( r'INSERT INTO task_action_timers VALUES (?, ?, ?, ?, ?, ?, ?, ?)', ( '1', 'foo', '["try_timers", "execution-retry"]', None, '[94608000.0]', 1, None, None, ), ), ] db_file = _setup_db(stmts) mock_pool = Mock() load_db_task_action_timers = partial( TaskPool.load_db_task_action_timers, mock_pool ) with CylcWorkflowDAO(db_file, create_tables=True) as dao: dao.select_task_action_timers(load_db_task_action_timers) cylc-flow-8.7.0/tests/unit/test_terminal.py0000664000175000017500000001410715264420307021164 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from optparse import OptionParser, Values import pytest from cylc.flow.exceptions import CylcError from cylc.flow.parsec.exceptions import ParsecError from cylc.flow.terminal import ( cli_function, prompt, should_use_color, ) # this puts Exception in globals() where we can easily find it later Exception = Exception SystemExit = SystemExit def get_option_parser(): """An option parser with no options.""" return OptionParser() @cli_function(get_option_parser) def cli(parser, opts, exc_class): """Dummy command line interface which raises an exception. Args: exc_class: The class of the exception to raise. """ if exc_class: raise globals()[exc_class]('message') @pytest.mark.parametrize( 'verbosity,exception_in,exception_out,return_code,stderr', [ # CylcError - "known" error pytest.param( # nicely formatted 0, CylcError, SystemExit, 1, 'CylcError: message\n', id='CylcError' ), pytest.param( # full traceback in debug mode 2, CylcError, CylcError, None, None, id='CylcError-debug' ), # ParsecError - "known" error pytest.param( # nicely formatted 0, ParsecError, SystemExit, 1, 'ParsecError: message\n', id='ParsecError' ), pytest.param( # full traceback in debug mode 2, ParsecError, ParsecError, None, None, id='ParsecError-debug' ), # Exception - "unknown" error pytest.param( # full traceback 0, Exception, Exception, None, None, id='Exception' ), pytest.param( # full traceback in debug mode 2, Exception, Exception, None, None, id='Exception-debug' ), # SystemExit - "unknown" error pytest.param( # full traceback 0, SystemExit, SystemExit, 1, 'ERROR: message\n', id='SystemExit' ), pytest.param( # full traceback in debug mode 2, SystemExit, SystemExit, 1, 'ERROR: message\n', id='SystemExit-debug' ), ] ) def test_cli( verbosity, exception_in, exception_out, return_code, stderr, monkeypatch, capsys ): """Test that the CLI formats exceptions appropriately. The idea here is that "known" errors (those which subclass CylcError or ParsecError) should be formatted nicely (as opposed to dumping the full traceback to stderr) in interactive mode. This behaviour can be overridden using --debug mode. In non-interactive mode we always print the full traceback for logging purposes. Other exceptions represent "unknown" errors which we would expect to occur. We should print the full traceback in these situations. """ monkeypatch.setattr('cylc.flow.flags.verbosity', verbosity) monkeypatch.setattr('cylc.flow.terminal.supports_color', lambda: False) with pytest.raises(exception_out) as exc_ctx: cli(exception_in.__name__) if return_code is not None: assert exc_ctx.value.args[0] == return_code if stderr is not None: assert capsys.readouterr()[1] == stderr @pytest.fixture def stdinput(monkeypatch): def _input(*lines): lines = list(lines) def __input(_message): try: return lines.pop(0) except IndexError: raise Exception('stdinput ran out of lines') monkeypatch.setattr( 'cylc.flow.terminal.input', __input, ) return _input def test_prompt(stdinput): """Test the prompt function with some simulated input.""" # test a multiple choice prompt stdinput('y') assert prompt('yes or no', ['y', 'n']) == 'y' stdinput('n') assert prompt('yes or no', ['y', 'n']) == 'n' # test a prompt with mapped return values stdinput('42') assert prompt('what is the answer', {'41': False, '42': True}) is True stdinput('41') assert prompt('what is the answer', {'41': False, '42': True}) is False # test incorrect input (should re-prompt until it gets a valid response) stdinput('40', '41', '42') assert prompt('what is the answer', ['42']) == '42' # test a prompt with a default stdinput('') assert prompt('whatever', ['x'], default='x') == 'x' # test a prompt with an input pre-process method thinggy stdinput('YES') assert prompt('yes yes yes no', ['yes', 'no'], process=str.lower) == 'yes' @pytest.mark.parametrize('opts, supported, expected', [ ({}, True, False), ({'color': 'never'}, True, False), ({'color': 'auto'}, True, True), ({'color': 'auto'}, False, False), ({'color': 'always'}, False, True), ]) def test_should_use_color( opts: dict, expected: bool, supported: bool, monkeypatch: pytest.MonkeyPatch ): monkeypatch.setattr('cylc.flow.terminal.supports_color', lambda: supported) options = Values(opts) assert should_use_color(options) == expected cylc-flow-8.7.0/tests/unit/cfgspec/0000775000175000017500000000000015264420307017347 5ustar alastairalastaircylc-flow-8.7.0/tests/unit/cfgspec/test_workflow.py0000664000175000017500000000545215264420307022640 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging import pytest from typing import Any, Dict, Optional from cylc.flow import CYLC_LOG from cylc.flow.cfgspec.workflow import warn_about_depr_platform from cylc.flow.exceptions import PlatformLookupError @pytest.mark.parametrize( 'runtime_cfg, fail_expected, expected_warning', [ pytest.param( { 'foo': {'script': 'true'} }, False, None, id="No platform setting" ), pytest.param( { 'foo': {'platform': 'fine'} }, False, None, id="Valid platform" ), pytest.param( { 'foo': {'platform': 'fine'}, 'bar': {'platform': '`not good`'} }, True, None, id="Invalid subshell notation" ), pytest.param( { 'foo': {'platform': 'fine'}, 'bar': { 'platform': '$(fine)', 'job': {'batch system': 'pbs'} } }, True, None, id="Platform/host conflict" ), pytest.param( { 'foo': {'platform': 'fine'}, 'bar': { 'job': {'batch system': 'pbs'} } }, False, "replace with [runtime][bar]platform", id="Deprecated settings" ) ] ) def test_warn_about_depr_platform( runtime_cfg: Dict[str, Any], fail_expected: bool, expected_warning: Optional[str], caplog: pytest.LogCaptureFixture): """Test warn_about_depr_platform()""" caplog.set_level(logging.WARNING, CYLC_LOG) cfg = {'runtime': runtime_cfg} if fail_expected: with pytest.raises(PlatformLookupError): warn_about_depr_platform(cfg) else: warn_about_depr_platform(cfg) if expected_warning: assert expected_warning in caplog.text else: assert caplog.record_tuples == [] cylc-flow-8.7.0/tests/unit/cfgspec/test_globalcfg.py0000664000175000017500000001460415264420307022705 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests for the Cylc GlobalConfig object.""" from typing import TYPE_CHECKING, Callable import pytest from cylc.flow.cfgspec.globalcfg import GlobalConfig, SPEC from cylc.flow.parsec.exceptions import ValidationError from cylc.flow.parsec.validate import cylc_config_validate if TYPE_CHECKING: from pathlib import Path TEST_CONF = ''' [platforms] [[foo]] hosts = of_morgoth [platform groups] [[BAR]] platforms = mario, sonic [task events] # Checking that config items that aren't platforms or platform groups # are not output. ''' @pytest.fixture def mock_global_config(tmp_path: 'Path', monkeypatch: pytest.MonkeyPatch): """Create a mock GlobalConfig object, given the global.cylc contents as a string.""" def _mock_global_config(cfg: str) -> GlobalConfig: glblcfg = GlobalConfig(SPEC, validator=cylc_config_validate) conf_path = tmp_path / GlobalConfig.CONF_BASENAME conf_path.write_text(cfg) monkeypatch.setenv("CYLC_CONF_PATH", str(conf_path.parent)) glblcfg.loadcfg(conf_path) return glblcfg return _mock_global_config def test_dump_platform_names(capsys, mock_global_config): """It dumps lists of platform names, nothing else.""" glblcfg: GlobalConfig = mock_global_config(TEST_CONF) glblcfg.dump_platform_names(glblcfg) stdout, _ = capsys.readouterr() expected = 'localhost\nfoo\nBAR\n' assert stdout == expected def test_dump_platform_details(capsys, mock_global_config): """It dumps lists of platform spec.""" glblcfg: GlobalConfig = mock_global_config(TEST_CONF) glblcfg.dump_platform_details(glblcfg) out, _ = capsys.readouterr() expected = ( '[platforms]\n [[foo]]\n hosts = of_morgoth\n' '[platform groups]\n [[BAR]]\n platforms = mario, sonic\n' ) assert expected == out def test_expand_commas(tmp_path: 'Path', mock_global_config: Callable): """It should expand comma separated platform and install target definitions.""" glblcfg: GlobalConfig = mock_global_config(''' [install] [[symlink dirs]] [[[foo, bar]]] run = /x [[[foo]]] share = /y [[[bar]]] share = /z [platforms] [[foo]] [[[meta]]] x = 1 [["bar"]] # double quoted name [[[meta]]] x = 2 [[baz, bar, pub]] # baz before bar to test order is handled correctly [[[meta]]] x = 3 [['pub']] # single quoted name [[[meta]]] x = 4 ''') glblcfg._expand_commas() # ensure the definition order is preserved assert glblcfg.get(['platforms']).keys() == [ 'localhost', 'foo', 'bar', 'baz', 'pub', ] # ensure platform sections are correctly deep-merged assert glblcfg.get(['platforms', 'foo', 'meta', 'x']) == '1' assert glblcfg.get(['platforms', 'bar', 'meta', 'x']) == '3' assert glblcfg.get(['platforms', 'baz', 'meta', 'x']) == '3' assert glblcfg.get(['platforms', 'pub', 'meta', 'x']) == '4' # ensure install target sections are correctly merged: assert glblcfg.get(["install", "symlink dirs", "foo", "run"]) == "/x" assert glblcfg.get(["install", "symlink dirs", "foo", "share"]) == "/y" assert glblcfg.get(["install", "symlink dirs", "bar", "run"]) == "/x" assert glblcfg.get(["install", "symlink dirs", "bar", "share"]) == "/z" @pytest.mark.parametrize( 'src_dir, err_expected', [ pytest.param( '/theoden/rohan', False, id="Abs path ok" ), pytest.param( 'theoden/rohan', True, id="Rel path bad" ), pytest.param( '~theoden/rohan', False, id="Starts with usr - ok" ), pytest.param( '$THEODEN/rohan', False, id="Starts with env var - ok" ), pytest.param( 'rohan/$THEODEN', True, id="Rel path with env var not at start - bad" ), ] ) def test_source_dir_validation( src_dir: str, err_expected: bool, tmp_path: 'Path', mock_global_config: Callable ): glblcfg: GlobalConfig = mock_global_config(f''' [install] source dirs = /denethor/gondor, {src_dir} ''') if err_expected: with pytest.raises(ValidationError) as excinfo: glblcfg.load() assert "must be an absolute path" in str(excinfo.value) else: glblcfg.load() def test_platform_ssh_forward_variables(mock_global_config): glblcfg: GlobalConfig = mock_global_config(''' [platforms] [[foo]] ssh forward environment variables = "FOO", "BAR" ''') assert glblcfg.get( ['platforms', 'foo', 'ssh forward environment variables'] ) == ["FOO", "BAR"] def test_reload( mock_global_config, tmp_path: 'Path', monkeypatch: pytest.MonkeyPatch ): # Load a config glblcfg: GlobalConfig = mock_global_config(''' [platforms] [[foo]] [[[meta]]] x = 1 ''') # Update the global config file and reload conf_path = tmp_path / GlobalConfig.CONF_BASENAME conf_path.write_text(''' [platforms] [[foo]] [[[meta]]] x = 2 ''') glblcfg.load() # Mock the global config singleton monkeypatch.setattr(GlobalConfig, "get_inst", lambda *a, **k: glblcfg) assert glblcfg.get(['platforms', 'foo', 'meta', 'x']) == '2' from cylc.flow.platforms import get_platform platform = get_platform("foo") assert platform['meta']['x'] == "2" cylc-flow-8.7.0/tests/unit/test_id_match.py0000664000175000017500000002011615264420307021116 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging from textwrap import dedent from types import SimpleNamespace from typing import TYPE_CHECKING, Set, Tuple, cast import pytest from cylc.flow import CYLC_LOG from cylc.flow.id import Tokens from cylc.flow.id_match import id_match from cylc.flow.config import WorkflowConfig if TYPE_CHECKING: from cylc.flow.id_match import TaskTokens def to_tokens(*ids): return cast('Set[TaskTokens]', {Tokens(id_, relative=True) for id_ in ids}) def to_string_ids(*ids): return {id_.relative_id for id_ in ids} def to_string_ids_with_selectors(*ids): return {id_.relative_id_with_selectors for id_ in ids} @pytest.fixture(scope='module') def mod_test_config(mod_tmp_path): path = mod_tmp_path / 'flow.cylc' with open(path, 'w+') as flow_cylc: flow_cylc.write(dedent(''' [scheduler] allow implicit tasks = True [scheduling] cycling mode = integer initial cycle point = 1 [[graph]] P1 = a => b => c => d P3 = z ''')) return WorkflowConfig('test', str(path), SimpleNamespace()) @pytest.fixture(scope='module') def mod_test_family_config(mod_tmp_path): """It should match family IDs.""" path = mod_tmp_path / 'flow.cylc' with open(path, 'w+') as flow_cylc: flow_cylc.write(dedent(''' [scheduling] cycling mode = integer initial cycle point = 1 [[graph]] P1 = a & b1 & b2 P2 = z1 & z2 3/P3 = z3 [runtime] [[a]] inherit = A [[b1, b2]] inherit = B [[A, B]] [[Z]] [[z1, z2, z3]] inherit = Z ''')) return WorkflowConfig('test', str(path), SimpleNamespace()) def _id_match( config: 'WorkflowConfig', pool: 'Set[TaskTokens]', ids: 'Set[TaskTokens]', only_match_pool: bool = False, ) -> 'Tuple[Set[str], Set[str]]': """Convenience function for testing, converts strings to tokens.""" matched, unmatched = id_match( config, pool, to_tokens(*ids), only_match_pool=only_match_pool, ) return ( to_string_ids(*matched), to_string_ids_with_selectors(*unmatched), ) @pytest.mark.parametrize( 'ids,matched,unmatched', [ ( {'1'}, {'1/a', '1/b', '1/c'}, set(), ), ( {'2'}, set(), {'2'} ), ( {'*'}, {'1/a', '1/b', '1/c'}, set(), ), ( {'1/*'}, {'1/a', '1/b', '1/c'}, set(), ), ( {'2/*'}, set(), {'2/*'} ), ( {'*/*'}, {'1/a', '1/b', '1/c'}, set(), ), ( {'*/a'}, {'1/a'}, set(), ), ( {'*/z'}, set(), {'*/z'} ), ( {'*/*:x'}, {'1/a', '1/b', '1/c'}, set(), ), ( {'*/*:y'}, set(), {'*/*:y'}, ), ] ) def test_match_task(mod_test_config, ids, matched, unmatched): """It should match task IDs.""" pool = to_tokens('1/a:x', '1/b:x', '1/c:x') _matched, _unmatched = _id_match( mod_test_config, pool, ids, only_match_pool=True, ) assert _matched == matched assert _unmatched == unmatched @pytest.mark.parametrize( 'ids,matched,unmatched', [ ( {'1/a'}, {'1/a'}, set(), ), ( {'1/*'}, {'1/a', '1/b'}, set(), ), ( {'1/*:x'}, {'1/a', '1/b'}, set(), ), ( {'1/*:y'}, set(), {'1/*:y'}, ), ( {'*/*:x'}, {'1/a', '1/b', '2/a'}, set(), ), ( {'1/z'}, set(), {'1/z'}, ), ( {'1'}, {'1/a', '1/b'}, set(), ), ( {'3'}, set(), {'3'}, ), ] ) def test_match_cycle(mod_test_config, ids, matched, unmatched): """It should match cycle IDs.""" pool = to_tokens('1/a:x', '1/b:x', '2/a:x') assert _id_match( mod_test_config, pool, ids, only_match_pool=True, ) == (matched, unmatched) @pytest.mark.parametrize( 'ids,matched,unmatched', [ ( {'1/root'}, {'1/a', '1/b', '1/c', '1/d', '1/z'}, set(), ), ( {'2/root'}, {'2/a', '2/b', '2/c', '2/d'}, # 1/z isn't in cycle 2 set(), ), ( {'*/root'}, # should match all active cycles (i.e. cycles 1 and 2) {'1/a', '1/b', '1/c', '1/d', '1/z', '2/a', '2/b', '2/c', '2/d'}, set(), ), ( {'1/[ad]'}, {'1/a', '1/d'}, set(), ), ( {'1/[!ad]'}, {'1/b', '1/c', '1/z'}, set(), ), ] ) def test_match_inactive(mod_test_config, ids, matched, unmatched): """It should match non-pool tasks""" assert _id_match( mod_test_config, to_tokens('1/a:running', '2/a:waiting'), # active cycles are 1 and 2 ids, ) == (matched, unmatched) @pytest.mark.parametrize( 'ids,matched,unmatched', [ ({'1/A'}, {'1/a'}, set()), ({'1/B'}, {'1/b1', '1/b2'}, set()), ({'1/C'}, set(), {'1/C'}), ({'1/root'}, {'1/a', '1/b1', '1/b2', '1/z1', '1/z2'}, set()), ] ) def test_match_family(mod_test_family_config, ids, matched, unmatched): """It should match family IDs.""" pool = to_tokens('1/a:x', '1/b1:x', '1/b2:x') assert _id_match(mod_test_family_config, pool, ids) == (matched, unmatched) @pytest.mark.parametrize( 'ids,matched,unmatched,warnings', [ # These IDs all reference 2/z, but, z are not valid for cycle 2: # explicit reference -> warning ({'2/z1'}, set(), {'2/z1'}, ('2/z1',)), ({'2/Z'}, set(), {'2/Z'}, ('2/z1', '2/z2', '2/z3')), # implicit reference -> no warning ({'*/Z'}, {'1/z1', '1/z2'}, set(), set()), ({'2/*'}, {'2/a', '2/b1', '2/b2'}, set(), set()), ({'2/root'}, {'2/a', '2/b1', '2/b2'}, set(), set()), ({'*/[XYZ]'}, {'1/z1', '1/z2'}, set(), set()), ] ) def test_match_warnings( mod_test_family_config, ids, matched, unmatched, warnings, caplog, ): """It should warn about invalid cycle points, only when relevant. See https://github.com/cylc/cylc-flow/issues/7184 """ pool = to_tokens('1/a:x', '1/b1:x', '1/b2:x', '2/a:x') caplog.clear() caplog.set_level(logging.WARNING, CYLC_LOG) assert _id_match(mod_test_family_config, pool, ids) == (matched, unmatched) assert caplog.messages == [ f'Invalid cycle point for task:' f' {Tokens(id_, relative=True)["task"]}' f', {Tokens(id_, relative=True)["cycle"]}' for id_ in warnings ] cylc-flow-8.7.0/tests/unit/test_xtrigger_mgr.py0000664000175000017500000002406415264420307022054 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging import pytest from cylc.flow import CYLC_LOG from cylc.flow.cycling.iso8601 import ISO8601Point, ISO8601Sequence, init from cylc.flow.exceptions import XtriggerConfigError from cylc.flow.id import Tokens from cylc.flow.subprocctx import SubFuncContext from cylc.flow.task_proxy import TaskProxy from cylc.flow.taskdef import TaskDef from cylc.flow.xtrigger_mgr import ( RE_STR_TMPL, XtriggerCollator ) def test_extract_templates(): """Test escaped templates in xtrigger arg string. They should be left alone and passed into the function as string literals, not identified as template args. """ assert ( RE_STR_TMPL.findall('%(cat)s, %(dog)s, %%(fish)s') == ['cat', 'dog'] ) def test_add_missing_func(): """Test for adding an xtrigger that can't be found.""" xtriggers = XtriggerCollator() xtrig = SubFuncContext( label="fooble", func_name="fooble123", # no such module func_args=["name", "age"], func_kwargs={"location": "soweto"} ) with pytest.raises( XtriggerConfigError, match=r"\[@xtrig\] fooble123\(.*\)\nNo module named 'fooble123'" ): xtriggers.add_trig("xtrig", xtrig, 'fdir') def test_add_xtrigger(): """Test for adding and validating an xtrigger.""" xtriggers = XtriggerCollator() xtrig = SubFuncContext( label="echo", func_name="echo", func_args=["name", "age"], func_kwargs={"location": "soweto"} ) with pytest.raises( XtriggerConfigError, match="Requires 'succeed=True/False' arg" ): xtriggers.add_trig("xtrig", xtrig, 'fdir') xtrig = SubFuncContext( label="echo", func_name="echo", func_args=["name", "age"], func_kwargs={"location": "soweto", "succeed": True} ) xtriggers.add_trig("xtrig", xtrig, 'fdir') assert xtrig == xtriggers.functx_map["xtrig"] def test_add_xtrigger_with_template_good(): """Test adding an xtrigger with a valid string template arg value.""" xtriggers = XtriggerCollator() xtrig = SubFuncContext( label="echo", func_name="echo", func_args=["name", "%(point)s"], # valid template func_kwargs={"location": "soweto", "succeed": True} ) xtriggers.add_trig("xtrig", xtrig, 'fdir') assert xtrig == xtriggers.functx_map["xtrig"] def test_add_xtrigger_with_template_bad(): """Test adding an xtrigger with an invalid string template arg value.""" xtriggers = XtriggerCollator() xtrig = SubFuncContext( label="echo", func_name="echo", func_args=["name", "%(point)s"], # invalid template: func_kwargs={"location": "%(what_is_this)s", "succeed": True} ) with pytest.raises( XtriggerConfigError, match="Illegal template in xtrigger: what_is_this" ): xtriggers.add_trig("xtrig", xtrig, 'fdir') def test_add_xtrigger_with_deprecated_params( caplog: pytest.LogCaptureFixture ): """It should flag deprecated template variables.""" xtriggers = XtriggerCollator() xtrig = SubFuncContext( label="echo", func_name="echo", func_args=[1, "name", "%(suite_name)s"], func_kwargs={"succeed": True} ) caplog.set_level(logging.WARNING, CYLC_LOG) xtriggers.add_trig("xtrig", xtrig, 'fdir') assert caplog.messages == [ 'Xtrigger "xtrig" uses deprecated template variables, which ' 'will be removed in Cylc 8.9: suite_name' ] def test_load_xtrigger_for_restart(xtrigger_mgr): """Test loading an xtrigger for restart. The function is loaded from database, where the value is formatted as JSON.""" row = "get_name", "{\"name\": \"function\"}" xtrigger_mgr.load_xtrigger_for_restart(row_idx=0, row=row) assert xtrigger_mgr.sat_xtrig["get_name"]["name"] == "function" def test_load_invalid_xtrigger_for_restart(xtrigger_mgr): """Test loading an invalid xtrigger for restart. It simulates that the DB has a value that is not valid JSON. """ row = "get_name", "{name: \"function\"}" # missing double quotes with pytest.raises(ValueError): xtrigger_mgr.load_xtrigger_for_restart(row_idx=0, row=row) def test_housekeeping_nothing_satisfied(xtrigger_mgr): """The housekeeping method makes sure only satisfied xtrigger function are kept.""" row = "get_name", "{\"name\": \"function\"}" # now XtriggerManager#sat_xtrigger will contain the get_name xtrigger xtrigger_mgr.add_xtriggers(XtriggerCollator()) xtrigger_mgr.load_xtrigger_for_restart(row_idx=0, row=row) assert xtrigger_mgr.sat_xtrig xtrigger_mgr.housekeep([]) assert not xtrigger_mgr.sat_xtrig def test_housekeeping_with_xtrigger_satisfied(xtrigger_mgr): """The housekeeping method makes sure only satisfied xtrigger function are kept.""" xtriggers = XtriggerCollator() xtrig = SubFuncContext( label="get_name", func_name="echo", func_args=[], func_kwargs={"succeed": True} ) xtriggers.add_trig("get_name", xtrig, 'fdir') xtrigger_mgr.add_xtriggers(xtriggers) xtrig.out = "[\"True\", {\"name\": \"Yossarian\"}]" tdef = TaskDef( name="foo", rtcfg={'completion': None}, start_point=1, initial_point=1, ) init() sequence = ISO8601Sequence('P1D', '2019') tdef.xtrig_labels[sequence] = ["get_name"] start_point = ISO8601Point('2019') itask = TaskProxy(Tokens('~user/workflow'), tdef, start_point) # pretend the function has been activated xtrigger_mgr.active.append(xtrig.get_signature()) xtrigger_mgr.callback(xtrig) assert xtrigger_mgr.sat_xtrig xtrigger_mgr.housekeep([itask]) # here we still have the same number as before assert xtrigger_mgr.sat_xtrig def test__call_xtriggers_async(xtrigger_mgr): """Test _call_xtriggers_async""" xtriggers = XtriggerCollator() # the echo1 xtrig (not satisfied) echo1_xtrig = SubFuncContext( label="echo1", func_name="echo", func_args=[], func_kwargs={"succeed": False} ) echo1_xtrig.out = "[\"True\", {\"name\": \"herminia\"}]" xtriggers.add_trig("echo1", echo1_xtrig, "fdir") # the echo2 xtrig (satisfied through callback later) echo2_xtrig = SubFuncContext( label="echo2", func_name="echo", func_args=[], func_kwargs={"succeed": True} ) echo2_xtrig.out = "[\"True\", {\"name\": \"herminia\"}]" xtriggers.add_trig("echo2", echo2_xtrig, "fdir") xtrigger_mgr.add_xtriggers(xtriggers) # create a task tdef = TaskDef( name="foo", rtcfg={'completion': None}, start_point=1, initial_point=1 ) init() sequence = ISO8601Sequence('P1D', '2000') tdef.xtrig_labels[sequence] = ["echo1", "echo2"] # cycle point for task proxy init() start_point = ISO8601Point('2019') # create task proxy itask = TaskProxy(Tokens('~user/workflow'), tdef, start_point) # we start with no satisfied xtriggers, and nothing active assert len(xtrigger_mgr.sat_xtrig) == 0 assert len(xtrigger_mgr.active) == 0 # after calling the first time, we get two active xtrigger_mgr.call_xtriggers_async(itask) assert len(xtrigger_mgr.sat_xtrig) == 0 assert len(xtrigger_mgr.active) == 2 # calling again does not change anything xtrigger_mgr.call_xtriggers_async(itask) assert len(xtrigger_mgr.sat_xtrig) == 0 assert len(xtrigger_mgr.active) == 2 # now we call callback manually as the proc_pool we passed is a mock # then both should be satisfied xtrigger_mgr.callback(echo1_xtrig) xtrigger_mgr.callback(echo2_xtrig) # so both were satisfied, and nothing is active assert len(xtrigger_mgr.sat_xtrig) == 2 assert len(xtrigger_mgr.active) == 0 # calling satisfy_xtriggers again still does not change anything xtrigger_mgr.call_xtriggers_async(itask) assert len(xtrigger_mgr.sat_xtrig) == 2 assert len(xtrigger_mgr.active) == 0 def test_callback_not_active(xtrigger_mgr): """Test callback with no active contexts.""" # calling callback with a SubFuncContext with none active # results in a ValueError get_name = SubFuncContext( label="get_name", func_name="get_name", func_args=[], func_kwargs={} ) with pytest.raises(ValueError): xtrigger_mgr.callback(get_name) def test_callback_invalid_json(xtrigger_mgr): """Test callback with invalid JSON.""" get_name = SubFuncContext( label="get_name", func_name="get_name", func_args=[], func_kwargs={} ) get_name.out = "{no_quotes: \"mom!\"}" xtrigger_mgr.active.append(get_name.get_signature()) xtrigger_mgr.callback(get_name) # this means that the xtrigger was not satisfied # TODO: this means site admins are only aware of this if they # look at the debug log. Is that OK? assert not xtrigger_mgr.sat_xtrig def test_callback(xtrigger_mgr): """Test callback.""" get_name = SubFuncContext( label="get_name", func_name="get_name", func_args=[], func_kwargs={} ) get_name.out = "[\"True\", \"1\"]" xtrigger_mgr.active.append(get_name.get_signature()) xtrigger_mgr.callback(get_name) # this means that the xtrigger was satisfied assert xtrigger_mgr.sat_xtrig cylc-flow-8.7.0/tests/unit/test_task_events_mgr.py0000664000175000017500000001046315264420307022545 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from typing import Optional from unittest.mock import Mock import pytest from cylc.flow.broadcast_mgr import BroadcastMgr from cylc.flow.task_events_mgr import TaskEventsManager from cylc.flow.task_proxy import TaskProxy from cylc.flow.taskdef import TaskDef @pytest.mark.parametrize( "broadcast, remote, platforms, expected", [ ("hpc1", "a", "b", "hpc1"), (None, "hpc1", "b", "hpc1"), (None, None, "hpc1", "hpc1"), (None, None, None, None), ] ) def test_get_remote_conf(broadcast, remote, platforms, expected): """Test TaskEventsManager._get_remote_conf().""" task_events_mgr = TaskEventsManager( None, None, None, None, None, None, None, None, None) task_events_mgr.broadcast_mgr = Mock( get_broadcast=lambda x: { "remote": { "host": broadcast } } ) itask = Mock( identity='foo.1', tdef=Mock( rtconfig={ 'remote': { 'host': remote } } ), platform={ 'host': platforms } ) assert task_events_mgr._get_remote_conf(itask, 'host') == expected @pytest.mark.parametrize( "broadcast, workflow, platforms, expected", [ ([800], [700], [600], [800]), (None, [700], [600], [700]), (None, None, [600], [600]), ] ) def test_get_workflow_platforms_conf(broadcast, workflow, platforms, expected): """Test TaskEventsManager._get_polling_interval_conf().""" task_events_mgr = TaskEventsManager( None, None, None, None, None, None, None, None, None) KEY = "execution polling intervals" task_events_mgr.broadcast_mgr = Mock( get_broadcast=lambda x: { KEY: broadcast } ) itask = Mock( identity='foo.1', tdef=Mock( rtconfig={ KEY: workflow } ), platform={ KEY: platforms } ) assert ( task_events_mgr._get_workflow_platforms_conf(itask, KEY) == expected ) @pytest.mark.parametrize( 'rt_val, schd_val, glbl_val, expected', [ ('rt', 'schd', 'glbl', 'rt'), (None, 'schd', 'glbl', 'schd'), (None, None, 'glbl', 'glbl'), (None, None, None, 'default'), ] ) def test_get_events_conf__mail_to_from( mock_glbl_cfg, rt_val: Optional[str], schd_val: Optional[str], glbl_val: Optional[str], expected: str ): """Test order of precedence for [mail]to/from.""" if glbl_val: mock_glbl_cfg( 'cylc.flow.task_events_mgr.glbl_cfg', f''' [scheduler] [[mail]] from = {glbl_val} to = {glbl_val} ''' ) mock_task = Mock( spec=TaskProxy, tdef=Mock( spec=TaskDef, rtconfig={ 'events': {}, 'mail': {'to': rt_val, 'from': rt_val} if rt_val else {}, }, ), ) mock_task_events_mgr = Mock( spec=TaskEventsManager, workflow_cfg={ 'scheduler': { 'mail': {'to': schd_val, 'from': schd_val}, }, } if schd_val else {}, broadcast_mgr=Mock( spec_set=BroadcastMgr, get_broadcast=lambda *a, **k: {}, ), ) for key in ('to', 'from'): assert TaskEventsManager._get_events_conf( mock_task_events_mgr, itask=mock_task, key=key, default='default' ) == expected cylc-flow-8.7.0/tests/unit/test_option_parsers.py0000664000175000017500000004356715264420307022434 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from contextlib import redirect_stdout import io import sys from types import SimpleNamespace from typing import List import pytest from pytest import param import cylc.flow.flags from cylc.flow.option_parsers import ( CylcOptionParser as COP, Options, combine_options, combine_options_pair, OptionSettings, cleanup_sysargv, filter_sysargv ) USAGE_WITH_COMMENT = "usage \n # comment" ARGS = 'args' KWARGS = 'kwargs' SOURCES = 'sources' USEIF = 'useif' @pytest.fixture(scope='module') def parser(): return COP( USAGE_WITH_COMMENT, argdoc=[('SOME_ARG', "Description of SOME_ARG")] ) @pytest.mark.parametrize( 'args,verbosity', [ ([], 0), (['-v'], 1), (['-v', '-v', '-v'], 3), (['-q'], -1), (['-q', '-q', '-q'], -3), (['-q', '-v', '-q'], -1), (['--debug'], 2), (['--debug', '-q'], 1), (['--debug', '-v'], 3), ] ) def test_verbosity( args: List[str], verbosity: int, parser: COP, monkeypatch: pytest.MonkeyPatch ) -> None: """-v, -q, --debug should be additive.""" # patch the cylc.flow.flags value so that it gets reset after the test monkeypatch.setattr('cylc.flow.flags.verbosity', None) opts, args = parser.parse_args(['default-arg'] + args) assert opts.verbosity == verbosity # test side-effect, the verbosity flag should be set assert cylc.flow.flags.verbosity == verbosity def test_help_color(monkeypatch: pytest.MonkeyPatch, parser: COP): """Test for colorized comments in 'cylc cmd --help --color=always'.""" # This colorization is done on the fly when help is printed. monkeypatch.setattr("sys.argv", ['cmd', 'foo', '--color=always']) parser.parse_args(None) assert parser.values.color == "always" f = io.StringIO() with redirect_stdout(f): parser.print_help() assert not (f.getvalue()).startswith("Usage: " + USAGE_WITH_COMMENT) def test_help_nocolor(monkeypatch: pytest.MonkeyPatch, parser: COP): """Test for no colorization in 'cylc cmd --help --color=never'.""" # This colorization is done on the fly when help is printed. monkeypatch.setattr(sys, "argv", ['cmd', 'foo', '--color=never']) parser.parse_args(None) assert parser.values.color == "never" f = io.StringIO() with redirect_stdout(f): parser.print_help() assert (f.getvalue()).startswith("Usage: " + USAGE_WITH_COMMENT) def test_Options_std_opts(): """Test Python Options API with standard options.""" parser = COP(USAGE_WITH_COMMENT, auto_add=True) MyOptions = Options(parser) MyValues = MyOptions(verbosity=1) assert MyValues.verbosity == 1 # Add overlapping args tomorrow @pytest.mark.parametrize( 'first, second, expect', [ param( [{ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'do'}}], [{ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'dont'}}], ( [{ ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'do', 'dont'}, USEIF: '' }] ), id='identical arg lists unchanged' ), param( [{ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'fall'}}], [{ ARGS: ['-f', '--foolish'], KWARGS: {'help': 'not identical'}, SOURCES: {'fold'}}], ( [ { ARGS: ['--foo'], KWARGS: {}, SOURCES: {'fall'}, USEIF: '' }, { ARGS: ['--foolish'], KWARGS: {'help': 'not identical'}, SOURCES: {'fold'}, USEIF: '' } ] ), id='different arg lists lose shared names' ), param( [{ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'cook'}}], [{ ARGS: ['-f', '--foo'], KWARGS: {'help': 'not identical', 'dest': 'foobius'}, SOURCES: {'bake'}, USEIF: '' }], None, id='different args identical arg list cause exception' ), param( [{ARGS: ['-g', '--goo'], KWARGS: {}, SOURCES: {'knit'}}], [{ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'feed'}}], [ { ARGS: ['-g', '--goo'], KWARGS: {}, SOURCES: {'knit'}, USEIF: '' }, { ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'feed'}, USEIF: '' }, ], id='all unrelated args added' ), param( [ {ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'work'}}, {ARGS: ['-r', '--redesdale'], KWARGS: {}, SOURCES: {'work'}} ], [ {ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'sink'}}, { ARGS: ['-b', '--buttered-peas'], KWARGS: {}, SOURCES: {'sink'} } ], [ { ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'work', 'sink'}, USEIF: '' }, { ARGS: ['-b', '--buttered-peas'], KWARGS: {}, SOURCES: {'sink'}, USEIF: '' }, { ARGS: ['-r', '--redesdale'], KWARGS: {}, SOURCES: {'work'}, USEIF: '' }, ], id='do not repeat args' ), param( [ { ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'push'} }, ], [], [ { ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'push'}, USEIF: '' }, ], id='one empty list is fine' ) ] ) def test_combine_options_pair(first, second, expect): """It combines sets of options""" first = [ OptionSettings(i[ARGS], sources=i[SOURCES], **i[KWARGS]) for i in first ] second = [ OptionSettings(i[ARGS], sources=i[SOURCES], **i[KWARGS]) for i in second ] if expect is not None: result = combine_options_pair(first, second) assert [i.__dict__ for i in result] == expect else: with pytest.raises(Exception, match='Clashing Options'): combine_options_pair(first, second) @pytest.mark.parametrize( 'inputs, expect', [ param( [ ([OptionSettings( ['-i', '--inflammable'], help='', sources={'wish'} )]), ([OptionSettings( ['-f', '--flammable'], help='', sources={'rest'} )]), ([OptionSettings( ['-n', '--non-flammable'], help='', sources={'swim'} )]), ], [ {ARGS: ['-i', '--inflammable']}, {ARGS: ['-f', '--flammable']}, {ARGS: ['-n', '--non-flammable']} ], id='merge three argsets no overlap' ), param( [ [ OptionSettings( ['-m', '--morpeth'], help='', sources={'stop'}), OptionSettings( ['-r', '--redesdale'], help='', sources={'stop'}), ], [ OptionSettings( ['-b', '--byker'], help='', sources={'walk'}), OptionSettings( ['-r', '--roxborough'], help='', sources={'walk'}), ], [ OptionSettings( ['-b', '--bellingham'], help='', sources={'leap'}), ] ], [ {ARGS: ['--bellingham']}, {ARGS: ['--roxborough']}, {ARGS: ['--redesdale']}, {ARGS: ['--byker']}, {ARGS: ['-m', '--morpeth']} ], id='merge three overlapping argsets' ), param( [ ([]), ( [ OptionSettings( ['-c', '--campden'], help='x', sources={'foo'}) ] ) ], [ {ARGS: ['-c', '--campden']} ], id="empty list doesn't clear result" ), ] ) def test_combine_options(inputs, expect): """It combines multiple input sets""" result = combine_options(*inputs) result_args = [i.args for i in result] # Order of args irrelevant to test for option in expect: assert option[ARGS] in result_args @pytest.mark.parametrize( 'argv_before, kwargs, expect', [ param( 'vip myworkflow -f something -b something_else --baz', { 'script_name': 'play', 'workflow_id': 'myworkflow', 'compound_script_opts': [ OptionSettings(['--foo', '-f']), OptionSettings(['--bar', '-b'], action='store'), OptionSettings(['--baz'], action='store_true'), ], 'script_opts': [ OptionSettings(['--foo', '-f']), ] }, 'play myworkflow -f something', id='remove some opts' ), param( 'vip myworkflow', { 'script_name': 'play', 'workflow_id': 'myworkflow', 'compound_script_opts': [ OptionSettings(['--foo', '-f']), OptionSettings(['--bar', '-b']), OptionSettings(['--baz']), ], 'script_opts': [] }, 'play myworkflow', id='no opts to keep' ), param( 'vip ./myworkflow --foo something', { 'script_name': 'play', 'workflow_id': 'myworkflow', 'compound_script_opts': [ OptionSettings(['--foo', '-f'])], 'script_opts': [ OptionSettings(['--foo', '-f']), ], 'source': './myworkflow', }, 'play --foo something myworkflow', id='replace path' ), param( 'vip --foo something', { 'script_name': 'play', 'workflow_id': 'myworkflow', 'compound_script_opts': [ OptionSettings(['--foo', '-f'])], 'script_opts': [ OptionSettings(['--foo', '-f']), ], 'source': './myworkflow', }, 'play --foo something myworkflow', id='no path given' ), param( 'vip -n myworkflow --no-run-name', { 'script_name': 'play', 'workflow_id': 'myworkflow', 'compound_script_opts': [ OptionSettings(['--workflow-name', '-n']), OptionSettings(['--no-run-name']), ], 'script_opts': [ OptionSettings(['--not-used']), ] }, 'play myworkflow', id='workflow-id-added' ), ] ) def test_cleanup_sysargv( monkeypatch: pytest.MonkeyPatch, argv_before: str, kwargs: dict, expect: str ): """It replaces the contents of sysargv with Cylc Play argv items. """ # Fake up sys.argv: for this test. dummy_cylc_path = ['/pathto/my/cylc/bin/cylc'] monkeypatch.setattr(sys, 'argv', dummy_cylc_path + argv_before.split()) # Fake options too: opts = SimpleNamespace(**{ i.args[0].replace('--', ''): i for i in kwargs['compound_script_opts'] }) kwargs.update({'options': opts}) if not kwargs.get('source', None): kwargs.update({'source': ''}) # Test the script: cleanup_sysargv(**kwargs) assert sys.argv == dummy_cylc_path + expect.split() @pytest.mark.parametrize( 'sysargs, simple, compound, expect', ( param( # Test for https://github.com/cylc/cylc-flow/issues/5905 '--no-run-name --workflow-name=name'.split(), ['--no-run-name'], ['--workflow-name'], [], id='--workflow-name=name' ), param( '--foo something'.split(), [], [], '--foo something'.split(), id='no-opts-removed' ), param( [], ['--foo'], ['--bar'], [], id='Null-check' ), param( '''--keep1 --keep2 42 --keep3=Hi --throw1 --throw2 84 --throw3=There '''.split(), ['--throw1'], '--throw2 --throw3'.split(), '--keep1 --keep2 42 --keep3=Hi'.split(), id='complex' ), param( "--foo 'foo=42' --bar='foo=94'".split(), [], ['--foo'], ['--bar=\'foo=94\''], id='--bar=\'foo=94\'' ) ) ) def test_filter_sysargv( sysargs, simple, compound, expect ): """It returns the subset of sys.argv that we ask for. n.b. The three most basic cases for this function are stored in its own docstring. """ assert filter_sysargv(sysargs, simple, compound) == expect class TestOptionSettings(): @staticmethod def test_init(): args = ['--foo', '-f'] kwargs = {'bar': 42} sources = {'touch'} useif = 'hello' result = OptionSettings( args, sources=sources, useif=useif, **kwargs) assert result.__dict__ == { 'kwargs': kwargs, 'sources': sources, 'useif': useif, 'args': args } @staticmethod @pytest.mark.parametrize( 'first, second, expect', ( param( (['--foo', '-f'], {'bar': 42}, {'touch'}, 'hello'), (['--foo', '-f'], {'bar': 42}, {'touch'}, 'hello'), True, id='Totally the same'), param( (['--foo', '-f'], {'bar': 42}, {'touch'}, 'hello'), (['--foo', '-f'], {'bar': 42}, {'wibble'}, 'byee'), True, id='Differing extras'), param( (['-f'], {'bar': 42}, {'touch'}, 'hello'), (['--foo', '-f'], {'bar': 42}, {'wibble'}, 'byee'), False, id='Not equal args'), ) ) def test___eq__args_intersection(first, second, expect): args, kwargs, sources, useif = first first = OptionSettings( args, sources=sources, useif=useif, **kwargs) args, kwargs, sources, useif = second second = OptionSettings( args, sources=sources, useif=useif, **kwargs) assert (first == second) == expect @staticmethod @pytest.mark.parametrize( 'first, second, expect', ( param( ['--foo', '-f'], ['--foo', '-f'], ['--foo', '-f'], id='Totally the same'), param( ['--foo', '-f'], ['--foolish', '-f'], ['-f'], id='Some overlap'), param( ['--foo', '-f'], ['--bar', '-b'], [], id='No overlap'), ) ) def test___and__(first, second, expect): first = OptionSettings(first) second = OptionSettings(second) assert sorted(first & second) == sorted(expect) @staticmethod @pytest.mark.parametrize( 'first, second, expect', ( param( ['--foo', '-f'], ['--foo', '-f'], [], id='Totally the same'), param( ['--foo', '-f'], ['--foolish', '-f'], ['--foo'], id='Some overlap'), param( ['--foolish', '-f'], ['--foo', '-f'], ['--foolish'], id='Some overlap not commuting'), param( ['--foo', '-f'], ['--bar', '-b'], ['--foo', '-f'], id='No overlap'), ) ) def test___sub__args_subtraction(first, second, expect): first = OptionSettings(first) second = OptionSettings(second) assert sorted(first - second) == sorted(expect) @staticmethod def test__in_list(): """It is in a list.""" first = OptionSettings(['--foo']) second = OptionSettings(['--foo']) third = OptionSettings(['--bar']) assert first._in_list([second, third]) is True cylc-flow-8.7.0/tests/unit/test_task_remote_mgr.py0000664000175000017500000002572215264420307022540 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from contextlib import suppress from pathlib import Path from time import sleep import pytest from typing import (Any, Optional) from unittest.mock import MagicMock, Mock from cylc.flow.exceptions import PlatformError from cylc.flow.network.client_factory import CommsMeth from cylc.flow.task_remote_mgr import ( REMOTE_FILE_INSTALL_DONE, REMOTE_INIT_IN_PROGRESS, TaskRemoteMgr) from cylc.flow.workflow_files import WorkflowFiles, get_workflow_srv_dir Fixture = Any @pytest.mark.parametrize( 'comms_meth, expected', [ (CommsMeth.SSH, True), (CommsMeth.ZMQ, True), (CommsMeth.POLL, False) ] ) def test__remote_init_items(comms_meth: CommsMeth, expected: bool): """Test _remote_init_items(). Should only includes files under .service/ """ id_ = 'barclay' mock_mgr = Mock(workflow=id_) srv_dir = get_workflow_srv_dir(id_) items = TaskRemoteMgr._remote_init_items(mock_mgr, comms_meth) if expected: assert items for src_path, dst_path in items: Path(src_path).relative_to(srv_dir) Path(dst_path).relative_to(WorkflowFiles.Service.DIRNAME) else: assert not items @pytest.mark.parametrize( 'install_target, skip_expected, expected_status', [('localhost', True, REMOTE_FILE_INSTALL_DONE), ('something_else', False, REMOTE_INIT_IN_PROGRESS)] ) def test_remote_init_skip( install_target: str, skip_expected: bool, expected_status: str, monkeypatch: Fixture): """Test the TaskRemoteMgr.remote_init() skips localhost install target. Params: install_target: The platform's install target. skip_expected: Whether remote init is expected to be skipped. expected_status: The expected value of TaskRemoteMgr.remote_init_map[install_target]. """ platform = { 'install target': install_target, 'communication method': CommsMeth.POLL, 'hosts': ['localhost'], 'selection': {'method': 'random'}, 'name': 'foo' } mock_task_remote_mgr = MagicMock(remote_init_map={}, bad_hosts=[]) mock_construct_ssh_cmd = Mock() monkeypatch.setattr('cylc.flow.task_remote_mgr.construct_ssh_cmd', mock_construct_ssh_cmd) for item in ( 'tarfile', 'get_remote_workflow_run_dir', 'get_dirs_to_symlink'): monkeypatch.setattr(f'cylc.flow.task_remote_mgr.{item}', MagicMock()) TaskRemoteMgr.remote_init(mock_task_remote_mgr, platform) call_expected = not skip_expected assert mock_task_remote_mgr._remote_init_items.called is call_expected assert mock_construct_ssh_cmd.called is call_expected assert mock_task_remote_mgr.proc_pool.put_command.called is call_expected status = mock_task_remote_mgr.remote_init_map[install_target] assert status == expected_status @pytest.mark.parametrize( 'install_target, load_type, expected', [ ('install_target', None, '03-start-install_target.log'), ('some_install_target', 'restart', '03-restart-some_install_target.log'), ('another_install_target', 'reload', '03-reload-another_install_target.log') ] ) def test_get_log_file_name(tmp_path: Path, install_target: str, load_type: Optional[str], expected: str): task_remote_mgr = TaskRemoteMgr('some_workflow', None, None, None, None) if load_type == 'restart': task_remote_mgr.is_restart = True elif load_type == 'reload': task_remote_mgr.is_reload = True # else load type is start (no flag required) run_dir = tmp_path log_dir = run_dir / 'some_workflow' / 'log' / 'remote-install' log_dir.mkdir(parents=True) for log_num in range(1, 3): Path(f"{log_dir}/{log_num:02d}-start-{install_target}.log").touch() sleep(0.1) log_name = task_remote_mgr.get_log_file_name( install_target, install_log_dir=log_dir) assert log_name == expected @pytest.mark.parametrize( 'platform_names, install_targets, glblcfg, expect', [ pytest.param( # Two platforms share an install target. Both are reachable. ['sir_handel', 'peter_sam'], ['mountain_railway'], ''' [platforms] [[peter_sam, sir_handel]] install target = mountain_railway ''', { 'targets': {'mountain_railway': ['peter_sam', 'sir_handel']}, 'unreachable': set() }, id='basic' ), pytest.param( # Two platforms share an install target. Both are unreachable. set(), ['mountain_railway'], ''' [platforms] [[peter_sam, sir_handel]] install target = mountain_railway ''', { 'targets': {'mountain_railway': []}, 'unreachable': {'mountain_railway'} }, id='platform_unreachable' ), pytest.param( # One of our install targets matches one of our platforms, # but only implicitly; i.e. the platform name is the same as the # install target name. ['sir_handel'], ['sir_handel'], ''' [platforms] [[sir_handel]] ''', { 'targets': {'sir_handel': ['sir_handel']}, 'unreachable': set() }, id='implicit-target' ), pytest.param( # One of our install targets matches one of our platforms, # but only implicitly, and the platform name is defined using a # regex. ['sir_handel42'], ['sir_handel42'], ''' [platforms] [[sir_handel..]] ''', { 'targets': {'sir_handel42': ['sir_handel42']}, 'unreachable': set() }, id='implicit-target-regex' ), pytest.param( # One of our install targets (rusty) has no defined platforms # causing a PlatformLookupError. ['duncan', 'rusty'], ['mountain_railway', 'rusty'], ''' [platforms] [[duncan]] install target = mountain_railway ''', { 'targets': {'mountain_railway': ['duncan']}, 'unreachable': {'rusty'} }, id='PlatformLookupError' ) ] ) def test_map_platforms_used_for_install_targets( mock_glbl_cfg, platform_names, install_targets, glblcfg, expect, caplog ): def flatten_install_targets_map(itm): result = {} for target, platforms in itm.items(): result[target] = sorted([p['name'] for p in platforms]) return result mock_glbl_cfg('cylc.flow.platforms.glbl_cfg', glblcfg) install_targets_map = TaskRemoteMgr._get_remote_tidy_targets( set(platform_names), set(install_targets)) with suppress(KeyError): install_targets_map.pop('localhost') assert ( expect['targets'] == flatten_install_targets_map(install_targets_map)) if expect['unreachable']: for unreachable in expect["unreachable"]: assert ( unreachable in caplog.records[0].msg) else: assert not caplog.records shared_eval_params = [ pytest.param( 'localhost', {}, 'localhost', id="localhost" ), pytest.param( '$(some-cmd)', {}, None, id="subshell_1st_eval" ), pytest.param( '$(some-cmd)', {'some-cmd': None}, None, id="subshell_awaiting_eval" ), pytest.param( '$(some-cmd)', {'some-cmd': 'isaac clarke'}, 'isaac clarke', id="subshell_finished_eval" ), pytest.param( '$SOME_ENV', {}, 'nolan stross', id="env_var" ), pytest.param( '$(env-cmd)', {'env-cmd': '$SOME_ENV'}, 'nolan stross', id="subshell_env_var" ), pytest.param( 'titan_station', {}, 'titan_station', id="verbatim_name" ), pytest.param( '', {}, 'localhost', id="empty" ), ] @pytest.fixture def task_remote_mgr_eval(monkeypatch: pytest.MonkeyPatch): """Fixture providing a task remote manager for eval_platform() & eval_host() tests.""" def _task_remote_mgr_eval(remote_cmd_map: dict) -> TaskRemoteMgr: monkeypatch.setenv('SOME_ENV', 'nolan stross') task_remote_mgr = TaskRemoteMgr( workflow='usg_ishimura', proc_pool=Mock(), bad_hosts=[], db_mgr=None, server=None ) task_remote_mgr.remote_command_map = remote_cmd_map return task_remote_mgr return _task_remote_mgr_eval @pytest.mark.parametrize( 'eval_str, remote_cmd_map, expected', [ *shared_eval_params, pytest.param( 'localhost_tau_volantis', {}, 'localhost_tau_volantis', id="name_begin_w_localhost" ) ] ) def test_eval_platform( eval_str: str, remote_cmd_map: dict, expected: Optional[str], task_remote_mgr_eval, ): task_remote_mgr: TaskRemoteMgr = task_remote_mgr_eval(remote_cmd_map) assert task_remote_mgr.eval_platform(eval_str) == expected def test_eval_platform_bad(task_remote_mgr_eval): exc_msg = "foreign contaminant detected" task_remote_mgr: TaskRemoteMgr = task_remote_mgr_eval( {'cmd': PlatformError(exc_msg, 'aegis_vii')} ) with pytest.raises(PlatformError, match=exc_msg): task_remote_mgr.eval_platform('$(cmd)') @pytest.mark.parametrize( 'eval_str, remote_cmd_map, expected', [ *shared_eval_params, pytest.param( 'localhost4.localdomain4', {}, 'localhost', id="localhost_variant" ), pytest.param( '`other cmd`', {'other cmd': 'nicole brennan'}, 'nicole brennan', id="backticks" ), ] ) def test_eval_host( eval_str: str, remote_cmd_map: dict, expected: Optional[str], task_remote_mgr_eval, ): task_remote_mgr: TaskRemoteMgr = task_remote_mgr_eval(remote_cmd_map) assert task_remote_mgr.eval_host(eval_str) == expected cylc-flow-8.7.0/tests/unit/test_job_runner_mgr.py0000664000175000017500000000554215264420307022364 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.job_runner_mgr import ( JobRunnerManager, JOB_FILES_REMOVED_MESSAGE) jrm = JobRunnerManager() SAMPLE_STATUS = """ ignore me, I have no = sign CYLC_JOB_RUNNER_NAME=pbs CYLC_JOB_ID=2361713 CYLC_JOB_RUNNER_SUBMIT_TIME=2025-01-28T14:46:04Z CYLC_JOB_PID=2361713 CYLC_JOB_INIT_TIME=2025-01-28T14:46:05Z CYLC_MESSAGE=2025-01-28T14:46:05Z|INFO|sleep 31 CYLC_JOB_RUNNER_EXIT_POLLED=2025-01-28T14:46:08Z CYLC_JOB_EXIT=SUCCEEDED CYLC_JOB_EXIT_TIME=2025-01-28T14:46:38Z """ def test__job_poll_status_files(tmp_path): """Good Path: A valid job.status files exists""" (tmp_path / 'sub').mkdir() (tmp_path / 'sub' / 'job.status').write_text(SAMPLE_STATUS) ctx = jrm._jobs_poll_status_files(str(tmp_path), 'sub') assert ctx.job_runner_name == 'pbs' assert ctx.job_id == '2361713' assert ctx.job_runner_exit_polled == 1 assert ctx.pid == '2361713' assert ctx.time_submit_exit == '2025-01-28T14:46:04Z' assert ctx.time_run == '2025-01-28T14:46:05Z' assert ctx.time_run_exit == '2025-01-28T14:46:38Z' assert ctx.run_status == 0 assert ctx.messages == ['2025-01-28T14:46:05Z|INFO|sleep 31'] def test__job_poll_status_files_task_failed(tmp_path): """Good Path: A valid job.status files exists""" (tmp_path / 'sub').mkdir() (tmp_path / 'sub' / 'job.status').write_text("CYLC_JOB_EXIT=FOO") ctx = jrm._jobs_poll_status_files(str(tmp_path), 'sub') assert ctx.run_status == 1 assert ctx.run_signal == 'FOO' def test__job_poll_status_files_deleted_logdir(): """The log dir has been deleted whilst the task is still active. Return the context with the message that the task has failed. """ ctx = jrm._jobs_poll_status_files('foo', 'bar') assert ctx.run_signal == JOB_FILES_REMOVED_MESSAGE assert ctx.run_status == 1 assert ctx.job_runner_exit_polled == 1 def test__job_poll_status_files_ioerror(tmp_path, capsys): """There is no readable file. """ (tmp_path / 'sub').mkdir() jrm._jobs_poll_status_files(str(tmp_path), 'sub') cap = capsys.readouterr() assert '[Errno 2] No such file or directory' in cap.err cylc-flow-8.7.0/tests/unit/test_context_node.py0000664000175000017500000001071215264420307022040 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from textwrap import dedent import pytest from cylc.flow.context_node import ContextNode with ContextNode('a') as a: with ContextNode('b') as b: c = ContextNode('c') d = ContextNode('d') def test_context_node_tree(): """Ensure parents and children are correctly organised.""" assert a.name == 'a' assert a._parent is None assert a._children == {'b': b, 'd': d} assert b.name == 'b' assert b._parent == a assert b._children == {'c': c} assert c.name == 'c' assert c._parent == b assert c._children is None def test_data_state(): """Ensure the in-class state remains intact.""" # DATA should start clean assert ContextNode.DATA == {} # nodes add themselves to DATA then remove themselves from it with ContextNode('foo'): pass # so DATA should end clean assert ContextNode.DATA == {} # however erroneously creating leaf-nodes outside of a context # will make a mess a = ContextNode('a') assert ContextNode.DATA == {a: set()} # just to make sure this doesn't result in hard-to-debug errors # we make sure this doesn't prevent us creating new trees with ContextNode('foo') as foo: bar = ContextNode('bar') assert ContextNode.DATA == {a: set(), foo: {a}} assert list(foo.walk()) == [(0, foo), (1, bar)] # finally clean up for niceness del ContextNode.DATA[a] del ContextNode.DATA[foo] def test_context_iter(): """Test iterating over child nodes.""" assert list(a) == [b, d] assert list(b) == [c] assert list(c) == [] def test_context_contains(): """Test `node.__contains__`.""" assert 'b' in a assert 'c' in b assert 'c' not in a def test_context_getitem(): """Test `node.__getitem__`.""" assert a['b'] == b assert b['c'] == c with pytest.raises(KeyError): assert b['d'] with pytest.raises(TypeError): assert c['b'] def test_context_str(): """Test the string representation of a node.""" assert str(a) == 'a' assert str(b) == 'b' assert str(c) == 'c' def test_context_repr(): """Test the Python representation of a node.""" assert repr(a) == 'a' assert repr(b) == 'a/b' assert repr(c) == 'a/b/c' def test_context_walk(): """Test walking the tree.""" assert list(a.walk()) == [ (0, a), (1, b), (2, c), (1, d) ] assert list(b.walk()) == [ (0, b), (1, c), ] assert list(c.walk()) == [ (0, c) ] def test_context_walk_depth(): assert list(a.walk(depth=1)) == [ (0, a), (1, b), (1, d) ] def test_context_tree(): """Test the string representation of the tree.""" assert a.tree() == dedent(''' a b c d ''').strip() assert b.tree() == dedent(''' b c ''').strip() assert c.tree() == dedent(''' c ''').strip() def test_context_is_root(): """Test root node detection.""" assert a.is_root() assert not b.is_root() assert not c.is_root() def test_context_is_leaf(): """Test leaf node detection.""" assert not a.is_leaf() assert not b.is_leaf() assert c.is_leaf() def test_context_parents(): """Test linearised ancestry.""" assert list(a.parents()) == [] assert list(b.parents()) == [a] assert list(c.parents()) == [b, a] def test_re_enter(): """It should be able to re __enter__ multiple times.""" # create a node with one child with ContextNode('x') as x: ContextNode('y') assert 'y' in x # go back and add another child retrospectively with x: ContextNode('z') assert 'y' in x assert 'z' in x cylc-flow-8.7.0/tests/unit/jinja2_filters/0000775000175000017500000000000015264420307020642 5ustar alastairalastaircylc-flow-8.7.0/tests/unit/jinja2_filters/test_duration_as.py0000664000175000017500000000251115264420307024562 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) 2008-2019 Earth Sciences New Zealand # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.jinja.filters.duration_as import duration_as import pytest @pytest.mark.parametrize( 'duration,fmt,result', [ pytest.param('PT1H', 's', 3600, id='PT1H->s'), pytest.param('PT1H', 'm', 60, id='PT1H->m'), pytest.param('PT1H', 'h', 1, id='PT1H->h'), pytest.param('PT1H', 'd', 1 / 24, id='PT1H->d'), pytest.param('PT1H', 'w', 1 / (24 * 7), id='PT1H->w'), pytest.param('P7D', 'd', 7, id='P7D->d'), pytest.param('P7D', 's', 604800, id='P7D->s'), ] ) def test_all(duration, fmt, result): assert duration_as(duration, fmt) == result cylc-flow-8.7.0/tests/unit/test_rundb.py0000664000175000017500000001607315264420307020467 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import contextlib import json from pathlib import Path import sqlite3 import sys from typing import ( List, Optional, Tuple, ) import unittest from unittest import mock import pytest from cylc.flow.flow_mgr import FlowNums from cylc.flow.rundb import CylcWorkflowDAO from cylc.flow.util import serialise_set class TestRunDb(unittest.TestCase): def setUp(self): self.dao = CylcWorkflowDAO(':memory:') self.mocked_connection = mock.Mock() self.dao.connect = mock.MagicMock(return_value=self.mocked_connection) get_select_task_job = [ ["cycle", "name", "NN"], ["cycle", "name", None], ["cycle", "name", "02"], ] def test_select_task_job(self): """Test the rundb CylcWorkflowDAO select_task_job method""" columns = self.dao.tables[CylcWorkflowDAO.TABLE_TASK_JOBS].columns[3:] expected_values = [[2 for _ in columns]] self.mocked_connection.execute.return_value = expected_values # parameterized test for cycle, name, submit_num in self.get_select_task_job: returned_values = self.dao.select_task_job(cycle, name, submit_num) for column in columns: self.assertEqual(2, returned_values[column.name]) def test_select_task_job_sqlite_error(self): """Test when the rundb CylcWorkflowDAO select_task_job method raises a SQLite exception, the method returns None""" self.mocked_connection.execute.side_effect = sqlite3.DatabaseError r = self.dao.select_task_job("it'll", "raise", "an error!") self.assertIsNone(r) @contextlib.contextmanager def create_temp_db(): """Create and tidy a temporary database for testing purposes.""" conn = sqlite3.connect(':memory:') yield conn conn.close() # doesn't raise error on re-invocation def test_operational_error(tmp_path, caplog): """Test logging on operational error.""" # create a db object db_file = tmp_path / 'db' with CylcWorkflowDAO(db_file) as dao: # stage some stuff dao.add_delete_item(CylcWorkflowDAO.TABLE_TASK_JOBS) dao.add_insert_item(CylcWorkflowDAO.TABLE_TASK_JOBS, ['pub']) dao.add_update_item( CylcWorkflowDAO.TABLE_TASK_JOBS, ({'pub': None}, {}) ) # connect the to DB dao.connect() # then delete the file - this will result in an OperationalError db_file.unlink() # execute & commit the staged items with pytest.raises(sqlite3.OperationalError): dao.execute_queued_items() # ensure that the failed transaction is logged for debug purposes assert len(caplog.messages) == 1 message = caplog.messages[0] assert 'An error occurred when writing to the database' in message assert 'DELETE FROM task_jobs' in message assert 'INSERT OR REPLACE INTO task_jobs' in message assert 'UPDATE task_jobs' in message assert 'The error was: no such table: task_jobs' in message if sys.version_info.major == 3 and sys.version_info.minor >= 11: assert 'SQLite error code: 1' in message assert 'SQLite error name: SQLITE_ERROR' in message else: assert 'SQLite error code: Not available' in message assert 'SQLite error name: Not available' in message def test_table_creation(tmp_path: Path): """Test tables are NOT created by default.""" db_file = tmp_path / 'db' stmt = "SELECT name FROM sqlite_master WHERE type='table'" with CylcWorkflowDAO(db_file) as dao: tables = list(dao.connect().execute(stmt)) assert tables == [] with CylcWorkflowDAO(db_file, create_tables=True) as dao: tables = [i[0] for i in dao.connect().execute(stmt)] assert CylcWorkflowDAO.TABLE_WORKFLOW_PARAMS in tables def test_context_manager_exit( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ): """Test connection is closed even if an exception occurs somewhere.""" db_file = tmp_path / 'db' mock_close = mock.Mock() with monkeypatch.context() as mp: mp.setattr(CylcWorkflowDAO, 'close', mock_close) with CylcWorkflowDAO(db_file) as dao, pytest.raises(RuntimeError): mock_close.assert_not_called() raise RuntimeError('mock err') mock_close.assert_called_once() # Close connection for real: dao.close() @pytest.mark.parametrize( 'values, expected', [ pytest.param( [ ({1, 2}, '2021-01-01T00:00:00'), ({3, 4}, '2021-01-01T00:00:02'), ({5, 6}, '2021-01-01T00:00:01'), ], {3, 4}, id="basic" ), pytest.param( [ ({2}, '2021-01-01T00:00:00'), (set(), '2021-01-01T00:00:01'), (set(), '2021-01-01T00:00:02'), ], {2}, id="ignore flow=none" ), pytest.param( [ (set(), '2021-01-01T00:00:01'), (set(), '2021-01-01T00:00:02'), ], None, id="all flow=none" ), ], ) def test_select_latest_flow_nums( values: List[Tuple[FlowNums, str]], expected: Optional[FlowNums] ): with CylcWorkflowDAO(':memory:') as dao: conn = dao.connect() conn.execute( "CREATE TABLE task_states (flow_nums TEXT, time_created TEXT)" ) for (fnums, timestamp) in values: conn.execute( 'INSERT INTO task_states VALUES (?, ?)', (serialise_set(fnums), timestamp) ) conn.commit() assert dao.select_latest_flow_nums() == expected def test_select_task_outputs__identical_outputs(): """It yields all entries with different flow nums, even if they have identical outputs. """ outputs_str = json.dumps({'f': 'foo', 'g': 'goo'}) with CylcWorkflowDAO(':memory:') as dao: dao.create_tables() conn = dao.connect() conn.executemany( "INSERT INTO task_outputs VALUES (?, ?, ?, ?)", [ ("1", "task_a", serialise_set({1}), outputs_str), ("1", "task_a", serialise_set(set()), outputs_str), ], ) conn.commit() assert list(dao.select_task_outputs("task_a", "1")) == [ (outputs_str, {1}), (outputs_str, set()), ] cylc-flow-8.7.0/tests/unit/README.md0000664000175000017500000000204715264420307017217 0ustar alastairalastair# Unit Tests This directory contains Cylc unit tests. ## How To Run These Tests ```console $ pytest tests/u $ pytest tests/u -n 5 # run up to 5 tests in parallel $ pytest tests/u --dist=no -n0 # turn off xdist (allows --pdb etc) ``` ## What Are Unit Tests Unit tests test the smallest possible units of functionality, typically methods or functions. The interaction of components is mitigated by mocking input objects. ## Guidelines Don't write integration tests here: * If your test requires any of the fixtures in the integration tests then it is an integration test. * If your test sees logic flow through multiple modules it's not a unit test. * If you are constructing computationally expensive objects it's unlikely to be a unit test. ## Doctests Doctests are Python interactive shell examples embedded in the docstrings of functions/methods, that can provide some level of unit testing. E.g. ```python def factorial(n): """ Example: >>> factorial(7) 5040 """ ``` To run doctests: ```console $ pytest cylc/flow ``` cylc-flow-8.7.0/tests/unit/test_task_trigger.py0000664000175000017500000001355115264420307022040 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.cycling.loader import ( get_point, get_sequence, ) from cylc.flow.task_outputs import TaskOutputs from cylc.flow.task_trigger import ( Dependency, TaskTrigger, ) def test_check_with_cycle_point(): task_trigger = TaskTrigger( 'fake_task_name', 1, 'fakeOutput', None, None, None, None) actual = str(task_trigger) expected = 'fake_task_name[1]:fakeOutput' assert actual == expected def test_check_with_no_cycle_point_with_offset(): task_trigger = TaskTrigger( 'fake_task_name', 2, 'fakeOutput', None, None, None, None) actual = str(task_trigger) expected = 'fake_task_name[2]:fakeOutput' assert actual == expected def test_check_with_no_cycle_point_or_offset(): task_trigger = TaskTrigger( 'fake_task_name', None, 'fakeOutput', None, None, None, None) actual = str(task_trigger) expected = 'fake_task_name:fakeOutput' assert actual == expected def test_check_for_false_suicide(): task_trigger = TaskTrigger( 'fake_task_name', 1, 'fakeOutput', None, None, None, None) dependency = Dependency( [task_trigger, '&', task_trigger], [task_trigger], False) actual = str(dependency) expected = ( '( fake_task_name[1]:fakeOutput ) ( & ) ( fake_task_name[1]' ':fakeOutput )') assert actual == expected def test_check_for_true_suicide(): task_trigger = TaskTrigger( 'fake_task_name', None, 'fakeOutput', None, None, None, None) dependency = Dependency( [task_trigger, '&', task_trigger], [task_trigger], True) actual = str(dependency) expected = ( '! ( fake_task_name:fakeOutput ) ( & ) ( fake_task_name:fakeOutput )') assert actual == expected def test_check_for_list_of_lists_exp(): task_trigger = TaskTrigger( 'fake_task_name', None, 'fakeOutput', None, None, None, None) dependency = Dependency( [ task_trigger, '&', ['task', '&', 'another_task'] ], [task_trigger], False ) actual = str(dependency) expected = ( "( fake_task_name:fakeOutput ) ( & ) ['task', '&', 'another_task']") assert actual == expected def test_check_trigger_name(): assert not TaskOutputs.is_valid_std_name("Elephant") def test_get_parent_point(set_cycling_type): set_cycling_type() one = get_point('1') two = get_point('2') trigger = TaskTrigger('name', None, 'output') assert trigger.get_parent_point(one) == one trigger = TaskTrigger('name', one, 'output', offset_is_absolute=True) assert trigger.get_parent_point(None) == one trigger = TaskTrigger('name', '+P1', 'output', initial_point=one) assert trigger.get_parent_point(one) == two trigger = TaskTrigger( 'name', '+P1', 'output', offset_is_from_icp=True, initial_point=one) assert trigger.get_parent_point(two) == two assert trigger.get_parent_point(one) == two def test_get_child_point(set_cycling_type): set_cycling_type() zero = get_point('0') one = get_point('1') two = get_point('2') p1 = get_sequence('P1', one) trigger = TaskTrigger('name', None, 'output') assert trigger.get_child_point(one, p1) == one assert trigger.get_child_point(two, p1) == two trigger = TaskTrigger('name', '+P1', 'output', offset_is_absolute=True) assert trigger.get_child_point(None, p1) == one trigger = TaskTrigger('name', '+P1', 'output', offset_is_from_icp=True) assert trigger.get_child_point(None, p1) == one trigger = TaskTrigger('name', '+P1', 'output', offset_is_irregular=True) assert trigger.get_child_point(one, p1) == zero trigger = TaskTrigger('name', '-P1', 'output', offset_is_irregular=True) assert trigger.get_child_point(one, p1) == two trigger = TaskTrigger('name', '+P1', 'output') assert trigger.get_child_point(one, None) == zero trigger = TaskTrigger('name', '-P1', 'output') assert trigger.get_child_point(one, None) == two def test_get_point(set_cycling_type): set_cycling_type() one = get_point('1') two = get_point('2') trigger = TaskTrigger('name', '1', 'output', offset_is_absolute=True) assert trigger.get_point(None) == one trigger = TaskTrigger( 'name', '+P1', 'output', offset_is_from_icp=True, initial_point=one) assert trigger.get_point(None) == two trigger = TaskTrigger('name', '+P1', 'output') assert trigger.get_point(one) == two trigger = TaskTrigger('name', None, 'output') assert trigger.get_point(one) == one def test_str(set_cycling_type): set_cycling_type() trigger = TaskTrigger('name', '1', 'output', offset_is_absolute=True) assert str(trigger) == 'name[1]:output' trigger = TaskTrigger('name', '+P1', 'output') assert str(trigger) == 'name[+P1]:output' trigger = TaskTrigger('name', None, 'output') assert str(trigger) == 'name:output' def test_eq(): args = ('foo', '+P1', 'succeeded') assert TaskTrigger(*args) == TaskTrigger(*args) assert TaskTrigger(*args) != TaskTrigger( *args, initial_point=IntegerPoint('1') ) cylc-flow-8.7.0/tests/unit/test_id.py0000664000175000017500000003203015264420307017740 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test the Cylc universal identifier system.""" import pytest from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.cycling.iso8601 import ISO8601Point from cylc.flow.id import ( LEGACY_CYCLE_SLASH_TASK, LEGACY_TASK_DOT_CYCLE, RELATIVE_ID, UNIVERSAL_ID, Tokens, quick_relative_id, ) @pytest.mark.parametrize( 'identifier', [ '', '~', '~user//cycle' '~flow:state', 'flow:flow_sel:flow_sel', ] ) def test_univseral_id_illegal(identifier): """Test illegal formats of the universal identifier.""" assert UNIVERSAL_ID.match(identifier) is None @pytest.mark.parametrize( 'identifier', [ '~user', '~user/', '~user/workflow', '~user/workflow//', '~user/workflow:workflow_sel', '~user/workflow:workflow_sel//', '~user/workflow:workflow_sel//cycle', '~user/workflow:workflow_sel//cycle/', '~user/workflow:workflow_sel//cycle:cycle_sel', '~user/workflow:workflow_sel//cycle:cycle_sel/', '~user/workflow:workflow_sel//cycle:cycle_sel/task', '~user/workflow:workflow_sel//cycle:cycle_sel/task/', '~user/workflow:workflow_sel//cycle:cycle_sel/task:task_sel', '~user/workflow:workflow_sel//cycle:cycle_sel/task:task_sel/', '~user/workflow:workflow_sel//cycle:cycle_sel/task:task_sel/job', ( '~user/workflow:workflow_sel//cycle:cycle_sel/task:task_sel/job' ':job_sel' ), 'workflow', 'workflow//', 'workflow:workflow_sel', 'workflow:workflow_sel//', 'workflow:workflow_sel//cycle', 'workflow:workflow_sel//cycle/', 'workflow:workflow_sel//cycle:cycle_sel', 'workflow:workflow_sel//cycle:cycle_sel/', 'workflow:workflow_sel//cycle:cycle_sel/task', 'workflow:workflow_sel//cycle:cycle_sel/task/', 'workflow:workflow_sel//cycle:cycle_sel/task:task_sel', 'workflow:workflow_sel//cycle:cycle_sel/task:task_sel/', 'workflow:workflow_sel//cycle:cycle_sel/task:task_sel/job', 'workflow:workflow_sel//cycle:cycle_sel/task:task_sel/job:job_sel' ] ) def test_universal_id_matches(identifier): """test every legal format of the universal identifier.""" # fmt: off expected_tokens = { 'user': 'user' if 'user' in identifier else None, 'workflow': 'workflow' if 'workflow' in identifier else None, 'workflow_sel': 'workflow_sel' if 'workflow_sel' in identifier else None, 'cycle': 'cycle' if 'cycle' in identifier else None, 'cycle_sel': 'cycle_sel' if 'cycle_sel' in identifier else None, 'task': 'task' if 'task' in identifier else None, 'task_sel': 'task_sel' if 'task_sel' in identifier else None, 'job': 'job' if 'job' in identifier else None, 'job_sel': 'job_sel' if 'job_sel' in identifier else None } # fmt: on match = UNIVERSAL_ID.match(identifier) assert match assert match.groupdict() == expected_tokens @pytest.mark.parametrize( 'identifier', [ '~user/a/b/c', '~user/a/b/c//', '~user/a/b/c:workflow_sel', '~user/a/b/c:workflow_sel//', '~user/a/b/c:workflow_sel//cycle', '~user/a/b/c:workflow_sel//cycle/', '~user/a/b/c:workflow_sel//cycle:cycle_sel', '~user/a/b/c:workflow_sel//cycle:cycle_sel/', '~user/a/b/c:workflow_sel//cycle:cycle_sel/task', '~user/a/b/c:workflow_sel//cycle:cycle_sel/task/', '~user/a/b/c:workflow_sel//cycle:cycle_sel/task:task_sel', '~user/a/b/c:workflow_sel//cycle:cycle_sel/task:task_sel/', '~user/a/b/c:workflow_sel//cycle:cycle_sel/task:task_sel/job', ( '~user/a/b/c:workflow_sel//cycle:cycle_sel/task:task_sel/job' ':job_sel' ), 'a/b/c', 'a/b/c//', 'a/b/c:workflow_sel', 'a/b/c:workflow_sel//', 'a/b/c:workflow_sel//cycle', 'a/b/c:workflow_sel//cycle/', 'a/b/c:workflow_sel//cycle:cycle_sel', 'a/b/c:workflow_sel//cycle:cycle_sel/', 'a/b/c:workflow_sel//cycle:cycle_sel/task', 'a/b/c:workflow_sel//cycle:cycle_sel/task/', 'a/b/c:workflow_sel//cycle:cycle_sel/task:task_sel', 'a/b/c:workflow_sel//cycle:cycle_sel/task:task_sel/', 'a/b/c:workflow_sel//cycle:cycle_sel/task:task_sel/job', 'a/b/c:workflow_sel//cycle:cycle_sel/task:task_sel/job:job_sel' ] ) def test_universal_id_matches_hierarchical(identifier): """Test the UID with hierarchical workflow IDs.""" # fmt: off expected_tokens = { 'user': 'user' if 'user' in identifier else None, 'workflow': 'a/b/c', # the hierarchical workflow ID 'workflow_sel': 'workflow_sel' if 'workflow_sel' in identifier else None, 'cycle': 'cycle' if 'cycle' in identifier else None, 'cycle_sel': 'cycle_sel' if 'cycle_sel' in identifier else None, 'task': 'task' if 'task' in identifier else None, 'task_sel': 'task_sel' if 'task_sel' in identifier else None, 'job': 'job' if 'job' in identifier else None, 'job_sel': 'job_sel' if 'job_sel' in identifier else None } # fmt: on match = UNIVERSAL_ID.match(identifier) assert match assert match.groupdict() == expected_tokens @pytest.mark.parametrize( 'identifier', [ '', '~', ':', 'workflow//cycle', 'task:task_sel:task_sel', 'cycle/task' '//', '//~', '//:', '//workflow//cycle', '//cycle/task:task_sel:task_sel' ] ) def test_relative_id_illegal(identifier): """Test illegal formats of the universal identifier.""" assert RELATIVE_ID.match(identifier) is None @pytest.mark.parametrize( 'identifier', [ '//cycle', '//cycle/', '//cycle:cycle_sel', '//cycle:cycle_sel/', '//cycle:cycle_sel/task', '//cycle:cycle_sel/task/', '//cycle:cycle_sel/task:task_sel', '//cycle:cycle_sel/task:task_sel/', '//cycle:cycle_sel/task:task_sel/job', '//cycle:cycle_sel/task:task_sel/job:job_sel', ] ) def test_relative_id_matches(identifier): """test every legal format of the relative identifier.""" expected_tokens = { 'cycle': 'cycle' if 'cycle' in identifier else None, 'cycle_sel': 'cycle_sel' if 'cycle_sel' in identifier else None, 'task': 'task' if 'task' in identifier else None, 'task_sel': 'task_sel' if 'task_sel' in identifier else None, 'job': 'job' if 'job' in identifier else None, 'job_sel': 'job_sel' if 'job_sel' in identifier else None } match = RELATIVE_ID.match(identifier) assert match assert match.groupdict() == expected_tokens @pytest.mark.parametrize( 'identifier', [ '', '~', '/', ':', 'task.cycle', # the first digit of the cycle should be a number '//task.123', # don't match the new format 'task.cycle/job', 'task:task_sel.123' # selector should suffix the cycle ] ) def test_legacy_task_dot_cycle_illegal(identifier): """Test illegal formats of the legacy task.cycle identifier.""" assert LEGACY_TASK_DOT_CYCLE.match(identifier) is None @pytest.mark.parametrize( 'identifier,expected_tokens', [ ( 'task.1', # integer cycles can be one character long {'task': 'task', 'cycle': '1', 'task_sel': None} ), ( 't.a.s.k.123', {'task': 't.a.s.k', 'cycle': '123', 'task_sel': None} ), ( 'task.123:task_sel', {'task': 'task', 'cycle': '123', 'task_sel': 'task_sel'} ), ] ) def test_legacy_task_dot_cycle_matches(identifier, expected_tokens): match = LEGACY_TASK_DOT_CYCLE.match(identifier) assert match assert match.groupdict() == expected_tokens @pytest.mark.parametrize( 'identifier', [ '', '~', '/', ':', 'cycle/task', # the first digit of the cycle should be a number '//123/task', # don't match the new format 'cycle/task/job' ] ) def test_legacy_cycle_slash_task_illegal(identifier): """Test illegal formats of the legacy cycle/task identifier.""" assert LEGACY_CYCLE_SLASH_TASK.match(identifier) is None @pytest.mark.parametrize( 'identifier,expected_tokens', [ ( '123/task', {'task': 'task', 'cycle': '123', 'task_sel': None} ), ( '123/t.a.s.k', {'task': 't.a.s.k', 'cycle': '123', 'task_sel': None} ), ( '123/task:task_sel', {'task': 'task', 'cycle': '123', 'task_sel': 'task_sel'} ) ] ) def test_legacy_cycle_slash_task_matches(identifier, expected_tokens): match = LEGACY_CYCLE_SLASH_TASK.match(identifier) assert match assert match.groupdict() == expected_tokens def test_tokens(): # tested mainly in doctests Tokens('a') with pytest.raises(ValueError): Tokens('a', 'b') Tokens(cycle='a') with pytest.raises(ValueError): Tokens(foo='a') Tokens().duplicate(cycle='a') with pytest.raises(ValueError): Tokens(foo='a') # test equality assert Tokens('a') == Tokens('a') assert Tokens('a') != Tokens('b') assert Tokens('a', relative=True) == Tokens('a', relative=True) assert Tokens('a', relative=True) != Tokens('b', relative=True) assert Tokens() != Tokens('a') assert Tokens(workflow='a') == Tokens('a') # test equality with non Tokens objects assert Tokens('a') != 'a' assert not Tokens('a') == 'a' assert Tokens('a') != 1 assert not Tokens('a') == 1 tokens = Tokens('a//b') new_tokens = tokens.duplicate(cycle='c', task='d') assert new_tokens == Tokens('a//c/d') with pytest.raises(Exception): tokens.update({'foo': 'c'}) with pytest.raises(Exception): tokens['cycle'] = 'a' # test gt/lt assert sorted( tokens.id for tokens in [ Tokens('~u/c'), Tokens('~u/b//1'), Tokens('~u/a'), Tokens('~u/b'), Tokens('~u/b//2'), ] ) == ['~u/a', '~u/b', '~u/b//1', '~u/b//2', '~u/c'] def test_no_look_behind(): """Ensure the UID pattern does not use lookbehinds. Ideally this pattern should be work for both cylc-flow and cylc-ui. 2022-01-11: * Lookbehind support is at ~75% * https://caniuse.com/js-regexp-lookbehind """ assert '?<=' not in UNIVERSAL_ID.pattern assert '?. """Test subscriber module components.""" from cylc.flow.network.subscriber import process_delta_msg def test_process_delta_msg(): """Test delta message processing.""" # test non-key not_topic, not_delta = process_delta_msg(b'foo', b'bar', None) assert not_topic == 'foo' assert not_delta == b'bar' cylc-flow-8.7.0/tests/unit/network/test_publisher.py0000664000175000017500000000206015264420307023032 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.network.publisher import serialize_data def test_serialize_data(): str1 = 'hello' assert serialize_data(str1, None) == str1 assert serialize_data(str1, 'encode', 'utf-8') == str1.encode('utf-8') assert serialize_data(str1, bytes, 'utf-8') == bytes(str1, 'utf-8') cylc-flow-8.7.0/tests/unit/network/test__init__.py0000664000175000017500000000437215264420307022445 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test __init__.py for network interfaces to Cylc scheduler objects.""" import pytest import cylc.flow from cylc.flow.exceptions import CylcVersionError from cylc.flow.network import get_location from cylc.flow.workflow_files import ContactFileFields BASE_CONTACT_DATA = { ContactFileFields.HOST: 'foo', ContactFileFields.PORT: '42', } @pytest.fixture() def mpatch_get_fqdn_by_host(monkeypatch): """Monkeypatch function used the same by all tests.""" monkeypatch.setattr( cylc.flow.network, 'get_fqdn_by_host', lambda _: 'myhost.x.y.z' ) def test_get_location_ok(monkeypatch, mpatch_get_fqdn_by_host): """It passes when information is available.""" contact_data = BASE_CONTACT_DATA.copy() contact_data[ContactFileFields.PUBLISH_PORT] = '8042' contact_data[ContactFileFields.VERSION] = cylc.flow.__version__ monkeypatch.setattr( cylc.flow.network, 'load_contact_file', lambda _: contact_data ) assert get_location('_') == ( 'myhost.x.y.z', 42, 8042, cylc.flow.__version__ ) def test_get_location_old_contact_file(monkeypatch, mpatch_get_fqdn_by_host): """It Fails because it's not a Cylc 8 workflow.""" contact_data = BASE_CONTACT_DATA.copy() contact_data['CYLC_SUITE_PUBLISH_PORT'] = '8042' contact_data['CYLC_VERSION'] = '5.1.2' monkeypatch.setattr( cylc.flow.network, 'load_contact_file', lambda _: contact_data ) with pytest.raises(CylcVersionError, match=r'.*5.1.2.*'): get_location('_') cylc-flow-8.7.0/tests/unit/network/test_multi.py0000664000175000017500000001147215264420307022176 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from functools import partial import pytest from cylc.flow.exceptions import CylcError, WorkflowStopped from cylc.flow.network.multi import _report, _process_response from cylc.flow.terminal import DIM def response(success, msg, operation='set'): return { operation: { 'result': [{'id': '~user/workflow', 'response': [success, msg]}] } } def test_report_valid(monkeypatch): """It should report command outcome.""" monkeypatch.setattr('cylc.flow.flags.verbosity', 0) # fail case assert _report(response(False, 'MyError')) == ( None, 'MyError', False, ) # success case assert _report(response(True, '12345')) == ( 'Command queued', None, True, ) # success case (debug mode) monkeypatch.setattr('cylc.flow.flags.verbosity', 1) assert _report(response(True, '12345')) == ( f'Command queued <{DIM}>id=12345', None, True, ) def test_report_invalid(monkeypatch): """It should report invalid responses. Tests that the code behaves as well as can be expected when confronted with responses which should not be possible. """ # test "None" response monkeypatch.setattr('cylc.flow.flags.verbosity', 0) assert _report({'set': None}) == ( None, 'Error processing command:' "\n TypeError: 'NoneType' object is not subscriptable", False, ) # test "None" response in debug mode monkeypatch.setattr('cylc.flow.flags.verbosity', 2) assert _report({'set': None}) == ( None, 'Error processing command:' "\n TypeError: 'NoneType' object is not subscriptable" # the response should be output in debug mode "\n response={'set': None}", False, ) # test multiple mutations in one operation (not supported) monkeypatch.setattr('cylc.flow.flags.verbosity', 0) assert _report( { **response(True, '12345'), **response(True, '23456', 'trigger'), } ) == ( None, 'Error processing command:' '\n NotImplementedError:' ' Cannot process multiple mutations in one operation.', False, ) # test zero mutations in the operation assert _report( {} ) == ( None, 'Error processing command:' '\n Exception: {}', False, ) def test_process_response(monkeypatch): """It should handle exceptions and return processed results.""" def report(exception_class, _response): raise exception_class('xxx') class Foo(Exception): pass # WorkflowStopped -> fail case monkeypatch.setattr('cylc.flow.flags.verbosity', 0) assert _process_response(partial(report, WorkflowStopped), {}) == ( None, 'WorkflowStopped: xxx is not running', False, ) # WorkflowStopped -> success case for this command monkeypatch.setattr('cylc.flow.flags.verbosity', 0) assert _process_response( partial(report, WorkflowStopped), {}, # this overrides the default interpretation of "WorkflowStopped" as a # fail case success_exceptions=(WorkflowStopped,), ) == ( 'WorkflowStopped: xxx is not running', None, True, # success outcome ) # CylcError -> expected error, log it monkeypatch.setattr('cylc.flow.flags.verbosity', 0) assert _process_response(partial(report, CylcError), {}) == ( None, 'CylcError: xxx', False, ) # CylcError -> expected error, raise it (debug mode) monkeypatch.setattr('cylc.flow.flags.verbosity', 2) with pytest.raises(CylcError): _process_response(partial(report, CylcError), {}) # Exception -> unexpected error, raise it monkeypatch.setattr('cylc.flow.flags.verbosity', 0) with pytest.raises(Foo): _process_response(partial(report, Foo), {}) cylc-flow-8.7.0/tests/unit/network/test_schema.py0000664000175000017500000001072415264420307022303 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from dataclasses import dataclass from inspect import isclass import re import graphene import pytest from cylc.flow.cfgspec.workflow import SPEC as WORKFLOW_SPEC from cylc.flow.network.schema import ( RUNTIME_FIELD_TO_CFG_MAP, Mutations, Runtime, WorkflowStopMode, runtime_schema_to_cfg, sort_elements, SortArgs, ) from cylc.flow.workflow_status import StopMode, WorkflowStatus @dataclass class DummyObject: value: int @pytest.mark.parametrize( 'elements,sort_args,expected_result', [ # sort asc by key ( [DummyObject(1), DummyObject(3), DummyObject(2)], { 'keys': ['value'], 'reverse': False # NOTE: GraphQL ensures reverse is not None! }, [DummyObject(1), DummyObject(2), DummyObject(3)] ), # sort desc by key ( [DummyObject(1), DummyObject(3), DummyObject(2)], { 'keys': ['value'], 'reverse': True }, [DummyObject(3), DummyObject(2), DummyObject(1)] ), # raise error when no keys given ( [DummyObject(1), DummyObject(3), DummyObject(2)], { 'keys': [], 'reverse': True }, ValueError ), # raise error when any of the keys given are not in the schema ( [DummyObject(1), DummyObject(3), DummyObject(2)], { 'keys': ['value', 'river_name'], 'reverse': True }, ValueError ) ] ) def test_sort_args(elements, sort_args, expected_result): """Test the sorting function used by the schema.""" sort = SortArgs() sort.keys = sort_args['keys'] sort.reverse = sort_args['reverse'] args = { 'sort': sort } if isclass(expected_result): with pytest.raises(expected_result): sort_elements(elements, args) else: sort_elements(elements, args) assert elements == expected_result def test_runtime_field_to_cfg_map(): """Ensure the Runtime type's fields can be mapped back to the workflow config.""" assert set(RUNTIME_FIELD_TO_CFG_MAP) == set(Runtime._meta.fields) for cfg_name in RUNTIME_FIELD_TO_CFG_MAP.values(): assert WORKFLOW_SPEC.get('runtime', '__MANY__', cfg_name) @pytest.mark.parametrize('runtime_dict,expected', [ pytest.param( {'run_mode': 'Skip'}, {'run mode': 'skip'}, id='edit-runtime' ), pytest.param( {'run mode': 'skip'}, {'run mode': 'skip'}, id='broadcast' ), ]) def test_runtime_schema_to_cfg(runtime_dict, expected): """Test this function can handle Edit Runtime submitted values as well as normal broadcast values.""" assert runtime_schema_to_cfg(runtime_dict) == expected @pytest.mark.parametrize('mutation', ( pytest.param(attr, id=name) for name, attr in Mutations.__dict__.items() if isinstance(attr, graphene.Field) )) def test_mutations_valid_for(mutation): """Check that all mutations have a "Valid for" in their description. This is needed by the UI to disable mutations that are not valid for the workflow state. """ match = re.search( r'Valid for:\s(.*)\sworkflows.', mutation.description ) assert match valid_states = set(match.group(1).split(', ')) assert valid_states assert not valid_states.difference(i.value for i in WorkflowStatus) @pytest.mark.parametrize('wflow_stop_mode', list(WorkflowStopMode)) def test_stop_mode_enum(wflow_stop_mode): """Check that WorkflowStopMode is a subset of StopMode.""" assert StopMode(wflow_stop_mode.value) assert wflow_stop_mode.description cylc-flow-8.7.0/tests/unit/network/test_graphql.py0000664000175000017500000001413515264420307022501 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from pytest import param from graphql import ( TypeInfo, TypeInfoVisitor, parse, visit ) from cylc.flow.data_messages_pb2 import PbTaskProxy, PbPrerequisite from cylc.flow.network.graphql import ( CylcVisitor, null_setter, strip_null, async_next, NULL_VALUE, grow_tree ) from cylc.flow.network.schema import schema TASK_PROXY_PREREQS = PbTaskProxy() TASK_PROXY_PREREQS.prerequisites.append(PbPrerequisite(expression="foo")) @pytest.mark.parametrize( 'query,' 'variables,' 'search_arg,' 'expected_result', [ pytest.param( ''' query ($workflowID: ID) { workflows (ids: [$workflowID]) { id } } ''', { 'workflowID': 'cylc|workflow' }, { 'ids': ['cylc|workflow'], }, True, id="simple query with correct variables" ), pytest.param( ''' query ($workflowID: ID) { ...WorkflowData } fragment WorkflowData on workflows { workflows (ids: [$workflowID]) { id } } ''', { 'workflowID': 'cylc|workflow' }, { 'ids': ['cylc|workflow'], }, True, id="query with a fragment and correct variables" ), pytest.param( ''' query ($workflowID: ID) { workflows (ids: [$workflowID]) { id } } ''', { 'workflowID': 'cylc|workflow' }, { 'ids': None, }, False, id="correct variable definition, but missing variable in " "provided values" ), pytest.param( ''' query ($workflowID: ID) { workflows (ids: [$workflowID]) { id } } ''', { 'workflowID': 'cylc|workflow' }, { 'idfsdf': ['cylc|workflow'], }, False, id="correct variable definition, but wrong search argument" ) ] ) def test_query_variables( query: str, variables: dict, search_arg: dict, expected_result: bool, ): """Test that query variables are parsed and found correctly. Args: query: a valid GraphQL query (using our schema) variables: map with variable values for the query search_arg: argument and value to search for expected_result: was the argument and value found """ document = parse(query) type_info = TypeInfo(schema.graphql_schema) cylc_visitor = CylcVisitor( type_info, variables, search_arg ) visit( document, TypeInfoVisitor( type_info, cylc_visitor ), None ) assert expected_result == cylc_visitor.arg_flag @pytest.mark.parametrize( 'pre_result,' 'expected_result', [ ( 'foo', 'foo' ), ( [], NULL_VALUE ), ( {}, NULL_VALUE ), ( TASK_PROXY_PREREQS.prerequisites, TASK_PROXY_PREREQS.prerequisites ), ( PbTaskProxy().prerequisites, NULL_VALUE ) ] ) def test_null_setter(pre_result, expected_result): """Test the null setting of different data types/results.""" post_result = null_setter(pre_result) assert post_result == expected_result @pytest.mark.parametrize( 'pre_result,' 'expected_result', [ ( 'foo', 'foo' ), ( [NULL_VALUE], [] ), ( {'nothing': NULL_VALUE}, {}, ), ( TASK_PROXY_PREREQS.prerequisites, TASK_PROXY_PREREQS.prerequisites ), ] ) async def test_strip_null(pre_result, expected_result): """Test the null stripping of different result data/types.""" # non-async post_result = async_next(strip_null, pre_result) assert post_result == expected_result async def async_result(result): return result # async async_post_result = async_next(strip_null, async_result(pre_result)) assert await async_post_result == expected_result @pytest.mark.parametrize( 'expect, tree, path, leaves', [ param( {'foo': {'bar': {'baz': {}}}}, {}, ['foo', 'bar', 'baz'], None, id='fill-empty-tree' ), param( {'bar': {'baz': {}}}, {'bar': {'baz': {}}}, ['bar', 'baz'], None, id='keep-full-tree' ), param( {'foo': {'bar': {}, 'qux': {}}}, {'foo': {'bar': {}}}, ['foo', 'qux'], None, id='add-new-branch' ), param( {'leaves': {'foo': {}}}, {}, [], {'foo': {}}, id='add-leaves' ) ] ) def test_grow_tree(expect, tree, path, leaves): grow_tree(tree, path, leaves) assert tree == expect cylc-flow-8.7.0/tests/unit/network/test_scan.py0000664000175000017500000001044415264420307021766 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test scan filters and data provision stuff.""" from pathlib import Path import re from textwrap import dedent from cylc.flow.network.scan import ( api_version, contact_info, cylc_version, filter_name, graphql_query, validate_contact_info ) from cylc.flow.workflow_files import ( ContactFileFields, WorkflowFiles ) SRV_DIR = Path(WorkflowFiles.Service.DIRNAME) CONTACT = Path(WorkflowFiles.Service.CONTACT) def test_filter_name_preprocess(): """It should combine provided patterns and compile them.""" pipe = filter_name('^f', '^c') assert pipe.args[0] == re.compile('(^f|^c)') async def test_filter_name(): """It should filter flows by registration name.""" pipe = filter_name('^f') assert await pipe.func( {'name': 'foo'}, *pipe.args ) assert not await pipe.func( {'name': 'bar'}, *pipe.args ) async def test_cylc_version(): """It should filter flows by cylc version.""" version = ContactFileFields.VERSION pipe = cylc_version('>= 8.0a1, < 9') assert await pipe.func( {version: '8.0a1'}, *pipe.args ) pipe = cylc_version('>= 8.0a1, < 9') assert not await pipe.func( {version: '7.8.4'}, *pipe.args ) async def test_api_version(): """It should filter flows by api version.""" version = ContactFileFields.API pipe = api_version('>= 4, < 5') assert await pipe.func( {version: '4'}, *pipe.args ) pipe = api_version('>= 4, < 5') assert not await pipe.func( {version: '5'}, *pipe.args ) async def test_contact_info(tmp_path): """It should load info from the contact file.""" # create a dummy flow Path(tmp_path, 'foo', SRV_DIR).mkdir(parents=True) # write a contact file with some junk in it with open(Path(tmp_path, 'foo', SRV_DIR, CONTACT), 'w+') as contact: contact.write(dedent(''' foo=1 bar=2 baz=3 ''').strip()) # create a flow dict as returned by scan flow = { 'name': 'foo', 'path': tmp_path / 'foo' } # ensure the contact fields get added to the flow dict assert await contact_info.func(flow) == { **flow, 'foo': '1', 'bar': '2', 'baz': '3' } def test_graphql_query_preproc(): """It should format graphql query fragments from the input data.""" pipe = graphql_query(['a', 'b', 'c']) assert pipe.args[0] == dedent(''' a b c ''') pipe = graphql_query({'a': None, 'b': None, 'c': None}) assert pipe.args[0] == dedent(''' a b c ''') pipe = graphql_query({'a': None, 'b': ['ba', 'bb'], 'c': None}) assert pipe.args[0] == dedent(''' a c b { ba bb } ''') async def test_validate_contact_file(tmp_path): """Ensure rejection for missing fields""" flow = { 'name': 'foo', 'path': tmp_path / 'foo', } # contact_info has already loaded info from contact file (missing fields) assert await validate_contact_info.func(flow) is False async def test_validate_contact_file_no_missing_fields(tmp_path): """Ensure rejection for missing fields""" version = ContactFileFields.API name = ContactFileFields.NAME host = ContactFileFields.HOST flow = { 'name': 'foo', 'path': tmp_path / 'foo', version: 1, name: 'moo', host: 'hosty' } assert await validate_contact_info.func(flow) == flow cylc-flow-8.7.0/tests/unit/filetree.py0000664000175000017500000001463315264420307020115 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Utilities for testing workflow directory structure. (There should be no tests in this module.) A filetree is represented by a dict like so: { # Dirs are represented by dicts (which are also sub-filetrees): 'dir': { 'another-dir': { # Files are represented by None: 'file.txt': None } }, # Symlinks are represented by the Symlink class, with the target # represented by the relative path from the tmp_path directory: 'symlink': Symlink('dir/another-dir') } """ from pathlib import Path, PosixPath from typing import Any, Dict, List class Symlink(PosixPath): """A class to represent a symlink target.""" ... def create_filetree( filetree: Dict[str, Any], location: Path, root: Path ) -> None: """Create the directory structure represented by the filetree dict. Args: filetree: The filetree to create. location: The absolute path in which to create the filetree. root: The top-level dir from which relative symlink targets are located (typically tmp_path). """ for name, entry in filetree.items(): path = location / name if isinstance(entry, dict): path.mkdir(exist_ok=True) create_filetree(entry, path, root) elif isinstance(entry, Symlink): path.symlink_to(root / entry) else: path.touch() def get_filetree_as_list( filetree: Dict[str, Any], location: Path ) -> List[str]: """Return a list of the paths in a filetree. Args: filetree: The filetree to listify. location: The absolute path to the filetree. """ ret: List[str] = [] for name, entry in filetree.items(): path = location / name ret.append(str(path)) if isinstance(entry, dict): ret.extend(get_filetree_as_list(entry, path)) return ret FILETREE_1 = { 'cylc-run': { 'foo': { 'bar': { '.service': { 'db': None, }, 'flow.cylc': None, 'log': Symlink('sym/cylc-run/foo/bar/log'), 'mirkwood': Symlink('you-shall-not-pass/mirkwood'), 'rincewind.txt': Symlink('you-shall-not-pass/rincewind.txt'), }, }, }, 'sym': { 'cylc-run': { 'foo': { 'bar': { 'log': { 'darmok': Symlink('you-shall-not-pass/darmok'), 'temba.txt': Symlink('you-shall-not-pass/temba.txt'), 'bib': { 'fortuna.txt': None, }, }, }, }, }, }, 'you-shall-not-pass': { # Nothing in here should get deleted 'darmok': { 'jalad.txt': None, }, 'mirkwood': { 'spiders.txt': None, }, 'rincewind.txt': None, 'temba.txt': None, }, } FILETREE_2 = { 'cylc-run': {'foo': {'bar': Symlink('sym-run/cylc-run/foo/bar')}}, 'sym-run': { 'cylc-run': { 'foo': { 'bar': { '.service': { 'db': None, }, 'flow.cylc': None, 'share': Symlink('sym-share/cylc-run/foo/bar/share'), }, }, }, }, 'sym-share': { 'cylc-run': { 'foo': { 'bar': { 'share': { 'cycle': Symlink( 'sym-cycle/cylc-run/foo/bar/share/cycle' ), }, }, }, }, }, 'sym-cycle': { 'cylc-run': { 'foo': { 'bar': { 'share': { 'cycle': { 'macklunkey.txt': None, }, }, }, }, }, }, 'you-shall-not-pass': {}, } FILETREE_3 = { 'cylc-run': { 'foo': { 'bar': Symlink('sym-run/cylc-run/foo/bar'), }, }, 'sym-run': { 'cylc-run': { 'foo': { 'bar': { '.service': { 'db': None, }, 'flow.cylc': None, 'share': { 'cycle': Symlink( 'sym-cycle/cylc-run/foo/bar/share/cycle' ), }, }, }, }, }, 'sym-cycle': { 'cylc-run': { 'foo': { 'bar': { 'share': { 'cycle': { 'sokath.txt': None, }, }, }, }, }, }, 'you-shall-not-pass': {}, } FILETREE_4 = { 'cylc-run': { 'foo': { 'bar': { '.service': { 'db': None, }, 'flow.cylc': None, 'share': { 'cycle': Symlink('sym-cycle/cylc-run/foo/bar/share/cycle'), }, }, }, }, 'sym-cycle': { 'cylc-run': { 'foo': { 'bar': { 'share': { 'cycle': { 'kiazi.txt': None, }, }, }, }, }, }, 'you-shall-not-pass': {}, } cylc-flow-8.7.0/tests/unit/test_scheduler.py0000664000175000017500000001311715264420307021327 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests for Cylc scheduler server.""" import logging from secrets import token_hex from time import time from types import SimpleNamespace from typing import List from unittest.mock import ( MagicMock, Mock, ) import pytest from cylc.flow import CYLC_LOG from cylc.flow.exceptions import InputError from cylc.flow.scheduler import Scheduler from cylc.flow.scheduler_cli import RunOptions from cylc.flow.task_pool import TaskPool from cylc.flow.task_proxy import TaskProxy from cylc.flow.workflow_status import AutoRestartMode @pytest.mark.parametrize( 'opts_to_test, is_restart, err_msg', [ pytest.param( ['icp', 'startcp', 'starttask'], True, "option --{} is not valid for restart", id="start opts on restart" ), pytest.param( ['icp', 'startcp', 'starttask'], False, "option --{}=reload is not valid", id="start opts =reload" ), pytest.param( ['fcp', 'stopcp'], False, "option --{}=reload is only valid for restart", id="end opts =reload when not restart" ), ] ) def test_check_startup_opts( opts_to_test: List[str], is_restart: bool, err_msg: str ) -> None: """Test Scheduler._check_startup_opts()""" for opt in opts_to_test: mocked_scheduler = Mock(is_restart=is_restart) mocked_scheduler.options = SimpleNamespace(**{opt: 'reload'}) with pytest.raises(InputError) as excinfo: Scheduler._check_startup_opts(mocked_scheduler) assert err_msg.format(opt) in str(excinfo) @pytest.mark.parametrize( 'auto_restart_time, expected', [ (-1, True), (0, True), (1, False), (None, False), ] ) def test_should_auto_restart_now( auto_restart_time, expected, monkeypatch: pytest.MonkeyPatch ): """Test Scheduler.should_auto_restart_now().""" time_now = time() monkeypatch.setattr('cylc.flow.scheduler.time', lambda: time_now) if auto_restart_time is not None: auto_restart_time += time_now mock_schd = Mock(spec=Scheduler, auto_restart_time=auto_restart_time) assert Scheduler.should_auto_restart_now(mock_schd) == expected def test_release_tasks_to_run__auto_restart(): """Test that Scheduler.release_tasks_to_run() works as expected during auto restart.""" mock_schd = MagicMock( auto_restart_time=(time() - 100), auto_restart_mode=AutoRestartMode.RESTART_NORMAL, is_paused=False, stop_mode=None, pool=Mock( spec=TaskPool, get_tasks=lambda: [Mock(spec=TaskProxy)] ), workflow='parachutes', options=RunOptions(), task_job_mgr=MagicMock() ) Scheduler.release_tasks_to_run(mock_schd) # Should not actually release any more tasks, just submit the # preparing ones mock_schd.pool.release_queued_tasks.assert_not_called() Scheduler.start_job_submission(mock_schd, mock_schd.pool.get_tasks()) mock_schd.submit_task_jobs.assert_called() async def test_auto_restart_DNS_error(mock_glbl_cfg, caplog, log_filter): """Ensure that DNS errors in host selection are caught.""" # fake a "get address info" error # this error can occur due to an unknown host resulting from broken # DNS or an invalid host name in the global config hostname = f'elephant_{token_hex(4)}' mock_glbl_cfg( 'cylc.flow.host_select.glbl_cfg', f''' [scheduler] [[run hosts]] available = {hostname} ''' ) schd = Mock( workflow='myworkflow', options=RunOptions(abort_if_any_task_fails=False), INTERVAL_AUTO_RESTART_ERROR=0, ) caplog.set_level(logging.ERROR, CYLC_LOG) assert not await Scheduler.workflow_auto_restart(schd, max_retries=2) assert log_filter(contains=hostname) async def test_auto_restart_popen_error(monkeypatch, caplog, log_filter): """Ensure that subprocess errors are handled.""" async def _select_workflow_host(cached=False): # mock a host-select return value return ('foo', 'foo') monkeypatch.setattr( 'cylc.flow.scheduler.select_workflow_host', _select_workflow_host, ) def _popen(*args, **kwargs): # mock an auto-restart command failure return Mock( wait=lambda: 1, communicate=lambda: ('mystdout', 'mystderr'), ) monkeypatch.setattr( 'cylc.flow.scheduler.Popen', _popen, ) schd = Mock( workflow='myworkflow', options=RunOptions(abort_if_any_task_fails=False), INTERVAL_AUTO_RESTART_ERROR=0, ) caplog.set_level(logging.ERROR, CYLC_LOG) assert not await Scheduler.workflow_auto_restart(schd, max_retries=2) assert log_filter(contains='mystderr') cylc-flow-8.7.0/tests/unit/test_flow_mgr.py0000664000175000017500000000567515264420307021177 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Unit tests for FlowManager.""" import pytest import datetime import logging from cylc.flow.flow_mgr import FlowMgr from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager from cylc.flow import CYLC_LOG FAKE_NOW = datetime.datetime(2020, 12, 25, 17, 5, 55) FAKE_NOW_ISO = FAKE_NOW.isoformat() @pytest.fixture def patch_datetime_now(monkeypatch): class mydatetime: @classmethod def now(cls, tz=None): return FAKE_NOW monkeypatch.setattr(datetime, 'datetime', mydatetime) def test_all( patch_datetime_now, caplog: pytest.LogCaptureFixture, ): """Test flow number management.""" db_mgr = WorkflowDatabaseManager() flow_mgr = FlowMgr(db_mgr) caplog.set_level(logging.DEBUG, CYLC_LOG) meta = "the quick brown fox" assert flow_mgr.get_flow(None, meta) == 1 msg1 = f"flow: 1 ({meta}) {FAKE_NOW_ISO}" assert f"New {msg1}" in caplog.messages # automatic: expect 2 meta = "jumped over the lazy dog" assert flow_mgr.get_flow(None, meta) == 2 msg2 = f"flow: 2 ({meta}) {FAKE_NOW_ISO}" assert f"New {msg2}" in caplog.messages # give flow 2: not a new flow meta = "jumped over the moon" assert flow_mgr.get_flow(2, meta) == 2 msg3 = f"flow: 2 ({meta}) {FAKE_NOW_ISO}" assert f"New {msg3}" not in caplog.messages assert ( f"Ignoring flow metadata \"{meta}\": 2 is not a new flow" in caplog.messages ) # give flow 4: new flow meta = "jumped over the cheese" assert flow_mgr.get_flow(4, meta) == 4 msg4 = f"flow: 4 ({meta}) {FAKE_NOW_ISO}" assert f"New {msg4}" in caplog.messages # automatic: expect 3 meta = "jumped over the log" assert flow_mgr.get_flow(None, meta) == 3 msg5 = f"flow: 3 ({meta}) {FAKE_NOW_ISO}" assert f"New {msg5}" in caplog.messages # automatic: expect 5 (skip over 4) meta = "crawled under the log" assert flow_mgr.get_flow(None, meta) == 5 msg6 = f"flow: 5 ({meta}) {FAKE_NOW_ISO}" assert f"New {msg6}" in caplog.messages flow_mgr._log() assert ( "Flows:\n" f"{msg1}\n" f"{msg2}\n" f"{msg4}\n" f"{msg5}\n" f"{msg6}" ) in caplog.messages cylc-flow-8.7.0/tests/unit/test_task_state_prop.py0000664000175000017500000000273415264420307022556 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.task_state import ( TASK_STATUS_SUBMIT_FAILED, TASK_STATUS_FAILED ) from cylc.flow.task_state_prop import extract_group_state def test_extract_group_state_childless(): assert extract_group_state(child_states=[]) is None @pytest.mark.parametrize("child_states, is_stopped, expected", [ ( [TASK_STATUS_SUBMIT_FAILED, TASK_STATUS_FAILED], False, TASK_STATUS_SUBMIT_FAILED ), ( ["Who?", TASK_STATUS_FAILED], False, TASK_STATUS_FAILED ) ]) def test_extract_group_state_order(child_states, is_stopped, expected): assert extract_group_state( child_states=child_states, is_stopped=is_stopped ) == expected cylc-flow-8.7.0/tests/unit/test_util.py0000664000175000017500000000166415264420307020332 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.util import deserialise_set def test_deserialise_set(): actual = deserialise_set('["2", "3"]') expected = {'2', '3'} assert actual == expected cylc-flow-8.7.0/tests/unit/test_param_expand.py0000664000175000017500000004112515264420307022010 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import unittest import pytest from pytest import param from cylc.flow.exceptions import ParamExpandError from cylc.flow.param_expand import NameExpander, GraphExpander class TestParamExpand(unittest.TestCase): """Unit tests for the parameter expansion module.""" def setUp(self): """Create some parameters and templates for use in tests.""" params_map = {'a': [-3, -1], 'i': [0, 1], 'j': [0, 1, 2], 'k': [0, 1]} # k has template is deliberately bad templates = { 'a': '_a%(a)d', 'i': '_i%(i)d', 'j': '_j%(j)d', 'k': '_k%(z)d'} self.name_expander = NameExpander((params_map, templates)) self.graph_expander = GraphExpander((params_map, templates)) def test_name_one_param(self): """Test name expansion and returned value for a single parameter.""" self.assertEqual( self.name_expander.expand('foo'), [('foo_j0', {'j': 0}), ('foo_j1', {'j': 1}), ('foo_j2', {'j': 2})] ) def test_name_two_params(self): """Test name expansion and returned values for two parameters.""" self.assertEqual( self.name_expander.expand('foo'), [('foo_i0_j0', {'i': 0, 'j': 0}), ('foo_i0_j1', {'i': 0, 'j': 1}), ('foo_i0_j2', {'i': 0, 'j': 2}), ('foo_i1_j0', {'i': 1, 'j': 0}), ('foo_i1_j1', {'i': 1, 'j': 1}), ('foo_i1_j2', {'i': 1, 'j': 2})] ) def test_name_two_names(self): """Test name expansion for two names.""" self.assertEqual( self.name_expander.expand('foo, bar'), [('foo_i0', {'i': 0}), ('foo_i1', {'i': 1}), ('bar_j0', {'j': 0}), ('bar_j1', {'j': 1}), ('bar_j2', {'j': 2})] ) def test_name_specific_val_1(self): """Test singling out a specific value, in name expansion.""" self.assertEqual( self.name_expander.expand('foo'), [('foo_i0', {'i': 0})] ) def test_name_specific_val_2(self): """Test specific value in the first parameter of a pair.""" self.assertEqual( self.name_expander.expand('foo'), [('foo_i0_j0', {'i': 0, 'j': 0}), ('foo_i0_j1', {'i': 0, 'j': 1}), ('foo_i0_j2', {'i': 0, 'j': 2})] ) def test_name_specific_val_3(self): """Test specific value in the second parameter of a pair.""" self.assertEqual( self.name_expander.expand('foo'), [('foo_i0_j1', {'i': 0, 'j': 1}), ('foo_i1_j1', {'i': 1, 'j': 1})] ) def test_name_fail_bare_value(self): """Test foo<0,j> fails.""" # It should be foo. self.assertRaises(ParamExpandError, self.name_expander.expand, 'foo<0,j>') def test_name_fail_undefined_param(self): """Test that an undefined parameter gets failed.""" # m is not defined. self.assertRaises(ParamExpandError, self.name_expander.expand, 'foo') def test_name_fail_param_value_too_high(self): """Test that an out-of-range parameter gets failed.""" # i stops at 3. self.assertRaises(ParamExpandError, self.name_expander.expand, 'foo') def test_name_multiple(self): """Test expansion of two names, with one and two parameters.""" self.assertEqual( self.name_expander.expand('foo, bar'), [('foo_i0', {'i': 0}), ('foo_i1', {'i': 1}), ('bar_i0_j0', {'i': 0, 'j': 0}), ('bar_i0_j1', {'i': 0, 'j': 1}), ('bar_i0_j2', {'i': 0, 'j': 2}), ('bar_i1_j0', {'i': 1, 'j': 0}), ('bar_i1_j1', {'i': 1, 'j': 1}), ('bar_i1_j2', {'i': 1, 'j': 2})] ) def test_graph_expand_1(self): """Test graph expansion with two parameters each side of an arrow.""" self.assertEqual( self.graph_expander.expand("bar=>baz"), set(["bar_i0_j1=>baz_i0_j1", "bar_i1_j2=>baz_i1_j2", "bar_i0_j2=>baz_i0_j2", "bar_i1_j1=>baz_i1_j1", "bar_i1_j0=>baz_i1_j0", "bar_i0_j0=>baz_i0_j0"]) ) def test_graph_expand_2(self): """Test graph expansion to 'branch and merge' a workflow.""" self.assertEqual( self.graph_expander.expand("pre=>bar=>baz=>post"), set(["pre=>bar_i0=>baz_i0_j1=>post", "pre=>bar_i1=>baz_i1_j2=>post", "pre=>bar_i0=>baz_i0_j2=>post", "pre=>bar_i1=>baz_i1_j1=>post", "pre=>bar_i1=>baz_i1_j0=>post", "pre=>bar_i0=>baz_i0_j0=>post"]) ) def test_graph_expand_3(self): """Test graph expansion -ve integers.""" self.assertEqual( self.graph_expander.expand("bar"), set(["bar_a-1", "bar_a-3"])) def test_graph_expand_offset_1(self): """Test graph expansion with a -ve offset.""" self.assertEqual( self.graph_expander.expand("bar=>baz"), set(["bar_i-32768_j0=>baz_i0_j0", "bar_i-32768_j1=>baz_i0_j1", "bar_i-32768_j2=>baz_i0_j2", "bar_i0_j0=>baz_i1_j0", "bar_i0_j1=>baz_i1_j1", "bar_i0_j2=>baz_i1_j2"]) ) def test_graph_expand_offset_2(self): """Test graph expansion with a +ve offset.""" self.assertEqual( self.graph_expander.expand("baz=>baz"), set(["baz_i0=>baz_i1", "baz_i1=>baz_i-32768"]) ) def test_graph_expand_specific(self): """Test graph expansion with a specific value.""" self.assertEqual( self.graph_expander.expand("bar=>baz"), set(["bar_i1_j0=>baz_i0_j0", "bar_i1_j1=>baz_i0_j1", "bar_i1_j2=>baz_i0_j2", "bar_i1_j0=>baz_i1_j0", "bar_i1_j1=>baz_i1_j1", "bar_i1_j2=>baz_i1_j2"]) ) def test_graph_fail_bare_value(self): """Test that a bare parameter value fails in the graph.""" self.assertRaises(ParamExpandError, self.graph_expander.expand, 'foo<0,j>=>bar') def test_graph_fail_undefined_param(self): """Test that an undefined parameter value fails in the graph.""" self.assertRaises(ParamExpandError, self.graph_expander.expand, 'foo=>bar') def test_graph_fail_param_value_too_high(self): """Test that an out-of-range parameter value fails in the graph.""" self.assertRaises(ParamExpandError, self.graph_expander.expand, 'foo') def test_template_fail_missing_param(self): """Test a template string specifying a non-existent parameter.""" self.assertRaises( ParamExpandError, self.name_expander.expand, 'foo') self.assertRaises( ParamExpandError, self.graph_expander.expand, 'foo') @staticmethod def _param_expand_params(): """Test data for test_parameter_graph_mixing_offset_and_conditional. params_map, templates, expanded_str, expanded_values params_map : map of parameters used in the graph expression templates : parameters template expanded_str : graph string, using params/template expanded_values: values expected to exist after params expanded """ return ( # original case from #2608 ( {'m': ["cat", "dog"]}, {'m': '_%(m)s'}, "foo & baz => foo", [ 'foo_-32768 & baz => foo_cat', 'foo_cat & baz => foo_dog' ] ), # cases from comments from #2608 # see cylc/cylc-flow/pull/3452#issuecomment-670782800 ( # single element, so bar does not exist {'m': ["cat"]}, {'m': '_%(m)s'}, "foo & bar & baz => qux", [ "foo & bar_-32768 & baz => qux" ] ), # cases from comments from #2608 # see cylc/cylc-flow/pull/3452#issuecomment-670776749 ( {'m': ["1", "2"]}, {'m': '_%(m)s'}, "foo => bar => baz", [ "foo_-32768 => bar_1 => baz", "foo_1 => bar_2 => baz" ] ), # cases from comments from #2608 # see cylc/cylc-flow/pull/3452#discussion_r430967867 ( {'m': ["cat", "dog"]}, {'m': '_%(m)s'}, "baz & foo & pub => foo", [ "baz & foo_-32768 & pub => foo_cat", "baz & foo_cat & pub => foo_dog" ] ), ( {'m': ["cat", "dog"]}, {'m': '_%(m)s'}, "bar & foo & pub & qux => foo", [ "bar & foo_-32768 & pub_-32768 & qux => foo_cat", "bar & foo_cat & pub_cat & qux => foo_dog" ] ), # GraphParser strips spaces! ( {'m': ["cat"]}, {'m': '_%(m)s'}, "foo&bar&baz=>qux", [ "foo&bar_-32768&baz=>qux" ] ), ( {'m': ["cat", "dog"]}, {'m': '_%(m)s'}, "foo&bar&baz=>qux", [ "foo&bar_-32768&baz=>qux", "foo&bar_cat&baz=>qux" ] ), # must support & and | in graph expressions ( {'m': ["cat", "dog"]}, {'m': '_%(m)s'}, "foo|bar|baz=>qux", [ "foo|bar_-32768|baz=>qux", "foo|bar_cat|baz=>qux" ] ), ( {'m': ["cat", "dog"]}, {'m': '_%(m)s'}, "foo&bar|baz=>qux", [ "foo&bar_-32768|baz=>qux", "foo&bar_cat|baz=>qux" ] ), ( {'m': ["cat", "dog"]}, {'m': '_%(m)s'}, "foo&bar|baz=>qux", [ "foo&bar_-32768|baz=>qux", "foo&bar_cat|baz=>qux" ] ), ( {'m': ["cat"]}, {'m': '_%(m)s'}, "foo => bar => baz", [ "foo=>bar_-32768=>baz" ] ) ) def test_parameter_graph_mixing_offset_and_conditional(self): """Test for bug reported in issue #2608 on GitHub.""" for test_case in self._param_expand_params(): params_map, templates, expanded_str, expanded_values = \ test_case graph_expander = GraphExpander((params_map, templates)) # Ignore white spaces. expanded = [expanded.replace(' ', '') for expanded in graph_expander.expand(expanded_str)] self.assertEqual( len(expanded_values), len(expanded), f"Invalid length for expected {expanded_values} and " f"{expanded}") # When testing, we don't really care for white spaces,as they # are removed in the GraphParser anyway. That's why we have # ''.replace(' ', ''). for expected in expanded_values: self.assertTrue( expected.replace(' ', '') in expanded, f"Expected value {expected.replace(' ', '')} " f"not in {expanded}") class myParam(): def __init__( self, raw_str, parameter_values=None, templates=None, raises=None, id_=None, expect=None, ): """Ease of reading wrapper for pytest.param Args: expect: Output of expand_parent_params() raw_str: The parent_params input string. parameter_values """ parameter_values = parameter_values if parameter_values else {} templates = templates if templates else {} self.raises = raises self.expect = expect self.raw_str = raw_str self.parameter_values = parameter_values self.templates = templates self.parameters = ((parameter_values, templates)) self.name_expander = NameExpander(self.parameters) self.id_ = 'raises:' + id_ if raises else id_ def get(self): return param(self, id=self.id_) @pytest.mark.parametrize( "param", ( myParam( expect=(None, 'no_params_here'), raw_str='no_params_here', id_='basic' ).get(), myParam( expect=({'bar': 1}, 'bar1'), raw_str='', parameter_values={'bar': 1}, templates={'bar': 'bar%(bar)s'}, id_='one-valid_-param' ).get(), myParam( expect=({'bar': 1}, 'foo_bar1_baz'), raw_str='foobaz', parameter_values={'bar': 1}, templates={'bar': '_bar%(bar)s_'}, id_='one-valid_-param' ).get(), myParam( raw_str='foobaz', parameter_values={'qux': 2}, templates={'bar': '_bar%(bar)s_'}, raises=(ParamExpandError, 'parameter \'bar\' undefined'), id_='one-invalid_-param' ).get(), myParam( expect=({'bar': 1, 'baz': 42}, 'foo_bar1_baz42'), raw_str='foo', parameter_values={'bar': 1, 'baz': 42}, templates={'bar': '_bar%(bar)s', 'baz': '_baz%(baz)s'}, id_='two-valid_-param' ).get(), myParam( expect=({'bar': 1, 'baz': 42}, 'foo_bar1qux_baz42'), raw_str='fooqux', parameter_values={'bar': 1, 'baz': 42}, templates={'bar': '_bar%(bar)s', 'baz': '_baz%(baz)s'}, id_='two-valid_-param-sep-brackets', ).get(), myParam( raw_str='foobaz', raises=(ParamExpandError, '^parameter offsets illegal here'), id_='offsets-illegal' ).get(), myParam( expect=({'bar': 1}, 'foo_bar1_baz'), raw_str='foobaz', parameter_values={'bar': [1, 2]}, templates={'bar': '_bar%(bar)s_'}, id_='value-set' ).get(), myParam( raw_str='foobaz', parameter_values={'bar': [1, 2]}, raises=(ParamExpandError, '^illegal'), id_='illegal-value' ).get(), myParam( expect=({'bar': 1}, 'foo_bar1_baz'), raw_str='foobaz', raises=(ParamExpandError, '^parameter \'bar\' undefined'), id_='parameter-undefined' ).get(), ) ) def test_expand_parent_params(param): if not param.raises: # Good Path tests: result = param.name_expander.expand_parent_params( param.raw_str, param.parameter_values, 'Errortext') assert result == param.expect else: # Bad path tests: with pytest.raises(param.raises[0], match=param.raises[1]): param.name_expander.expand_parent_params( param.raw_str, param.parameter_values, 'Errortext') cylc-flow-8.7.0/tests/unit/test_clean.py0000664000175000017500000010545215264420307020437 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio from copy import deepcopy from glob import iglob import logging import os from pathlib import Path from secrets import token_hex import shutil from typing import ( TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple, Type, ) from unittest import mock import pytest from cylc.flow import ( CYLC_LOG, clean as cylc_clean, ) from cylc.flow.clean import ( _clean_using_glob, clean, glob_in_run_dir, init_clean, ) from cylc.flow.exceptions import ( CylcError, InputError, SchedulerAlive, ServiceFileError, WorkflowFilesError, ) from cylc.flow.pathutil import parse_rm_dirs from cylc.flow.scripts.clean import CleanOptions from cylc.flow.workflow_files import ( WorkflowFiles, get_symlink_dirs, ) from .filetree import ( FILETREE_1, FILETREE_2, FILETREE_3, FILETREE_4, Symlink, create_filetree, get_filetree_as_list, ) if TYPE_CHECKING: from .conftest import MonkeyMock NonCallableFixture = Any # global.cylc[install]scan depth for these tests: MAX_SCAN_DEPTH = 3 @pytest.fixture def glbl_cfg_max_scan_depth(mock_glbl_cfg: Callable) -> None: mock_glbl_cfg( 'cylc.flow.workflow_files.glbl_cfg', f''' [install] max depth = {MAX_SCAN_DEPTH} ''' ) @pytest.mark.parametrize( 'id_, stopped, err, err_msg', [ ('foo/..', True, WorkflowFilesError, "cannot be a path that points to the cylc-run directory or above"), ('foo/../..', True, WorkflowFilesError, "cannot be a path that points to the cylc-run directory or above"), ('foo', False, ServiceFileError, "Cannot clean running workflow"), ] ) def test_clean_check__fail( id_: str, stopped: bool, err: Type[Exception], err_msg: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: """Test that _clean_check() fails appropriately. Params: id_: Workflow name. stopped: Whether the workflow is stopped when _clean_check() is called. err: Expected error class. err_msg: Message that is expected to be in the exception. """ def mocked_detect_old_contact_file(*a, **k): if not stopped: raise SchedulerAlive('Mocked error') monkeypatch.setattr( 'cylc.flow.clean.detect_old_contact_file', mocked_detect_old_contact_file ) with pytest.raises(err) as exc: cylc_clean._clean_check(CleanOptions(), id_, tmp_path) assert err_msg in str(exc.value) @pytest.mark.parametrize( 'db_platforms, opts, clean_called, remote_clean_called', [ pytest.param( ['localhost', 'localhost'], {}, True, False, id="Only platform in DB is localhost" ), pytest.param( ['horse'], {}, True, True, id="Remote platform in DB" ), pytest.param( ['horse'], {'local_only': True}, True, False, id="Local clean only" ), pytest.param( ['horse'], {'remote_only': True}, False, True, id="Remote clean only" ) ] ) async def test_init_clean( db_platforms: list[str], opts: dict[str, Any], clean_called: bool, remote_clean_called: bool, monkeypatch: pytest.MonkeyPatch, tmp_run_dir: Callable ) -> None: """Test the init_clean() function logic. Params: db_platforms: Platform names that would be loaded from the database. opts: Any options passed to the cylc clean CLI. clean_called: If a local clean is expected to go ahead. remote_clean_called: If a remote clean is expected to go ahead. """ id_ = 'foo/bar/' rdir = tmp_run_dir(id_, installed=True) Path(rdir, WorkflowFiles.Service.DIRNAME, WorkflowFiles.Service.DB).touch() monkeypatch.setattr('cylc.flow.clean.clean', mock_clean := mock.Mock()) monkeypatch.setattr( 'cylc.flow.clean._remote_clean_cmd', mock_remote_clean_cmd := mock.AsyncMock(), ) monkeypatch.setattr( 'cylc.flow.clean.get_platforms_from_db', lambda x: set(db_platforms) ) monkeypatch.setattr( 'cylc.flow.clean.get_install_targets_map', lambda p_names: {p: [{}] for p in p_names if p != 'localhost'} ) await init_clean(id_, opts=CleanOptions(**opts)) assert mock_clean.called is clean_called assert mock_remote_clean_cmd.called is remote_clean_called async def test_init_clean__no_dir( monkeymock: 'MonkeyMock', tmp_run_dir: Callable, caplog: pytest.LogCaptureFixture ) -> None: """Test init_clean() when the run dir doesn't exist""" caplog.set_level(logging.INFO, CYLC_LOG) mock_clean = monkeymock('cylc.flow.clean.clean') mock_remote_clean = monkeymock('cylc.flow.clean.remote_clean') await init_clean('foo/bar', opts=CleanOptions()) assert "No directory to clean" in caplog.text assert mock_clean.called is False assert mock_remote_clean.called is False async def test_init_clean__no_db( monkeymock: 'MonkeyMock', tmp_run_dir: Callable, caplog: pytest.LogCaptureFixture ) -> None: """Test init_clean() when the workflow database doesn't exist""" caplog.set_level(logging.INFO, CYLC_LOG) tmp_run_dir('bespin') mock_clean = monkeymock('cylc.flow.clean.clean') mock_remote_clean = monkeymock('cylc.flow.clean.remote_clean') await init_clean('bespin', opts=CleanOptions()) assert ( "No workflow database for bespin - will only clean locally" ) in caplog.text assert mock_clean.called is True assert mock_remote_clean.called is False async def test_init_clean__remote_only_no_db( monkeymock: 'MonkeyMock', tmp_run_dir: Callable ) -> None: """Test remote-only init_clean() when the workflow DB doesn't exist""" tmp_run_dir('hoth') mock_clean = monkeymock('cylc.flow.clean.clean') mock_remote_clean = monkeymock('cylc.flow.clean.remote_clean') with pytest.raises(ServiceFileError) as exc: await init_clean('hoth', opts=CleanOptions(remote_only=True)) assert ( "No workflow database for hoth - cannot perform remote clean" ) in str(exc.value) assert mock_clean.called is False assert mock_remote_clean.called is False async def test_init_clean__running_workflow( monkeypatch: pytest.MonkeyPatch, tmp_run_dir: Callable ) -> None: """Test init_clean() fails when workflow is still running""" def mock_err(*args, **kwargs): raise SchedulerAlive("Mocked error") monkeypatch.setattr('cylc.flow.clean.detect_old_contact_file', mock_err) tmp_run_dir('yavin') with pytest.raises(ServiceFileError) as exc: await init_clean('yavin', opts=CleanOptions()) assert "Cannot clean running workflow" in str(exc.value) @pytest.mark.parametrize( 'rm_dirs, expected_clean, expected_remote_clean', [(None, None, []), (["r2d2:c3po"], {"r2d2", "c3po"}, ["r2d2:c3po"])] ) async def test_init_clean__rm_dirs( rm_dirs: list[str] | None, expected_clean: set[str], expected_remote_clean: list[str], monkeypatch: pytest.MonkeyPatch, tmp_run_dir: Callable ) -> None: """Test init_clean() with the --rm option. Params: rm_dirs: Dirs given by --rm option. expected_clean: The dirs that are expected to be passed to clean(). expected_remote_clean: The dirs that are expected to be passed to remote_clean(). """ id_ = 'dagobah' run_dir: Path = tmp_run_dir(id_) Path( run_dir, WorkflowFiles.Service.DIRNAME, WorkflowFiles.Service.DB ).touch() monkeypatch.setattr('cylc.flow.clean.clean', mock_clean := mock.Mock()) monkeypatch.setattr( 'cylc.flow.clean.remote_clean', mock_remote_clean := mock.AsyncMock() ) platforms = {'platform_one'} monkeypatch.setattr('cylc.flow.clean.get_platforms_from_db', lambda x: platforms) opts = CleanOptions(rm_dirs=rm_dirs) if rm_dirs else CleanOptions() await init_clean(id_, opts=opts) mock_clean.assert_called_with(id_, run_dir, expected_clean) mock_remote_clean.assert_called_with( id_, platforms, opts.remote_timeout, expected_remote_clean ) @pytest.mark.parametrize( 'id_, symlink_dirs, rm_dirs, expected_deleted, expected_remaining', [ pytest.param( 'foo/bar', {}, None, ['cylc-run/foo'], ['cylc-run'], id="Basic clean" ), pytest.param( 'foo/bar/baz', { 'log': 'sym-log', 'share': 'sym-share', 'share/cycle': 'sym-cycle', 'work': 'sym-work' }, None, ['cylc-run/foo', 'sym-log/cylc-run/foo', 'sym-share/cylc-run/foo', 'sym-cycle/cylc-run/foo', 'sym-work/cylc-run/foo'], ['cylc-run', 'sym-log/cylc-run', 'sym-share/cylc-run', 'sym-cycle/cylc-run', 'sym-work/cylc-run'], id="Symlink dirs" ), pytest.param( 'foo', { 'run': 'sym-run', 'log': 'sym-log', 'share': 'sym-share', 'share/cycle': 'sym-cycle', 'work': 'sym-work' }, None, ['cylc-run/foo', 'sym-run/cylc-run/foo', 'sym-log/cylc-run/foo', 'sym-share/cylc-run/foo', 'sym-cycle/cylc-run/foo', 'sym-work/cylc-run/foo'], ['cylc-run', 'sym-run/cylc-run', 'sym-log/cylc-run', 'sym-share/cylc-run', 'sym-cycle/cylc-run', 'sym-work'], id="Symlink dirs including run dir" ), pytest.param( 'foo', {}, {'log', 'share'}, ['cylc-run/foo/log', 'cylc-run/foo/share'], ['cylc-run/foo/work'], id="Targeted clean" ), pytest.param( 'foo', {'log': 'sym-log'}, {'log'}, ['cylc-run/foo/log', 'sym-log/cylc-run/foo'], ['cylc-run/foo/work', 'cylc-run/foo/share/cycle', 'sym-log/cylc-run'], id="Targeted clean with symlink dirs" ), pytest.param( 'foo', {}, {'share/cy*'}, ['cylc-run/foo/share/cycle'], ['cylc-run/foo/log', 'cylc-run/foo/work', 'cylc-run/foo/share'], id="Targeted clean with glob" ), pytest.param( 'foo', {'log': 'sym-log'}, {'w*', 'wo*', 'l*', 'lo*'}, ['cylc-run/foo/work', 'cylc-run/foo/log', 'sym-log/cylc-run/foo'], ['cylc-run/foo/share', 'cylc-run/foo/share/cycle'], id="Targeted clean with degenerate glob" ), ] ) def test_clean( id_: str, symlink_dirs: Dict[str, str], rm_dirs: Optional[Set[str]], expected_deleted: List[str], expected_remaining: List[str], tmp_path: Path, tmp_run_dir: Callable ) -> None: """Test the clean() function. Params: id_: Workflow name. symlink_dirs: As you would find in the global config under [symlink dirs][platform]. rm_dirs: As passed to clean(). expected_deleted: Dirs (relative paths under tmp_path) that are expected to be cleaned. expected_remaining: Any dirs (relative paths under tmp_path) that are not expected to be cleaned. """ # --- Setup --- run_dir: Path = tmp_run_dir(id_) if 'run' in symlink_dirs: target = tmp_path / symlink_dirs['run'] / 'cylc-run' / id_ target.mkdir(parents=True) shutil.rmtree(run_dir) run_dir.symlink_to(target) symlink_dirs.pop('run') for symlink_name, target_name in symlink_dirs.items(): target = tmp_path / target_name / 'cylc-run' / id_ / symlink_name target.mkdir(parents=True) symlink = run_dir / symlink_name symlink.symlink_to(target) for d_name in ('log', 'share', 'share/cycle', 'work'): if d_name not in symlink_dirs: (run_dir / d_name).mkdir() for rel_path in [*expected_deleted, *expected_remaining]: assert (tmp_path / rel_path).exists() # --- The actual test --- cylc_clean.clean(id_, run_dir, rm_dirs) for rel_path in expected_deleted: assert (tmp_path / rel_path).exists() is False assert (tmp_path / rel_path).is_symlink() is False for rel_path in expected_remaining: assert (tmp_path / rel_path).exists() def test_clean__broken_symlink_run_dir( tmp_path: Path, tmp_run_dir: Callable ) -> None: """Test clean() successfully remove a run dir that is a broken symlink.""" # Setup id_ = 'foo/bar' run_dir: Path = tmp_run_dir(id_) target = tmp_path.joinpath('rabbow/cylc-run', id_) target.mkdir(parents=True) shutil.rmtree(run_dir) run_dir.symlink_to(target) target.rmdir() assert run_dir.parent.exists() is True # cylc-run/foo should exist # Test cylc_clean.clean(id_, run_dir) assert run_dir.parent.exists() is False # cylc-run/foo should be gone assert target.parent.exists() is False # rabbow/cylc-run/foo too def test_clean__bad_symlink_dir_wrong_type( tmp_path: Path, tmp_run_dir: Callable ) -> None: """Test clean() raises error when a symlink dir actually points to a file instead of a dir""" id_ = 'foo' run_dir: Path = tmp_run_dir(id_) symlink = run_dir.joinpath('log') target = tmp_path.joinpath('sym-log', 'cylc-run', id_, 'meow.txt') target.parent.mkdir(parents=True) target.touch() symlink.symlink_to(target) with pytest.raises(WorkflowFilesError) as exc: cylc_clean.clean(id_, run_dir) assert "Invalid symlink at" in str(exc.value) assert symlink.exists() is True def test_clean__bad_symlink_dir_wrong_form( tmp_path: Path, tmp_run_dir: Callable ) -> None: """Test clean() raises error when a symlink dir points to an unexpected dir""" run_dir: Path = tmp_run_dir('foo') symlink = run_dir.joinpath('log') target = tmp_path.joinpath('sym-log', 'oops', 'log') target.mkdir(parents=True) symlink.symlink_to(target) with pytest.raises(WorkflowFilesError) as exc: cylc_clean.clean('foo', run_dir) assert 'should end with "cylc-run/foo/log"' in str(exc.value) assert symlink.exists() is True @pytest.mark.parametrize('pattern', ['thing/', 'thing/*']) def test_clean__rm_dir_not_file(pattern: str, tmp_run_dir: Callable): """Test clean() does not remove a file when the rm_dir glob pattern would match a dir only.""" id_ = 'foo' run_dir: Path = tmp_run_dir(id_) a_file = run_dir.joinpath('thing') a_file.touch() rm_dirs = parse_rm_dirs([pattern]) cylc_clean.clean(id_, run_dir, rm_dirs) assert a_file.exists() @pytest.fixture def filetree_for_testing_cylc_clean(tmp_path: Path): """Fixture that creates a filetree from the given dict, and returns which files are expected to be deleted and which aren't. See tests/unit/filetree.py Args: id_: Workflow name. initial_filetree: The filetree before cleaning. filetree_left_behind: The filetree that is expected to be left behind after cleaning, excluding the 'you-shall-not-pass/' directory, which is always expected to be left behind. Returns: run_dir: Workflow run dir. files_to_delete: List of files that are expected to be deleted. files_not_to_delete: List of files that are not expected to be deleted. """ def _filetree_for_testing_cylc_clean( id_: str, initial_filetree: Dict[str, Any], filetree_left_behind: Dict[str, Any] ) -> Tuple[Path, List[str], List[str]]: create_filetree(initial_filetree, tmp_path, tmp_path) files_not_to_delete = [ os.path.normpath(i) for i in iglob(str(tmp_path / 'you-shall-not-pass/**'), recursive=True) ] files_not_to_delete.extend( get_filetree_as_list(filetree_left_behind, tmp_path) ) files_to_delete = list( set(get_filetree_as_list(initial_filetree, tmp_path)).difference( files_not_to_delete ) ) run_dir = tmp_path / 'cylc-run' / id_ return run_dir, files_to_delete, files_not_to_delete return _filetree_for_testing_cylc_clean @pytest.mark.parametrize( 'pattern, initial_filetree, filetree_left_behind', [ pytest.param( '**', FILETREE_1, { 'cylc-run': {'foo': {}}, 'sym': {'cylc-run': {'foo': {'bar': {}}}} } ), pytest.param( '*/**', FILETREE_1, { 'cylc-run': {'foo': {'bar': { '.service': {'db': None}, 'flow.cylc': None, 'rincewind.txt': Symlink('whatever') }}}, 'sym': {'cylc-run': {'foo': {'bar': {}}}} } ), pytest.param( '**/*.txt', FILETREE_1, { 'cylc-run': {'foo': {'bar': { '.service': {'db': None}, 'flow.cylc': None, 'log': Symlink('whatever'), 'mirkwood': Symlink('whatever') }}}, 'sym': {'cylc-run': {'foo': {'bar': { 'log': { 'darmok': Symlink('whatever'), 'bib': {} } }}}} } ) ] ) def test__clean_using_glob( pattern: str, initial_filetree: Dict[str, Any], filetree_left_behind: Dict[str, Any], filetree_for_testing_cylc_clean: Callable ) -> None: """Test _clean_using_glob(), particularly that it does not follow and delete symlinks (apart from the standard symlink dirs). Params: pattern: The glob pattern to test. initial_filetree: The filetree to test against. files_left_behind: The filetree expected to remain after _clean_using_glob() is called (excluding /you-shall-not-pass, which is always expected to remain). """ # --- Setup --- run_dir: Path files_to_delete: List[str] files_not_to_delete: List[str] run_dir, files_to_delete, files_not_to_delete = ( filetree_for_testing_cylc_clean( 'foo/bar', initial_filetree, filetree_left_behind) ) # --- Test --- _clean_using_glob(run_dir, pattern, symlink_dirs=['log']) for file in files_not_to_delete: assert os.path.exists(file) is True for file in files_to_delete: assert os.path.lexists(file) is False @pytest.mark.parametrize( 'rm_dirs, initial_filetree, filetree_left_behind', [ pytest.param( {'**'}, FILETREE_1, { 'cylc-run': {}, 'sym': {'cylc-run': {}} }, id="filetree1 **" ), pytest.param( {'*/**'}, FILETREE_1, { 'cylc-run': {'foo': {'bar': { '.service': {'db': None}, 'flow.cylc': None, 'rincewind.txt': Symlink('whatever') }}}, 'sym': {'cylc-run': {}} }, id="filetree1 */**" ), pytest.param( {'**/*.txt'}, FILETREE_1, { 'cylc-run': {'foo': {'bar': { '.service': {'db': None}, 'flow.cylc': None, 'log': Symlink('whatever'), 'mirkwood': Symlink('whatever') }}}, 'sym': {'cylc-run': {'foo': {'bar': { 'log': { 'darmok': Symlink('whatever'), 'bib': {} } }}}} }, id="filetree1 **/*.txt" ), pytest.param( {'**/cycle'}, FILETREE_2, { 'cylc-run': {'foo': { 'bar': Symlink('sym-run/cylc-run/foo/bar') }}, 'sym-run': {'cylc-run': {'foo': {'bar': { '.service': {'db': None}, 'flow.cylc': None, 'share': Symlink('sym-share/cylc-run/foo/bar/share') }}}}, 'sym-share': {'cylc-run': {'foo': {'bar': { 'share': {} }}}}, 'sym-cycle': {'cylc-run': {}} }, id="filetree2 **/cycle" ), pytest.param( {'share'}, FILETREE_2, { 'cylc-run': {'foo': { 'bar': Symlink('sym-run/cylc-run/foo/bar') }}, 'sym-run': {'cylc-run': {'foo': {'bar': { '.service': {'db': None}, 'flow.cylc': None, }}}}, 'sym-share': {'cylc-run': {}}, 'sym-cycle': {'cylc-run': {}}, }, id="filetree2 share" ), pytest.param( {'**'}, FILETREE_2, { 'cylc-run': {}, 'sym-run': {'cylc-run': {}}, 'sym-share': {'cylc-run': {}}, 'sym-cycle': {'cylc-run': {}} }, id="filetree2 **" ), pytest.param( {'*'}, FILETREE_2, { 'cylc-run': {'foo': { 'bar': Symlink('sym-run/cylc-run/foo/bar') }}, 'sym-run': {'cylc-run': {'foo': {'bar': { '.service': {'db': None}, }}}}, 'sym-share': {'cylc-run': {}}, 'sym-cycle': {'cylc-run': {}}, }, id="filetree2 *" ), pytest.param( # Check https://bugs.python.org/issue35201 has no effect {'non-exist/**'}, FILETREE_2, FILETREE_2, id="filetree2 non-exist/**" ), pytest.param( {'**'}, FILETREE_3, { 'cylc-run': {}, 'sym-run': {'cylc-run': {}}, 'sym-cycle': {'cylc-run': {}}, }, id="filetree3 **" ), pytest.param( {'**'}, FILETREE_4, { 'cylc-run': {}, 'sym-cycle': {'cylc-run': {}}, }, id="filetree4 **" ) ], ) def test_clean__targeted( rm_dirs: Set[str], initial_filetree: Dict[str, Any], filetree_left_behind: Dict[str, Any], caplog: pytest.LogCaptureFixture, tmp_run_dir: Callable, filetree_for_testing_cylc_clean: Callable ) -> None: """Test clean(), particularly that it does not follow and delete symlinks (apart from the standard symlink dirs). This is similar to test__clean_using_glob(), but the filetree expected to remain after cleaning is different due to the tidy up of empty dirs. Params: rm_dirs: The glob patterns to test. initial_filetree: The filetree to test against. files_left_behind: The filetree expected to remain after clean() is called (excluding /you-shall-not-pass, which is always expected to remain). """ # --- Setup --- caplog.set_level(logging.DEBUG, CYLC_LOG) id_ = 'foo/bar' run_dir: Path files_to_delete: List[str] files_not_to_delete: List[str] run_dir, files_to_delete, files_not_to_delete = ( filetree_for_testing_cylc_clean( id_, initial_filetree, filetree_left_behind) ) # --- Test --- cylc_clean.clean(id_, run_dir, rm_dirs) for file in files_not_to_delete: assert os.path.exists(file) is True for file in files_to_delete: assert os.path.lexists(file) is False @pytest.mark.parametrize( 'rm_dirs', [ [".."], ["foo:.."], ["foo/../../meow"] ] ) async def test_init_clean__targeted_bad( rm_dirs: List[str], tmp_run_dir: Callable, monkeymock: 'MonkeyMock' ): """Test init_clean() fails when abusing --rm option.""" tmp_run_dir('chalmers') mock_clean = monkeymock('cylc.flow.clean.clean') mock_remote_clean = monkeymock('cylc.flow.clean.remote_clean') with pytest.raises(InputError) as exc_info: await init_clean('chalmers', opts=CleanOptions(rm_dirs=rm_dirs)) assert "cannot take paths that point to the run directory or above" in str( exc_info.value ) mock_clean.assert_not_called() mock_remote_clean.assert_not_called() PLATFORMS = { 'enterprise': { 'hosts': ['kirk', 'picard'], 'install target': 'picard', }, 'voyager': { 'hosts': ['janeway'], 'install target': 'janeway', }, 'stargazer': { 'hosts': ['picard'], 'install target': 'picard', }, 'exeter': { 'hosts': ['localhost'], 'install target': 'localhost', } } for k, v in PLATFORMS.items(): v['name'] = k v['selection'] = {'method': 'definition order'} def mock_platform_from_name(name: str) -> dict: return deepcopy(PLATFORMS[name]) @pytest.mark.parametrize( ('platform_names', 'failed_hosts', 'expected_hosts', 'exc_expected', 'expected_err_msgs'), [ pytest.param( ['exeter'], {}, set(), False, [], id="Only localhost install target - no remote clean" ), pytest.param( ['exeter', 'enterprise'], {}, {'kirk'}, False, [], id="Localhost and remote install target" ), pytest.param( ['enterprise', 'stargazer', 'voyager'], {}, {'kirk', 'janeway'}, False, [], id="Only remote install targets" ), pytest.param( ['stargazer', 'enterprise', 'voyager'], {'picard': 255}, {'kirk', 'picard', 'janeway'}, False, [], id="Platform with 1 uncontactable, 1 successful host" ), pytest.param( ['enterprise', 'stargazer', 'voyager'], {'kirk': 255, 'picard': 255}, {'kirk', 'picard', 'janeway'}, True, ["picard:"], id="Platform with no contactable hosts" ), pytest.param( ['enterprise', 'voyager'], {'kirk': 255, 'picard': 255, 'janeway': 255}, {'kirk', 'picard', 'janeway'}, True, ["picard:", "janeway:"], id="No contactable hosts on any platform" ), pytest.param( ['enterprise', 'stargazer'], {'kirk': 1}, {'kirk'}, True, ["picard:"], id="Remote clean fails for non-SSH reason - no retry" ), ] ) async def test_remote_clean( platform_names: list[str], failed_hosts: dict[str, int], expected_hosts: set[str], exc_expected: bool, expected_err_msgs: list[str], monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, log_filter: Callable, ) -> None: """Test remote_clean() logic. Params: platform_names: These platforms will be considered to exist. failed_hosts: Any hosts that clean will artificially fail on in this test case. The key is the host and the value is the remote clean cmd return code. expected_hosts: If specified, all the hosts that the remote clean cmd is expected to be submitted to. exc_expected: If a CylcError is expected to be raised. expected_err_msgs: List of error messages expected to be in the log. """ # ----- Setup ----- caplog.set_level(logging.DEBUG, CYLC_LOG) monkeypatch.setattr( 'cylc.flow.clean.platform_from_name', mock_platform_from_name ) monkeypatch.setattr( 'cylc.flow.clean.construct_ssh_cmd', lambda *a, host, **k: ['echo', host], ) called_hosts = set() async def mock_create_subprocess_exec(*cmd, **k): host = cmd[-1] called_hosts.add(host) return mock.Mock( spec=asyncio.subprocess.Process, returncode=failed_hosts.get(host, 0), communicate=mock.AsyncMock( return_value=(b'Mocked stdout', b'Mocked stderr') ), ) monkeypatch.setattr( 'cylc.flow.clean.asyncio.create_subprocess_exec', mock_create_subprocess_exec, ) # ----- Test ----- id_ = 'foo' rm_dirs = ["whatever"] if exc_expected: with pytest.raises(CylcError) as exc: await cylc_clean.remote_clean( id_, platform_names, timeout='irrelevant', rm_dirs=rm_dirs ) assert f"Remote clean for {id_} did not complete" in ( exc_msg := str(exc.value) ) for msg in expected_err_msgs: assert msg in exc_msg else: await cylc_clean.remote_clean( id_, platform_names, timeout='irrelevant', rm_dirs=rm_dirs ) for msg in expected_err_msgs: assert log_filter(logging.ERROR, msg) assert called_hosts == expected_hosts async def test_remote_clean__timeout( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ): """Test remote_clean() gives a sensible error message for return code 124. """ caplog.set_level(logging.ERROR, CYLC_LOG) monkeypatch.setattr( 'cylc.flow.clean.asyncio.create_subprocess_exec', mock.AsyncMock( return_value=mock.Mock( spec=asyncio.subprocess.Process, returncode=124, communicate=mock.AsyncMock(return_value=(b'', b'')), ) ), ) monkeypatch.setattr( 'cylc.flow.clean.construct_ssh_cmd', lambda *a, **k: ['echo', 'mock'] ) monkeypatch.setattr( 'cylc.flow.clean.platform_from_name', mock_platform_from_name ) with pytest.raises(CylcError) as e: await cylc_clean.remote_clean( 'blah', platform_names=['stargazer'], timeout='blah' ) assert "cylc clean timed out" in str(e.value) # No need to log the remote clean cmd etc. for timeout assert "ssh" not in caplog.text.lower() assert "stderr" not in caplog.text.lower() async def test_remote_clean__platform_no_longer_exists( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ): """Test remote_clean() gives a sensible error message when a job ran on a platform that no longer exists in the global config.""" caplog.set_level(logging.WARNING, CYLC_LOG) monkeypatch.setattr( 'cylc.flow.clean.asyncio.create_subprocess_exec', mock_create_subprocess_exec := mock.AsyncMock(), ) await cylc_clean.remote_clean( 'blah', platform_names=[f'nonexist_{token_hex(4)}'], timeout='blah' ) assert "No matching platform" in caplog.text assert not mock_create_subprocess_exec.called def test_clean_top_level(tmp_run_dir: Callable): """Test that cleaning last remaining run dir inside a workflow dir removes the top level dir if it's empty (excluding _cylc-install).""" # Setup id_ = 'blue/planet/run1' run_dir: Path = tmp_run_dir(id_, installed=True, named=True) cylc_install_dir = run_dir.parent / WorkflowFiles.Install.DIRNAME assert cylc_install_dir.is_dir() runN_symlink = run_dir.parent / WorkflowFiles.RUN_N assert runN_symlink.exists() # Test clean(id_, run_dir) assert not run_dir.parent.parent.exists() # Now check that if the top level dir is not empty, it doesn't get removed run_dir: Path = tmp_run_dir(id_, installed=True, named=True) jellyfish_file = (run_dir.parent / 'jellyfish.txt') jellyfish_file.touch() clean(id_, run_dir) assert cylc_install_dir.is_dir() assert jellyfish_file.exists() @pytest.mark.parametrize( 'pattern, filetree, expected_matches', [ pytest.param( '**', FILETREE_1, ['cylc-run/foo/bar', 'cylc-run/foo/bar/log'], id="filetree1 **" ), pytest.param( '*', FILETREE_1, ['cylc-run/foo/bar/flow.cylc', 'cylc-run/foo/bar/log', 'cylc-run/foo/bar/mirkwood', 'cylc-run/foo/bar/rincewind.txt'], id="filetree1 *" ), pytest.param( '**/*.txt', FILETREE_1, ['cylc-run/foo/bar/log/bib/fortuna.txt', 'cylc-run/foo/bar/log/temba.txt', 'cylc-run/foo/bar/rincewind.txt'], id="filetree1 **/*.txt" ), pytest.param( '**', FILETREE_2, ['cylc-run/foo/bar', 'cylc-run/foo/bar/share', 'cylc-run/foo/bar/share/cycle'], id="filetree2 **" ), pytest.param( 'share', FILETREE_2, ['cylc-run/foo/bar/share'], id="filetree2 share" ), pytest.param( '**', FILETREE_3, ['cylc-run/foo/bar', 'cylc-run/foo/bar/share/cycle'], id="filetree3 **" ), pytest.param( '**/s*', FILETREE_3, ['cylc-run/foo/bar/share', 'cylc-run/foo/bar/share/cycle/sokath.txt'], id="filetree3 **/s*" ), pytest.param( '**', FILETREE_4, ['cylc-run/foo/bar', 'cylc-run/foo/bar/share/cycle'], id="filetree4 **" ), ] ) def test_glob_in_run_dir( pattern: str, filetree: Dict[str, Any], expected_matches: List[str], tmp_path: Path, tmp_run_dir: Callable ) -> None: """Test that glob_in_run_dir() returns the minimal set of results with no redundant paths. """ # Setup cylc_run_dir: Path = tmp_run_dir() id_ = 'foo/bar' run_dir = cylc_run_dir / id_ create_filetree(filetree, tmp_path, tmp_path) symlink_dirs = [run_dir / i for i in get_symlink_dirs(id_, run_dir)] expected = [tmp_path / i for i in expected_matches] # Test assert glob_in_run_dir(run_dir, pattern, symlink_dirs) == expected cylc-flow-8.7.0/tests/unit/test_config_upgrader.py0000664000175000017500000001026115264420307022504 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # Tests that configs can be upgraded from earlier versions of Cylc. import logging import pytest from cylc.flow.cfgspec.workflow import ( upg, upgrade_param_env_templates, ) from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults as ord_dict @pytest.mark.parametrize( 'cfg, expected', [ ( # No clashes - order is important: { 'parameter environment templates': ord_dict([ ('FOO', 'jupiter'), ('MOO', 'pluto') ]), 'environment': ord_dict([ ('BAR', 'neptune'), ('BAZ', 'ares') ]) }, { 'environment': ord_dict([ ('FOO', 'jupiter'), ('MOO', 'pluto'), ('BAR', 'neptune'), ('BAZ', 'ares') ]) } ), ( # Clashes - environment wins: { 'parameter environment templates': ord_dict([ ('FOO', 'jupiter'), ('BAR', 'neptune') ]), 'environment': ord_dict([ ('FOO', 'zeus'), ('BAR', 'poseidon') ]) }, { 'environment': ord_dict([ ('FOO', 'zeus'), ('BAR', 'poseidon') ]) } ), ( # No environment section: { 'parameter environment templates': ord_dict([ ('FOO', 'jupiter'), ('BAR', 'neptune') ]) }, { 'environment': ord_dict([ ('FOO', 'jupiter'), ('BAR', 'neptune') ]) } ) ] ) def test_upgrade_param_env_templates(cfg, expected, log_filter): """Test that the deprecated [runtime][X][parameter environment templates] contents are prepended to [runtime][X][environment], in the correct order""" def _cfg(dic): """Return OrderedDictWithDefaults config populated with values from dic (dictionary)""" result = ord_dict({ 'runtime': ord_dict({ '': ord_dict({ 'script': 'echo whatever' }) }) }) if 'parameter environment templates' in dic: result['runtime']['']['parameter environment templates'] = ( dic['parameter environment templates'] ) if 'environment' in dic: result['runtime']['']['environment'] = dic['environment'] return result config = _cfg(cfg) upgrade_param_env_templates(config, 'flow.cylc') assert config == _cfg(expected) assert log_filter( logging.WARNING, contains=( "support for parameter environment templates will be " "removed at Cylc 8.9" ) ) @pytest.mark.parametrize( 'macp, rlim', [(16, 'P15'), ('', '')] ) def test_upgrade_max_active_cycle_points(macp, rlim): """Test that `max active cycle points` is correctly upgraded to `runahead limit`.""" cfg = { 'scheduling': {'max active cycle points': macp} } expected = { 'scheduling': {'runahead limit': rlim} } upg(cfg, 'flow.cylc') assert cfg == expected cylc-flow-8.7.0/tests/unit/test_subprocpool.py0000664000175000017500000003276315264420307021730 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from pathlib import Path from types import SimpleNamespace from tempfile import ( NamedTemporaryFile, SpooledTemporaryFile, TemporaryFile, TemporaryDirectory, ) import pytest from cylc.flow import LOG from cylc.flow.id import Tokens from cylc.flow.cycling.iso8601 import ISO8601Point from cylc.flow.task_events_mgr import TaskJobLogsRetrieveContext from cylc.flow.subprocctx import SubProcContext from cylc.flow.subprocpool import ( SubProcPool, _XTRIG_FUNC_CACHE, get_xtrig_func, ) from cylc.flow.task_outputs import ( TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_SUBMIT_FAILED, TASK_OUTPUT_SUCCEEDED, TASK_OUTPUT_FAILED, TASK_OUTPUT_EXPIRED, ) from cylc.flow.task_proxy import TaskProxy def test_get_temporary_file(): """Test SubProcPool.get_temporary_file.""" assert isinstance(SubProcPool.get_temporary_file(), SpooledTemporaryFile) def test_run_command_returns_0(): """Test basic usage, command returns 0""" ctx = SubProcContext('truth', ['true']) SubProcPool.run_command(ctx) assert ctx.err == '' assert ctx.out == '' assert ctx.ret_code == 0 def test_run_command_returns_1(): """Test basic usage, command returns 1""" ctx = SubProcContext('lies', ['false']) SubProcPool.run_command(ctx) assert ctx.err == '' assert ctx.out == '' assert ctx.ret_code == 1 def test_run_command_writes_to_out(): """Test basic usage, command writes to STDOUT""" ctx = SubProcContext('parrot', ['echo', 'pirate', 'urrrr']) SubProcPool.run_command(ctx) assert ctx.err == '' assert ctx.out == 'pirate urrrr\n' assert ctx.ret_code == 0 def test_run_command_writes_to_err(): """Test basic usage, command writes to STDERR""" ctx = SubProcContext( 'parrot2', ['bash', '--noprofile', '--norc', '-c', 'echo pirate errrr >&2'] ) SubProcPool.run_command(ctx) assert 'pirate errrr\n' assert ctx.out == '' assert ctx.ret_code == 0 def test_run_command_with_stdin_from_str(): """Test STDIN from string""" ctx = SubProcContext('meow', ['cat'], stdin_str='catches mice.\n') SubProcPool.run_command(ctx) assert ctx.err == '' assert ctx.out == 'catches mice.\n' assert ctx.ret_code == 0 def test_run_command_with_stdin_from_unicode(): """Test STDIN from string with Unicode""" ctx = SubProcContext('meow', ['cat'], stdin_str='喵\n') SubProcPool.run_command(ctx) assert ctx.err == '' assert ctx.out == '喵\n' assert ctx.ret_code == 0 def test_run_command_with_stdin_from_handle(): """Test STDIN from a single opened file handle""" handle = TemporaryFile() handle.write('catches mice.\n'.encode('UTF-8')) handle.seek(0) ctx = SubProcContext('meow', ['cat'], stdin_files=[handle]) SubProcPool.run_command(ctx) assert ctx.err == '' assert ctx.out == 'catches mice.\n' assert ctx.ret_code == 0 handle.close() def test_run_command_with_stdin_from_path(): """Test STDIN from a single file path""" handle = NamedTemporaryFile() handle.write('catches mice.\n'.encode('UTF-8')) handle.seek(0) ctx = SubProcContext('meow', ['cat'], stdin_files=[handle.name]) SubProcPool.run_command(ctx) assert ctx.err == '' assert ctx.out == 'catches mice.\n' assert ctx.ret_code == 0 handle.close() def test_run_command_with_stdin_from_handles(): """Test STDIN from multiple file handles""" handles = [] for txt in ['catches mice.\n', 'eat fish.\n']: handle = TemporaryFile() handle.write(txt.encode('UTF-8')) handle.seek(0) handles.append(handle) ctx = SubProcContext('meow', ['cat'], stdin_files=handles) SubProcPool.run_command(ctx) assert ctx.err == '' assert ctx.out == 'catches mice.\neat fish.\n' assert ctx.ret_code == 0 for handle in handles: handle.close() def test_run_command_with_stdin_from_paths(): """Test STDIN from multiple file paths""" handles = [] for txt in ['catches mice.\n', 'eat fish.\n']: handle = NamedTemporaryFile() handle.write(txt.encode('UTF-8')) handle.seek(0) handles.append(handle) ctx = SubProcContext( 'meow', ['cat'], stdin_files=[handle.name for handle in handles] ) SubProcPool.run_command(ctx) assert ctx.err == '' assert ctx.out == 'catches mice.\neat fish.\n' assert ctx.ret_code == 0 for handle in handles: handle.close() def test_xfunction(): """Test xtrigger function import.""" with TemporaryDirectory() as temp_dir: python_dir = Path(temp_dir, "lib", "python") python_dir.mkdir(parents=True) the_answer_file = python_dir / "the_answer.py" with the_answer_file.open(mode="w") as f: f.write("""the_answer = lambda: 42""") f.flush() f_name = "the_answer" fn = get_xtrig_func(f_name, f_name, temp_dir) result = fn() assert 42 == result def test_xfunction_cache(): """Test xtrigger function import cache.""" with TemporaryDirectory() as temp_dir: python_dir = Path(temp_dir, "lib", "python") python_dir.mkdir(parents=True) amandita_file = python_dir / "amandita.py" with amandita_file.open(mode="w") as f: f.write("""choco = lambda: 'chocolate'""") f.flush() m_name = "amandita" # module f_name = "choco" # function fn = get_xtrig_func(m_name, f_name, temp_dir) result = fn() assert 'chocolate' == result # is in the cache assert (m_name, f_name) in _XTRIG_FUNC_CACHE # returned from cache assert fn, get_xtrig_func(m_name, f_name == temp_dir) def test_xfunction_import_error(): """Test for error on importing a xtrigger function. To prevent the test eventually failing if the test function is added and successfully imported, we use an invalid module name as per Python spec. """ with TemporaryDirectory() as temp_dir, pytest.raises(ModuleNotFoundError): get_xtrig_func("invalid-module-name", "func-name", temp_dir) def test_xfunction_attribute_error(): """Test for error on looking for an attribute in a xtrigger script.""" with TemporaryDirectory() as temp_dir: python_dir = Path(temp_dir, "lib", "python") python_dir.mkdir(parents=True) the_answer_file = python_dir / "the_sword.py" with the_answer_file.open(mode="w") as f: f.write("""the_droid = lambda: 'excalibur'""") f.flush() f_name = "the_sword" with pytest.raises(AttributeError): get_xtrig_func(f_name, f_name, temp_dir) @pytest.fixture def mock_ctx(): def inner_(ret_code=None, host=None, cmd_key=None, cmd=None): """Provide a SimpleNamespace which looks like a ctx object.""" inputs = locals() defaults = { 'ret_code': 255, 'host': 'mouse', 'cmd_key': 'my-command', 'cmd': ['bistromathic', 'take-off'], } for key in inputs: if inputs[key] is None: inputs[key] = defaults[key] ctx = SimpleNamespace( cmd=inputs['cmd'], timestamp=None, ret_code=inputs['ret_code'], host=inputs['host'], cmd_key=inputs['cmd_key'], ) return ctx yield inner_ def _test_callback(ctx, foo=''): """Very Simple test callback function""" LOG.error(f'callback called.{foo}') def _test_callback_255(ctx, foo=''): """Very Simple test callback function""" LOG.error(f'255 callback called.{foo}') @pytest.mark.parametrize( 'expect, ret_code, cmd_key', [ pytest.param('callback called', 0, 'ssh something', id="return 0"), pytest.param('callback called', 1, 'ssh something', id="return 1"), pytest.param( 'platform: None - Could not connect to mouse.', 255, 'ssh', id="return 255", ), pytest.param( 'platform: localhost - Could not connect to mouse.', 255, TaskJobLogsRetrieveContext(['ssh', 'something'], None, None), id="return 255 (log-ret)", ), ], ) def test__run_command_exit(caplog, mock_ctx, expect, ret_code, cmd_key): """It runs a callback""" ctx = mock_ctx(ret_code=ret_code, cmd_key=cmd_key, cmd=['ssh']) SubProcPool._run_command_exit( ctx, callback=_test_callback, callback_255=_test_callback_255 ) assert expect in caplog.records[0].msg if ret_code == 255: assert '255 callback called.' in caplog.records[1].msg def test__run_command_exit_no_255_callback(caplog, mock_ctx): """It runs the vanilla callback if no 255 callback provided""" SubProcPool._run_command_exit(mock_ctx(), callback=_test_callback) assert 'callback called' in caplog.records[0].msg def test__run_command_exit_no_gettable_platform(caplog, mock_ctx): """It logs being unable to select a platform""" ret_ctx = TaskJobLogsRetrieveContext( platform_name='rhenas', max_size=256, key='rhenas' ) ctx = mock_ctx(cmd_key=ret_ctx, cmd=['ssh'], ret_code=255) SubProcPool._run_command_exit(ctx, callback=_test_callback) assert 'platform: rhenas' in caplog.records[0].msg def test__run_command_exit_no_255_args(caplog, mock_ctx): """It runs the 255 callback with the args of the callback if no callback 255 args provided. """ SubProcPool._run_command_exit( mock_ctx(cmd=['ssh', 'Zaphod']), callback=_test_callback, callback_args=['Zaphod'], callback_255=_test_callback_255, ) assert '255' in caplog.records[1].msg def test__run_command_exit_add_to_badhosts(mock_ctx): """It updates the list of badhosts""" badhosts = {'foo', 'bar'} SubProcPool._run_command_exit( mock_ctx(cmd=['ssh']), bad_hosts=badhosts, callback=print, callback_args=['Welcome to Magrathea'], ) assert badhosts == {'foo', 'bar', 'mouse'} def test__run_command_exit_add_to_badhosts_log(caplog, mock_ctx): """It gets platform name from the callback args.""" badhosts = {'foo', 'bar'} SubProcPool._run_command_exit( mock_ctx(cmd=['ssh']), bad_hosts=badhosts, callback=lambda x, t: print(str(x)), callback_args=[ TaskProxy( Tokens('~u/w//c/t/2'), SimpleNamespace( name='t', dependencies={}, sequential='', external_triggers=[], xtrig_labels={}, expiration_offset=None, outputs={ TASK_OUTPUT_SUBMITTED: [None, None], TASK_OUTPUT_SUBMIT_FAILED: [None, None], TASK_OUTPUT_SUCCEEDED: [None, None], TASK_OUTPUT_FAILED: [None, None], TASK_OUTPUT_EXPIRED: [None, None], }, graph_children={}, rtconfig={'platform': 'foo'}, ), ISO8601Point('1990'), ) ], ) assert 'platform: foo' in caplog.records[0].message assert badhosts == {'foo', 'bar', 'mouse'} def test__run_command_exit_rsync_fails(mock_ctx): """It updates the list of badhosts""" badhosts = {'foo', 'bar'} ctx = mock_ctx(cmd=['rsync'], ret_code=42, cmd_key='file-install') SubProcPool._run_command_exit( ctx=ctx, bad_hosts=badhosts, callback=print, callback_args=[ { 'name': 'Magrathea', 'ssh command': 'ssh', 'rsync command': 'rsync command', }, 'Welcome to Magrathea', ], ) assert badhosts == {'foo', 'bar', 'mouse'} @pytest.mark.parametrize( 'expect, ctx_kwargs', [ (True, {'cmd': ['ssh'], 'ret_code': 255}), (False, {'cmd': ['foo'], 'ret_code': 255}), (False, {'cmd': ['ssh'], 'ret_code': 42}), ], ) def test_ssh_255_fail(mock_ctx, expect, ctx_kwargs): """It knows when a ctx has failed""" output = SubProcPool.ssh_255_fail(mock_ctx(**ctx_kwargs)) assert output == expect @pytest.mark.parametrize( 'expect, ctx_kwargs', [ (True, {'cmd': ['rsync'], 'ret_code': 99, 'host': 'not_local'}), (True, {'cmd': ['rsync'], 'ret_code': 255, 'host': 'not_local'}), (False, {'cmd': ['make it-so'], 'ret_code': 255, 'host': 'not_local'}), (False, {'cmd': ['rsync'], 'ret_code': 125, 'host': 'localhost'}), ], ) def test_rsync_255_fail(mock_ctx, expect, ctx_kwargs): """It knows when a ctx has failed""" output = SubProcPool.rsync_255_fail( mock_ctx(**ctx_kwargs), {'ssh command': 'ssh', 'rsync command': 'rsync command'}, ) assert output == expect cylc-flow-8.7.0/tests/unit/test_graph_parser.py0000664000175000017500000006663015264420307022036 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Unit tests for the GraphParser.""" from itertools import product from types import SimpleNamespace from typing import Dict, List import pytest from pytest import param from cylc.flow.exceptions import GraphParseError, ParamExpandError from cylc.flow.graph_parser import GraphParser from cylc.flow.task_outputs import ( TASK_OUTPUT_FAILED, TASK_OUTPUT_STARTED, TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_SUCCEEDED, ) @pytest.mark.parametrize( 'graph', [ 't1 => & t2', 't1 => t2 &', '& t1 => t2', 't1 & => t2', 't1 => => t2' ] ) def test_parse_graph_fails_null_task_name(graph): """Test fail null task names.""" with pytest.raises(GraphParseError) as cm: GraphParser().parse_graph(graph) assert "Null task name in graph:" in str(cm.value) @pytest.mark.parametrize('seq', ('&', '|', '=>')) @pytest.mark.parametrize( 'graph, expected_err', [ [ "{0} b", "Leading {0}" ], [ "a {0}", "Dangling {0}" ], [ "{0} b {0} c", "Leading {0}" ], [ "a {0} b {0}", "Dangling {0}" ] ] ) def test_graph_syntax_errors_2(seq, graph, expected_err): """Test various graph syntax errors.""" graph = graph.format(seq) expected_err = expected_err.format(seq) with pytest.raises(GraphParseError) as cm: GraphParser().parse_graph(graph) assert ( expected_err in str(cm.value) ) @pytest.mark.parametrize( 'graph, expected_err', [ ( "a b => c", "Bad graph node format" ), ( "a => b c", "Bad graph node format" ), ( "!foo => bar", "Suicide markers must be on the right of a trigger:" ), ( "( foo & bar => baz", 'Mismatched parentheses in: "(foo&bar"' ), ( "a => b & c)", 'Mismatched parentheses in: "b&c)"' ), ( "(a => b & c)", 'Mismatched parentheses in: "(a"' ), ( "(a => b[+P1]", 'Mismatched parentheses in: "(a"' ), ( """(a | b & c) => d foo => bar (a | b & c) => !d""", "can't trigger both d and !d" ), ( "a => b | c", "Illegal OR on right side" ), ( "foo && bar => baz", "The graph AND operator is '&'" ), ( "foo || bar => baz", "The graph OR operator is '|'" ), param( # See https://github.com/cylc/cylc-flow/issues/5844 "foo => bar[1649]", 'Invalid cycle point offsets only on right', id='no-cycle-point-RHS' ), # See https://github.com/cylc/cylc-flow/issues/6523 # For the next 4 tests: # NB "foo:succeeded" now explicit on the right (this used to test # inferred "foo:succeeded" from "=> foo", which we no longer infer). param( # Yes I know it's circular, but it's here to # demonstrate that the test below is broken: "foo:finished => foo:succeeded", 'Output foo:succeeded can\'t be both required and optional', id='finish-implies-success-optional' ), param( "foo[-P1]:finish => foo:succeeded", 'Output foo:succeeded can\'t be both required and optional', id='finish-implies-success-optional-offset' ), param( "foo[-P1]:succeeded | foo[-P1]:failed => bar", # order of outputs varies in the error message 'must both be optional if both are used', id='succeed-or-failed-mustbe-optional' ), param( "foo[-P1]:succeeded? | foo[-P1]:failed? => foo:succeeded", 'Output foo:succeeded can\'t be both required and optional', id='succeed-or-failed-implies-success-optional' ), ] ) def test_graph_syntax_errors(graph, expected_err): """Test various graph syntax errors.""" with pytest.raises(GraphParseError) as cm: GraphParser().parse_graph(graph) assert expected_err in str(cm.value) def test_parse_graph_simple(): """Test parsing graphs.""" # added white spaces and comments to show that these change nothing gp = GraphParser() gp.parse_graph('a => b\n \n# this is a comment\n') original = gp.original triggers = gp.triggers families = gp.family_map assert ( original == {'a': {'': ''}, 'b': {'a:succeeded': 'a:succeeded'}} ) assert ( triggers == { 'a': {'': ([], False)}, 'b': {'a:succeeded': (['a:succeeded'], False)} } ) assert not families @pytest.mark.parametrize( 'graph, expect', [ param( 'a => b\n=> c', SimpleNamespace( original={ 'c': {'b:succeeded': 'b:succeeded'}, 'b': {'a:succeeded': 'a:succeeded'}, 'a': {'': ''} }, triggers={ 'c': {'b:succeeded': (['b:succeeded'], False)}, 'b': {'a:succeeded': (['a:succeeded'], False)}, 'a': {'': ([], False)} }, families={} ), id='line break on =>' ), param( 'a & b\n& c', SimpleNamespace( original={'b': {'': ''}, 'a': {'': ''}, 'c': {'': ''}}, triggers={ 'b': {'': ([], False)}, 'a': {'': ([], False)}, 'c': {'': ([], False)} }, families={} ), id='line break on &' ), param( 'a | b\n| c', SimpleNamespace( original={'b': {'': ''}, 'c': {'': ''}, 'a': {'': ''}}, triggers={ 'b': {'': ([], False)}, 'c': {'': ([], False)}, 'a': {'': ([], False)} }, families={} ), id='line break on |' ) ] ) def test_parse_graph_simple_with_break_line_01(graph, expect): """Test parsing graphs.""" parser = GraphParser() parser.parse_graph(graph) assert parser.original == expect.original assert parser.triggers == expect.triggers assert not parser.family_map def test_parse_graph_simple_with_break_line_02(): """Test parsing graphs.""" gp = GraphParser() gp.parse_graph( 'a => b\n' '=> c =>\n' 'd' ) original = gp.original triggers = gp.triggers families = gp.family_map assert original['a'] == {'': ''} assert original['b'] == {'a:succeeded': 'a:succeeded'} assert original['c'] == {'b:succeeded': 'b:succeeded'} assert original['d'] == {'c:succeeded': 'c:succeeded'} assert triggers['a'] == {'': ([], False)} assert triggers['b'] == {'a:succeeded': (['a:succeeded'], False)} assert triggers['c'] == {'b:succeeded': (['b:succeeded'], False)} assert triggers['d'] == {'c:succeeded': (['c:succeeded'], False)} assert not families def test_parse_graph_with_parameters(): """Test parsing graphs with parameters.""" parameterized_parser = GraphParser( None, ({'city': ['la_paz']}, {'city': '_%(city)s'})) parameterized_parser.parse_graph('a => b') original = parameterized_parser.original triggers = parameterized_parser.triggers families = parameterized_parser.family_map assert ( original == {'a': {'': ''}, 'b_la_paz': {'a:succeeded': 'a:succeeded'}} ) assert ( triggers == { 'a': {'': ([], False)}, 'b_la_paz': {'a:succeeded': (['a:succeeded'], False)} } ) assert not families def test_parse_graph_with_invalid_parameters(): """Test parsing graphs with invalid parameters.""" parameterized_parser = GraphParser( None, ({'city': ['la_paz']}, {'city': '_%(city)s'})) with pytest.raises(ParamExpandError): # no state in the parameters list parameterized_parser.parse_graph('a => b') def test_inter_workflow_dependence_simple(): """Test invalid inter-workflow dependence""" gp = GraphParser() gp.parse_graph( """ a => b c => d """ ) assert ( gp.original == { 'a': {'': ''}, 'b': {'a:succeeded': 'a:succeeded'}, 'c': {'': ''}, 'd': {'c:succeeded': 'c:succeeded'} } ) assert ( gp.triggers == { 'a': {'': ([], False)}, 'c': {'': ([], False)}, 'b': {'a:succeeded': (['a:succeeded'], False)}, 'd': {'c:succeeded': (['c:succeeded'], False)} } ) assert ( gp.workflow_state_polling_tasks == { 'a': ( 'WORKFLOW', 'TASK', 'failed', '' ), # Default to "succeeded" is done in config module. 'c': ( 'WORKFLOW', 'TASK', None, '' ) } ) assert not gp.family_map def test_line_continuation(): """Test syntax-driven line continuation.""" graph1 = "a => b => c" graph2 = """a => b => c""" graph3 = """a => b => c""" gp1 = GraphParser() gp1.parse_graph(graph1) gp2 = GraphParser() gp2.parse_graph(graph2) gp3 = GraphParser() gp3.parse_graph(graph3) res = { 'a': {'': ([], False)}, 'c': {'b:succeeded': (['b:succeeded'], False)}, 'b': {'a:succeeded': (['a:succeeded'], False)} } assert res == gp1.triggers assert gp1.triggers == gp2.triggers assert gp1.triggers == gp3.triggers graph = """foo => bar a => b =>""" gp = GraphParser() pytest.raises(GraphParseError, gp.parse_graph, graph) graph = """ => a => b foo => bar""" gp = GraphParser() pytest.raises(GraphParseError, gp.parse_graph, graph) @pytest.mark.parametrize( 'graph1, graph2', [ [ "foo => bar", # default trigger "foo:succeed => bar" ], [ "foo => bar", # default trigger "foo:succeeded => bar" ], [ "foo => bar", # repeat trigger """foo => bar foo => bar""" ], [ "foo:finished => bar", # finish trigger "(foo:succeed? | foo:fail?) => bar" ], [ """ bar foo => bar:succeed => baz # ignore qualifier on RHS """, """ foo => bar bar:succeed => baz """ ], [ """ foo => bar[1649] => baz """, """ foo => bar[1649] bar[1649] => baz """ ], ] ) def test_trigger_equivalence(graph1, graph2): gp1 = GraphParser() gp1.parse_graph(graph1) gp2 = GraphParser() gp2.parse_graph(graph2) assert gp1.triggers == gp2.triggers @pytest.mark.parametrize( 'fam_map, fam_graph, member_graph', [ [ {'FAM': ['m1', 'm2'], 'BAM': ['b1', 'b2']}, "FAM:succeed-all => BAM", """(m1 & m2) => b1 (m1 & m2) => b2""" ], [ {'FAM': ['m1', 'm2']}, "pre => FAM", """pre => m1 pre => m2""" ], [ {'FAM': ['m1', 'm2']}, "FAM:succeed-all => post", "(m1 & m2) => post" ], [ {'FAM': ['m1', 'm2']}, "FAM:succeed-any => post", "(m1 | m2) => post", ], [ {'FAM': ['m1', 'm2'], 'BAM': ['b1', 'b2']}, "FAM:fail-any => BAM", """(m1:fail | m2:fail) => b1 (m1:fail | m2:fail) => b2""" ], [ {'FAM': ['m1', 'm2']}, "FAM:finish-all => post", "((m1? | m1:fail?) & (m2? | m2:fail?)) => post" ] ] ) def test_family_trigger_equivalence(fam_map, fam_graph, member_graph): """Test family trigger semantics.""" gp1 = GraphParser(fam_map) gp1.parse_graph(fam_graph) gp2 = GraphParser() gp2.parse_graph(member_graph) assert gp1.triggers == gp2.triggers def test_parameter_expand(): """Test graph parameter expansion.""" fam_map = { 'FAM_m0': ['fa_m0', 'fb_m0'], 'FAM_m1': ['fa_m1', 'fb_m1'], } params = {'m': ['0', '1'], 'n': ['0', '1']} templates = {'m': '_m%(m)s', 'n': '_n%(n)s'} gp1 = GraphParser(fam_map, (params, templates)) gp1.parse_graph(""" pre => foo => bar bar => baz # specific case bar => bar # inter-chunk """) gp2 = GraphParser() gp2.parse_graph(""" pre => foo_m0_n0 => bar_n0 pre => foo_m0_n1 => bar_n1 pre => foo_m1_n0 => bar_n0 pre => foo_m1_n1 => bar_n1 bar_n0 => baz bar_n0 => bar_n1 """) assert gp1.triggers == gp2.triggers def test_parameter_specific(): """Test graph parameter expansion with a specific value.""" params = {'i': ['0', '1'], 'j': ['0', '1', '2']} templates = {'i': '_i%(i)s', 'j': '_j%(j)s'} gp1 = GraphParser(family_map=None, parameters=(params, templates)) gp1.parse_graph("bar => baz\nfoo => qux") gp2 = GraphParser() gp2.parse_graph(""" foo_i1_j0 => qux foo_i1_j1 => qux foo_i1_j2 => qux bar_i0_j0 => baz_i1_j0 bar_i0_j1 => baz_i1_j1 bar_i0_j2 => baz_i1_j2""") assert gp1.triggers == gp2.triggers def test_parameter_offset(): """Test graph parameter expansion with an offset.""" params = {'i': ['0', '1'], 'j': ['0', '1', '2']} templates = {'i': '_i%(i)s', 'j': '_j%(j)s'} gp1 = GraphParser(family_map=None, parameters=(params, templates)) gp1.parse_graph("bar => baz") gp2 = GraphParser() gp2.parse_graph(""" bar_i0_j0 => baz_i1_j0 bar_i0_j1 => baz_i1_j1 bar_i0_j2 => baz_i1_j2""") assert gp1.triggers == gp2.triggers def test_conditional(): """Test generation of conditional triggers.""" gp1 = GraphParser() gp1.parse_graph("(foo:start | bar) => baz") res = { 'baz': { '(foo:started|bar:succeeded)': ( ['foo:started', 'bar:succeeded'], False) }, 'foo': { '': ([], False) }, 'bar': { '': ([], False) } } assert res == gp1.triggers == res @pytest.mark.parametrize( 'graph', [ "foo[-P1Y] => bar", "foo:fail => bar", "foo:fail[-P1Y] => bar", "foo[-P1Y]:fail => bar", "foo[-P1Y]:fail => bar", "foo:fail[-P1Y] => bar", "foo:fail[-P1Y] => bar", ":fail[-P1Y] => bar", "[-P1Y] => bar", "[-P1Y]:fail => bar", "bar => foo:fail[-P1Y]", "foo[-P1Y]baz => bar" ] ) def test_bad_node_syntax(graph): """Test that badly formatted graph nodes are detected. The correct format is: NAME()([CYCLE-POINT-OFFSET])(:TRIGGER-TYPE)") """ params = {'m': ['0', '1'], 'n': ['0', '1']} templates = {'m': '_m%(m)s', 'n': '_n%(n)s'} gp = GraphParser(parameters=(params, templates)) with pytest.raises(GraphParseError) as cm: gp.parse_graph(graph) assert "Bad graph node format" in str(cm.value) def test_spaces_between_tasks_fails(): """Test that is rejected (i.e. no & or | in between)""" gp = GraphParser() pytest.raises( GraphParseError, gp.parse_graph, "foo bar=> baz") pytest.raises( GraphParseError, gp.parse_graph, "foo&bar=> ba z") pytest.raises( GraphParseError, gp.parse_graph, "foo 123=> bar") pytest.raises( GraphParseError, gp.parse_graph, "foo - 123 baz=> bar") def test_spaces_between_parameters_fails(): """Test that are rejected (i.e. no comma)""" gp = GraphParser() pytest.raises( GraphParseError, gp.parse_graph, " => baz") pytest.raises( GraphParseError, gp.parse_graph, " => baz") pytest.raises( GraphParseError, gp.parse_graph, " => baz") def test_spaces_between_parameters_passes(): """Test that works, with spaces around the -+ signs""" params = {'m': ['0', '1', '2']} templates = {'m': '_m%(m)s'} gp = GraphParser(parameters=(params, templates)) gp.parse_graph(" => ") gp.parse_graph(" => ") gp.parse_graph(" => ") gp.parse_graph(" => ") gp.parse_graph(" => ") gp.parse_graph(" => ") def test_spaces_in_trigger_fails(): """Test that 'task:a- b' are rejected""" gp = GraphParser() pytest.raises( GraphParseError, gp.parse_graph, "FOO:custom -trigger => baz") pytest.raises( GraphParseError, gp.parse_graph, "FOO:custom- trigger => baz") pytest.raises( GraphParseError, gp.parse_graph, "FOO:custom - trigger => baz") def test_parameter_graph_mixing_offset_and_conditional(): """Test for bug reported in issue #2608 on GitHub: https://github.com/cylc/cylc-flow/issues/2608""" params = {'m': ["cat", "dog"]} templates = {'m': '_%(m)s'} gp = GraphParser(parameters=(params, templates)) gp.parse_graph("foo & baz => foo") triggers = { 'foo_cat': { '': ( [], False ), 'baz:succeeded': ( ['baz:succeeded'], False ) }, 'foo_dog': { 'foo_cat:succeeded': ( ['foo_cat:succeeded'], False ), 'baz:succeeded': ( ['baz:succeeded'], False ) }, 'baz': { '': ([], False) } } assert gp.triggers == triggers def test_param_expand_graph_parser(): """Test to validate that the graph parser removes out-of-edge nodes: https://github.com/cylc/cylc-flow/pull/3452#issuecomment-677165000""" params = {'m': ["cat"]} templates = {'m': '_%(m)s'} gp = GraphParser(parameters=(params, templates)) gp.parse_graph("foo => bar => baz") triggers = { 'foo': { '': ([], False) } } assert gp.triggers == triggers @pytest.mark.parametrize( 'expect', ('&', '|', '=>') ) def test_parse_graph_fails_with_continuation_at_last_line(expect): """Fails if last line contains a continuation char. """ parser = GraphParser() with pytest.raises(GraphParseError) as raised: parser.parse_graph(f't1 => t2 {expect}') assert isinstance(raised.value, GraphParseError) assert f'Dangling {expect}' in raised.value.args[0] @pytest.mark.parametrize( 'before, after', product(['&', '|', '=>'], repeat=2) ) def test_parse_graph_fails_with_too_many_continuations(before, after): """Fails if one line ends with continuation char and the next line _also_ starts with one. """ parser = GraphParser() with pytest.raises(GraphParseError) as raised: parser.parse_graph(f'foo & bar {before}\n{after}baz') assert isinstance(raised.value, GraphParseError) assert 'Consecutive lines end and start' in raised.value.args[0] def test_task_optional_outputs(): """Test optional outputs are correctly parsed from graph. This checks "task_output_opt" dict which holds output optionality inferred from the graph. Note since https://github.com/cylc/cylc-flow/pull/6999 we no longer infer optionality from *implicit* outputs on RHS of triggers, i.e. "a => b" does not imply b:succeeded is a required output. """ OPTIONAL = True REQUIRED = False gp = GraphParser() gp.parse_graph( """ a1 => b1 # does not imply b1:succeeded ... a2:succeed => b2:succeeded a3:succeed => b3:succeed c1? => d1? c2:succeed? => d2? c3:succeed? => d3:succeed? x:fail? => y foo:finish => bar """ ) for i in range(1, 4): for task in (f'a{i}', f'b{i}'): if task != "b1": assert ( gp.task_output_opt[(task, TASK_OUTPUT_SUCCEEDED)] == (REQUIRED, False, True) ) for task in (f'c{i}', f'd{i}'): assert ( gp.task_output_opt[(task, TASK_OUTPUT_SUCCEEDED)] == (OPTIONAL, True, True) ) assert ( gp.task_output_opt[('x', TASK_OUTPUT_FAILED)] == (OPTIONAL, True, True) ) assert ( gp.task_output_opt[('foo', TASK_OUTPUT_SUCCEEDED)] == (OPTIONAL, True, True) ) assert ( gp.task_output_opt[('foo', TASK_OUTPUT_FAILED)] == (OPTIONAL, True, True) ) @pytest.mark.parametrize( 'qual, task_output', [ ('start', TASK_OUTPUT_STARTED), ('succeed', TASK_OUTPUT_SUCCEEDED), ('fail', TASK_OUTPUT_FAILED), ('submit', TASK_OUTPUT_SUBMITTED), ] ) def test_family_optional_outputs(qual, task_output): """Test member output optionality inferred from family triggers.""" fam_map = { 'FAM': ['f1', 'f2'], 'BAM': ['b1', 'b2'], } gp = GraphParser(fam_map) gp.parse_graph( f""" # required FAM:{qual}-all => foo # optional member f2:{task_output}? # required BAM:{qual}-any => bar """ ) # -all for member in ['f1', 'f2']: optional = (member == 'f2') assert gp.task_output_opt[(member, task_output)][0] == optional # -any optional = False for member in ['b1', 'b2']: assert gp.task_output_opt[(member, task_output)][0] == optional def test_cannot_be_required(): """Is should not allow :expired or :submit-failed to be required. See proposal point 4: https://cylc.github.io/cylc-admin/proposal-optional-output-extension.html#proposal """ gp = GraphParser({}) # outputs can be optional gp.parse_graph('a:expired? => b') gp.parse_graph('a:submit-failed? => b') # but cannot be required with pytest.raises(GraphParseError, match='must be optional'): gp.parse_graph('a:expired => b') with pytest.raises(GraphParseError, match='must be optional'): gp.parse_graph('a:submit-failed => b') @pytest.mark.parametrize( 'graph, error', [ [ """FAM:succeed-all => foo FAM:fail-all => foo""", ("must both be optional if both are used (via family trigger" " defaults") ], [ """FAM:succeed-all => foo FAM:succeed-any? => bar""", ("can't default to both optional and required (via family trigger" " defaults)") ], [ "FAM:blargh-all => foo", # LHS "Illegal family trigger" ], [ "foo => FAM:blargh-all", # RHS "Illegal family trigger" ], [ "FAM => foo", # bare family on LHS "Family trigger required: FAM => foo" ], [ "FAM:expire-all => foo", "must be optional" ], ] ) def test_family_trigger_errors(graph, error): """Test errors via bad family triggers and member output optionality.""" fam_map = { 'FAM': ['f1', 'f2'] } gp = GraphParser(fam_map) with pytest.raises(GraphParseError) as cm: gp.parse_graph(graph) assert error in str(cm.value) @pytest.mark.parametrize( 'graph, c8error', [ [ """a:x => b a:x? => c""", "Output a:x can't be both required and optional", ], [ """a? => c a => b""", "Output a:succeeded can't be both required and optional", ], [ """a => c a:fail => b""", ("must both be optional if both are used"), ], [ """a:fail? => b a => c""", ("must both be optional if both are used"), ], [ "a:finish? => b", "Pseudo-output a:finished can't be optional", ], [ "a:expire => b", "must be optional", ], ] ) def test_task_optional_output_errors_order( graph, c8error, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch ): """Test optional output errors are raised as expected.""" gp = GraphParser() with pytest.raises(GraphParseError) as cm: gp.parse_graph(graph) assert c8error in str(cm.value) @pytest.mark.parametrize( 'ftrig', GraphParser.fam_to_mem_trigger_map.keys() ) def test_fail_family_triggers_on_tasks(ftrig): gp = GraphParser() with pytest.raises(GraphParseError) as cm: gp.parse_graph(f"foo:{ftrig} => bar") assert ( str(cm.value).startswith( "family trigger on non-family namespace" ) ) @pytest.mark.parametrize( 'graph, expected_triggers', [ param( 'a => b & c', {'a': [''], 'b': ['a:succeeded'], 'c': ['a:succeeded']}, id="simple" ), param( 'a => (b & c)', {'a': [''], 'b': ['a:succeeded'], 'c': ['a:succeeded']}, id="simple w/ parentheses" ), param( 'a => (b & (c & d))', { 'a': [''], 'b': ['a:succeeded'], 'c': ['a:succeeded'], 'd': ['a:succeeded'], }, id="more parentheses" ), ] ) def test_RHS_AND(graph: str, expected_triggers: Dict[str, List[str]]): """Test '&' operator on right hand side of trigger expression.""" gp = GraphParser() gp.parse_graph(graph) triggers = { task: list(trigs.keys()) for task, trigs in gp.triggers.items() } assert triggers == expected_triggers @pytest.mark.parametrize( 'args, err', ( # No error if offset in NON-terminal RHS: param((('a', 'b[-P42M]'), {}, set(), set()), None), # Check the left hand side if this has a non-terminal RHS: param( (('a &', 'b[-P42M]'), {}, set(), set()), 'Null task name in graph' ), ), ) def test_proc_dep_pair(args, err): """ Unit tests for _proc_dep_pair. """ gp = GraphParser() if err: with pytest.raises(GraphParseError, match=err): gp._proc_dep_pair(*args) else: assert gp._proc_dep_pair(*args) is None cylc-flow-8.7.0/tests/unit/run_modes/0000775000175000017500000000000015264420307017730 5ustar alastairalastaircylc-flow-8.7.0/tests/unit/run_modes/__init__.py0000664000175000017500000000000015264420307022027 0ustar alastairalastaircylc-flow-8.7.0/tests/unit/run_modes/test_dummy.py0000664000175000017500000000265315264420307022502 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests for utilities supporting dummy mode. """ import pytest from cylc.flow.run_modes.dummy import build_dummy_script @pytest.mark.parametrize( 'fail_one_time_only', (True, False) ) def test_build_dummy_script(fail_one_time_only): rtc = { 'outputs': {'foo': '1', 'bar': '2'}, 'simulation': { 'fail try 1 only': fail_one_time_only, 'fail cycle points': '1', } } result = build_dummy_script(rtc, 60) assert result.split('\n') == [ 'sleep 60', "cylc message '1'", "cylc message '2'", f"cylc__job__dummy_result {str(fail_one_time_only).lower()}" " 1 || exit 1" ] cylc-flow-8.7.0/tests/unit/run_modes/test_simulation_units.py0000664000175000017500000001222715264420307024753 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests for utilities supporting simulation and skip modes """ import pytest from pytest import param from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.cycling.iso8601 import ISO8601Point from cylc.flow.run_modes.simulation import ( disable_platforms, get_simulated_run_len, parse_fail_cycle_points, sim_task_failed, ) @pytest.mark.parametrize( 'execution_time_limit, speedup_factor, default_run_length', ( param(None, None, 'PT1H', id='default-run-length'), param(None, 10, 'PT1H', id='speedup-factor-alone'), param('PT1H', None, 'PT1H', id='execution-time-limit-alone'), param('P1D', 24, 'PT1M', id='speed-up-and-execution-tl'), ) ) def test_get_simulated_run_len( execution_time_limit, speedup_factor, default_run_length ): """Test the logic of the presence or absence of config items. Avoid testing the correct working of DurationParser. """ rtc = { 'execution time limit': execution_time_limit, 'simulation': { 'speedup factor': speedup_factor, 'default run length': default_run_length, 'time limit buffer': 'PT0S', } } assert get_simulated_run_len(rtc) == 3600 @pytest.mark.parametrize( 'rtc, expect', ( ({'platform': 'skarloey'}, 'localhost'), ({'remote': {'host': 'rheneas'}}, 'localhost'), ({'job': {'batch system': 'loaf'}}, 'localhost'), ) ) def test_disable_platforms(rtc, expect): """A sampling of items FORBIDDEN_WITH_PLATFORMS are removed from a config passed to this method. """ disable_platforms(rtc) assert rtc['platform'] == expect subdicts = [v for v in rtc.values() if isinstance(v, dict)] for subdict in subdicts: for k, val in subdict.items(): if k != 'platform': assert val is None @pytest.mark.parametrize( 'args, cycling, fallback', ( param((['2', '4'], ['']), 'integer', False, id='int.valid'), param((['garbage'], []), 'integer', True, id='int.invalid'), param((['20200101T0000Z'], []), 'iso8601', False, id='iso.valid'), param((['garbage'], []), 'iso8601', True, id='iso.invalid'), ), ) def test_parse_fail_cycle_points( caplog, set_cycling_type, args, cycling, fallback ): """Tests for parse_fail_cycle points. """ set_cycling_type(cycling) if fallback: expect = args[1] check_log = True else: expect = args[0] check_log = False if cycling == 'integer': assert parse_fail_cycle_points(*args) == [ IntegerPoint(i) for i in expect ] else: assert parse_fail_cycle_points(*args) == [ ISO8601Point(i) for i in expect ] if check_log: assert "Incompatible" in caplog.messages[0] assert cycling in caplog.messages[0].lower() @pytest.mark.parametrize( 'conf, point, try_, expect', ( param( {'fail cycle points': [], 'fail try 1 only': True}, ISO8601Point('1'), 1, False, id='defaults' ), param( {'fail cycle points': None, 'fail try 1 only': False}, ISO8601Point('1066'), 1, True, id='fail-all' ), param( { 'fail cycle points': [ ISO8601Point('1066'), ISO8601Point('1067')], 'fail try 1 only': False }, ISO8601Point('1067'), 1, True, id='point-in-failCP' ), param( { 'fail cycle points': [ ISO8601Point('1066'), ISO8601Point('1067')], 'fail try 1 only': True }, ISO8601Point('1000'), 1, False, id='point-notin-failCP' ), param( {'fail cycle points': None, 'fail try 1 only': True}, ISO8601Point('1066'), 2, False, id='succeed-attempt2' ), param( {'fail cycle points': None, 'fail try 1 only': False}, ISO8601Point('1066'), 7, True, id='fail-attempt7' ), ) ) def test_sim_task_failed( conf, point, try_, expect, set_cycling_type ): set_cycling_type('iso8601') assert sim_task_failed(conf, point, try_) == expect cylc-flow-8.7.0/tests/unit/run_modes/test_skip_units.py0000664000175000017500000001104015264420307023525 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Unit tests for utilities supporting skip modes """ import logging import pytest from pytest import param, raises from types import SimpleNamespace from cylc.flow.exceptions import WorkflowConfigError from cylc.flow.run_modes.skip import ( check_task_skip_config, process_outputs, skip_mode_validate, ) @pytest.mark.parametrize( 'conf', ( param({}, id='no-skip-config'), param({'skip': {'outputs': []}}, id='no-skip-outputs'), param({'skip': {'outputs': ['foo1', 'failed']}}, id='ok-skip-outputs'), ) ) def test_good_check_task_skip_config(conf): """It returns none if the problems this function checks are not present. """ tdef = SimpleNamespace(rtconfig=conf) tdef.name = 'foo' assert check_task_skip_config(tdef) is None def test_raises_check_task_skip_config(): """It raises an error if succeeded and failed are set. """ tdef = SimpleNamespace( rtconfig={'skip': {'outputs': ['foo1', 'failed', 'succeeded']}} ) tdef.name = 'foo' with raises(WorkflowConfigError, match='succeeded AND failed'): check_task_skip_config(tdef) @pytest.mark.parametrize( 'outputs, required, expect', ( param([], [], ['succeeded'], id='implicit-succeded'), param( ['succeeded'], ['succeeded'], ['succeeded'], id='explicit-succeded' ), param(['submitted'], [], ['succeeded'], id='only-1-submit'), param( ['foo', 'bar', 'baz', 'qux'], ['bar', 'qux'], ['bar', 'qux', 'succeeded'], id='required-only' ), param( ['foo', 'baz'], ['bar', 'qux'], ['succeeded'], id='no-required' ), param( ['failed'], [], ['failed'], id='explicit-failed' ), ) ) def test_process_outputs(outputs, required, expect): """Check that skip outputs: 1. Doesn't send submitted twice. 2. Sends every required output. 3. If failed is set send failed 4. If failed in not set send succeeded. """ # Create a mocked up task-proxy: rtconf = {'skip': {'outputs': outputs}} itask = SimpleNamespace( tdef=SimpleNamespace(rtconfig=rtconf), state=SimpleNamespace( outputs=SimpleNamespace( iter_required_messages=lambda *a, **k: iter(required), _message_to_trigger={v: v for v in required}, ) ), ) assert process_outputs(itask, rtconf) == {'submitted', 'started'}.union( expect ) def test_skip_mode_validate(caplog, log_filter): """It logs a message if we've set a task config to nonlive mode. (And not otherwise) Point 3 from the skip mode proposal https://github.com/cylc/cylc-admin/blob/master/docs/proposal-skip-mode.md | If the run mode is set to simulation or skip in the workflow | configuration, then cylc validate and cylc lint should produce | warning (similar to development features in other languages / systems). Edit: Warning demoted to info to prevent the orange warning triangles popping up in the GUI every time a workflow with configured skip tasks is started. See: https://github.com/cylc/cylc-flow/pull/6854 """ caplog.set_level(logging.INFO) taskdefs = { f'{run_mode}_task': SimpleNamespace( rtconfig={'run mode': run_mode}, name=f'{run_mode}_task' ) for run_mode in ['live', 'skip'] } skip_mode_validate(taskdefs) assert len(caplog.records) == 1 assert log_filter( level=logging.INFO, exact_match=( "The following tasks are set to run in skip mode:\n" " * skip_task" ), log=caplog ) cylc-flow-8.7.0/tests/unit/run_modes/test_run_modes.py0000664000175000017500000000301615264420307023334 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Tests for utilities supporting run modes. """ import pytest from cylc.flow.run_modes import RunMode def test_run_mode_desc(): """All run mode labels have descriptions.""" for mode in RunMode: assert mode.describe() def test_get_default_live(): """RunMode.get() => live""" assert RunMode.get({}) == RunMode.LIVE @pytest.mark.parametrize('str_', ('LIVE', 'Dummy', 'SkIp', 'siMuLATioN')) def test__missing_(str_): """The RunMode enumeration works when fed a string in the wrong case""" assert RunMode(str_).value == str_.lower() def test__missing_still_doesnt_work(): """ The RunMode enumeration raises an error when fed a string that is not a mode. """ with pytest.raises(ValueError): RunMode('garbage') cylc-flow-8.7.0/tests/unit/test_taskdef.py0000664000175000017500000002036715264420307020777 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from pytest import param from cylc.flow.config import WorkflowConfig from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.cycling.iso8601 import ISO8601Point from cylc.flow.taskdef import TaskTuple, generate_graph_parents def test_generate_graph_parents_1(tmp_flow_config): # noqa: F811 """Test that parents are only generated from valid recurrences.""" id_ = 'pan-galactic' flow_file = tmp_flow_config( id_, """ [scheduler] UTC mode = True [scheduling] initial cycle point = 2023 [[graph]] R1 = run_once_at_midnight T00 = run_once_at_midnight[-PT0H] => every_cycle T03 = run_once_at_midnight[-PT3H] => every_cycle T06 = run_once_at_midnight[-PT6H] => every_cycle [runtime] [[every_cycle, run_once_at_midnight]] """ ) cfg = WorkflowConfig(workflow=id_, fpath=flow_file, options=None) # Each instance of every_cycle should have a parent only at T00. for point in [ ISO8601Point('20230101T00'), ISO8601Point('20230101T03'), ISO8601Point('20230101T06') ]: assert generate_graph_parents( cfg.taskdefs['every_cycle'], point, cfg.taskdefs ) == [ ("run_once_at_midnight", ISO8601Point('20230101T0000Z'), False), ] def test_generate_graph_parents_2(tmp_flow_config): # noqa: F811 """Test inferred parents are valid w.r.t to their own recurrences.""" id_ = 'gargle-blaster' flow_file = tmp_flow_config( id_, """ [scheduling] cycling mode = integer [[graph]] P1 = "foo[-P1] => foo" [runtime] [[foo]] """ ) cfg = WorkflowConfig(workflow=id_, fpath=flow_file, options=None) # Each instance of every_cycle should have a parent only at T00. assert generate_graph_parents( cfg.taskdefs['foo'], IntegerPoint("1"), cfg.taskdefs ) == [] # No parents at first point. assert generate_graph_parents( cfg.taskdefs['foo'], IntegerPoint("2"), cfg.taskdefs ) == [ ("foo", IntegerPoint('1'), False), ] def test_generate_graph_parents__sequential(tmp_flow_config): """It chooses the latest previous point when using old-style sequential.""" flow_file = tmp_flow_config( id_ := 'gargle-blaster', """ [scheduler] allow implicit tasks = True [scheduling] initial cycle point = 2010-01-01 [[special tasks]] sequential = foo [[graph]] P1D = foo P1M = foo """ ) cfg = WorkflowConfig(workflow=id_, fpath=flow_file, options=None) parents = generate_graph_parents( cfg.taskdefs['foo'], ISO8601Point('2010-02-01'), cfg.taskdefs ) assert list(parents) == [ TaskTuple('foo', ISO8601Point('2010-01-31'), False), ] @pytest.mark.parametrize( "task, point, expected", [ param( 'foo', IntegerPoint("1"), ['0/foo'], id='it.gets-prerequisites', ), param( 'multiple_pre', IntegerPoint("2"), ['2/food', '2/fool', '2/foolhardy', '2/foolish'], id='it.gets-multiple-prerequisites', ), param( 'foo', IntegerPoint("3"), [], id='it.only-returns-for-valid-points', ), param( 'bar', IntegerPoint("2"), [], id='it.does-not-return-suicide-prereqs', ), ], ) def test_get_prereqs(tmp_flow_config, task, point, expected): # noqa: F811 """Test that get_prereqs() returns the correct prerequisites for a task.""" id_ = 'gargle-blaster' flow_file = tmp_flow_config( id_, """ [scheduler] allow implicit tasks = True [scheduling] final cycle point = 2 cycling mode = integer [[graph]] P1 = ''' foo[-P1] => foo bar:fail? => !bar food & fool => multiple_pre foolish | foolhardy => multiple_pre ''' """ ) cfg = WorkflowConfig(workflow=id_, fpath=flow_file, options=None) taskdef = cfg.taskdefs[task] point = IntegerPoint(point) res = sorted([ condition.get_id() for pre in taskdef.get_prereqs(point) for condition in pre.keys() ]) assert res == expected def test_get_xtrigs(tmp_flow_config): id = 'foo' flow_file = tmp_flow_config( id, """ [scheduler] allow implicit tasks = True [scheduling] initial cycle point = 1 final cycle point = 16 cycling mode = integer [[xtriggers]] xt_once = xrandom(1) xt_every = xrandom(1) xt_odd = xrandom(1) xt_final = xrandom(1) [[graph]] R1 = @xt_once => foo P1 = @xt_every => foo P2 = @xt_odd => foo R1/$ = @xt_final => foo """ ) cfg = WorkflowConfig(workflow=id, fpath=flow_file, options=None) taskdef = cfg.taskdefs['foo'] assert taskdef.get_xtrigs(IntegerPoint('1')) == { 'xt_once', 'xt_odd', 'xt_every' } assert taskdef.get_xtrigs(IntegerPoint('2')) == {'xt_every'} assert taskdef.get_xtrigs(IntegerPoint('3')) == {'xt_odd', 'xt_every'} assert taskdef.get_xtrigs(IntegerPoint('16')) == {'xt_final', 'xt_every'} @pytest.mark.parametrize( "task, point, expected", [ param( 'foo', IntegerPoint("1"), ['foo[-P1]:succeeded'], id='it.gets-triggers', ), param( 'multiple_pre', IntegerPoint("2"), ['food:succeeded', 'fool:succeeded', 'foolhardy:succeeded', 'foolish:succeeded'], id='it.gets-multiple-triggers', ), param( 'foo', IntegerPoint("3"), [], id='it.only-returns-triggers-for-valid-points', ), param( 'bar', IntegerPoint("2"), [], id='it.does-not-return-suicide-triggers', ), ], ) def test_get_triggers(tmp_flow_config, task, point, expected): # noqa: F811 """Test that get_triggers() returns the correct triggers for a task. """ id_ = 'gargle-blaster' flow_file = tmp_flow_config( id_, """ [scheduler] allow implicit tasks = True [scheduling] final cycle point = 2 cycling mode = integer [[graph]] P1 = ''' foo[-P1] => foo bar:fail? => !bar food & fool => multiple_pre foolish | foolhardy => multiple_pre ''' """ ) cfg = WorkflowConfig(workflow=id_, fpath=flow_file, options=None) taskdef = cfg.taskdefs[task] point = IntegerPoint(point) res = sorted([str(t) for t in taskdef.get_triggers(point)]) assert res == expected cylc-flow-8.7.0/tests/unit/test_host_select_remote.py0000664000175000017500000001425715264420307023246 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test the cylc.flow.host_select module with hosts. NOTE: These tests require a remote host to work with and are skipped unless one is provided. NOTE: These are functional tests, for unit tests see the docstrings in the host_select module. """ from shlex import quote import socket from subprocess import call, DEVNULL import pytest from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.exceptions import HostSelectException from cylc.flow.host_select import ( select_host, select_workflow_host ) from cylc.flow.hostuserutil import get_fqdn_by_host local_host, local_host_alises, _ = socket.gethostbyname_ex('localhost') local_host_fqdn = get_fqdn_by_host(local_host) try: # get a suitable remote host for running tests on # NOTE: do NOT copy this testing approach in other python tests remote_platform = glbl_cfg().get( ['platforms', '_remote_background_shared_tcp', 'hosts'], [] )[0] # don't run tests unless host is contactable if call( ['ssh', quote(remote_platform), 'hostname'], stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL ): raise KeyError('remote platform') # get the fqdn for this host remote_platform_fqdn = get_fqdn_by_host(remote_platform) except (KeyError, IndexError): pytest.skip('Remote test host not available', allow_module_level=True) remote_platform = None async def test_remote_select(): """Test host selection works with remote host names.""" assert await select_host([remote_platform]) == ( remote_platform, remote_platform_fqdn ) async def test_remote_blacklict(): """Test that blacklisting works with remote host names.""" # blacklist by fqdn with pytest.raises(HostSelectException): await select_host( [remote_platform], blacklist=[remote_platform] ) # blacklist by short name with pytest.raises(HostSelectException): await select_host( [remote_platform], blacklist=[remote_platform_fqdn] ) # make extra sure filters are really being applied for _ in range(10): assert await select_host( [remote_platform, local_host], blacklist=[remote_platform] ) == (local_host, local_host_fqdn) async def test_remote_rankings(): """Test that ranking evaluation works on hosts (via SSH).""" assert await select_host( [remote_platform], ranking_string=''' # if this test fails due to race conditions # then you have bigger issues than a test failure virtual_memory().available > 1 getloadavg()[0] < 500 cpu_count() > 1 disk_usage('/').free > 1 ''' ) == (remote_platform, remote_platform_fqdn) async def test_remote_exclude(monkeypatch): """Ensure that hosts get excluded if they don't meet the rankings. Already tested elsewhere but this double-checks that it works if more than one host is provided to choose from.""" async def mocked_get_metrics(hosts, metrics, _=None): # pretend that ssh to remote_platform failed return {f'{local_host_fqdn}': {('cpu_count',): 123}} monkeypatch.setattr( 'cylc.flow.host_select._get_metrics', mocked_get_metrics ) assert await select_host( [local_host, remote_platform], ranking_string=''' cpu_count() ''' ) == (local_host, local_host_fqdn) async def test_remote_workflow_host_select(mock_glbl_cfg): """test [scheduler][run hosts]available""" mock_glbl_cfg( 'cylc.flow.host_select.glbl_cfg', f''' [scheduler] [[run hosts]] available = {remote_platform} ''' ) assert await select_workflow_host() == ( remote_platform, remote_platform_fqdn, ) async def test_remote_workflow_host_condemned(mock_glbl_cfg): """test [scheduler][run hosts]condemned hosts""" mock_glbl_cfg( 'cylc.flow.host_select.glbl_cfg', f''' [scheduler] [[run hosts]] available = {remote_platform}, {local_host} condemned = {remote_platform} ''' ) for _ in range(10): assert await select_workflow_host() == (local_host, local_host_fqdn) async def test_remote_workflow_host_rankings(mock_glbl_cfg): """test [scheduler][run hosts]rankings""" mock_glbl_cfg( 'cylc.flow.host_select.glbl_cfg', f''' [scheduler] [[run hosts]] available = {remote_platform} ranking = """ # if this test fails due to race conditions # then you are very lucky virtual_memory().available > 123456789123456789 cpu_count() > 512 disk_usage('/').free > 123456789123456789 """ ''' ) with pytest.raises(HostSelectException) as excinfo: await select_workflow_host() # ensure that host selection actually evaluated rankings assert set(excinfo.value.data[remote_platform_fqdn]) - {'returncode'} == { 'virtual_memory().available > 123456789123456789', 'cpu_count() > 512', "disk_usage('/').free > 123456789123456789" } # ensure that none of the rankings passed assert not any(excinfo.value.data[remote_platform_fqdn].values()) cylc-flow-8.7.0/tests/unit/test_task_pool.py0000664000175000017500000000522315264420307021343 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from typing import List from unittest.mock import Mock import pytest from cylc.flow.flow_mgr import FlowNums from cylc.flow.prerequisite import SatisfiedState from cylc.flow.task_pool import TaskPool @pytest.mark.parametrize('pool, db_fnums, expected', [ pytest.param( [{1, 2}, {2, 3}], {5, 6}, {1, 2, 3}, id="all-active" ), pytest.param( [set(), set()], {5, 6}, {5, 6}, id="from-db" ), pytest.param( [set()], set(), {1}, id="fallback" # see https://github.com/cylc/cylc-flow/pull/6445 ), ]) def test_get_active_flow_nums( pool: List[FlowNums], db_fnums: FlowNums, expected ): mock_task_pool = Mock( get_tasks=lambda: [Mock(flow_nums=fnums) for fnums in pool], ) mock_task_pool.workflow_db_mgr.pri_dao.select_latest_flow_nums = ( lambda: db_fnums ) assert TaskPool._get_active_flow_nums(mock_task_pool) == expected @pytest.mark.parametrize('output_msg, flow_nums, db_flow_nums, expected', [ ('foo', set(), {1}, False), ('foo', set(), set(), False), ('foo', {1, 3}, {1}, 'satisfied from database'), ('goo', {1, 3}, {1, 2}, 'satisfied from database'), ('foo', {1, 3}, set(), False), ('foo', {2}, {1}, False), ('foo', {2}, {1, 2}, 'satisfied from database'), ('f', {1}, {1}, False), ]) def test_check_output( output_msg: str, flow_nums: set, db_flow_nums: set, expected: SatisfiedState, ): """It checks DB for matching output msg and intersection of flow_nums.""" def select_task_outputs(*a, **k): yield ('{"f": "foo", "g": "goo"}', db_flow_nums) mock_task_pool = Mock() mock_task_pool.workflow_db_mgr.pri_dao.select_task_outputs = ( select_task_outputs ) assert TaskPool.check_task_output( mock_task_pool, '2000', 'haddock', output_msg, flow_nums ) == expected cylc-flow-8.7.0/tests/unit/test_hostuserutil.py0000664000175000017500000000440615264420307022124 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import os import re import pytest from cylc.flow.hostuserutil import ( get_fqdn_by_host, get_host, get_user, get_user_home, is_remote_host, is_remote_user ) def test_is_remote_user_on_current_user(): """is_remote_user with current user.""" assert not is_remote_user(None) assert not is_remote_user(os.getenv('USER')) def test_is_remote_host_on_localhost(monkeypatch): """is_remote_host with localhost.""" assert not is_remote_host(None) assert not is_remote_host('localhost') assert not is_remote_host('localhost4.localhost42') assert not is_remote_host(os.getenv('HOSTNAME')) assert not is_remote_host(get_host()) def test_get_fqdn_by_host_on_bad_host(): """get_fqdn_by_host bad host. Warning: This test can fail due to ISP/network configuration (for example ISP may reroute failed DNS to custom search page) e.g: https://www.virginmedia.com/help/advanced-network-error-search """ bad_host = 'nosuchhost.nosuchdomain.org' with pytest.raises(IOError) as exc: get_fqdn_by_host(bad_host) assert re.match( r"(\[Errno -2\] Name or service|" r"\[Errno 8\] nodename nor servname provided, or)" r" not known: '{}'".format(bad_host), str(exc.value) ) assert exc.value.filename == bad_host def test_get_user(): """get_user.""" assert os.getenv('USER') == get_user() def test_get_user_home(): """get_user_home.""" assert os.getenv('HOME') == get_user_home() cylc-flow-8.7.0/tests/unit/test_time_parser.py0000664000175000017500000003010115264420307021653 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test Cylc recurring date/time syntax parsing.""" import pytest from unittest.mock import Mock from metomi.isodatetime.data import Duration from cylc.flow.exceptions import CylcTimeSyntaxError from cylc.flow.time_parser import ( CylcTimeParser, UTC_UTC_OFFSET_HOURS_MINUTES, ) @pytest.fixture def parsers(): _start_point = "19991226T0930Z" # Note: the following timezone will be Z-ified *after* truncation # or offsets are applied. _end_point = "20010506T1200+0200" return { 0: CylcTimeParser( _start_point, _end_point, CylcTimeParser.initiate_parsers( assumed_time_zone=UTC_UTC_OFFSET_HOURS_MINUTES ) ), 2: CylcTimeParser( _start_point, _end_point, CylcTimeParser.initiate_parsers( num_expanded_year_digits=2, assumed_time_zone=UTC_UTC_OFFSET_HOURS_MINUTES ) ) } @pytest.mark.parametrize( 'expression, num_expanded_year_digits, ctrl_data, ctrl_results', [ ( "R5/T06/04T1230-0300", 0, "R5/19991227T0600Z/20010604T1530Z", ["19991227T0600Z", "20010604T1530Z", "20021112T0100Z", "20040420T1030Z", "20050927T2000Z"] ), ( "R2/-0450000101T04Z/+0020630405T2200-05", 2, "R2/-0450000101T0400Z/+0020630406T0300Z", ["-0450000101T0400Z", "+0020630406T0300Z"] ), ( "R10/T12+P10D/-P1Y", 0, "R10/20000105T1200Z/20000506T1000Z", ["20000105T1200Z", "20000506T1000Z", "20000905T0800Z", "20010105T0600Z", "20010507T0400Z", "20010906T0200Z", "20020106T0000Z", "20020507T2200Z", "20020906T2000Z", "20030106T1800Z"] ) ] ) def test_first_recurrence_format( expression, num_expanded_year_digits, ctrl_data, ctrl_results, parsers ): """Test the first ISO 8601 recurrence format.""" parser = parsers[num_expanded_year_digits] recurrence = parser.parse_recurrence(expression)[0] test_data = str(recurrence) assert test_data == ctrl_data test_results = [str(res) for res in recurrence] assert test_results == ctrl_results def test_third_recurrence_format(parsers): """Test the third ISO 8601 recurrence format.""" tests = [("T06", "R/19991227T0600Z/P1D"), ("T1230", "R/19991226T1230Z/P1D"), ("04T00", "R/20000104T0000Z/P1M"), ("01T06/PT2H", "R/20000101T0600Z/PT2H"), ("0501T/P4Y3M", "R/20000501T0000Z/P4Y3M"), ("P1YT5H", "R/19991226T0930Z/P1YT5H", ["19991226T0930Z", "20001226T1430Z"]), ("PT5M", "R/19991226T0930Z/PT5M"), ("R/T-40", "R/19991226T0940Z/PT1H"), ("R/150401T", "R/20150401T0000Z/P100Y"), ("R5/T04-0300", "R5/19991227T0700Z/P1D"), ("R2/T23Z/", "R2/19991226T2300Z/P1D"), ("R/19991226T2359Z/P1W", "R/19991226T2359Z/P1W"), ("R/-P1W/P1W", "R/19991219T0930Z/P1W"), ("R/+P1D/P1Y", "R/19991227T0930Z/P1Y"), ("R/T06-P1D/PT1H", "R/19991226T0600Z/PT1H"), ("R/T12/PT10,5H", "R/19991226T1200Z/PT10,5H"), ("R/T12/PT10.5H", "R/19991226T1200Z/PT10,5H"), ("R5/+P10Y5M4D/PT2H", "R5/20100529T0930Z/PT2H"), ("R30/-P200D/P10D", "R30/19990609T0930Z/P10D"), ("R10/T06/PT2H", "R10/19991227T0600Z/PT2H", ["19991227T0600Z", "19991227T0800Z", "19991227T1000Z", "19991227T1200Z", "19991227T1400Z", "19991227T1600Z", "19991227T1800Z", "19991227T2000Z", "19991227T2200Z", "19991228T0000Z"]), ("R//P1Y", "R/19991226T0930Z/P1Y", ["19991226T0930Z", "20001226T0930Z"]), ("R5//P1D", "R5/19991226T0930Z/P1D", ["19991226T0930Z", "19991227T0930Z", "19991228T0930Z", "19991229T0930Z", "19991230T0930Z"]), ("R1", "R1/19991226T0930Z/P0Y", ["19991226T0930Z"])] for test in tests: if len(test) == 2: expression, ctrl_data = test ctrl_results = None else: expression, ctrl_data, ctrl_results = test recurrence = (parsers[0].parse_recurrence(expression))[0] test_data = str(recurrence) assert test_data == ctrl_data if ctrl_results is None: continue test_results = [] for i, test_result in enumerate(recurrence): if i > len(ctrl_results) - 1: break assert str(test_result) == ctrl_results[i] test_results.append(str(test_result)) assert test_results == ctrl_results def test_fourth_recurrence_format(parsers): """Test the fourth ISO 8601 recurrence format.""" tests = [("PT6H/20000101T0500Z", "R25/19991226T0500Z/PT6H"), ("P12D/+P2W", "R44/19991221T1000Z/P12D"), ("R2/P1W/-P1M1D", "R2/P1W/20010405T1000Z"), ("R3/P6D/T12+02", "R3/P6D/20010506T1000Z"), ("R4/P6DT12H/01T00+02", "R4/P6DT12H/20010531T2200Z"), ("R5/P1D/20010506T1200+0200", "R5/P1D/20010506T1000Z"), ("R6/PT5M/+PT2M", "R6/PT5M/20010506T1002Z"), ("R7/P20Y/-P20Y", "R7/P20Y/19810506T1000Z"), ("R8/P3YT2H/T18-02", "R8/P3YT2H/20010506T2000Z"), ("R9/PT3H/31T", "R9/PT3H/20010531T0000Z"), ("R10/P1Y/", "R10/P1Y/20010506T1000Z"), ("R3/P2Y/02T", "R3/P2Y/20010602T0000Z"), ("R/P2Y", "R2/19990506T1000Z/P2Y"), ("R48/PT2H", "R48/PT2H/20010506T1000Z"), ("R100/P21Y/", "R100/P21Y/20010506T1000Z")] for test in tests: if len(test) == 2: expression, ctrl_data = test ctrl_results = None else: expression, ctrl_data, ctrl_results = test recurrence = (parsers[0].parse_recurrence( expression))[0] test_data = str(recurrence) assert test_data == ctrl_data if ctrl_results is None: continue test_results = [] for i, test_result in enumerate(recurrence): assert str(test_result) == ctrl_results[i] test_results.append(str(test_result)) assert test_results == ctrl_results def test_inter_cycle_timepoints(parsers): """Test the inter-cycle point parsing.""" task_cycle_time = parsers[0].parse_timepoint("20000101T00Z") tests = [("T06", "20000101T0600Z", 0), ("-PT6H", "19991231T1800Z", 0), ("+P5Y2M", "20050301T0000Z", 0), ("0229T", "20000229T0000Z", 0), ("+P54D", "20000224T0000Z", 0), ("T12+P5W", "20000205T1200Z", 0), ("-P1Y", "19990101T0000Z", 0), ("-9999990101T00Z", "-9999990101T0000Z", 2), ("20050601T2359+0200", "20050601T2159Z", 0)] for expression, ctrl_data, num_expanded_year_digits in tests: parser = parsers[num_expanded_year_digits] test_data = str(parser.parse_timepoint( expression, context_point=task_cycle_time)) assert test_data == ctrl_data def test_interval(parsers): """Test the interval timepoint parsing (purposefully weak tests).""" tests = ["PT6H2M", "PT6H", "P2Y5D", "P5W", "PT12M2S", "PT65S", "PT2M", "P1YT567,4M"] for expression in tests: test_data = str(parsers[0].parse_interval(expression)) assert test_data == expression def test_parse_timepoint_invalid(parsers): """It should raise CylcTimeSyntaxError for invalid expressions.""" with pytest.raises(CylcTimeSyntaxError, match="not a valid"): parsers[0].parse_timepoint("not_a_date") def test_parse_timepoint_none(parsers): """It should raise CylcTimeSyntaxError when expr is None.""" with pytest.raises(CylcTimeSyntaxError, match="not a valid"): parsers[0].parse_timepoint(None) def test_parse_recurrence_invalid(parsers): """It should raise CylcTimeSyntaxError for unparsable expressions.""" with pytest.raises(CylcTimeSyntaxError, match="Could not parse"): parsers[0].parse_recurrence("///") def test_parse_recurrence_with_context(parsers): """It should use explicit context points when provided.""" parser = parsers[0] start = parser.parse_timepoint("20000101T00Z") end = parser.parse_timepoint("20010101T00Z") recurrence = parser.parse_recurrence( "R2/P1Y/", context_start_point=start, context_end_point=end, )[0] assert str(recurrence) == "R2/P1Y/20010101T0000Z" def test_get_interval_from_expression(parsers): """ Method should get a duration from an expression. Currently contains a bug, referenced here: https://github.com/cylc/cylc-flow/issues/2382#issuecomment-1651837780 Which results in unexpected behaviour around 31 day periods and month boundaries. """ parser = parsers[0] # when expr is provided, it should just parse it assert str(parser._get_interval_from_expression("P1D")) == "P1D" # when expr is None and context is None, return None assert parser._get_interval_from_expression(None, None) is None # when expr is None and context is not truncated, return None context = Mock(truncated=False) assert parser._get_interval_from_expression(None, context) is None # test each truncated property name cases = [ ("year_of_century", Duration(years=100)), ("year_of_decade", Duration(years=10)), ("month_of_year", Duration(years=1)), ("week_of_year", Duration(years=1)), ("day_of_year", Duration(years=1)), ("day_of_month", Duration(months=1)), ("day_of_week", Duration(days=7)), ("hour_of_day", Duration(days=1)), ("minute_of_hour", Duration(hours=1)), ("second_of_minute", Duration(minutes=1)), ] for prop_name, expected in cases: context = Mock(truncated=True) context.get_largest_truncated_property_name.return_value = prop_name result = parser._get_interval_from_expression(None, context) assert result == expected, f"Failed for {prop_name}" # when prop_name doesn't match anything, return None context = Mock(truncated=True) context.get_largest_truncated_property_name.return_value = "unknown_prop" assert parser._get_interval_from_expression(None, context) is None def test_get_min_from_expression_unresolvable(parsers): """It should skip points that cannot be resolved (cpoint is None).""" parser = parsers[0] # "+P1M" with context=None results in cpoint=None (pure offset, no # context to resolve against), so it should be skipped. # "20000101T0000Z" resolves fine and should be selected as the min. result = parser._get_min_from_expression( "min(+P1M, 20000101T0000Z)", context=None ) assert result == "20000101T0000Z" def test_get_point_from_expression_truncated_isodatetime_error(parsers): """ Method should raise CylcTimeSyntaxError for invalid truncated expressions. """ parser = parsers[0] # "99T25" matches the TRUNCATED_REC_MAP regex, # but "---99T25" is not a valid truncated ISO point (no day 99, hour 25) with pytest.raises(CylcTimeSyntaxError): parser._get_point_from_expression( "99T25", context=None, allow_truncated=True ) cylc-flow-8.7.0/tests/unit/cycling/0000775000175000017500000000000015264420307017365 5ustar alastairalastaircylc-flow-8.7.0/tests/unit/cycling/test_cycling.py0000664000175000017500000000461615264420307022435 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.cycling import ( SequenceBase, IntervalBase, PointBase, parse_exclusion, ) def test_simple_abstract_class_test(): """Cannot instantiate abstract classes, they must be defined in the subclasses""" with pytest.raises(TypeError): SequenceBase('sequence-string', 'context_string') with pytest.raises(TypeError): IntervalBase('value') with pytest.raises(TypeError): PointBase('value') def test_parse_exclusion_simple(): """Tests the simple case of exclusion parsing""" expression = "PT1H!20000101T02Z" sequence, exclusion = parse_exclusion(expression) assert sequence == "PT1H" assert exclusion == ['20000101T02Z'] def test_parse_exclusions_list(): """Tests the simple case of exclusion parsing""" expression = "PT1H!(T03, T06, T09)" sequence, exclusion = parse_exclusion(expression) assert sequence == "PT1H" assert exclusion == ['T03', 'T06', 'T09'] def test_parse_exclusions_list_spaces(): """Tests the simple case of exclusion parsing""" expression = "PT1H! (T03, T06, T09) " sequence, exclusion = parse_exclusion(expression) assert sequence == "PT1H" assert exclusion == ['T03', 'T06', 'T09'] @pytest.mark.parametrize( 'expression', [ 'T01/PT1H!(T06, T09), PT5M', 'T01/PT1H!T03, PT17H, (T06, T09), PT5M', 'T01/PT1H! PT8H, (T06, T09)', 'T01/PT1H! T03, T06, T09', 'T01/PT1H !T03 !T06', ], ) def test_parse_bad_exclusion(expression): """Tests incorrectly formatted exclusions""" with pytest.raises(Exception): parse_exclusion(expression) cylc-flow-8.7.0/tests/unit/cycling/__init__.py0000664000175000017500000000140715264420307021500 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . cylc-flow-8.7.0/tests/unit/cycling/test_integer.py0000664000175000017500000002343515264420307022442 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from cylc.flow.cycling.integer import ( IntegerSequence, IntegerPoint, IntegerInterval, IntervalParsingError, SequenceParsingError, ) from cylc.flow.cycling.iso8601 import ( ISO8601Point, ISO8601Interval, ) from cylc.flow.exceptions import CyclerTypeError def test_exclusions_simple(): """Test the generation of points for integer sequences with exclusions. """ sequence = IntegerSequence('R/P1!3', 1, 5) output = [] point = sequence.get_start_point() while point: output.append(point) point = sequence.get_next_point(point) assert [int(out) for out in output] == [1, 2, 4, 5] def test_multiple_exclusions_simple(): """Tests the multiple exclusion syntax for integer notation""" sequence = IntegerSequence('R/P1!(2,3,7)', 1, 10) output = [] point = sequence.get_start_point() while point: output.append(point) point = sequence.get_next_point(point) assert [int(out) for out in output] == [1, 4, 5, 6, 8, 9, 10] # duplicate excluded points should be ignored sequence1 = IntegerSequence('R/P1!(2,2,2)', 1, 10) sequence2 = IntegerSequence('R/P1!(2)', 1, 10) assert ( sequence1.exclusions.exclusion_points ) == sequence2.exclusions.exclusion_points def test_multiple_exclusions_integer_sequence(): """Tests the multiple exclusion syntax for integer notation""" sequence = IntegerSequence('P1 ! P2', 1, 10) output = [] point = sequence.get_start_point() while point: output.append(point) point = sequence.get_next_point(point) assert [int(out) for out in output] == [2, 4, 6, 8, 10] assert sequence.exclusions[0] == IntegerSequence('P2', 1, 10) def test_multiple_exclusions_integer_sequence2(): """Tests the multiple exclusion syntax for integer notation""" sequence = IntegerSequence('P1 ! +P1/P2', 1, 10) output = [] point = sequence.get_start_point() while point: output.append(point) point = sequence.get_next_point(point) assert [int(out) for out in output] == [1, 3, 5, 7, 9] def test_multiple_exclusions_integer_sequence3(): """Tests the multiple exclusion syntax for integer notation""" sequence = IntegerSequence('P1 ! (P2, 6, 8) ', 1, 10) output = [] point = sequence.get_start_point() while point: output.append(point) point = sequence.get_next_point(point) assert [int(out) for out in output] == [2, 4, 10] def test_multiple_exclusions_integer_sequence_weird_valid_formatting(): """Tests the multiple exclusion syntax for integer notation""" sequence = IntegerSequence('P1 !(P2, 6,8) ', 1, 10) output = [] point = sequence.get_start_point() while point: output.append(point) point = sequence.get_next_point(point) assert [int(out) for out in output] == [2, 4, 10] def test_multiple_exclusions_integer_sequence_invalid_formatting(): """Tests the multiple exclusion syntax for integer notation""" with pytest.raises(Exception): IntegerSequence('P1 !(6,8), P2 ', 1, 10) def test_multiple_exclusions_extensive(): """Tests IntegerSequence methods for sequences with multi-exclusions""" points = [IntegerPoint(i) for i in range(10)] sequence = IntegerSequence('R/P1!(2,3,7)', 1, 10) assert not sequence.is_on_sequence(points[3]) assert not sequence.is_valid(points[3]) assert sequence.get_prev_point(points[3]) == points[1] assert sequence.get_prev_point(points[4]) == points[1] assert sequence.get_nearest_prev_point(points[3]) == points[1] assert sequence.get_nearest_prev_point(points[4]) == points[1] assert sequence.get_next_point(points[3]) == points[4] assert sequence.get_next_point(points[2]) == points[4] assert sequence.get_next_point_on_sequence(points[3]) == points[4] assert sequence.get_next_point_on_sequence(points[6]) == points[8] sequence = IntegerSequence('R/P1!(1,3,4)', 1, 10) assert sequence.get_first_point(points[1]) == points[2] assert sequence.get_first_point(points[0]) == points[2] assert sequence.get_start_point() == points[2] sequence = IntegerSequence('R/P1!(8,9,10)', 1, 10) assert sequence.get_stop_point() == points[7] def test_exclusions_extensive(): """Test IntegerSequence methods for sequences with exclusions.""" point_0 = IntegerPoint(0) point_1 = IntegerPoint(1) point_2 = IntegerPoint(2) point_3 = IntegerPoint(3) point_4 = IntegerPoint(4) sequence = IntegerSequence('R/P1!3', 1, 5) assert not sequence.is_on_sequence(point_3) assert not sequence.is_valid(point_3) assert sequence.get_prev_point(point_3) == point_2 assert sequence.get_prev_point(point_4) == point_2 assert sequence.get_nearest_prev_point(point_3) == point_2 assert sequence.get_nearest_prev_point(point_3) == point_2 assert sequence.get_next_point(point_3) == point_4 assert sequence.get_next_point(point_2) == point_4 assert sequence.get_next_point_on_sequence(point_3) == point_4 assert sequence.get_next_point_on_sequence(point_2) == point_4 sequence = IntegerSequence('R/P1!1', 1, 5) assert sequence.get_first_point(point_1) == point_2 assert sequence.get_first_point(point_0) == point_2 assert sequence.get_start_point() == point_2 sequence = IntegerSequence('R/P1!5', 1, 5) assert sequence.get_stop_point() == point_4 def test_simple(): """Run some simple tests for integer cycling.""" sequence = IntegerSequence('R/1/P3', 1, 10) start = sequence.p_start stop = sequence.p_stop # Test point generation forwards. point = start output = [] while point and stop and point <= stop: output.append(point) point = sequence.get_next_point(point) assert [int(out) for out in output] == [1, 4, 7, 10] # Test point generation backwards. point = stop output = [] while point and start and point >= start: output.append(point) point = sequence.get_prev_point(point) assert [int(out) for out in output] == [10, 7, 4, 1] # Test sequence comparison assert IntegerSequence('R/1/P2', 1, 10) == IntegerSequence('R/1/P2', 1, 10) assert IntegerSequence('R/1/P2', 1, 10) == IntegerSequence( 'R/-1/P2', 1, 10 ) assert IntegerSequence('R/1/P2', 1, 10) == IntegerSequence('R/1/P2', 1, 10) def test_interval_parsing_error(): """It should reject invalid intervals.""" with pytest.raises(IntervalParsingError): IntegerInterval(42) with pytest.raises(IntervalParsingError): IntegerInterval('forty two') def test_sequence_parsing_error(): with pytest.raises(SequenceParsingError): IntegerSequence('zz0+za', 1) def test_interval_arithmetic(): """It should do basic maths on integer intervals.""" a = IntegerInterval('P2') b = IntegerInterval('P3') p = IntegerPoint('3') assert a + b == IntegerInterval('P5') assert a + p == IntegerPoint('5') assert a - b == IntegerInterval('-P1') assert a - p == IntegerPoint('-1') assert abs(IntegerInterval('-P2')) == IntegerInterval('P2') assert a + IntegerInterval.get_null_offset() == a with pytest.raises(CyclerTypeError): IntegerPoint(1000) - ISO8601Point('1000') with pytest.raises(CyclerTypeError): IntegerPoint(1000) + ISO8601Point('1000') with pytest.raises(CyclerTypeError): IntegerInterval('P1') - ISO8601Interval('P1Y') with pytest.raises(CyclerTypeError): IntegerInterval('P1') + ISO8601Interval('P1Y') def test_async_expr(): """The async expression should run once and only once.""" point = IntegerPoint('5') sequence = IntegerSequence(IntegerSequence.get_async_expr(point), 1, 10) assert sequence.get_next_point(IntegerPoint('1')) == point assert sequence.get_next_point(IntegerPoint('5')) is None def test_point_comparisons(): # basic comparisons assert IntegerPoint(1) == IntegerPoint(1) assert IntegerPoint(1) < IntegerPoint(2) assert IntegerPoint(1) <= IntegerPoint(2) assert IntegerPoint(2) >= IntegerPoint(1) assert IntegerPoint(1) != IntegerPoint(2) assert IntegerInterval('P1') == IntegerInterval('P1') assert IntegerInterval('P1') < IntegerInterval('P2') assert IntegerInterval('P1') <= IntegerInterval('P2') assert IntegerInterval('P2') >= IntegerInterval('P1') assert IntegerInterval('P1') != IntegerInterval('P2') # None comparisons work counter intuitively # (reason unknown) assert IntegerPoint(1) < None assert None > IntegerPoint(1) assert IntegerInterval('P1') < None assert None > IntegerInterval('P1') # compare against other PointBase implementations assert ISO8601Point('1000') > IntegerPoint(1000) assert IntegerPoint(1000) < ISO8601Point('1000') assert ISO8601Interval('P1Y') > IntegerInterval('P1') assert IntegerInterval('P1') < ISO8601Interval('P1Y') def test_string_representations(): p = IntegerPoint(1) assert str(p) == '1' assert repr(p) == '1' i = IntegerInterval('P1') assert str(i) == 'P1' assert repr(i) == '' cylc-flow-8.7.0/tests/unit/cycling/test_util.py0000664000175000017500000000261015264420307021752 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test cycling utils.""" import pytest from cylc.flow.cycling.util import add_offset def test_add_offset(): """Test socket start.""" orig_point = '20200202T0000Z' plus_offset = '+PT02H02M' assert str(add_offset(orig_point, plus_offset)) == '20200202T0202Z' minus_offset = '-P1MT22H59M' assert str(add_offset(orig_point, minus_offset)) == '20200101T0101Z' assert str( add_offset(orig_point, minus_offset, dmp_fmt="CCYY-MM-DDThh:mmZ") ) == '2020-01-01T01:01Z' bad_offset = '+foo' with pytest.raises(ValueError, match=r'ERROR, bad offset format'): add_offset(orig_point, bad_offset) cylc-flow-8.7.0/tests/unit/cycling/test_iso8601.py0000664000175000017500000007631015264420307022116 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from datetime import ( datetime, timezone, ) import pytest from cylc.flow.cycling.iso8601 import ( ISO8601Interval, ISO8601Point, ISO8601Sequence, ingest_time, ) from cylc.flow.cycling.loader import ISO8601_CYCLING_TYPE def test_exclusions_simple(set_cycling_type): """Test the generation of points for sequences with exclusions.""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") sequence = ISO8601Sequence("PT1H!20000101T02Z", "20000101T00Z") output = [] point = sequence.get_start_point() count = 0 while point and count < 4: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] assert output == [ "20000101T0000Z", "20000101T0100Z", "20000101T0300Z", "20000101T0400Z", ] def test_exclusions_offset(set_cycling_type): """Test the generation of points for sequences with exclusions that have an offset on the end""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") sequence = ISO8601Sequence("PT1H!20000101T00Z+PT1H", "20000101T00Z") output = [] point = sequence.get_start_point() count = 0 while point and count < 4: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] assert output == [ "20000101T0000Z", "20000101T0200Z", "20000101T0300Z", "20000101T0400Z", ] def test_multiple_exclusions_complex1(set_cycling_type): """Tests sequences that have multiple exclusions and a more complicated format""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") # A sequence that specifies a dep start time sequence = ISO8601Sequence( "20000101T01Z/PT1H!20000101T02Z", "20000101T01Z" ) output = [] point = sequence.get_start_point() count = 0 # We are going to make four sequence points while point and count < 4: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] # We should expect one of the hours to be excluded: T02 assert output == [ "20000101T0100Z", "20000101T0300Z", "20000101T0400Z", "20000101T0500Z", ] def test_multiple_exclusions_complex2(set_cycling_type): """Tests sequences that have multiple exclusions and a more complicated format""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") # A sequence that specifies a dep start time sequence = ISO8601Sequence( "20000101T01Z/PT1H!" "(20000101T02Z,20000101T03Z)", "20000101T00Z", "20000101T05Z", ) output = [] point = sequence.get_start_point() count = 0 # We are going to make four sequence points while point and count < 3: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] # We should expect two of the hours to be excluded: T02, T03 assert output == [ "20000101T0100Z", "20000101T0400Z", "20000101T0500Z", ] def test_multiple_exclusions_simple(set_cycling_type): """Tests generation of points for sequences with multiple exclusions""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") sequence = ISO8601Sequence( "PT1H!(20000101T02Z,20000101T03Z)", "20000101T00Z" ) output = [] point = sequence.get_start_point() count = 0 # We are going to make four sequence points while point and count < 4: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] # We should expect two of the hours to be excluded: T02 and T03 assert output == [ "20000101T0000Z", "20000101T0100Z", "20000101T0400Z", "20000101T0500Z", ] def test_advanced_exclusions_partial_datetime1(set_cycling_type): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") # Run 3-hourly but not at 06:00 (from the ICP) sequence = ISO8601Sequence("PT3H!T06", "20000101T00Z") output = [] point = sequence.get_start_point() count = 0 # We are going to make ten sequence points while point and count < 10: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] # We should expect every T06 to be excluded assert output == [ "20000101T0000Z", "20000101T0300Z", "20000101T0900Z", "20000101T1200Z", "20000101T1500Z", "20000101T1800Z", "20000101T2100Z", "20000102T0000Z", "20000102T0300Z", "20000102T0900Z", ] def test_advanced_exclusions_partial_datetime2(set_cycling_type): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") # Run hourly but not at 00:00, 06:00, 12:00, 18:00 sequence = ISO8601Sequence("T-00!(T00, T06, T12, T18)", "20000101T00Z") output = [] point = sequence.get_start_point() count = 0 # We are going to make 18 sequence points while point and count < 18: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] # We should expect T00, T06, T12, and T18 to be excluded assert output == [ "20000101T0100Z", "20000101T0200Z", "20000101T0300Z", "20000101T0400Z", "20000101T0500Z", "20000101T0700Z", "20000101T0800Z", "20000101T0900Z", "20000101T1000Z", "20000101T1100Z", "20000101T1300Z", "20000101T1400Z", "20000101T1500Z", "20000101T1600Z", "20000101T1700Z", "20000101T1900Z", "20000101T2000Z", "20000101T2100Z", ] def test_advanced_exclusions_partial_datetime3(set_cycling_type): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") # Run 5 minutely but not at 15 minutes past the hour from ICP sequence = ISO8601Sequence("PT5M!T-15", "20000101T00Z") output = [] point = sequence.get_start_point() count = 0 # We are going to make 15 sequence points while point and count < 15: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] # We should expect xx:15 (15 minutes past the hour) to be excluded assert output == [ "20000101T0000Z", "20000101T0005Z", "20000101T0010Z", "20000101T0020Z", "20000101T0025Z", "20000101T0030Z", "20000101T0035Z", "20000101T0040Z", "20000101T0045Z", "20000101T0050Z", "20000101T0055Z", "20000101T0100Z", "20000101T0105Z", "20000101T0110Z", "20000101T0120Z", ] def test_advanced_exclusions_partial_datetime4(set_cycling_type): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") # Run daily at 00:00 except on Mondays sequence = ISO8601Sequence("T00!W-1T00", "20170422T00Z") output = [] point = sequence.get_start_point() count = 0 # We are going to make 19 sequence points while point and count < 9: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] # We should expect Monday 24th April and Monday 1st May # to be excluded. assert output == [ "20170422T0000Z", "20170423T0000Z", "20170425T0000Z", "20170426T0000Z", "20170427T0000Z", "20170428T0000Z", "20170429T0000Z", "20170430T0000Z", "20170502T0000Z", ] def test_exclusions_to_string(set_cycling_type): set_cycling_type(ISO8601_CYCLING_TYPE, "Z") # Check that exclusions are not included where they should not be. basic = ISO8601Sequence("PT1H", "2000", "2001") assert "!" not in str(basic) # Check that exclusions are parsable. sequence = ISO8601Sequence("PT1H!(20000101T10Z, PT6H)", "2000", "2001") sequence2 = ISO8601Sequence(str(sequence), "2000", "2001") assert sequence == sequence2 def test_advanced_exclusions_sequences1(set_cycling_type): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") # Run hourly from the ICP but not 3-hourly sequence = ISO8601Sequence("PT1H!PT3H", "20000101T01Z") output = [] point = sequence.get_start_point() count = 0 # We are going to make six sequence points while point and count < 6: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] # We should expect to see hourly from ICP but not 3 hourly assert output == [ "20000101T0200Z", "20000101T0300Z", "20000101T0500Z", "20000101T0600Z", "20000101T0800Z", "20000101T0900Z", ] def test_advanced_exclusions_sequences2(set_cycling_type): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") # Run hourly on the hour but not 3 hourly on the hour sequence = ISO8601Sequence("T-00!T-00/PT3H", "20000101T00Z") output = [] point = sequence.get_start_point() count = 0 # We are going to make six sequence points while point and count < 6: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] assert output == [ "20000101T0100Z", "20000101T0200Z", "20000101T0400Z", "20000101T0500Z", "20000101T0700Z", "20000101T0800Z", ] def test_advanced_exclusions_sequences3(set_cycling_type): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") # Run daily at 12:00 except every 3rd day sequence = ISO8601Sequence("T12!P3D", "20000101T12Z") output = [] point = sequence.get_start_point() count = 0 # We are going to make six sequence points while point and count < 6: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] assert output == [ "20000102T1200Z", "20000103T1200Z", "20000105T1200Z", "20000106T1200Z", "20000108T1200Z", "20000109T1200Z", ] def test_advanced_exclusions_sequences4(set_cycling_type): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") # Run every hour from 01:00 excluding every 3rd hour. sequence = ISO8601Sequence("T01/PT1H!+PT3H/PT3H", "20000101T01Z") output = [] point = sequence.get_start_point() count = 0 # We are going to make six sequence points while point and count < 6: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] assert output == [ "20000101T0100Z", "20000101T0200Z", "20000101T0300Z", "20000101T0500Z", "20000101T0600Z", "20000101T0800Z", ] def test_advanced_exclusions_sequences5(set_cycling_type): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") # Run every hour from 01:00 excluding every 3rd hour. sequence = ISO8601Sequence("T-00 ! 2000", "20000101T00Z") output = [] point = sequence.get_start_point() count = 0 # We are going to make six sequence points while point and count < 6: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] assert output == [ "20000101T0100Z", "20000101T0200Z", "20000101T0300Z", "20000101T0400Z", "20000101T0500Z", "20000101T0600Z", ] def test_advanced_exclusions_sequences_mix_points_sequences(set_cycling_type): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") # Run every hour from 01:00 excluding every 3rd hour. sequence = ISO8601Sequence("T-00 ! (2000, PT2H)", "20000101T00Z") output = [] point = sequence.get_start_point() count = 0 # We are going to make six sequence points while point and count < 6: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] assert output == [ "20000101T0100Z", "20000101T0300Z", "20000101T0500Z", "20000101T0700Z", "20000101T0900Z", "20000101T1100Z", ] def test_advanced_exclusions_sequences_implied_start_point(set_cycling_type): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") # Run every hour from 01:00 excluding every 3rd hour. sequence = ISO8601Sequence("T05/PT1H!PT3H", "20000101T00Z") output = [] point = sequence.get_start_point() count = 0 # We are going to make six sequence points while point and count < 6: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] assert output == [ "20000101T0600Z", "20000101T0700Z", "20000101T0900Z", "20000101T1000Z", "20000101T1200Z", "20000101T1300Z", ] def test_point_init(): """It should error if inited with anything other than a string.""" ISO8601Point('1000') with pytest.raises(TypeError): ISO8601Point(1000) with pytest.raises(TypeError): ISO8601Interval(1000) def test_exclusions_sequences_points(set_cycling_type): """Test ISO8601Sequence methods for sequences with exclusions""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") # Run every hour from 01:00 excluding every 3rd hour sequence = ISO8601Sequence("T01/PT1H!PT3H", "20000101T01Z") point_0 = ISO8601Point("20000101T00Z") point_1 = ISO8601Point("20000101T01Z") point_2 = ISO8601Point("20000101T02Z") point_3 = ISO8601Point("20000101T03Z") point_4 = ISO8601Point("20000101T04Z") assert point_0 not in sequence.exclusions assert point_1 in sequence.exclusions assert sequence.is_on_sequence(point_2) assert sequence.is_on_sequence(point_3) assert not sequence.is_on_sequence(point_4) assert point_4 in sequence.exclusions def test_exclusions_extensive(set_cycling_type): """Test ISO8601Sequence methods for sequences with exclusions""" set_cycling_type(ISO8601_CYCLING_TYPE, "+05") sequence = ISO8601Sequence( "PT1H!20000101T02+05", "20000101T00", "20000101T05" ) point_0 = ISO8601Point("20000101T0000+05") point_1 = ISO8601Point("20000101T0100+05") point_2 = ISO8601Point("20000101T0200+05") # The excluded point. point_3 = ISO8601Point("20000101T0300+05") assert not sequence.is_on_sequence(point_2) assert not sequence.is_valid(point_2) assert sequence.get_prev_point(point_2) == point_1 assert sequence.get_prev_point(point_3) == point_1 assert sequence.get_prev_point(point_2) == point_1 assert sequence.get_nearest_prev_point(point_3) == point_1 assert sequence.get_next_point(point_1) == point_3 assert sequence.get_next_point(point_2) == point_3 sequence = ISO8601Sequence("PT1H!20000101T00+05", "20000101T00+05") assert sequence.get_first_point(point_0) == point_1 assert sequence.get_start_point() == point_1 def test_multiple_exclusions_extensive(set_cycling_type): """Test ISO8601Sequence methods for sequences with multiple exclusions""" set_cycling_type(ISO8601_CYCLING_TYPE, "+05") sequence = ISO8601Sequence( "PT1H!(20000101T02,20000101T03)", "20000101T00", "20000101T06" ) point_0 = ISO8601Point("20000101T0000+05") point_1 = ISO8601Point("20000101T0100+05") point_2 = ISO8601Point("20000101T0200+05") # First excluded point point_3 = ISO8601Point("20000101T0300+05") # Second excluded point point_4 = ISO8601Point("20000101T0400+05") # Check the excluded points are not on the sequence assert not sequence.is_on_sequence(point_2) assert not sequence.is_on_sequence(point_3) assert not sequence.is_valid(point_2) # Should be excluded assert not sequence.is_valid(point_3) # Should be excluded # Check that we can correctly retrieve previous points assert sequence.get_prev_point(point_2) == point_1 # Should skip two excluded points assert sequence.get_prev_point(point_4) == point_1 assert sequence.get_prev_point(point_2) == point_1 assert sequence.get_nearest_prev_point(point_4) == point_1 assert sequence.get_next_point(point_1) == point_4 assert sequence.get_next_point(point_3) == point_4 sequence = ISO8601Sequence("PT1H!20000101T00+05", "20000101T00") # Check that the first point is after 00. assert sequence.get_first_point(point_0) == point_1 assert sequence.get_start_point() == point_1 # Check a longer list of exclusions # Also note you can change the format of the exclusion list # (removing the parentheses) sequence = ISO8601Sequence( "PT1H!(20000101T02+05, 20000101T03+05," "20000101T04+05)", "20000101T00", "20000101T06", ) assert sequence.get_prev_point(point_3) == point_1 assert sequence.get_prev_point(point_4) == point_1 def test_get_nearest_prev_point_exclusions(set_cycling_type): """Another test for get_nearest_prev_point() for sequences with exclusions. https://github.com/cylc/cylc-flow/issues/7268 """ set_cycling_type(ISO8601_CYCLING_TYPE) P = ISO8601Point seq = ISO8601Sequence('T00 ! 2025-01-01T00Z', '2025-01-01T00Z') assert seq.get_nearest_prev_point(P('2025-01-01T09Z')) is None assert seq.get_nearest_prev_point(P('2025-01-02T09Z')) == P( '2025-01-02T00Z' ) def test_exclusion_zero_duration_warning(set_cycling_type, caplog, log_filter): """It should not log zero-duration warnings for exclusion points. Exclusions may either be sequences or points. We first attempt to parse them as sequences, if this fails, we attempt to parse them as points. The zero-duration recurrence warning would be logged if we attempted to parse a point as a sequence. To avoid spurious warnings this should be turned off for exclusion parsing. """ # parsing a point as a sequences causes a zero-duration warning set_cycling_type(ISO8601_CYCLING_TYPE, "+05") with pytest.raises(Exception): ISO8601Sequence('3000', '2999') assert log_filter(contains='zero-duration') # parsing a point in an exclusion should not caplog.clear() ISO8601Sequence('P1Y ! 3000', '2999') assert not log_filter(contains='zero-duration') def test_simple(set_cycling_type): """Run some simple tests for date-time cycling.""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") p_start = ISO8601Point("20100808T00") p_stop = ISO8601Point("20100808T02") i = ISO8601Interval("PT6H") assert p_start - i == ISO8601Point("20100807T18") assert p_stop + i == ISO8601Point("20100808T08") sequence = ISO8601Sequence( "PT10M", str(p_start), str(p_stop), ) point = sequence.get_next_point(ISO8601Point("20100808T0000")) assert point == ISO8601Point("20100808T0010") output = [] # Test point generation forwards. while point and point < p_stop: output.append(point) assert sequence.is_on_sequence(point) point = sequence.get_next_point(point) assert [str(out) for out in output] == [ "20100808T0010Z", "20100808T0020Z", "20100808T0030Z", "20100808T0040Z", "20100808T0050Z", "20100808T0100Z", "20100808T0110Z", "20100808T0120Z", "20100808T0130Z", "20100808T0140Z", "20100808T0150Z", ] assert point == ISO8601Point("20100808T0200") # Test point generation backwards. output = [] while point and point >= p_start: output.append(point) assert sequence.is_on_sequence(point) point = sequence.get_prev_point(point) assert [str(out) for out in output] == [ "20100808T0200Z", "20100808T0150Z", "20100808T0140Z", "20100808T0130Z", "20100808T0120Z", "20100808T0110Z", "20100808T0100Z", "20100808T0050Z", "20100808T0040Z", "20100808T0030Z", "20100808T0020Z", "20100808T0010Z", "20100808T0000Z", ] assert not sequence.is_on_sequence(ISO8601Point("20100809T0005")) @pytest.mark.parametrize( 'value, expected', [ ('next(T2100Z)', '20100808T2100Z'), ('next(T00)', '20100809T0000Z'), ('next(T-15)', '20100808T1615Z'), ('next(T-45)', '20100808T1545Z'), ('next(-10)', '21100101T0000Z'), ('next(-1008)', '21100801T0000Z'), ('next(--10)', '20101001T0000Z'), ('next(--0325)', '20110325T0000Z'), ('next(---10)', '20100810T0000Z'), ('next(---05T1200Z)', '20100905T1200Z'), ('next(--08-08)', '20110808T0000Z'), ('next(T15)', '20100809T1500Z'), ('next(T-41)', '20100808T1541Z'), ] ) def test_next_simple(value: str, expected: str, set_cycling_type): """Test the generation of CP using 'next' from single input.""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") my_now = "2010-08-08T15:41Z" assert ingest_time(value, my_now) == expected @pytest.mark.parametrize( 'value, expected', [ ('previous(T2100Z)', '20100807T2100Z'), ('previous(T00)', '20100808T0000Z'), ('previous(T-15)', '20100808T1515Z'), ('previous(T-45)', '20100808T1445Z'), ('previous(-10)', '20100101T0000Z'), ('previous(-1008)', '20100801T0000Z'), ('previous(--10)', '20091001T0000Z'), ('previous(--0325)', '20100325T0000Z'), ('previous(---10)', '20100710T0000Z'), ('previous(---05T1200Z)', '20100805T1200Z'), ('previous(--08-08)', '20100808T0000Z'), ('previous(T15)', '20100808T1500Z'), ('previous(T-41)', '20100808T1441Z'), ] ) def test_previous_simple(value: str, expected: str, set_cycling_type): """Test the generation of CP using 'previous' from single input.""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") my_now = "2010-08-08T15:41Z" assert ingest_time(value, my_now) == expected def test_sequence(set_cycling_type): """Test the generation of CP from list input.""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") my_now = "20100808T1540Z" sequence = ( "next(T-00;T-15;T-30;T-45)", # 20100808T1545Z "previous(T-00;T-15;T-30;T-45)", # 20100808T1530Z "next(T00;T06;T12;T18)", # 20100808T1800Z "previous(T00;T06;T12;T18)", ) # 20100808T1200Z output = [] for point in sequence: output.append(ingest_time(point, my_now)) assert output == [ "20100808T1545Z", "20100808T1530Z", "20100808T1800Z", "20100808T1200Z", ] def test_offset_simple(set_cycling_type): """Test the generation of offset CP.""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") my_now = "20100808T1540Z" sequence = ( "PT15M", # 20100808T1555Z "-PT30M", # 20100808T1510Z "PT1H", # 20100808T1640Z "-PT18H", # 20100807T2140Z "P3D", # 20100811T1540Z "-P2W", # 20100725T1540Z "P6M", # 20110208T1540Z "-P1M", # 20100708T1540Z "P1Y", # 20110808T1540Z "-P5Y", # 20050808T1540Z ) output = [] for point in sequence: output.append(ingest_time(point, my_now)) assert output == [ "20100808T1555Z", "20100808T1510Z", "20100808T1640Z", "20100807T2140Z", "20100811T1540Z", "20100725T1540Z", "20110208T1540Z", "20100708T1540Z", "20110808T1540Z", "20050808T1540Z", ] def test_offset(set_cycling_type): """Test the generation of offset CP with 'next' and 'previous'.""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") my_now = "20100808T1540Z" sequence = ( "next(T06) +P1D", # 20100810T0600Z "previous(T-30) -PT12H", # 20100808T0330Z "next(T00;T06;T12;T18) -P1W", # 20100801T1800Z "previous(T00;T06;T12;T18) +PT1H", # 20100808T1300Z ) output = [] for point in sequence: output.append(ingest_time(point, my_now)) assert output == [ "20100810T0600Z", "20100808T0330Z", "20100801T1800Z", "20100808T1300Z", ] def test_weeks_days(set_cycling_type): """Test the generation of CP with day-of-week, ordinal day, and week (with day-of-week specified).""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") my_now = "20100808T1540Z" sequence = ( "next(-W-1)", # 20100809T0000Z "previous(-W-4)", # 20100805T0000Z "next(-010)", # 20110110T0000Z "previous(-101)", # 20100411T0000Z "next(-W40-1)", # 20101004T0000Z "previous(-W05-1)", # 20100201T0000Z "next(-W05-5)", # 20110204T0000Z "previous(-W40-4)", # 20091001T0000Z ) output = [] for point in sequence: output.append(ingest_time(point, my_now)) assert output == [ "20100809T0000Z", "20100805T0000Z", "20110110T0000Z", "20100411T0000Z", "20101004T0000Z", "20100201T0000Z", "20110204T0000Z", "20091001T0000Z", ] @pytest.mark.parametrize( 'value, expected', [ ('next(T-00)', '20180314T1600Z'), ('previous(T-00)', '20180314T1500Z'), ('next(T-00; T-15; T-30; T-45)', '20180314T1515Z'), ('previous(T-00; T-15; T-30; T-45)', '20180314T1500Z'), ('next(T00)', '20180315T0000Z'), ('previous(T00)', '20180314T0000Z'), ('next(T06:30Z)', '20180315T0630Z'), ('previous(T06:30) -P1D', '20180313T0630Z'), ('next(T00; T06; T12; T18)', '20180314T1800Z'), ('previous(T00; T06; T12; T18)', '20180314T1200Z'), ('next(T00; T06; T12; T18)+P1W', '20180321T1800Z'), ('PT1H', '20180314T1612Z'), ('-P1M', '20180214T1512Z'), ('next(-00)', '21000101T0000Z'), ('previous(--01)', '20180101T0000Z'), ('next(---01)', '20180401T0000Z'), ('previous(--1225)', '20171225T0000Z'), ('next(-2006)', '20200601T0000Z'), ('previous(-W101)', '20180305T0000Z'), ('next(-W-1; -W-3; -W-5)', '20180316T0000Z'), ('next(-001; -091; -181; -271)', '20180401T0000Z'), ('previous(-365T12Z)', '20171231T1200Z'), ] ) def test_user_guide_examples(value: str, expected: str, set_cycling_type): """Test the offset CP examples in the Cylc user guide. https://cylc.github.io/cylc-doc/stable/html/user-guide/writing-workflows/scheduling.html """ set_cycling_type(ISO8601_CYCLING_TYPE, "Z") my_now = "2018-03-14T15:12Z" assert ingest_time(value, my_now) == expected def test_next_simple_no_now(set_cycling_type): """Test the generation of CP using 'next' with no value for `now`.""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") my_now = None point = "next(T00Z)+P1D" output = ingest_time(point, my_now) current_time = datetime.now(timezone.utc) # my_now is None, but ingest_time will have used a similar time, and # the returned value must be after current_time output_time = datetime.strptime(output, "%Y%m%dT%H%M%z") assert current_time < output_time def test_integer_cycling_is_returned(set_cycling_type): """Test that when integer points are given, the same value is returned.""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") integer_point = "1" assert integer_point, ingest_time(integer_point is None) def test_expanded_dates_are_returned(set_cycling_type): """Test that when expanded dates are given, the same value is returned.""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") expanded_date = "+0100400101T0000Z" assert expanded_date, ingest_time(expanded_date is None) def test_timepoint_truncated(set_cycling_type): """Test that when a timepoint is given, and is truncated, then the value is added to `now`.""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") my_now = "2018-03-14T15:12Z" timepoint_truncated = "T15:00Z" # 20180315T1500Z output = ingest_time(timepoint_truncated, my_now) assert output == "20180315T1500Z" def test_timepoint(set_cycling_type): """Test that when a timepoint is given, and is not truncated, the same value is returned.""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") my_now = "2018-03-14T15:12Z" timepoint_truncated = "19951231T0630" # 19951231T0630 output = ingest_time(timepoint_truncated, my_now) assert output == "19951231T0630" @pytest.mark.parametrize( "_input, errortext", ( ( "next (T-00, T-30)", r"Invalid offset.* Offset lists are semicolon separated", ), ("next (wildebeest)", "Invalid ISO 8601 date"), ), ) def test_validate_fails_comma_sep_offset_list( _input, errortext, set_cycling_type ): """It raises an exception if validating a list separated by commas""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") with pytest.raises(Exception, match=errortext): ingest_time(_input) cylc-flow-8.7.0/tests/unit/test_task_message.py0000664000175000017500000000263415264420307022021 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from socket import gaierror import pytest from cylc.flow.task_message import send_messages async def test_send_messages_err( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture ): """If an error occurs while initializing the client, it should be printed. """ exc_msg = 'Relic malfunction detected' def mock_get_client(*a, **k): raise gaierror(-2, exc_msg) monkeypatch.setattr('cylc.flow.task_message.get_client', mock_get_client) await send_messages( 'arasaka', '1/v/01', [['INFO', 'silverhand']], '2077-01-01T00:00:00Z' ) assert f"gaierror: [Errno -2] {exc_msg}" in capsys.readouterr().err cylc-flow-8.7.0/tests/unit/test_task_id.py0000664000175000017500000000541115264420307020765 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import unittest import pytest from cylc.flow.task_id import TaskID from cylc.flow.cycling.iso8601 import ISO8601Point class TestTaskId(unittest.TestCase): def test_get(self): self.assertEqual("a.1", TaskID.get("a", 1)) self.assertEqual("a._1", TaskID.get("a", "_1")) self.assertEqual( "WTASK.20101010T101010", TaskID.get("WTASK", "20101010T101010")) def test_split(self): self.assertEqual(["a", '1'], TaskID.split("a.1")) self.assertEqual(["a", '_1'], TaskID.split("a._1")) self.assertEqual( ["WTAS", '20101010T101010'], TaskID.split("WTAS.20101010T101010")) def test_is_valid_name(self): for name in [ "abc", "123", "____", "_", "a_b", "a_1", "1_b", "ABC" ]: self.assertTrue(TaskID.is_valid_name(name)) for name in [ "a.1", None, "%abc", "", " " ]: self.assertFalse(TaskID.is_valid_name(name)) def test_is_valid_id(self): for id1 in [ "a.1", "_.098098439535$#%#@!#~" ]: self.assertTrue(TaskID.is_valid_id(id1)) for id2 in [ "abc", "123", "____", "_", "a_b", "a_1", "1_b", "ABC", "a.A A" ]: self.assertFalse(TaskID.is_valid_id(id2)) def test_is_valid_id_2(self): # TBD: a.A A is invalid for valid_id, but valid for valid_id_2? # TBD: a/a.a is OK? for id1 in [ "a.1", "_.098098439535$#%#@!#~", "a/1", "_/098098439535$#%#@!#~", "a.A A", "a/a.a" ]: self.assertTrue(TaskID.is_valid_id_2(id1)) for id2 in [ "abc", "123", "____", "_", "a_b", "a_1", "1_b", "ABC" ]: self.assertFalse(TaskID.is_valid_id_2(id2)) def test_get_standardised_point_string(set_cycling_type): """Test that invalid point string raises ValueError.""" set_cycling_type(ISO8601Point.TYPE) with pytest.raises(ValueError, match="Invalid cycle point:"): TaskID.get_standardised_point_string("aaa") cylc-flow-8.7.0/tests/unit/test_platforms.py0000664000175000017500000004351515264420307021365 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # Tests for the platform lookup. from typing import ( Any, Dict, List, Optional, ) import pytest from cylc.flow.exceptions import ( GlobalConfigError, PlatformLookupError, ) from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults from cylc.flow.platforms import ( _platform_name_from_job_info, _validate_single_host, generic_items_match, get_install_target_from_platform, get_install_target_to_platforms_map, get_platform, get_platform_deprecated_settings, is_platform_definition_subshell, platform_from_name, ) from cylc.flow.run_modes import JOBLESS_MODES PLATFORMS = { 'desktop[0-9]{2}|laptop[0-9]{2}': { 'job runner': 'background' }, 'sugar': { 'hosts': 'localhost', 'job runner': 'slurm', }, 'hpc-no-logs': { 'hosts': ['hpc1', 'hpc2'], 'job runner': 'pbs', 'retrieve job logs': False }, 'hpc-logs': { 'hosts': ['hpc1', 'hpc2'], 'job runner': 'pbs', 'retrieve job logs': True }, 'hpc1-bg': { 'hosts': 'hpc1', 'job runner': 'background', }, 'hpc2-bg': { 'hosts': 'hpc2', 'job runner': 'background' }, 'localhost': { 'hosts': 'localhost', 'job runner': 'background' } } PLATFORMS_NO_UNIQUE = { 'sugar': { 'hosts': 'localhost', 'job runner': 'slurm' }, 'pepper': { 'hosts': ['hpc1', 'hpc2'], 'job runner': 'slurm' }, } PLATFORMS_WITH_RE = { 'hpc.*': {'hosts': 'hpc1', 'job runner': 'background'}, 'h.*': {'hosts': 'hpc3'}, r'vld\d{2,3}, anselm\d{4}': {}, 'nu.*': { 'job runner': 'slurm', 'hosts': ['localhost'] }, 'localhost': { 'hosts': 'localhost', 'job runner': 'background' } } PLATFORMS_TREK = { 'enterprise': { 'hosts': ['kirk', 'picard'], 'install target': 'picard', 'name': 'enterprise' }, 'voyager': { 'hosts': ['janeway'], 'install target': 'janeway', 'name': 'voyager' }, 'stargazer': { 'hosts': ['picard'], 'install target': 'picard', 'name': 'stargazer' } } PLATFORMS_INVALID = { 'enterprise': { 'hosts': ['kirk', 'picard'], 'install target': 'picard', 'job runner': 'background' # requires one host }, 'voyager': { 'hosts': ['janeway', 'seven-of-nine'], 'install target': 'janeway', 'job runner': 'at' # requires one host } } # ---------------------------------------------------------------------------- # Tests of platform_from_name # ---------------------------------------------------------------------------- @pytest.mark.parametrize( "PLATFORMS, platform, expected", [ (PLATFORMS_WITH_RE, "nutmeg", { "job runner": "slurm", "name": "nutmeg", "hosts": ['localhost'] }), (PLATFORMS_WITH_RE, "vld798", ["vld798"]), (PLATFORMS_WITH_RE, "vld56", ["vld56"]), ( PLATFORMS_NO_UNIQUE, "sugar", { "hosts": "localhost", "job runner": "slurm", "name": "sugar" }, ), ( PLATFORMS, None, { "hosts": "localhost", "name": "localhost", "job runner": "background" }, ), (PLATFORMS, "laptop22", { "job runner": "background", "name": "laptop22", "hosts": ["laptop22"] }), ( PLATFORMS, "hpc1-bg", { "hosts": "hpc1", "job runner": "background", "name": "hpc1-bg" }, ), (PLATFORMS_WITH_RE, "hpc2", {"hosts": "hpc3", "name": "hpc2"}), ], ) def test_basic(PLATFORMS, platform, expected): # n.b. The name field of the platform is set in the Globalconfig object # if the name is 'localhost', so we don't test for it here. platform = platform_from_name(platform_name=platform, platforms=PLATFORMS) if isinstance(expected, dict): assert platform == expected else: assert platform["hosts"] == expected def test_platform_not_there(): with pytest.raises(PlatformLookupError): platform_from_name('moooo', PLATFORMS) @pytest.mark.parametrize( 'platform', [ {i: j} for i, j in PLATFORMS_INVALID.items() ] ) def test_invalid_platforms(platform): with pytest.raises(GlobalConfigError): _validate_single_host(platform) def test_similar_but_not_exact_match(): with pytest.raises(PlatformLookupError): platform_from_name('vld1', PLATFORMS_WITH_RE) # ---------------------------------------------------------------------------- # Tests of platform_name_from_job_info # ---------------------------------------------------------------------------- # Basic tests that we can select sensible platforms @pytest.mark.parametrize( 'job, remote, returns', [ # Can we return a sensible platform for desktop 42 ( {}, {'host': 'desktop42'}, 'desktop42' ), # Basic test where the user hasn't submitted anything and the task # returns to default, i.e. localhost. ( {'batch system': 'background'}, {'retrieve job logs retry delays': None}, 'localhost' ), # Check that when the user asks for batch system = slurm alone # they get system = sugar ( {'batch system': 'slurm'}, {'host': ''}, 'sugar' ), # Check that when users asks for hpc1 and pbs they get a platform # with hpc1 in its list of hosts ( {'batch system': 'pbs'}, {'host': 'hpc1'}, 'hpc-logs' ), # When the user asks for hpc1 without specifying pbs user gets platform # hpc bg1 ( {'batch system': 'background'}, {'host': 'hpc1'}, 'hpc1-bg' ), # Check that None as a value is handled correctly ( {'batch system': None}, {'host': 'hpc1-bg'}, 'hpc1-bg' ), ( # Check that failure to set any items will return localhost {'batch system': None}, {'host': None}, 'localhost' ), ( # Check that all generic items are matched {'batch system': 'pbs'}, {'host': 'hpc1', 'retrieve job logs': False}, 'hpc-no-logs' ), ] ) def test_platform_name_from_job_info_basic(job, remote, returns): assert _platform_name_from_job_info(PLATFORMS, job, remote) == returns def test_platform_name_from_job_info_evaluated_hostname(): result = _platform_name_from_job_info( PLATFORMS, {'batch system': 'background'}, {'host': '$(cat tiddles)'}, evaluated_host='hpc2', ) assert result == 'hpc2-bg' def test_platform_name_from_job_info_ordered_dict_comparison(): """Check that we are only comparing set items in OrderedDictWithDefaults. """ job = {'batch system': 'background', 'Made up key': 'Zaphod'} remote = {'host': 'hpc1', 'Made up key': 'Arthur'} # Set up a fake OrderedDictWith a fake unset default. platform = OrderedDictWithDefaults() platform.defaults_ = {k: None for k in PLATFORMS['hpc1-bg'].keys()} platform.defaults_['Made up key'] = {} platform.update(PLATFORMS['hpc1-bg']) platforms = {'hpc1-bg': platform, 'dobbie': PLATFORMS['sugar']} assert _platform_name_from_job_info(platforms, job, remote) == 'hpc1-bg' # Cases where the error ought to be raised because no matching platform should # be found. @pytest.mark.parametrize( 'job, remote', [ # Check for error when the user asks for slurm on host desktop01 ( {'batch system': 'slurm'}, {'host': 'desktop01'}, ), # ('hpc1', 'slurm', 'error'), ( {'batch system': 'slurm'}, {'host': 'hpc1'}, ), # Localhost doesn't support pbs ( {'batch system': 'pbs'}, {}, ), ] ) def test_reverse_PlatformLookupError(job, remote): with pytest.raises(PlatformLookupError): _platform_name_from_job_info(PLATFORMS, job, remote) # An example of a global config with two Spice systems available @pytest.mark.parametrize( 'job, remote, returns', [ ( {'batch system': 'slurm'}, {'host': 'sugar'}, 'sugar' ), ( {'batch system': 'slurm'}, {}, 'sugar' ), ( {'batch system': 'slurm'}, {'host': 'pepper'}, 'pepper' ) ] ) def test_platform_name_from_job_info_two_spices( job, remote, returns ): platforms = { 'sugar': { 'hosts': ['sugar', 'localhost'], 'job runner': 'slurm', }, 'pepper': { 'job runner': 'slurm', 'hosts': 'pepper' }, } assert _platform_name_from_job_info(platforms, job, remote) == returns # An example of two platforms with the same hosts and job runner settings # but some other setting different @pytest.mark.parametrize( 'job, remote, returns', [ ( { 'job runner': 'background', 'job runner command template': '', 'shell': '/bin/fish' }, { 'host': 'desktop01', 'owner': '', 'workflow definition directory': '', 'retrieve job logs': '', 'retrieve job logs max size': '', 'retrieve job logs retry delays': 'None' }, 'my-platform-with-fish' ), ] ) def test_platform_name_from_job_info_similar_platforms( job, remote, returns ): platforms = { 'my-platform-with-bash': { 'hosts': 'desktop01', 'shell': '/bin/bash', 'job runner': 'background' }, # An extra platform to check that we only pick up the first match 'my-platform-with-fish-not-this-one': { 'hosts': 'desktop01', 'shell': '/bin/fish', 'job runner': 'background' }, 'my-platform-with-fish': { 'hosts': 'desktop01', 'shell': '/bin/fish', 'job runner': 'background' }, } assert _platform_name_from_job_info(platforms, job, remote) == returns # ----------------------------------------------------------------------------- # Tests for getting install target info @pytest.mark.parametrize( 'platform, expected', [ ({'name': 'rick', 'install target': 'desktop'}, 'desktop'), ({'name': 'morty', 'install target': ''}, 'morty') ] ) def test_get_install_target_from_platform(platform, expected): """Test that get_install_target_from_platform works as expected.""" assert get_install_target_from_platform(platform) == expected @pytest.mark.parametrize( 'platform_names, expected_map', [ ( ['enterprise', 'stargazer'], { 'picard': [ PLATFORMS_TREK['enterprise'], PLATFORMS_TREK['stargazer'] ] }, ), ( ['enterprise', 'voyager', 'enterprise'], { 'picard': [ PLATFORMS_TREK['enterprise'] ], 'janeway': [ PLATFORMS_TREK['voyager'] ] }, ), ( ['enterprise', 'starkiller'], # ignores non-existent platform { 'picard': [ PLATFORMS_TREK['enterprise'] ], }, ), ] ) def test_get_install_target_to_platforms_map( platform_names: list[str], expected_map: dict[str, Any], monkeypatch: pytest.MonkeyPatch, ): """Test that get_install_target_to_platforms_map works as expected.""" monkeypatch.setattr('cylc.flow.platforms.platform_from_name', lambda x: platform_from_name(x, PLATFORMS_TREK)) result = get_install_target_to_platforms_map(platform_names) # Sort the maps: for _map in (result, expected_map): for install_target in _map: _map[install_target] = sorted( _map[install_target], key=lambda k: k['name'] ) assert result == expected_map @pytest.mark.parametrize('mode', sorted(JOBLESS_MODES)) def test_platform_from_name__jobless_modes(mode): result = platform_from_name(mode) assert result['name'] == 'localhost' @pytest.mark.parametrize('mode', sorted(JOBLESS_MODES)) def test_get_install_target_to_platforms_map__jobless_modes(mode): result = get_install_target_to_platforms_map([mode]) assert list(result) == ['localhost'] assert len(result['localhost']) == 1 assert result['localhost'][0]['hosts'] == ['localhost'] assert result['localhost'][0]['install target'] == 'localhost' @pytest.mark.parametrize( 'platform, job, remote, expect', [ ( # Default, no old settings. {'ship': 'Enterprise'}, {}, {}, True ), ( {'captain': 'Kirk'}, {'captain': 'Picard'}, {}, False ), ( {'captain': 'Sisko'}, {}, {'captain': 'Janeway'}, False ), ( {'captain': 'Picard', 'ship': 'Enterprise'}, {'captain': 'Picard'}, {'ship': 'Enterprise'}, True ), ( {'captain': 'Picard', 'ship': 'Enterprise'}, {'captain': 'Picard'}, {'ship': 'Defiant'}, False ), ( {'captain': 'Picard', 'ship': 'Enterprise'}, {'captain': 'Picard'}, {}, True ) ] ) def test_generic_items_match(platform, job, remote, expect): assert generic_items_match(platform, job, remote) == expect @pytest.mark.parametrize( 'task_conf, expected', [ pytest.param( { 'remote': { 'host': 'cylcdevbox', 'retrieve job logs': True }, 'job': { 'batch system': 'pbs', 'batch submit command template': 'meow' } }, [ '[runtime][task][job]batch submit command template = meow', '[runtime][task][remote]retrieve job logs = True', '[runtime][task][remote]host = cylcdevbox', '[runtime][task][job]batch system = pbs' ], id="All are deprecated settings" ), pytest.param( { 'remote': {'host': 'localhost'}, 'job': { 'batch system': 'pbs', 'batch submit command template': None } }, ['[runtime][task][job]batch system = pbs'], id="Exclusions are excluded" ), pytest.param( { 'environment filter': { 'include': ['frodo', 'sam'] } }, [], id="No deprecated settings" ) ] ) def test_get_platform_deprecated_settings( task_conf: Dict[str, Any], expected: List[str] ): output = get_platform_deprecated_settings(task_conf, task_name='task') assert set(output) == set(expected) @pytest.mark.parametrize( 'plat_val, expected, err_msg', [('normal', False, None), ('$(yes)', True, None), ('`echo ${chamber}`', None, "backticks are not supported")] ) def test_is_platform_definition_subshell( plat_val: str, expected: Optional[bool], err_msg: Optional[str]): if err_msg: with pytest.raises(PlatformLookupError) as exc: is_platform_definition_subshell(plat_val) assert err_msg in str(exc.value) else: assert is_platform_definition_subshell(plat_val) is expected def test_get_platform_from_OrderedDictWithDefaults(mock_glbl_cfg): """Get platform works with OrderedDictWithDefaults. Most tests use dictionaries to check platforms functionality. This one was added to catch an issue where the behaviour of dict.get != OrderedDictWithDefaults.get. See - https://github.com/cylc/cylc-flow/issues/4979 """ mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[skarloey]] hosts = foo, bar job runner = slurm ''' ) task_conf = OrderedDictWithDefaults() task_conf.defaults_ = OrderedDictWithDefaults([ ('job', OrderedDictWithDefaults([ ('batch system', 'slurm') ])), ('remote', OrderedDictWithDefaults([ ('host', 'foo') ])), ]) result = get_platform(task_conf)['name'] assert result == 'skarloey' cylc-flow-8.7.0/tests/unit/test_scheduler_cli.py0000664000175000017500000002314315264420307022156 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from contextlib import contextmanager from secrets import token_hex import sqlite3 from unittest.mock import AsyncMock, Mock import pytest from cylc.flow.exceptions import ( HostSelectException, ServiceFileError, ) from cylc.flow.scheduler_cli import ( RunOptions, _distribute, _version_check, ) @pytest.fixture def stopped_workflow_db(tmp_path): """Returns a workflow DB with the `cylc_version` set to the provided string. def test_x(stopped_workflow_db): db_file = stopped_workflow_db(version) """ def _stopped_workflow_db(version): db_file = tmp_path / 'db' conn = sqlite3.connect(db_file) conn.execute(''' CREATE TABLE workflow_params(key TEXT, value TEXT, PRIMARY KEY(key)) ''') conn.execute( ''' INSERT INTO workflow_params VALUES (?, ?) ''', ('cylc_version', version) ) conn.commit() conn.close() return db_file return _stopped_workflow_db @pytest.fixture def set_cylc_version(monkeypatch): """Set the cylc.flow.__version__ attribute. def test_x(set_cylc_version): set_cylc_version('1.2.3') """ def _set_cylc_version(version): monkeypatch.setattr( 'cylc.flow.scheduler_cli.__version__', version, ) return _set_cylc_version @pytest.fixture def answer(monkeypatch): """Answer a `cylc play` CLI prompt. def test_x(answer): answer(users_response) It also adds an assert on the number of times the prompt interface was called. 0 if response is None, else 1. """ @contextmanager def _answer(response): calls = 0 def prompt(*args, **kwargs): nonlocal calls calls += 1 return response monkeypatch.setattr( 'cylc.flow.scheduler_cli.prompt', prompt, ) yield expected_calls = 1 if response is None: expected_calls = 0 assert calls == expected_calls return _answer @pytest.fixture def interactive(monkeypatch): monkeypatch.setattr( 'cylc.flow.scheduler_cli.is_terminal', lambda: True, ) @pytest.fixture def non_interactive(monkeypatch): monkeypatch.setattr( 'cylc.flow.scripts.reinstall.is_terminal', lambda: False, ) @pytest.mark.parametrize( 'before, after, downgrade, response, outcome', [ # no change ('8.0.0', '8.0.0', False, None, True), # upgrading ('8.0rc4.dev', '8.0.0', False, None, True), ('8.0.0', '8.0.1', False, None, True), ('8.0.0', '8.1.0', False, False, False), ('8.0.0', '8.1.0', False, True, True), ('8.0.0', '9.0.0', False, False, False), ('8.0.0', '9.0.0', False, True, True), # downgrading ('8.1.1', '8.1.0', False, None, False), ('8.1.1', '8.1.0', True, None, True), ('8.1.0', '8.0.0', False, None, False), ('8.1.0', '8.0.0', True, None, True), ('8.1.0', '8.0rc4.dev', True, None, True), ('9.1.0', '8.0.0', False, None, False), ('9.1.0', '8.0.0', True, None, True), # truncated versions ('8.1.1', '8', False, None, False), ('9.1.1', '8', True, None, True), ], ) def test_version_check_interactive( stopped_workflow_db, set_cylc_version, interactive, answer, before, after, response, downgrade, outcome, ): """It should check compatibility with the Cylc version of the prior run. When workflows are restarted we need to perform some checks to make sure it is safe and sensible to restart with this version of Cylc. Pytest Params: before: The Cylc version the workflow ran with previously. after: The version of Cylc being used to restart the workflow. downgrade: The --downgrade option of `cylc play`. response: The user's response the any CLI prompts. If `None` it will assert that no prompts were raised. outcome: The response of _version_check, True means safe to restart. """ db_file = stopped_workflow_db(before) set_cylc_version(after) with answer(response): assert ( _version_check( db_file, RunOptions(downgrade=downgrade) ) is outcome ) def test_version_check_interactive_upgrade( stopped_workflow_db, set_cylc_version, interactive, answer, ): """If a user interactively upgrades, it should set the upgrade option.""" db_file = stopped_workflow_db('8.0.0') set_cylc_version('8.1.0') opts = RunOptions() assert opts.upgrade is False with answer(True): assert _version_check(db_file, opts) is True assert opts.upgrade is True def test_version_check_non_interactive( stopped_workflow_db, set_cylc_version, non_interactive, ): """It should not prompt in non-interactive mode. * The --upgrade argument should permit upgrade. * The --downgrade argument should permit downgrade. """ # upgrade db_file = stopped_workflow_db('8.0.0') set_cylc_version('8.1.0') assert _version_check(db_file, RunOptions()) is False assert ( _version_check(db_file, RunOptions(upgrade=True)) is True ) # CLI --upgrade # downgrade db_file.unlink() db_file = stopped_workflow_db('8.1.0') set_cylc_version('8.0.0') assert _version_check(db_file, RunOptions()) is False assert ( _version_check(db_file, RunOptions(downgrade=True)) is True ) # CLI --downgrade def test_version_check_incompat(tmp_path): """It should fail for a corrupted or invalid database file.""" db_file = tmp_path / 'db' # invalid DB file db_file.touch() with pytest.raises(ServiceFileError): _version_check(db_file, RunOptions()) def test_version_check_no_db(tmp_path): """It should pass if there is no DB file (e.g. on workflow first start).""" db_file = tmp_path / 'db' # non-existent file assert _version_check(db_file, RunOptions()) @pytest.mark.parametrize( 'cli_colour, is_terminal, distribute_colour', [ ('never', True, '--color=never'), ('auto', True, '--color=always'), ('always', True, '--color=always'), ('never', False, '--color=never'), ('auto', False, '--color=never'), ('always', False, '--color=never'), ] ) async def test_distribute_colour( monkeypatch: pytest.MonkeyPatch, cli_colour, is_terminal, distribute_colour, ): """It should start detached workflows with the correct --colour option. The is_terminal test will fail for detached scheduler processes which means that the colour formatting will be stripped for startup. This includes the Cylc header logo and any warnings/errors raised during config parsing. In order to preserver colour formatting we must set the `--colour` arg to `always` when we want the detached process to start in colour mode. See https://github.com/cylc/cylc-flow/issues/5159 """ _is_terminal = Mock(return_value=is_terminal) monkeypatch.setattr('cylc.flow.scheduler_cli.is_terminal', _is_terminal) _cylc_server_cmd = AsyncMock(return_value=0) monkeypatch.setattr( 'cylc.flow.scheduler_cli.cylc_server_cmd', _cylc_server_cmd ) opts = RunOptions(host='myhost', color=cli_colour) with pytest.raises(SystemExit) as excinfo: await _distribute('foo', 'foo/run1', opts) assert excinfo.value.code == 0 assert distribute_colour in _cylc_server_cmd.call_args[0][0] async def test_distribute_upgrade( monkeypatch: pytest.MonkeyPatch ): """It should start detached workflows with the --upgrade option if the user has interactively chosen to upgrade (typed 'y' at prompt). """ monkeypatch.setattr( 'sys.argv', ['cylc', 'play', 'foo'] # no upgrade option here ) _cylc_server_cmd = AsyncMock(return_value=0) monkeypatch.setattr( 'cylc.flow.scheduler_cli.cylc_server_cmd', _cylc_server_cmd ) opts = RunOptions( host='myhost', upgrade=True, # added by interactive upgrade ) with pytest.raises(SystemExit) as excinfo: await _distribute('foo', 'foo/run1', opts) assert excinfo.value.code == 0 assert '--upgrade' in _cylc_server_cmd.call_args[0][0] async def test_distribute_invalid_host( mock_glbl_cfg, caplog: pytest.LogCaptureFixture ): """It handles a socket error when the host is invalid.""" mock_glbl_cfg( 'cylc.flow.host_select.glbl_cfg', f''' [scheduler] [[run hosts]] available = non_exist_{token_hex(4)} ''' ) with pytest.raises(HostSelectException): await _distribute('foo', 'foo/run1', RunOptions()) cylc-flow-8.7.0/tests/unit/main_loop/0000775000175000017500000000000015264420307017712 5ustar alastairalastaircylc-flow-8.7.0/tests/unit/main_loop/test_main_loop.py0000664000175000017500000001646215264420307023311 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import asyncio from collections import deque import logging import pytest from cylc.flow import CYLC_LOG from cylc.flow.exceptions import CylcError from cylc.flow.main_loop import ( CoroTypes, _wrapper, get_runners, load, ) from cylc.flow.main_loop.health_check import health_check as hc_during def test_load_plugins_blank(): """Test that log_plugins works when no plugins are requested.""" conf = { 'plugins': [] } assert load(conf) == { 'config': conf, 'state': {}, 'timings': {} } def test_load_plugins(): """Test the loading of a built-in plugin.""" conf = { 'plugins': ['health check'], 'health check': { 'interval': 1234 } } assert load(conf) == { CoroTypes.Periodic: { ('health check', 'health_check'): hc_during }, 'state': { 'health check': { } }, 'config': conf, 'timings': { ('health check', 'health_check'): deque([], maxlen=1) } } def test_wrapper_calls_function(): """Ensure the wrapper calls coroutines.""" flag = False async def test_coro(arg1, arg2): nonlocal flag assert arg1 == 'arg1' assert arg2 == 'arg2' flag = True coro = _wrapper( test_coro, 'arg1', 'arg2' ) asyncio.run(coro) assert flag def test_wrapper_logging(caplog): """Ensure the wrapper logs each coroutine call.""" async def test_coro(*_): pass coro = _wrapper( test_coro, None, None ) with caplog.at_level(logging.DEBUG, logger=CYLC_LOG): asyncio.run(coro) assert len(caplog.record_tuples) == 2 ( (run_log, run_level, run_msg), (end_log, end_level, end_msg) ) = caplog.record_tuples # we should have two messages, one sent before and one after # the function assert 'run' in run_msg assert 'end' in end_msg # both should contain the name of the function assert 'test_coro' in run_msg assert 'test_coro' in end_msg # and should be sent to the cylc logger at the debug level assert run_log == end_log == CYLC_LOG assert run_level == end_level == logging.DEBUG def test_wrapper_catches_exceptions(caplog): """Ensure the wrapper catches Exception instances and logs them.""" async def test_coro(*_): raise Exception('foo') coro = _wrapper( test_coro, None, None ) with caplog.at_level(logging.DEBUG, logger=CYLC_LOG): asyncio.run(coro) assert len(caplog.record_tuples) == 4 run, error, traceback, completed = caplog.record_tuples assert 'run' in run[2] assert error[1] == logging.ERROR assert traceback[1] == logging.ERROR assert 'foo' in traceback[2] assert completed[1] == logging.DEBUG def test_wrapper_passes_cylc_error(): """Ensure the wrapper does not catch CylcError instances.""" async def test_coro(*_): raise CylcError('foo') coro = _wrapper( test_coro, None, None ) with pytest.raises(CylcError): asyncio.run(coro) @pytest.fixture def basic_plugins(): calls = [] def capture(*args): calls.append(args) plugins = { 'config': { 'periodic plugin': { 'interval': 10 } }, 'timings': { ('periodic plugin', 'periodic_coro'): [], ('startup plugin', 'startup_coro'): [], }, 'state': { 'periodic plugin': { 'a': 1 }, 'startup plugin': { 'b': 2 } }, CoroTypes.Periodic: { ('periodic plugin', 'periodic_coro'): capture }, CoroTypes.StartUp: { ('startup plugin', 'startup_coro'): capture } } return (plugins, calls, capture) def test_get_runners_startup(basic_plugins): """IT should return runners for startup functions.""" plugins, calls, capture = basic_plugins runners = get_runners( plugins, CoroTypes.StartUp, 'scheduler object' ) assert len(runners) == 1 asyncio.run(runners[0]) assert calls == [('scheduler object', {'b': 2})] def test_get_runners_periodic(basic_plugins): """It should return runners for periodic functions.""" plugins, calls, capture = basic_plugins runners = get_runners( plugins, CoroTypes.Periodic, 'scheduler object' ) assert len(runners) == 1 asyncio.run(runners[0]) assert calls == [('scheduler object', {'a': 1})] def test_get_runners_periodic_debounce(basic_plugins): """It should run periodic functions based on the configured interval.""" plugins, calls, capture = basic_plugins # we should start with a blank timings object assert len(plugins['timings'][('periodic plugin', 'periodic_coro')]) == 0 runners = get_runners( plugins, CoroTypes.Periodic, 'scheduler object' ) assert len(runners) == 1 asyncio.run(runners[0]) assert calls == [('scheduler object', {'a': 1})] # the timings object should now contain the previous run assert len(plugins['timings'][('periodic plugin', 'periodic_coro')]) == 1 # the next run should be skipped because of the interval runners = get_runners( plugins, CoroTypes.Periodic, 'scheduler object' ) assert len(runners) == 0 # if we remove the interval the next run will not get skipped plugins['config']['periodic plugin']['interval'] = 0 runners = get_runners( plugins, CoroTypes.Periodic, 'scheduler object' ) assert len(runners) == 1 assert calls[-1] == ('scheduler object', {'a': 1}) # Clean up coroutines we didn't run for coro in runners: coro.close() def test_state(basic_plugins): """It should pass the same state object with each function call. * Run the same plugin function twice. * Ensure that the state object received by each call is the same object. """ plugins, calls, capture = basic_plugins runners = get_runners( plugins, CoroTypes.StartUp, 'scheduler object' ) assert len(runners) == 1 asyncio.run(*runners) assert len(calls) == 1 runners = get_runners( plugins, CoroTypes.StartUp, 'scheduler object' ) assert len(runners) == 1 asyncio.run(*runners) assert len(calls) == 2 (_, state1), (_, state2) = calls assert id(state1) == id(state2) cylc-flow-8.7.0/tests/unit/main_loop/test_log_tracemalloc.py0000664000175000017500000000516415264420307024460 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from types import SimpleNamespace from cylc.flow.main_loop.log_tracemalloc import take_snapshot, init, close_log import pytest # find the number of the MARKER line in this file MARKER_LINE = None with open(__file__, 'r') as this_file: for line_number, line in enumerate(this_file): print('$', line.strip()) print('#', line[-7:].strip()) if line[-7:] == 'MARKER\n': MARKER_LINE = line_number + 1 break @pytest.fixture async def state(tmp_path): """A clean state object for this plugin.""" state = {} await init(SimpleNamespace(workflow_run_dir=tmp_path), state) return state async def test_tracemalloc(tmp_path, state): """Test the tracemalloc plugin functionality.""" out_dir = tmp_path / 'tracemalloc' # test the empty state object assert state['itt'] == 0 assert len(list(out_dir.iterdir())) == 1 # the tracemalloc folder assert state['log'].closed is False # the log file is open # take a snapshot await take_snapshot(None, state, diff_filter=None) assert state['itt'] == 1, 'the iteration has been incremented' assert len(list(out_dir.iterdir())) == 2, 'dump file has been written' # allocate some memory _memory = [x for x in range(100)] # MARKER # take another snapshot await take_snapshot(None, state, diff_filter=None) assert state['itt'] == 2, 'the iteration has been incremented' assert len(list(out_dir.iterdir())) == 3, 'dump file has been written' # close the log file await close_log(None, state) assert state['log'].closed is True, 'log file has been closed' # ensure the allocated memory appears in the log file with open(out_dir / 'log', 'r') as tracemalloc_file: tracemalloc_log = tracemalloc_file.read() assert f'{__file__}:{MARKER_LINE}' in tracemalloc_log del _memory # make linters happy cylc-flow-8.7.0/tests/unit/main_loop/test_auto_restart.py0000664000175000017500000002460215264420307024043 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging from socket import gaierror from unittest.mock import Mock import pytest from cylc.flow import CYLC_LOG from cylc.flow.exceptions import ( CylcConfigError, CylcError, HostSelectException ) from cylc.flow.main_loop.auto_restart import ( _can_auto_restart, _set_auto_restart, _should_auto_restart, auto_restart, ) from cylc.flow.parsec.exceptions import ParsecError from cylc.flow.scheduler import Scheduler from cylc.flow.workflow_status import ( AutoRestartMode, StopMode ) async def test_can_auto_restart_pass(monkeypatch, caplog): """Test can_auto_restart for successful host selection.""" async def select_workflow_host(**_): return ('localhost', 'localhost') monkeypatch.setattr( 'cylc.flow.main_loop.auto_restart.select_workflow_host', select_workflow_host ) assert await _can_auto_restart() assert caplog.record_tuples == [] async def test_can_auto_restart_fail(monkeypatch, caplog): """Test can_auto_restart for unsuccessful host selection.""" async def select_workflow_host(**_): raise HostSelectException({}) monkeypatch.setattr( 'cylc.flow.main_loop.auto_restart.select_workflow_host', select_workflow_host ) with caplog.at_level(level=logging.DEBUG, logger=CYLC_LOG): assert not await _can_auto_restart() [(_, level, msg)] = caplog.record_tuples assert level == logging.CRITICAL assert 'No alternative host to restart workflow on' in msg async def test_can_auto_restart_fail_horribly( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ): """Test can_auto_restart for really unsuccessful host selection.""" async def select_workflow_host(**_): raise Exception('Unexpected error in host selection') monkeypatch.setattr( 'cylc.flow.main_loop.auto_restart.select_workflow_host', select_workflow_host ) with caplog.at_level(level=logging.ERROR, logger=CYLC_LOG): assert not await _can_auto_restart() assert 'Error in host selection' in caplog.text assert "Traceback (most recent call last):" in caplog.text @pytest.mark.parametrize( 'host, stop_mode, condemned_hosts,' ' auto_restart_time, should_auto_restart', [ ( # no reason to restart, no reason not to 'localhost', None, [], None, False ), ( # should restart but already stopping 'localhost', StopMode.AUTO, ['localhost'], None, False ), ( # stop restart but already auto-restarting 'localhost', StopMode.AUTO, ['localhost'], 12345, False ), ( # should restart 'localhost', None, ['localhost'], None, AutoRestartMode.RESTART_NORMAL ), ( # should force stop 'localhost', None, ['localhost!'], None, AutoRestartMode.FORCE_STOP ) ] ) def test_should_auto_restart( host, stop_mode, condemned_hosts, auto_restart_time, should_auto_restart, monkeypatch ): """Ensure the workflow only auto-restarts when appropriate.""" # factor out networking and FQDNs for testing purposes monkeypatch.setattr( 'cylc.flow.main_loop.auto_restart.get_fqdn_by_host', lambda x: x ) # mock a scheduler object scheduler = Mock( host=host, stop_mode=stop_mode, auto_restart_time=auto_restart_time ) # mock a workflow configuration object cfg = Mock() cfg.get = lambda x: condemned_hosts # test assert _should_auto_restart(scheduler, cfg) == should_auto_restart def test_should_auto_restart__condemned_host_down( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ): """Check we don't get any spurious error messages if a condemned host is unavailable. A condemned host being unavailable is expected! """ monkeypatch.setattr( 'cylc.flow.main_loop.auto_restart.get_fqdn_by_host', Mock(side_effect=gaierror('oopsie')) ) scheduler = Mock( spec=Scheduler, host='localhost', stop_mode=None, auto_restart_time=None, ) cfg = Mock(get=lambda x: ['condemned-host']) assert _should_auto_restart(scheduler, cfg) is False assert not caplog.records async def test_set_auto_restart_already_stopping(caplog): """Ensure restart isn't attempted if already stopping.""" scheduler = Mock( stop_mode=StopMode.AUTO ) with caplog.at_level(level=logging.DEBUG, logger=CYLC_LOG): assert await _set_auto_restart(scheduler) assert caplog.record_tuples == [] async def test_set_auto_restart_force_oveeride(caplog): """Ensure scheduled restart is cancelled for a force stop.""" scheduler = Mock( stop_mode=None, auto_restart_time=1234 ) with caplog.at_level(level=logging.DEBUG, logger=CYLC_LOG): assert await _set_auto_restart( scheduler, mode=AutoRestartMode.FORCE_STOP, ) assert len(caplog.record_tuples) == 2 [ (*_, msg1), (*_, msg2) ] = caplog.record_tuples assert 'This workflow will be shutdown' in msg1 assert 'Scheduled automatic restart canceled' in msg2 async def test_set_auto_restart_already_restarting(caplog): """Ensure restart isn't re-scheduled.""" scheduler = Mock( stop_mode=None, auto_restart_time=1234 ) with caplog.at_level(level=logging.DEBUG, logger=CYLC_LOG): assert await _set_auto_restart(scheduler) assert caplog.record_tuples == [] async def test_set_auto_restart_no_detach(caplog: pytest.LogCaptureFixture): """Ensure raises a CylcError (or subclass) if running in no-detach mode.""" scheduler = Mock( spec=Scheduler, stop_mode=None, auto_restart_time=None, options=Mock(no_detach=True) ) with caplog.at_level(level=logging.DEBUG, logger=CYLC_LOG): with pytest.raises(CylcError): await _set_auto_restart(scheduler) assert caplog.record_tuples == [] async def test_set_auto_restart_unable_to_restart(monkeypatch): """Ensure returns False if workflow is unable to restart""" called = False async def workflow_select_fail(**_): nonlocal called called = True # prevent this becoming a placebo return False monkeypatch.setattr( 'cylc.flow.main_loop.auto_restart._can_auto_restart', workflow_select_fail ) scheduler = Mock( stop_mode=None, auto_restart_time=None, options=Mock(no_detach=False) ) assert not await _set_auto_restart( scheduler ) assert called async def test_set_auto_restart_with_delay(monkeypatch, caplog): """Ensure workflows wait for a period before auto-restarting.""" called = False async def workflow_select_pass(**_): nonlocal called called = True # prevent this becoming a placebo return True monkeypatch.setattr( 'cylc.flow.main_loop.auto_restart._can_auto_restart', workflow_select_pass ) monkeypatch.setattr( # remove the random element of the restart delay 'cylc.flow.main_loop.auto_restart.random', lambda: 1 ) scheduler = Mock( stop_mode=None, auto_restart_time=None, options=Mock(no_detach=False) ) with caplog.at_level(level=logging.DEBUG, logger=CYLC_LOG): assert await _set_auto_restart( scheduler, restart_delay=1 ) [(*_, msg1), (*_, msg2)] = caplog.record_tuples assert 'will automatically restart' in msg1 assert 'will restart in 1s' in msg2 assert called async def test_set_auto_restart_without_delay(monkeypatch, caplog): """Ensure workflows auto-restart when no delay is provided.""" called = False async def workflow_select_pass(**_): nonlocal called called = True # prevent this becoming a placebo return True monkeypatch.setattr( 'cylc.flow.main_loop.auto_restart._can_auto_restart', workflow_select_pass ) scheduler = Mock( stop_mode=None, auto_restart_time=None, options=Mock(no_detach=False) ) with caplog.at_level(level=logging.DEBUG, logger=CYLC_LOG): assert await _set_auto_restart( scheduler ) [(*_, msg)] = caplog.record_tuples assert 'will automatically restart' in msg assert called @pytest.mark.parametrize('exc_class', [ParsecError, CylcConfigError]) async def test_log_config_error(caplog, log_filter, monkeypatch, exc_class): """It should log errors in the global config. When errors are present in the global config they should be caught and logged nicely rather than left to spill over as traceback in the log. """ # make the global config raise an error def global_config_load_error(*args, **kwargs): raise exc_class('something even more bizarrely inexplicable') monkeypatch.setattr( 'cylc.flow.main_loop.auto_restart.glbl_cfg', global_config_load_error, ) # call the auto_restart plugin, the error should be caught caplog.clear() assert await auto_restart(None, None) is False # the error should have been logged assert len(caplog.messages) == 1 assert 'an error in the global config' in caplog.messages[0] assert 'something even more bizarrely inexplicable' in caplog.messages[0] cylc-flow-8.7.0/tests/unit/main_loop/test_log_db.py0000664000175000017500000000257215264420307022557 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from textwrap import dedent from cylc.flow.main_loop.log_db import _format def test_format(): """It should indent, fix case and strip comments in SQL statements.""" assert _format(''' select a, b, c, d, e, f, g from table_1 left join table_2 where a = 1 and b = 2 and c = 3 # whatever ''') == dedent(''' SELECT a, b, c, d, e, f, g FROM table_1 LEFT JOIN table_2 WHERE a = 1 AND b = 2 AND c = 3 '''[1:]) cylc-flow-8.7.0/tests/unit/main_loop/test_log_main_loop.py0000664000175000017500000000357715264420307024155 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from collections import deque from pathlib import Path import pytest from cylc.flow.main_loop.log_main_loop import ( _normalise, _dump, _plot ) @pytest.fixture() def test_data(): return { # (plugin_name, coro_name): deque([(start_time, duration), ...]) ('foo', 'bar'): deque([(2, 1), (3, 2), (4, 3)]), ('baz', 'pub'): deque([(1, 4), (2, 5), (3, 6)]), } def test_normalise(test_data): """Ensure we correctly normalise the timings against the earliest time.""" assert _normalise(test_data) == { 'foo': [(1, 1), (2, 2), (3, 3)], 'baz': [(0, 4), (1, 5), (2, 6)], } def test_dump(test_data, tmp_path): """Ensure the data is serialisable.""" assert _dump(_normalise(test_data), tmp_path) assert list(tmp_path.iterdir()) == [ Path(tmp_path, 'cylc.flow.main_loop.log_main_loop.json') ] def test_plot(test_data, tmp_path): """Ensure the plotting mechanism doesn't raise errors.""" assert _plot(_normalise(test_data), tmp_path) assert list(tmp_path.iterdir()) == [ Path(tmp_path, 'cylc.flow.main_loop.log_main_loop.pdf') ] cylc-flow-8.7.0/tests/unit/main_loop/test_health_check.py0000664000175000017500000000433115264420307023726 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from unittest.mock import Mock import pytest from cylc.flow.main_loop import MainLoopPluginException from cylc.flow.main_loop.health_check import ( HealthCheckFailed, _check_contact_file, _check_workflow_run_dir, ) def test_check_workflow_run_dir(): """Ensure a missing workflow run dir raises HealthCheckFailed.""" sched = Mock(workflow_run_dir='/a/b/c/d/e') with pytest.raises(HealthCheckFailed): _check_workflow_run_dir(sched) def test_check_contact_file_data(monkeypatch): """Ensure differing contact file data raises HealthCheckFailed.""" contact_data = { 'a': 'beef', 'b': 2 } sched = Mock( workflow='foo', contact_data=dict(contact_data) ) monkeypatch.setattr( 'cylc.flow.main_loop.health_check.workflow_files.load_contact_file', lambda x: dict(contact_data) ) # pass _check_contact_file(sched) # fail contact_data['a'] = 'wellington' with pytest.raises(HealthCheckFailed): _check_contact_file(sched) def test_check_contact_file_io(monkeypatch): """Ensure unexpected IOError retrieving the contact file raises MainLoopPluginException.""" sched = Mock(workflow='foo') def whoopsie(*_): raise IOError('') monkeypatch.setattr( 'cylc.flow.main_loop.health_check.workflow_files.load_contact_file', whoopsie ) with pytest.raises(MainLoopPluginException): _check_contact_file(sched) cylc-flow-8.7.0/tests/unit/main_loop/test_log_memory.py0000664000175000017500000000542715264420307023504 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from pathlib import Path from unittest.mock import Mock import pytest try: from cylc.flow.main_loop.log_memory import ( _compute_sizes, _transpose, _dump, _plot ) except ModuleNotFoundError as exc: if exc.name == 'pympler': pytest.skip( 'pympler required for these tests', allow_module_level=True ) else: raise def test_compute_sizes(): """Test the interface for the calculation of instance attribute sizes.""" keys = { 'a': [], 'b': 42, 'c': 'beef wellington' } test_object = Mock(**keys) # no fields should be larger than 10kb sizes = _compute_sizes(test_object, 10000) sizes.pop('total') assert sizes == {} # all fields should be larger than 0kb ret = _compute_sizes(test_object, 0) assert { key for key, value in ret.items() # filter out mock fields if not key.startswith('_') and key not in ('method_calls', 'total') } == set(keys) @pytest.fixture() def test_data(): return [ (5, {'a': 1, 'b': 2, 'c': 3}), (6, {'a': 2, 'c': 4}), (7, {'a': 5, 'c': 2}) ] def test_transpose(test_data): """Test transposing the data from bin to series orientated.""" assert _transpose(test_data) == ( { # the keys are sorted by their last entry 'a': [1, 2, 5], 'c': [3, 4, 2], 'b': [2, -1, -1] # missing values become -1 }, [0, 1, 2] ) def test_dump(test_data, tmp_path): """Ensure the data is serialiseable.""" _dump(test_data, tmp_path) assert list(tmp_path.iterdir()) == [ Path(tmp_path, 'cylc.flow.main_loop.log_memory.json') ] def test_plot(test_data, tmp_path): """Ensure the plotting mechanism doesn't raise errors.""" fields, times = _transpose(test_data) _plot(fields, times, tmp_path) assert list(tmp_path.iterdir()) == [ Path(tmp_path, 'cylc.flow.main_loop.log_memory.pdf') ] cylc-flow-8.7.0/tests/unit/main_loop/test_log_data_store.py0000664000175000017500000000455315264420307024320 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from pathlib import Path import pytest try: from cylc.flow.main_loop.log_data_store import ( _dump, _plot, _iter_data_store ) except ModuleNotFoundError as exc: if exc.name == 'pympler': pytest.skip( 'pympler required for these tests', allow_module_level=True ) else: raise @pytest.fixture() def test_data(): return { 'times': [1, 2, 3], 'objects': { 'foo': [4, 5, 6] }, 'size': { 'foo': [7, 8, 9] } } def test_dump(test_data, tmp_path): """Ensure the data is serialiseable.""" assert _dump(test_data, tmp_path) assert list(tmp_path.iterdir()) == [ Path(tmp_path, 'cylc.flow.main_loop.log_data_store.json') ] def test_plot_no_data(tmp_path): """Ensure plotting skips if data insufficient.""" assert not _plot({'times': [1]}, tmp_path) def test_plot(test_data, tmp_path): """Ensure the plotting mechanism doesn't raise errors.""" pytest.importorskip('matplotlib', reason='requires matplotlib') assert _plot(test_data, tmp_path) assert list(tmp_path.iterdir()) == [ Path(tmp_path, 'cylc.flow.main_loop.log_data_store.pdf') ] def test_iter_data_store(): class DataStore: tracker = {'this': 'that'} data = {'x': {'a': 1, 'workflow': 2, 'c': 3}} ds = DataStore() assert ( list(_iter_data_store(ds)) ) == [ ('data_store_mgr (total)', ds), ('tracker', {'this': 'that'}), ('data.a', 1), ('data.workflow', [2]), ('data.c', 3) ] cylc-flow-8.7.0/tests/unit/test_resources.py0000664000175000017500000000472315264420307021366 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pytest from pathlib import Path from shlex import split from subprocess import run from cylc.flow.resources import ( RESOURCE_NAMES, get_resources, _backup, ) def test_get_resources_one(tmpdir): """Test extraction of a specific resource. Check that a file of the right name gets extracted. Do not check file content because there is no assurance that it will remain constant. """ get_resources('job.sh', tmpdir) assert (tmpdir / 'job.sh').isfile() @pytest.mark.parametrize( 'resource', [ r for r in list(RESOURCE_NAMES.keys()) if r[0] != '!' ] + ['tutorial/runtime-tutorial'] ) def test_get_resources_all(resource, tmpdir): get_resources(resource, tmpdir) assert (tmpdir / Path(resource).name).exists() def test_cli(tmpdir): result = run( split(f'cylc get-resources job.sh {str(tmpdir)}'), capture_output=True ) if result.returncode != 0: raise AssertionError( f'{result.stderr}' ) def test_backup(tmp_path, caplog): a = tmp_path / 'a' abc = tmp_path / 'a' / 'b' / 'c' abc.mkdir(parents=True) before = set(tmp_path.glob('*')) _backup(a) assert len(caplog.record_tuples) == 1 after = set(tmp_path.glob('*')) assert len(after - before) == 1 new = list(after - before)[0] assert new.name.startswith(a.name) new_abc = new / 'b' / 'c' assert new_abc.exists() def test_vim_deprecated(): """It fails, returning a warning if user asks for obsolete syntax file """ output = run( ['cylc', 'get-resources', 'syntax/cylc.vim'], capture_output=True ) assert 'has been replaced' in output.stderr.decode() cylc-flow-8.7.0/tests/unit/test_cylc_subproc.py0000664000175000017500000000774415264420307022051 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import unittest from cylc.flow.cylc_subproc import procopen from unittest.mock import call from testfixtures import compare from testfixtures.popen import PIPE, MockPopen # Method could be a function # pylint: disable=no-self-use class TestSubprocessSafe(unittest.TestCase): """Unit tests for the parameter procopen utility function""" def setUp(self): self.Popen = MockPopen() def test_sprocess_communicate_with_process(self): foo = ' foo' bar = ' bar' cmd = ["echo", "this is a command" + foo + bar] p = procopen(cmd, stdoutpipe=True) stdout, _ = p.communicate() compare(stdout, b"this is a command foo bar\n") def test_sprocess_communicate_with_input(self): command = "a command" Popen = MockPopen() Popen.set_command(command) # only static input used with simulated mockpopen # codacy mistakenly sees this as a call to popen process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) # nosec err, out = process.communicate('foo') compare([ # only static input used with simulated mockpopen # codacy mistakenly sees this as a call to popen call.Popen(command, shell=True, stderr=-1, stdout=-1), # nosec call.Popen_instance.communicate('foo'), ], Popen.mock.method_calls) return err, out def test_sprocess_safe_read_from_stdout_and_stderr(self): command = "a command" Popen = MockPopen() # only static input used with simulated mockpopen # codacy mistakenly sees this as a call to popen Popen.set_command(command, stdout=b'foo', stderr=b'bar') process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) # nosec compare(process.stdout.read(), b'foo') compare(process.stderr.read(), b'bar') compare([ call.Popen(command, shell=True, stderr=PIPE, # nosec stdout=PIPE), ], Popen.mock.method_calls) def test_sprocess_safe_write_to_stdin(self): command = "a command" Popen = MockPopen() Popen.set_command(command) # only static input used with simulated mockpopen # codacy mistakenly sees this as a call to popen process = Popen(command, stdin=PIPE, shell=True) # nosec process.stdin.write(command) process.stdin.close() compare([ # static input used with simulated mockpopen # codacy mistakenly sees this as a call to popen call.Popen(command, shell=True, stdin=PIPE), # nosec call.Popen_instance.stdin.write(command), call.Popen_instance.stdin.close(), ], Popen.mock.method_calls) def test_sprocess_safe_wait_and_return_code(self): command = "a command" Popen = MockPopen() Popen.set_command(command, returncode=3) process = Popen(command) compare(process.returncode, None) compare(process.wait(), 3) compare(process.returncode, 3) compare([ call.Popen(command), call.Popen_instance.wait(), ], Popen.mock.method_calls) cylc-flow-8.7.0/tests/unit/test_install.py0000664000175000017500000004521015264420307021016 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import os import shutil from pathlib import Path from typing import ( Any, Callable, Dict, Optional, Tuple, Type, Union, ) import pytest from cylc.flow.exceptions import ( InputError, WorkflowFilesError, ) from cylc.flow.workflow_files import ( WorkflowFiles, get_workflow_source_dir, ) from cylc.flow.install import ( NESTED_DIRS_MSG, check_nested_dirs, get_rsync_rund_cmd, get_run_dir_info, get_source_workflow_name, install_workflow, parse_cli_sym_dirs, reinstall_workflow, search_install_source_dirs, validate_source_dir, ) NonCallableFixture = Any # global.cylc[install]scan depth for these tests: MAX_SCAN_DEPTH = 3 @pytest.fixture def glbl_cfg_max_scan_depth(mock_glbl_cfg: Callable) -> None: mock_glbl_cfg( 'cylc.flow.install.glbl_cfg', f''' [install] max depth = {MAX_SCAN_DEPTH} ''' ) @pytest.mark.parametrize( 'workflow_name, err_expected', [ ('foo/' * (MAX_SCAN_DEPTH - 1), False), ('foo/' * MAX_SCAN_DEPTH, True) # /run1 takes it beyond max depth ] ) def test_install_workflow__max_depth( workflow_name: str, err_expected: bool, tmp_run_dir: Callable, tmp_src_dir: Callable, glbl_cfg_max_scan_depth: NonCallableFixture, prevent_symlinking, ): """Test that trying to install beyond max depth fails.""" src_dir = tmp_src_dir('bar') if err_expected: with pytest.raises(WorkflowFilesError) as exc_info: install_workflow(src_dir, workflow_name) assert "would exceed global.cylc[install]max depth" in str( exc_info.value ) else: install_workflow(src_dir, workflow_name) @pytest.mark.parametrize( 'flow_file, expected_exc', [ (WorkflowFiles.FLOW_FILE, WorkflowFilesError), (WorkflowFiles.SUITE_RC, WorkflowFilesError), (None, None) ] ) def test_install_workflow__next_to_flow_file( flow_file: Optional[str], expected_exc: Optional[Type[Exception]], tmp_run_dir: Callable, tmp_src_dir: Callable, prevent_symlinking, ): """Test that you can't install into a dir that contains a workflow file.""" # Setup cylc_run_dir: Path = tmp_run_dir() workflow_dir = cylc_run_dir / 'faden' workflow_dir.mkdir() src_dir: Path = tmp_src_dir('faden') if flow_file: (workflow_dir / flow_file).touch() # Test if expected_exc: with pytest.raises(expected_exc) as exc_info: install_workflow(src_dir, 'faden') assert "Nested run directories not allowed" in str(exc_info.value) else: install_workflow(src_dir, 'faden') def test_install_workflow__symlink_target_exists( tmp_path: Path, tmp_src_dir: Callable, tmp_run_dir: Callable, mock_glbl_cfg: Callable, ): """Test that you can't install workflow when run dir symlink dir target already exists.""" id_ = 'smeagol' src_dir: Path = tmp_src_dir(id_) sym_run = tmp_path / 'sym-run' sym_log = tmp_path / 'sym-log' mock_glbl_cfg( 'cylc.flow.pathutil.glbl_cfg', f''' [install] [[symlink dirs]] [[[localhost]]] run = {sym_run} log = {sym_log} ''' ) msg = "Symlink dir target already exists: .*{}" # Test: (sym_run / 'cylc-run' / id_ / 'run1').mkdir(parents=True) with pytest.raises(WorkflowFilesError, match=msg.format(sym_run)): install_workflow(src_dir) shutil.rmtree(sym_run) ( sym_log / 'cylc-run' / id_ / 'run1' / WorkflowFiles.LogDir.DIRNAME ).mkdir(parents=True) with pytest.raises(WorkflowFilesError, match=msg.format(sym_log)): install_workflow(src_dir) def test_check_nested_dirs( tmp_run_dir: Callable, glbl_cfg_max_scan_depth: NonCallableFixture ): """Test that check_nested_dirs() raises when a parent dir is a workflow directory.""" cylc_run_dir: Path = tmp_run_dir() test_dir = cylc_run_dir.joinpath('a/' * (MAX_SCAN_DEPTH + 3)) # note we check beyond max scan depth (because we're checking upwards) test_dir.mkdir(parents=True) # Parents are not run dirs - ok: check_nested_dirs(test_dir) # Parent contains a run dir but that run dir is not direct ancestor # of our test dir - ok: tmp_run_dir('a/Z') check_nested_dirs(test_dir) # Now make run dir out of parent - not ok: tmp_run_dir('a') with pytest.raises(WorkflowFilesError) as exc: check_nested_dirs(test_dir) assert str(exc.value) == NESTED_DIRS_MSG.format( dir_type='run', dest=test_dir, existing=(cylc_run_dir / 'a') ) @pytest.mark.parametrize( 'named_run', [True, False] ) @pytest.mark.parametrize( 'test_install_path, existing_install_path', [ pytest.param( f'{"child/" * (MAX_SCAN_DEPTH + 3)}', '', id="Check parents (beyond max scan depth)" ), pytest.param( '', f'{"child/" * MAX_SCAN_DEPTH}', id="Check children up to max scan depth" ) ] ) def test_check_nested_dirs_install_dirs( tmp_run_dir: Callable, glbl_cfg_max_scan_depth: NonCallableFixture, test_install_path: str, existing_install_path: str, named_run: bool ): """Test that check nested dirs looks both up and down a tree for WorkflowFiles.Install.DIRNAME. Params: test_install_path: Path relative to ~/cylc-run/thing where we are trying to install a workflow. existing_install_path: Path relative to ~/cylc-run/thing where there is an existing install dir. named_run: Whether the workflow we are trying to install has named/numbered run. """ cylc_run_dir: Path = tmp_run_dir() existing_install: Path = tmp_run_dir( f'thing/{existing_install_path}/run1', installed=True, named=True ).parent test_install_dir = cylc_run_dir / 'thing' / test_install_path test_run_dir = test_install_dir / 'run1' if named_run else test_install_dir with pytest.raises(WorkflowFilesError) as exc: check_nested_dirs(test_run_dir, test_install_dir) assert str(exc.value) == NESTED_DIRS_MSG.format( dir_type='install', dest=test_run_dir, existing=existing_install ) def test_get_workflow_source_dir_numbered_run(tmp_path): """Test get_workflow_source_dir returns correct source for numbered run""" cylc_install_dir = ( tmp_path / "cylc-run" / "flow-name" / "_cylc-install") cylc_install_dir.mkdir(parents=True) run_dir = (tmp_path / "cylc-run" / "flow-name" / "run1") run_dir.mkdir() source_dir = (tmp_path / "cylc-source" / "flow-name") source_dir.mkdir(parents=True) assert get_workflow_source_dir(run_dir) == (None, None) (cylc_install_dir / "source").symlink_to(source_dir) assert get_workflow_source_dir(run_dir) == ( str(source_dir), cylc_install_dir / "source") def test_get_workflow_source_dir_named_run(tmp_path): """Test get_workflow_source_dir returns correct source for named run""" cylc_install_dir = ( tmp_path / "cylc-run" / "flow-name" / "_cylc-install") cylc_install_dir.mkdir(parents=True) source_dir = (tmp_path / "cylc-source" / "flow-name") source_dir.mkdir(parents=True) (cylc_install_dir / "source").symlink_to(source_dir) assert get_workflow_source_dir( cylc_install_dir.parent) == ( str(source_dir), cylc_install_dir / "source") def test_reinstall_workflow(tmp_path, capsys): cylc_install_dir = ( tmp_path / "cylc-run" / "flow-name" / "_cylc-install") cylc_install_dir.mkdir(parents=True) source_dir = (tmp_path / "cylc-source" / "flow-name") source_dir.mkdir(parents=True) (source_dir / "flow.cylc").touch() (cylc_install_dir / "source").symlink_to(source_dir) run_dir = cylc_install_dir.parent reinstall_workflow(source_dir, "flow-name", run_dir) assert capsys.readouterr().out == ( f"REINSTALLED flow-name from {source_dir}\n") @pytest.mark.parametrize( 'filename, expected_err', [('flow.cylc', None), ('suite.rc', None), ('fluff.txt', (WorkflowFilesError, "Could not find workflow 'baa/baa'"))] ) def test_search_install_source_dirs( filename: str, expected_err: Optional[Tuple[Type[Exception], str]], tmp_path: Path, mock_glbl_cfg: Callable): """Test search_install_source_dirs(). Params: filename: A file to insert into one of the source dirs. expected_err: Exception and message expected to be raised. """ horse_dir = tmp_path / 'horse' horse_dir.mkdir() sheep_dir = tmp_path / 'sheep' source_dir = sheep_dir / 'baa' / 'baa' source_dir.mkdir(parents=True) source_dir_file = source_dir / filename source_dir_file.touch() mock_glbl_cfg( 'cylc.flow.install.glbl_cfg', f''' [install] source dirs = {horse_dir}, {sheep_dir} ''' ) if expected_err: err, msg = expected_err with pytest.raises(err) as exc: search_install_source_dirs('baa/baa') assert msg in str(exc.value) else: ret = search_install_source_dirs('baa/baa') assert ret == source_dir assert ret.is_absolute() def test_search_install_source_dirs_empty(mock_glbl_cfg: Callable): """Test search_install_source_dirs() when no source dirs configured.""" mock_glbl_cfg( 'cylc.flow.install.glbl_cfg', ''' [install] source dirs = ''' ) with pytest.raises(WorkflowFilesError) as exc: search_install_source_dirs('foo') assert str(exc.value) == ( "Cannot find workflow as 'global.cylc[install]source dirs' " "does not contain any paths") @pytest.mark.parametrize( 'path, expected', [ ('~/isla/nublar/dennis/nedry', 'dennis/nedry'), ('~/isla/sorna/paul/kirby', 'paul/kirby'), ('~/mos/eisley/owen/skywalker', 'skywalker') ] ) def test_get_source_workflow_name( path: str, expected: str, mock_glbl_cfg: Callable ): mock_glbl_cfg( 'cylc.flow.install.glbl_cfg', ''' [install] source dirs = ~/isla/nublar, ${HOME}/isla/sorna ''' ) assert get_source_workflow_name( Path(path).expanduser().resolve()) == expected def test_get_rsync_rund_cmd( tmp_src_dir: Callable, tmp_run_dir: Callable ): """Test rsync command for cylc install/reinstall excludes cylc dirs. """ src_dir = tmp_src_dir('foo') cylc_run_dir: Path = tmp_run_dir('rsync_flow', installed=True, named=False) for wdir in [ WorkflowFiles.WORK_DIR, WorkflowFiles.SHARE_DIR, WorkflowFiles.LogDir.DIRNAME, ]: cylc_run_dir.joinpath(wdir).mkdir(exist_ok=True) actual_cmd = get_rsync_rund_cmd(src_dir, cylc_run_dir) assert actual_cmd == [ 'rsync', '-a', '--checksum', '--out-format=%o %n%L', '--no-t', '--exclude=/log', '--exclude=/work', '--exclude=/share', '--exclude=/_cylc-install', '--exclude=/.service', f'{src_dir}/', f'{cylc_run_dir}/'] @pytest.mark.parametrize( 'args, expected_relink, expected_run_num, expected_run_dir', [ ( ['{cylc_run}/numbered', None, False], True, 1, '{cylc_run}/numbered/run1' ), ( ['{cylc_run}/named', 'dukat', False], False, None, '{cylc_run}/named/dukat' ), ( ['{cylc_run}/unnamed', None, True], False, None, '{cylc_run}/unnamed' ), ] ) def test_get_run_dir_info( args: list, expected_relink: bool, expected_run_num: Optional[int], expected_run_dir: Union[Path, str], tmp_run_dir: Callable ): """Test get_run_dir_info(). Params: args: Input args to function. expected_*: Expected return values. """ # Setup cylc_run_dir: Path = tmp_run_dir() sub = lambda x: Path(x.format(cylc_run=cylc_run_dir)) # noqa: E731 args[0] = sub(args[0]) expected_run_dir = sub(expected_run_dir) # Test assert get_run_dir_info(*args) == ( expected_relink, expected_run_num, expected_run_dir ) assert expected_run_dir.is_absolute() def test_get_run_dir_info__increment_run_num(tmp_run_dir: Callable): """Test that get_run_dir_info() increments run number and unlinks runN.""" # Setup cylc_run_dir: Path = tmp_run_dir() run_dir: Path = tmp_run_dir('gowron/run1') runN = run_dir.parent / WorkflowFiles.RUN_N assert os.path.lexists(runN) # Test assert get_run_dir_info(cylc_run_dir / 'gowron', None, False) == ( True, 2, cylc_run_dir / 'gowron' / 'run2' ) assert not os.path.lexists(runN) def test_get_run_dir_info__fail(tmp_run_dir: Callable): # Test that you can't install named runs when numbered runs exist cylc_run_dir: Path = tmp_run_dir() run_dir: Path = tmp_run_dir('martok/run1') with pytest.raises(WorkflowFilesError) as excinfo: get_run_dir_info(run_dir.parent, 'targ', False) assert "contains installed numbered runs" in str(excinfo.value) # Test that you can install numbered run in an empty dir base_dir = cylc_run_dir / 'odo' base_dir.mkdir() get_run_dir_info(base_dir, None, False) # But not when named runs exist tmp_run_dir('odo/meter') with pytest.raises(WorkflowFilesError) as excinfo: get_run_dir_info(base_dir, None, False) assert "contains an installed workflow" @pytest.mark.parametrize( 'symlink_dirs, err_msg, expected', [ ('log=$shortbread, share= $bourbon,share/cycle= $digestive, ', "There is an error in --symlink-dirs option:", None ), ('log=$shortbread share= $bourbon share/cycle= $digestive ', "There is an error in --symlink-dirs option:" " log=$shortbread share= $bourbon share/cycle= $digestive . " "Try entering option in the form --symlink-dirs=" "'log=$DIR, share=$DIR2, ...'", None ), ('run=$NICE, log= $Garibaldi, share/cycle=$RichTea', None, {'localhost': { 'run': '$NICE', 'log': '$Garibaldi', 'share/cycle': '$RichTea' }} ), ('some_other_dir=$bourbon', 'some_other_dir not a valid entry for --symlink-dirs', {'some_other_dir': '£bourbon'} ), ] ) def test_parse_cli_sym_dirs( symlink_dirs: str, err_msg: str, expected: Dict[str, Dict[str, Any]] ): """Test parse_cli_sym_dirs returns dict or correctly raises errors on cli symlink dir options""" if err_msg is not None: with pytest.raises(InputError) as exc: parse_cli_sym_dirs(symlink_dirs) assert err_msg in str(exc) else: actual = parse_cli_sym_dirs(symlink_dirs) assert actual == expected def test_validate_source_dir(tmp_run_dir: Callable, tmp_src_dir: Callable): cylc_run_dir: Path = tmp_run_dir() src_dir: Path = tmp_src_dir('ludlow') validate_source_dir(src_dir, 'ludlow') # Test that src dir must have flow file (src_dir / WorkflowFiles.FLOW_FILE).unlink() with pytest.raises(WorkflowFilesError): validate_source_dir(src_dir, 'ludlow') # Test that reserved dirnames not allowed in src dir src_dir = tmp_src_dir('roland') (src_dir / 'log').mkdir() with pytest.raises(WorkflowFilesError) as exc_info: validate_source_dir(src_dir, 'roland') assert "exists in source directory" in str(exc_info.value) # Test that src dir is allowed to be inside ~/cylc-run src_dir = cylc_run_dir / 'dieter' src_dir.mkdir() (src_dir / WorkflowFiles.FLOW_FILE).touch() validate_source_dir(src_dir, 'dieter') # Test that src dir is not allowed to be an installed dir. src_dir = cylc_run_dir / 'ajay' src_dir.mkdir() (src_dir / WorkflowFiles.Install.DIRNAME).mkdir() (src_dir / WorkflowFiles.FLOW_FILE).touch() with pytest.raises(WorkflowFilesError) as exc_info: validate_source_dir(src_dir, 'ajay') assert "exists in source directory" in str(exc_info.value) def test_install_workflow_failif_name_name(tmp_src_dir, tmp_run_dir): """If a run_name is given validate_workflow_name is called on the workflow and the run name in combination. """ src_dir: Path = tmp_src_dir('ludlow') # It only has a workflow name: with pytest.raises(WorkflowFilesError, match='can only contain'): install_workflow(src_dir, workflow_name='foo?') # It only has a run name: with pytest.raises(WorkflowFilesError, match='can only contain'): install_workflow(src_dir, run_name='foo?') # It has a legal workflow name, but an invalid run name: with pytest.raises(WorkflowFilesError, match='can only contain'): install_workflow(src_dir, workflow_name='foo', run_name='bar?') def test_install_workflow_failif_reserved_name(tmp_src_dir, tmp_run_dir): """Reserved names cause install validation failure. n.b. manually defined to avoid test dependency on workflow_files. """ src_dir = tmp_src_dir('ludlow') is_reserved = '(that filename is reserved)' reserved_names = { 'share', 'log', 'runN', 'suite.rc', 'work', '_cylc-install', 'flow.cylc', # .service fails because starting a workflow/run can't start with "." # And that check fails first. # '.service' } install_workflow(src_dir, workflow_name='ok', run_name='also_ok') for name in reserved_names: with pytest.raises(WorkflowFilesError, match=is_reserved): install_workflow(src_dir, workflow_name='ok', run_name=name) with pytest.raises(WorkflowFilesError, match=is_reserved): install_workflow(src_dir, workflow_name=name) with pytest.raises(WorkflowFilesError, match=is_reserved): install_workflow(src_dir, workflow_name=name, run_name='ok') cylc-flow-8.7.0/tests/f0000777000175000017500000000000015264420307017265 2functional/ustar alastairalastaircylc-flow-8.7.0/tests/functional/0000775000175000017500000000000015264420307017120 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-get-site-config/0000775000175000017500000000000015264420307023034 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-get-site-config/00-basic.t0000664000175000017500000000351515264420307024523 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test getting the global config . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-config" run_ok "${TEST_NAME}.validate" cylc config #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-get-items" run_fail "${TEST_NAME}.non-existent" \ cylc config --item='[this][doesnt]exist' #------------------------------------------------------------------------------- VAL1="$(cylc config -d --item '[platforms][localhost]use login shell')" VAL2="$(cylc config -d | sed -n '/\[\[localhost\]\]/,$p' | \ sed -n "0,/use login shell/s/^[ \t]*\(use login shell =.*\)/\1/p")" run_ok "${TEST_NAME_BASE}-check-output" \ test "use login shell = ${VAL1}" = "${VAL2}" #------------------------------------------------------------------------------- exit cylc-flow-8.7.0/tests/functional/cylc-get-site-config/03-host-bool-override.t0000664000175000017500000000233515264420307027167 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test site-config host item boolean override (GitHub #2282). . "$(dirname "$0")/test_header" set_test_number 3 create_test_global_config '' ' [platforms] [[localhost]] use login shell = True [[mytesthost]] use login shell = False' run_ok "${TEST_NAME_BASE}" \ cylc config --item='[platforms][mytesthost]use login shell' cmp_ok "${TEST_NAME_BASE}.stdout" <<<'False' cmp_ok "${TEST_NAME_BASE}.stderr" <'/dev/null' exit cylc-flow-8.7.0/tests/functional/cylc-get-site-config/02-jinja2.t0000664000175000017500000000224015264420307024613 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test site-config defaults actually validate (GitHub #1865). . "$(dirname "$0")/test_header" set_test_number 3 create_test_global_config '#!jinja2' ' {% set UTC_MODE = True %} [scheduler] UTC mode = {{UTC_MODE}}' run_ok "${TEST_NAME_BASE}" cylc config -d --item='[scheduler]UTC mode' cmp_ok "${TEST_NAME_BASE}.stdout" <<<'True' cmp_ok "${TEST_NAME_BASE}.stderr" <'/dev/null' exit cylc-flow-8.7.0/tests/functional/cylc-get-site-config/test_header0000777000175000017500000000000015264420307031351 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-get-site-config/04-homeless.t0000664000175000017500000000313415264420307025262 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Check undefined $HOME does not break: # a) use of $HOME in global config (GitHub #2895) # b) global config Jinja2 support (GitHub #5155) . "$(dirname "$0")/test_header" set_test_number 5 # shellcheck disable=SC2016 create_test_global_config '' ' [install] [[symlink dirs]] [[[localhost]]] run = $HOME/dr-malcolm ' run_ok "${TEST_NAME_BASE}" \ env -u HOME \ cylc config --item='[install][symlink dirs][localhost]run' cmp_ok "${TEST_NAME_BASE}.stdout" <<<"\$HOME/dr-malcolm" # The test global config is created with #!Jinja2 at the top, in case of any # Jinja2 code in global-tests.cylc. Parsec Jinja2 support uses $HOME to find # custom filters etc. GitHub #5155. for DIR in Filters Tests Globals; do grep_ok "\$HOME undefined: can't load ~/.cylc/Jinja2$DIR" "${TEST_NAME_BASE}.stderr" done cylc-flow-8.7.0/tests/functional/cylc-get-site-config/01-defaults.t0000664000175000017500000000231515264420307025247 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test site-config defaults actually validate (GitHub #1865). . "$(dirname "$0")/test_header" set_test_number 1 # Empty it (of non-default global-tests.cylc items, which would then be retrieved # by "cylc config" below). echo '' > "$CYLC_CONF_PATH/global.cylc" # Replace it entirely with system defaults. cylc config -d > "$CYLC_CONF_PATH/global.cylc" # Check that the new file parses OK. run_ok "${TEST_NAME_BASE}" cylc config cylc-flow-8.7.0/tests/functional/cylc-get-site-config/05-host-bool-override.t0000664000175000017500000000275615264420307027200 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test site-config host item boolean override (GitHub #2282). . "$(dirname "$0")/test_header" set_test_number 2 cat etc/global.cylc <<'__hi__' [platforms] [[desktop\d\d|laptop\d\d]] [[sugar]] login hosts = localhost job runner = slurm [[hpc]] login hosts = hpcl1, hpcl2 retrieve job logs = True job runner = pbs [[hpcl1-bg]] login hosts = hpcl1 retrieve job logs = True job runner = background [[hpcl2-bg]] login hosts = hpcl2 retrieve job logs = True job runner = background __hi__ export CYLC_CONF_PATH="${PWD}/etc" run_ok "${TEST_NAME_BASE}" cylc config cmp_ok "${TEST_NAME_BASE}.stderr" foo [runtime] [[start, foo]] script = true cylc-flow-8.7.0/tests/functional/intercycle/04-datetime-abs-2/0000775000175000017500000000000015264420307024200 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intercycle/04-datetime-abs-2/reference.log0000664000175000017500000000051315264420307026640 0ustar alastairalastairInitial point: 20000101T0000Z Final point: 20000103T0000Z 20000101T0000Z/start -triggered off [] 20000101T0000Z/foo -triggered off ['20000101T0000Z/start'] 20000102T0000Z/foo -triggered off ['20000101T0000Z/start'] 20000103T0000Z/foo -triggered off ['20000101T0000Z/start'] 20000102T1200Z/bar -triggered off ['20000102T0000Z/foo'] cylc-flow-8.7.0/tests/functional/intercycle/04-datetime-abs-2/flow.cylc0000664000175000017500000000054515264420307026027 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] initial cycle point = 20000101T00Z [[graph]] R1 = start T00 = start[^] => foo R1/20000102T12Z = foo[20000102T00Z] => bar [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/intercycle/03-datetime-abs.t0000664000175000017500000000201115264420307024217 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test dependency on absolute datetime point # https://github.com/cylc/cylc-flow/issues/1951 . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/intercycle/01-future.t0000664000175000017500000000171615264420307023203 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test future cycle dependencies. . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/intercycle/01-future/0000775000175000017500000000000015264420307023011 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intercycle/01-future/reference.log0000664000175000017500000000044115264420307025451 0ustar alastairalastairInitial point: 20120101T00 Final point: 20120101T18 20120101T06/a -triggered off [] 20120101T12/a -triggered off [] 20120101T00/b -triggered off ['20120101T06/a'] 20120101T06/b -triggered off ['20120101T12/a'] 20120101T18/a -triggered off [] 20120101T12/b -triggered off ['20120101T18/a'] cylc-flow-8.7.0/tests/functional/intercycle/01-future/flow.cylc0000664000175000017500000000040015264420307024626 0ustar alastairalastair[scheduler] cycle point format = %Y%m%dT%H [scheduling] initial cycle point = 20120101T00 final cycle point = 20120101T18 [[graph]] PT6H = a[+PT6H] => b R/+PT6H/PT6H = a [runtime] [[a,b]] script = "true" # fast cylc-flow-8.7.0/tests/functional/intercycle/04-datetime-abs-2.t0000664000175000017500000000240215264420307024363 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test dependency on absolute datetime point # https://github.com/cylc/cylc-flow/issues/1951 . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}" \ cylc play --reference-test --debug --no-detach --fcp=20000103 "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/intercycle/03-datetime-abs/0000775000175000017500000000000015264420307024040 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intercycle/03-datetime-abs/reference.log0000664000175000017500000000032415264420307026500 0ustar alastairalastairInitial point: 20100101T0000Z Final point: 20100102T0000Z 20100101T0600Z/fixed_cycle -triggered off [] 20100101T0000Z/init_cycle -triggered off [] 20100101T1200Z/foo -triggered off ['20100101T0600Z/fixed_cycle'] cylc-flow-8.7.0/tests/functional/intercycle/03-datetime-abs/flow.cylc0000664000175000017500000000057115264420307025666 0ustar alastairalastair[scheduler] UTC mode = True [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] initial cycle point = 20100101T0000Z final cycle point = +P1D [[graph]] R1 = init_cycle R1/20100101T0600Z = fixed_cycle T12 = fixed_cycle[20100101T0600Z] => foo [runtime] [[foo, fixed_cycle, init_cycle]] script = true cylc-flow-8.7.0/tests/functional/intercycle/05-datetime-abs-3.t0000664000175000017500000000201115264420307024361 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test dependency on absolute datetime point # https://github.com/cylc/cylc-flow/issues/1951 . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/intercycle/00-past/0000775000175000017500000000000015264420307022445 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intercycle/00-past/reference.log0000664000175000017500000000035215264420307025106 0ustar alastairalastairInitial point: 20120101T0000Z Final point: 20120101T1800Z 20120101T0000Z/A -triggered off [] 20120101T1200Z/A -triggered off [] 20120101T0600Z/B -triggered off ['20120101T0000Z/A'] 20120101T1800Z/B -triggered off ['20120101T1200Z/A'] cylc-flow-8.7.0/tests/functional/intercycle/00-past/flow.cylc0000664000175000017500000000062415264420307024272 0ustar alastairalastair[meta] title = "reference test workflow: zig-zag intercycle dependencies" description = """ Task A should only run at 0, 12 hours; Task B at 6, 18""" [scheduler] UTC mode = True [scheduling] initial cycle point = 20120101T00 final cycle point = 20120101T18 [[graph]] T00,T12 = "A" T06,T18 = "A[-PT6H] => B" [runtime] [[A, B]] script = "true" # fast cylc-flow-8.7.0/tests/functional/intercycle/00-past.t0000664000175000017500000000171415264420307022635 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test intercycle dependencies. . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/intercycle/test_header0000777000175000017500000000000015264420307027576 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/intercycle/05-datetime-abs-3/0000775000175000017500000000000015264420307024202 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intercycle/05-datetime-abs-3/reference.log0000664000175000017500000000032415264420307026642 0ustar alastairalastairInitial point: 20100101T0000Z Final point: 20100102T0000Z 20100101T0600Z/fixed_cycle -triggered off [] 20100101T0000Z/init_cycle -triggered off [] 20100101T1200Z/foo -triggered off ['20100101T0600Z/fixed_cycle'] cylc-flow-8.7.0/tests/functional/intercycle/05-datetime-abs-3/flow.cylc0000664000175000017500000000061115264420307026023 0ustar alastairalastair[scheduler] UTC mode = True [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] initial cycle point = 20100101T0000Z final cycle point = +P1D [[graph]] R1 = init_cycle R1/2010-01-01T06:00+00:00 = fixed_cycle T12 = fixed_cycle[2010-01-01T06:00+00:00] => foo [runtime] [[foo, fixed_cycle, init_cycle]] script = true cylc-flow-8.7.0/tests/functional/intercycle/02-integer-abs.t0000664000175000017500000000237215264420307024071 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test dependency on absolute integer point # https://github.com/cylc/cylc-flow/issues/1394 . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}" \ cylc play --reference-test --debug --no-detach --fcp=5 "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/cylc-clean/0000775000175000017500000000000015264420307021132 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-clean/05-old-remote-contact.t0000664000175000017500000000377515264420307025255 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # ----------------------------------------------------------------------------- # Test that cylc clean successfully removes the workflow on remote host even # when there is a leftover contact file with an unreachable host recorded in it export REQUIRE_PLATFORM='loc:remote fs:indep comms:tcp' . "$(dirname "$0")/test_header" set_test_number 3 SSH_CMD="$(cylc config -d -i "[platforms][${CYLC_TEST_PLATFORM}]ssh command") ${CYLC_TEST_HOST}" init_workflow "${TEST_NAME_BASE}" << __FLOW__ [scheduling] [[graph]] R1 = dilophosaurus [runtime] [[dilophosaurus]] platform = ${CYLC_TEST_PLATFORM} __FLOW__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "$WORKFLOW_NAME" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "$WORKFLOW_NAME" --no-detach # Create a fake old contact file on the remote host # shellcheck disable=SC2259 echo | $SSH_CMD "cat > \$HOME/cylc-run/${WORKFLOW_NAME}/.service/contact" << EOF CYLC_API=5 CYLC_VERSION=8.0.0 CYLC_WORKFLOW_COMMAND=echo Hello John CYLC_WORKFLOW_HOST=unreachable.isla_nublar.ingen CYLC_WORKFLOW_ID=${WORKFLOW_NAME} CYLC_WORKFLOW_PID=99999 CYLC_WORKFLOW_PORT=00000 EOF TEST_NAME="cylc-clean" run_ok "$TEST_NAME" cylc clean --remote "$WORKFLOW_NAME" dump_std "$TEST_NAME" purge cylc-flow-8.7.0/tests/functional/cylc-clean/00-basic.t0000664000175000017500000001232515264420307022620 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # ----------------------------------------------------------------------------- # Test the cylc clean command . "$(dirname "$0")/test_header" if ! command -v 'tree' >'/dev/null'; then skip_all '"tree" command not available' fi set_test_number 12 # Generate random name for symlink dirs to avoid any clashes with other tests SYM_NAME="$(mktemp -u)" SYM_NAME="${SYM_NAME##*tmp.}" create_test_global_config "" " [install] [[symlink dirs]] [[[localhost]]] run = ${TEST_DIR}/${SYM_NAME}/run log = ${TEST_DIR}/${SYM_NAME}/log log/job = ${TEST_DIR}/${SYM_NAME}/job share = ${TEST_DIR}/${SYM_NAME}/share share/cycle = ${TEST_DIR}/${SYM_NAME}/cycle work = ${TEST_DIR}/${SYM_NAME}/work " install_workflow "${TEST_NAME_BASE}" basic-workflow run_ok "${TEST_NAME_BASE}-val" cylc validate "$WORKFLOW_NAME" # Create a fake sibling workflow dir in the ${SYM_NAME}/log dir: mkdir "${TEST_DIR}/${SYM_NAME}/log/cylc-run/${CYLC_TEST_REG_BASE}/leave-me-alone" FUNCTIONAL_DIR="${TEST_SOURCE_DIR_BASE%/*}" # ----------------------------------------------------------------------------- TEST_NAME="run-dir-readlink-pre-clean" readlink "$WORKFLOW_RUN_DIR" > "${TEST_NAME}.stdout" cmp_ok "${TEST_NAME}.stdout" <<< "${TEST_DIR}/${SYM_NAME}/run/cylc-run/${WORKFLOW_NAME}" INSTALL_LOG_FILE=$(ls "${TEST_DIR}/${SYM_NAME}/log/cylc-run/${WORKFLOW_NAME}/log/install") TEST_NAME="test-dir-tree-pre-clean" run_ok "${TEST_NAME}" tree --noreport --charset=ascii "${TEST_DIR}/${SYM_NAME}/"*"/cylc-run/${CYLC_TEST_REG_BASE}" # Note: backticks need to be escaped in the heredoc cmp_ok "${TEST_NAME}.stdout" << __TREE__ ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} \`-- share \`-- cycle ${TEST_DIR}/${SYM_NAME}/job/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} \`-- log \`-- job ${TEST_DIR}/${SYM_NAME}/log/cylc-run/${CYLC_TEST_REG_BASE} |-- ${FUNCTIONAL_DIR} | \`-- cylc-clean | \`-- ${TEST_NAME_BASE} | \`-- log | |-- install | | \`-- ${INSTALL_LOG_FILE} | \`-- job -> ${TEST_DIR}/${SYM_NAME}/job/cylc-run/${WORKFLOW_NAME}/log/job \`-- leave-me-alone ${TEST_DIR}/${SYM_NAME}/run/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} |-- _cylc-install | \`-- source -> ${TEST_DIR}/${WORKFLOW_NAME} |-- flow.cylc |-- log -> ${TEST_DIR}/${SYM_NAME}/log/cylc-run/${WORKFLOW_NAME}/log |-- share -> ${TEST_DIR}/${SYM_NAME}/share/cylc-run/${WORKFLOW_NAME}/share \`-- work -> ${TEST_DIR}/${SYM_NAME}/work/cylc-run/${WORKFLOW_NAME}/work ${TEST_DIR}/${SYM_NAME}/share/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} \`-- share \`-- cycle -> ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${WORKFLOW_NAME}/share/cycle ${TEST_DIR}/${SYM_NAME}/work/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} \`-- work __TREE__ # ----------------------------------------------------------------------------- TEST_NAME="clean" run_ok "$TEST_NAME" cylc clean "$WORKFLOW_NAME" dump_std "$TEST_NAME" # ----------------------------------------------------------------------------- TEST_NAME="run-dir-not-exist-post-clean" exists_fail "$WORKFLOW_RUN_DIR" TEST_NAME="test-dir-tree-post-clean" run_ok "${TEST_NAME}" tree --noreport --charset=ascii "${TEST_DIR}/${SYM_NAME}/"*"/cylc-run/${CYLC_TEST_REG_BASE}" cmp_ok "${TEST_NAME}.stdout" << __TREE__ ${TEST_DIR}/${SYM_NAME}/log/cylc-run/${CYLC_TEST_REG_BASE} \`-- leave-me-alone __TREE__ # ----------------------------------------------------------------------------- TEST_NAME="clean-non-exist" run_ok "$TEST_NAME" cylc clean "$WORKFLOW_NAME" dump_std "$TEST_NAME" cmp_ok "${TEST_NAME}.stdout" << __EOF__ INFO - No directory to clean at ${WORKFLOW_RUN_DIR} __EOF__ # ----------------------------------------------------------------------------- TEST_NAME="clean-non-exist-pattern" run_ok "$TEST_NAME" cylc clean "nope*" dump_std "$TEST_NAME" cmp_ok "${TEST_NAME}.stderr" << __EOF__ WARNING - No stopped workflows matching nope* __EOF__ # ----------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/cylc-clean/02-targeted.t0000664000175000017500000001467315264420307023350 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # ----------------------------------------------------------------------------- # Test the cylc clean command . "$(dirname "$0")/test_header" if ! command -v 'tree' >'/dev/null'; then skip_all '"tree" command not available' fi set_test_number 16 # Generate random name for symlink dirs to avoid any clashes with other tests SYM_NAME="$(mktemp -u)" SYM_NAME="sym-${SYM_NAME##*tmp.}" create_test_global_config "" " [install] [[symlink dirs]] [[[localhost]]] run = ${TEST_DIR}/${SYM_NAME}/run log = ${TEST_DIR}/${SYM_NAME}/other share = ${TEST_DIR}/${SYM_NAME}/other work = ${TEST_DIR}/${SYM_NAME}/other # Need to override any symlink dirs set in global.cylc: share/cycle = " install_workflow "${TEST_NAME_BASE}" basic-workflow # Also create some other file touch "${WORKFLOW_RUN_DIR}/darmok.cylc" run_ok "${TEST_NAME_BASE}-val" cylc validate "$WORKFLOW_NAME" FUNCTIONAL_DIR="${TEST_SOURCE_DIR_BASE%/*}" # ----------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run-dir-readlink-pre-clean" readlink "$WORKFLOW_RUN_DIR" > "${TEST_NAME}.stdout" cmp_ok "${TEST_NAME}.stdout" <<< "${TEST_DIR}/${SYM_NAME}/run/cylc-run/${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-testdir-tree-pre-clean" run_ok "${TEST_NAME}" tree -L 5 --noreport --charset=ascii "${TEST_DIR}/${SYM_NAME}/"*"/cylc-run/${CYLC_TEST_REG_BASE}" # Note: backticks need to be escaped in the heredoc cmp_ok "${TEST_NAME}.stdout" << __TREE__ ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} |-- log | \`-- install |-- share \`-- work ${TEST_DIR}/${SYM_NAME}/run/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} |-- _cylc-install | \`-- source -> ${TEST_DIR}/${WORKFLOW_NAME} |-- darmok.cylc |-- flow.cylc |-- log -> ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${WORKFLOW_NAME}/log |-- share -> ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${WORKFLOW_NAME}/share \`-- work -> ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${WORKFLOW_NAME}/work __TREE__ # ----------------------------------------------------------------------------- # Clean the log dir only run_ok "${TEST_NAME_BASE}-targeted-clean-1" cylc clean "$WORKFLOW_NAME" \ --rm log TEST_NAME="${TEST_NAME_BASE}-testdir-tree-1" run_ok "${TEST_NAME}" tree -L 5 --noreport --charset=ascii "${TEST_DIR}/${SYM_NAME}/"*"/cylc-run/${CYLC_TEST_REG_BASE}" cmp_ok "${TEST_NAME}.stdout" << __TREE__ ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} |-- share \`-- work ${TEST_DIR}/${SYM_NAME}/run/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} |-- _cylc-install | \`-- source -> ${TEST_DIR}/${WORKFLOW_NAME} |-- darmok.cylc |-- flow.cylc |-- share -> ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${WORKFLOW_NAME}/share \`-- work -> ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${WORKFLOW_NAME}/work __TREE__ # ----------------------------------------------------------------------------- # Clean using a glob run_ok "${TEST_NAME_BASE}-targeted-clean-2" cylc clean "$WORKFLOW_NAME" \ --rm 'wo*' TEST_NAME="${TEST_NAME_BASE}-testdir-tree-2" run_ok "${TEST_NAME}" tree -L 5 --noreport --charset=ascii "${TEST_DIR}/${SYM_NAME}/"*"/cylc-run/${CYLC_TEST_REG_BASE}" # Note: when using glob, the symlink dir target is not deleted cmp_ok "${TEST_NAME}.stdout" << __TREE__ ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} \`-- share ${TEST_DIR}/${SYM_NAME}/run/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} |-- _cylc-install | \`-- source -> ${TEST_DIR}/${WORKFLOW_NAME} |-- darmok.cylc |-- flow.cylc \`-- share -> ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${WORKFLOW_NAME}/share __TREE__ # ----------------------------------------------------------------------------- # Clean the last remaining symlink dir run_ok "${TEST_NAME_BASE}-targeted-clean-3" cylc clean "$WORKFLOW_NAME" \ --rm 'share' TEST_NAME="${TEST_NAME_BASE}-testdir-tree-3" run_ok "${TEST_NAME}" tree -L 5 --noreport --charset=ascii "${TEST_DIR}/${SYM_NAME}/"*"/cylc-run/${CYLC_TEST_REG_BASE}" cmp_ok "${TEST_NAME}.stdout" << __TREE__ ${TEST_DIR}/${SYM_NAME}/run/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} |-- _cylc-install | \`-- source -> ${TEST_DIR}/${WORKFLOW_NAME} |-- darmok.cylc \`-- flow.cylc __TREE__ # ----------------------------------------------------------------------------- # Clean multiple things run_ok "${TEST_NAME_BASE}-targeted-clean-3" cylc clean "$WORKFLOW_NAME" \ --rm 'flow.cylc' --rm 'darmok.cylc' TEST_NAME="${TEST_NAME_BASE}-testdir-tree-3" run_ok "${TEST_NAME}" tree -L 5 --noreport --charset=ascii "${TEST_DIR}/${SYM_NAME}/"*"/cylc-run/${CYLC_TEST_REG_BASE}" cmp_ok "${TEST_NAME}.stdout" << __TREE__ ${TEST_DIR}/${SYM_NAME}/run/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} \`-- _cylc-install \`-- source -> ${TEST_DIR}/${WORKFLOW_NAME} __TREE__ # ----------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/cylc-clean/03-multi.t0000664000175000017500000000762715264420307022705 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # ----------------------------------------------------------------------------- # Test cleaning multiple run dirs . "$(dirname "$0")/test_header" if ! command -v 'tree' > /dev/null; then skip_all '"tree" command not available' fi set_test_number 18 RND_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" WORKFLOW_NAME="${RND_NAME}/cylc-clean" WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}" create_workflow() { mkdir -p "${TEST_DIR}/${WORKFLOW_NAME}" # make source dir touch "${TEST_DIR}/${WORKFLOW_NAME}/flow.cylc" cylc install "${TEST_DIR}/${WORKFLOW_NAME}" --workflow-name="${WORKFLOW_NAME}" } # ----------------------------------------------------------------------------- for _ in 1 2; do create_workflow done TEST_NAME="tree-pre-clean-1" run_ok "${TEST_NAME}" tree --noreport --charset=ascii -L 1 "${HOME}/cylc-run/${WORKFLOW_NAME}" # Note: backticks need to be escaped in the heredoc cmp_ok "${TEST_NAME}.stdout" << __TREE__ ${HOME}/cylc-run/${WORKFLOW_NAME} |-- _cylc-install |-- run1 |-- run2 \`-- runN -> run2 __TREE__ # Test trying to clean multiple run dirs without --yes fails: run_fail "${TEST_NAME_BASE}-no" cylc clean "$WORKFLOW_NAME" exists_ok "${WORKFLOW_RUN_DIR}/run1" exists_ok "${WORKFLOW_RUN_DIR}/run2" # Should work with --yes (removes top level dir too): run_ok "${TEST_NAME_BASE}-yes" cylc clean -y "$WORKFLOW_NAME" exists_fail "${RUN_DIR}/${RND_NAME}" # ----------------------------------------------------------------------------- # Should continue cleaning a list of workflows even if one fails. for _ in 1 2; do create_workflow done TEST_NAME="tree-pre-clean-2" run_ok "${TEST_NAME}" tree --noreport --charset=ascii -L 1 "${HOME}/cylc-run/${WORKFLOW_NAME}" cmp_ok "${TEST_NAME}.stdout" << __TREE__ ${HOME}/cylc-run/${WORKFLOW_NAME} |-- _cylc-install |-- run1 |-- run2 \`-- runN -> run2 __TREE__ mkdir "${WORKFLOW_RUN_DIR}/run1/.service" echo 'x' > "${WORKFLOW_RUN_DIR}/run1/.service/db" # corrupted db! TEST_NAME="${TEST_NAME_BASE}-yes-no" run_fail "${TEST_NAME}" \ cylc clean -y "$WORKFLOW_NAME/run1" "$WORKFLOW_NAME/run2" grep_ok "file is not a database" "${TEST_NAME}.stderr" -e TEST_NAME="tree-post-clean-2" run_ok "${TEST_NAME}" tree --noreport --charset=ascii -L 1 "${HOME}/cylc-run/${WORKFLOW_NAME}" cmp_ok "${TEST_NAME}.stdout" << __TREE__ ${HOME}/cylc-run/${WORKFLOW_NAME} |-- _cylc-install \`-- run1 __TREE__ purge "$WORKFLOW_NAME" # ----------------------------------------------------------------------------- # Should not clean top level dir if not empty. create_workflow touch "${WORKFLOW_RUN_DIR}/jellyfish.txt" TEST_NAME="tree-pre-clean-3" run_ok "${TEST_NAME}" tree --noreport --charset=ascii -L 1 "${HOME}/cylc-run/${WORKFLOW_NAME}" cmp_ok "${TEST_NAME}.stdout" << __TREE__ ${HOME}/cylc-run/${WORKFLOW_NAME} |-- _cylc-install |-- jellyfish.txt |-- run1 \`-- runN -> run1 __TREE__ run_ok "${TEST_NAME}" cylc clean -y "$WORKFLOW_NAME" TEST_NAME="tree-post-clean-3" run_ok "${TEST_NAME}" tree --noreport --charset=ascii -L 1 "${HOME}/cylc-run/${WORKFLOW_NAME}" cmp_ok "${TEST_NAME}.stdout" << __TREE__ ${HOME}/cylc-run/${WORKFLOW_NAME} |-- _cylc-install \`-- jellyfish.txt __TREE__ purge cylc-flow-8.7.0/tests/functional/cylc-clean/06-nfs.t0000664000175000017500000000534115264420307022333 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # ----------------------------------------------------------------------------- # This test covers the interaction between "cylc clean" and "cylc cat-log -m t" # on the NFS filesystem. The "cat-log" should not block the "clean". # # Tests: https://github.com/cylc/cylc-flow/pull/5359 # # If you try to delete a file that is stored on NFS, which is open for reading # by another process (e.g. `tail -f`), NFS will remove the file, but put a # ".nfs" file in its place. This ".nfs" file will cause "rm" operations on # the directory containing the NFS files to fail with one of two errors: # * https://docs.python.org/3/library/errno.html#errno.EBUSY # * https://docs.python.org/3/library/errno.html#errno.ENOTEMPTY # # To prevent "cylc cat-log -m t" which calls "tail -f" from blocking # "cylc clean" commands, we retry the "rm" operation with a delay. This # allows the "tail -f" to fail and release its file lock allowing the # "rm" to pass on a subsequent attempt. . "$(dirname "$0")/test_header" set_test_number 2 # install a blank source workflow init_workflow "${TEST_NAME_BASE}" <<< '# blank workflow' # add a scheduler log file with something written to it WORKFLOW_LOG_DIR="${WORKFLOW_RUN_DIR}/log/scheduler" mkdir -p "$WORKFLOW_LOG_DIR" LOG_STUFF='foo bar baz' echo "${LOG_STUFF}" > "${WORKFLOW_LOG_DIR}/01-start-01.log" # start cat-log running - this runs "tail -f" cylc cat-log -m t "$WORKFLOW_NAME" > out 2>err & PID="$!" # wait for tail to start poll pgrep -P "$PID" tail # try to clean the workflow run_ok "${TEST_NAME_BASE}-clean" cylc clean -y "${WORKFLOW_NAME}" # the tail command should have detected that the file isn't there any more # and exited -> cat-log should have exited poll_pid_done "$PID" # ensure the log dir was removed correctly # run_ok "${TEST_NAME_BASE}-dir-removed" [[ ! -d "${WORKFLOW_LOG_DIR}" ]] TEST_NAME="${TEST_NAME_BASE}-log-dir-removed" if [[ -d "${WORKFLOW_LOG_DIR}" ]]; then fail "${TEST_NAME}" else ok "${TEST_NAME}" fi purge cylc-flow-8.7.0/tests/functional/cylc-clean/04-runN.t0000664000175000017500000000236215264420307022465 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # ----------------------------------------------------------------------------- # Test cleaning latest run using /runN . "$(dirname "$0")/test_header" set_test_number 5 install_workflow "$TEST_NAME_BASE" basic-workflow true exists_ok "${WORKFLOW_RUN_DIR}/run1" exists_ok "${WORKFLOW_RUN_DIR}/runN" run_ok "${TEST_NAME_BASE}-clean" cylc clean "${WORKFLOW_NAME}/runN" exists_fail "${WORKFLOW_RUN_DIR}/run1" exists_fail "${WORKFLOW_RUN_DIR}/runN" purge cylc-flow-8.7.0/tests/functional/cylc-clean/basic-workflow/0000775000175000017500000000000015264420307024063 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-clean/basic-workflow/flow.cylc0000664000175000017500000000013315264420307025703 0ustar alastairalastair[scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = darmok cylc-flow-8.7.0/tests/functional/cylc-clean/test_header0000777000175000017500000000000015264420307027447 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-clean/01-remote.t0000664000175000017500000001263515264420307023037 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # ----------------------------------------------------------------------------- # Test that cylc clean successfully removes the workflow on remote host export REQUIRE_PLATFORM='loc:remote fs:indep' . "$(dirname "$0")/test_header" SSH_CMD="$(cylc config -d -i "[platforms][${CYLC_TEST_PLATFORM}]ssh command") ${CYLC_TEST_HOST}" if ! $SSH_CMD command -v 'tree' > '/dev/null'; then skip_all "'tree' command not available on remote host ${CYLC_TEST_HOST}" fi set_test_number 10 # Generate random name for symlink dirs to avoid any clashes with other tests SYM_NAME="$(mktemp -u)" SYM_NAME="${SYM_NAME##*tmp.}" create_test_global_config "" " [install] [[symlink dirs]] [[[${CYLC_TEST_INSTALL_TARGET}]]] run = ${TEST_DIR}/${SYM_NAME}/run log = ${TEST_DIR}/${SYM_NAME}/other log/job = ${TEST_DIR}/${SYM_NAME}/job share = ${TEST_DIR}/${SYM_NAME}/other share/cycle = ${TEST_DIR}/${SYM_NAME}/cycle work = ${TEST_DIR}/${SYM_NAME}/other " init_workflow "${TEST_NAME_BASE}" << __FLOW__ [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = santa [runtime] [[root]] platform = ${CYLC_TEST_PLATFORM} script = touch flow.cylc # testing that remote clean does not scan for workflows __FLOW__ FUNCTIONAL_DIR="${TEST_SOURCE_DIR_BASE%/*}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "$WORKFLOW_NAME" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "$WORKFLOW_NAME" --no-detach # Create a fake sibling workflow dir: $SSH_CMD mkdir "${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${CYLC_TEST_REG_BASE}/leave-me-alone" # ----------------------------------------------------------------------------- TEST_NAME="run-dir-readlink-pre-clean.remote" $SSH_CMD readlink "\$HOME/cylc-run/${WORKFLOW_NAME}" > "${TEST_NAME}.stdout" cmp_ok "${TEST_NAME}.stdout" <<< "${TEST_DIR}/${SYM_NAME}/run/cylc-run/${WORKFLOW_NAME}" TEST_NAME="test-dir-tree-pre-clean.remote" # shellcheck disable=SC2086 run_ok "${TEST_NAME}" $SSH_CMD tree -L 8 --noreport --charset=ascii \ "${TEST_DIR}/${SYM_NAME}/"'*'"/cylc-run/${CYLC_TEST_REG_BASE}" # Note: backticks need to be escaped in the heredoc cmp_ok "${TEST_NAME}.stdout" << __TREE__ ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${CYLC_TEST_REG_BASE} |-- ${FUNCTIONAL_DIR} | \`-- cylc-clean | \`-- ${TEST_NAME_BASE} | \`-- share | \`-- cycle | \`-- 1 \`-- leave-me-alone ${TEST_DIR}/${SYM_NAME}/job/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} \`-- log \`-- job \`-- 1 \`-- santa |-- 01 \`-- NN -> 01 ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} |-- log | \`-- job -> ${TEST_DIR}/${SYM_NAME}/job/cylc-run/${WORKFLOW_NAME}/log/job |-- share | \`-- cycle -> ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${WORKFLOW_NAME}/share/cycle \`-- work \`-- 1 \`-- santa \`-- flow.cylc ${TEST_DIR}/${SYM_NAME}/run/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean \`-- ${TEST_NAME_BASE} |-- log -> ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${WORKFLOW_NAME}/log |-- share -> ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${WORKFLOW_NAME}/share \`-- work -> ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${WORKFLOW_NAME}/work __TREE__ # ----------------------------------------------------------------------------- TEST_NAME="cylc-clean-ok" run_ok "$TEST_NAME" cylc clean "$WORKFLOW_NAME" --timeout PT2M # (timeout opt is covered by unit tests but no harm double-checking here) dump_std "$TEST_NAME" TEST_NAME="run-dir-not-exist-post-clean.local" # (Could use the function `exists_ok` here instead, but this keeps it consistent with the remote test below) if [[ ! -e "$WORKFLOW_RUN_DIR" ]]; then ok "$TEST_NAME" else fail "$TEST_NAME" fi TEST_NAME="run-dir-not-exist-post-clean.remote" if $SSH_CMD [[ ! -a "\$HOME/cylc-run/${WORKFLOW_NAME}" ]]; then ok "$TEST_NAME" else fail "$TEST_NAME" fi TEST_NAME="test-dir-tree-post-clean.remote" # shellcheck disable=SC2086 run_ok "${TEST_NAME}" $SSH_CMD tree --noreport --charset=ascii \ "${TEST_DIR}/${SYM_NAME}/"'*'"/cylc-run/${CYLC_TEST_REG_BASE}" # Note: backticks need to be escaped in the heredoc cmp_ok "${TEST_NAME}.stdout" << __TREE__ ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${CYLC_TEST_REG_BASE} \`-- leave-me-alone __TREE__ purge cylc-flow-8.7.0/tests/functional/pause-resume/0000775000175000017500000000000015264420307021533 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/pause-resume/00-workflow/0000775000175000017500000000000015264420307023622 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/pause-resume/00-workflow/reference.log0000664000175000017500000000024315264420307026262 0ustar alastairalastair20140101T00/pause_resume -triggered off [] 20140101T00/foo -triggered off ['20140101T00/pause_resume'] 20140101T00/bar -triggered off ['20140101T00/pause_resume'] cylc-flow-8.7.0/tests/functional/pause-resume/00-workflow/flow.cylc0000664000175000017500000000117615264420307025452 0ustar alastairalastair[meta] title = "pause/resume test workflow" # https://github.com/cylc/cylc-flow/pull/843 # See also tests/f/hold-release/00-workflow [scheduler] cycle point format = %Y%m%dT%H [scheduling] initial cycle point = 20140101T00 final cycle point = 20140101T00 [[graph]] R1 = "pause_resume => foo & bar" T00, T06 = "bar" [runtime] [[pause_resume]] script = """ wait cylc pause "${CYLC_WORKFLOW_ID}" cylc__job__poll_grep_workflow_log 'Command "pause" actioned' cylc play "${CYLC_WORKFLOW_ID}" """ [[foo,bar]] script = true cylc-flow-8.7.0/tests/functional/pause-resume/12-pause-then-retry.t0000775000175000017500000000175615264420307025370 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test workflow pause => retry and submit-retry => workflow resume . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/pause-resume/21-client/0000775000175000017500000000000015264420307023231 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/pause-resume/21-client/reference.log0000664000175000017500000000006715264420307025675 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/pause-resume/21-client/flow.cylc0000664000175000017500000000041115264420307025050 0ustar alastairalastair#!jinja2 [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = P1M [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = true cylc-flow-8.7.0/tests/functional/pause-resume/12-pause-then-retry/0000775000175000017500000000000015264420307025167 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/pause-resume/12-pause-then-retry/reference.log0000664000175000017500000000034615264420307027633 0ustar alastairalastairFinal point: 1 1/t-submit-retry-able -triggered off [] 1/t-retry-able -triggered off [] 1/t-pause -triggered off ['1/t-retry-able', '1/t-submit-retry-able'] 1/t-retry-able -triggered off [] 1/t-submit-retry-able -triggered off [] cylc-flow-8.7.0/tests/functional/pause-resume/12-pause-then-retry/flow.cylc0000664000175000017500000000370615264420307027020 0ustar alastairalastair[meta] title = Test: run task - pause workflow - task job retry - resume workflow [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT3M [scheduling] [[graph]] R1 = """ t-retry-able:start => t-pause t-submit-retry-able:submit => t-pause """ [runtime] [[t-pause]] script = """ cylc pause "${CYLC_WORKFLOW_ID}" cylc__job__poll_grep_workflow_log 'Command "pause" actioned' # Poll t-submit-retry-able, should return submit-fail cylc poll "${CYLC_WORKFLOW_ID}//*/t-submit-retry-able" # Allow t-retry-able to continue rm -f "${CYLC_WORKFLOW_RUN_DIR}/file" cylc__job__poll_grep_workflow_log -E \ '1/t-retry-able/01:running.* => waiting' cylc__job__poll_grep_workflow_log -E \ '1/t-submit-retry-able/01:submitted.* => waiting' # Resume the workflow cylc play "${CYLC_WORKFLOW_ID}" cylc__job__poll_grep_workflow_log -E \ '1/t-retry-able:waiting.* => waiting\(queued\)' cylc__job__poll_grep_workflow_log -E \ '1/t-submit-retry-able:waiting.* => waiting\(queued\)' """ [[t-retry-able]] script = """ if ((CYLC_TASK_SUBMIT_NUMBER == 1)); then touch "${CYLC_WORKFLOW_RUN_DIR}/file" while [[ -e "${CYLC_WORKFLOW_RUN_DIR}/file" ]]; do sleep 1 done false fi """ execution retry delays = PT5S [[t-submit-retry-able]] init-script = """ if [[ "${CYLC_TASK_JOB}" == *"/01" ]]; then cylc__job__disable_fail_signals ERR EXIT exit 1 fi """ script = true submission retry delays = PT5S cylc-flow-8.7.0/tests/functional/pause-resume/00-workflow.t0000664000175000017500000000176315264420307024016 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test workflow pause and resume, with cycling and async tasks present. . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/pause-resume/01-beyond-stop/0000775000175000017500000000000015264420307024214 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/pause-resume/01-beyond-stop/reference.log0000664000175000017500000000022115264420307026650 0ustar alastairalastair20140101T00/foo -triggered off [] 20140101T00/pause_resume -triggered off ['20140101T00/foo'] 20140101T00/bar -triggered off ['20140101T00/foo'] cylc-flow-8.7.0/tests/functional/pause-resume/01-beyond-stop/flow.cylc0000664000175000017500000000146315264420307026043 0ustar alastairalastair[meta] title = "pause/resume a workflow with tasks held beyond workflow stop point" description = """ Resuming a paused workflow should not release tasks beyond the workflow stop point. """ # https://github.com/cylc/cylc-flow/pull/1144 # See also tests/f/hold-release/01-beyond-stop [scheduler] cycle point format = %Y%m%dT%H [scheduling] initial cycle point = 20140101T00 stop after cycle point = 20140101T00 [[graph]] R1 = foo => pause_resume T00 = foo => bar [runtime] [[pause_resume]] # When this task runs foo will be held beyond the workflow stop point. script = """ cylc pause $CYLC_WORKFLOW_ID cylc play $CYLC_WORKFLOW_ID """ [[foo]] script = true [[bar]] script = true cylc-flow-8.7.0/tests/functional/pause-resume/test_header0000777000175000017500000000000015264420307032425 2../../functional/lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/pause-resume/02-paused-not-stalled.t0000664000175000017500000000307015264420307025644 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that abort on stall does not apply to a paused workflow # See also tests/functional/events/25-held-not-stalled.t . "$(dirname "$0")/test_header" set_test_number 2 init_workflow "${TEST_NAME_BASE}" << __FLOW__ [scheduler] [[events]] abort on inactivity timeout = False abort on stall timeout = True stall timeout = PT0S inactivity timeout handlers = cylc play '%(workflow)s' inactivity timeout = PT5S [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = true __FLOW__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --pause --no-detach "${WORKFLOW_NAME}" purge cylc-flow-8.7.0/tests/functional/pause-resume/01-beyond-stop.t0000664000175000017500000000175215264420307024406 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test pause and resume, with tasks present beyond stop point. . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/pause-resume/21-client.t0000775000175000017500000000312015264420307023415 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test resume paused workflow using the "cylc client" command. . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" cylc play --reference-test --pause --debug --no-detach "${WORKFLOW_NAME}" \ 1>"${TEST_NAME_BASE}.out" 2>&1 & CYLC_RUN_PID=$! poll_workflow_running read -r -d '' resume <<_args_ {"request_string": " mutation { resume(workflows: [\"${WORKFLOW_NAME}\"]){ result } } ", "variables": null} _args_ # shellcheck disable=SC2086 run_ok "${TEST_NAME_BASE}-client" \ cylc client "${WORKFLOW_NAME}" 'graphql' < <(echo ${resume}) run_ok "${TEST_NAME_BASE}-run" wait "${CYLC_RUN_PID}" purge exit cylc-flow-8.7.0/tests/functional/job-kill/0000775000175000017500000000000015264420307020623 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-kill/03-slurm/0000775000175000017500000000000015264420307022205 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-kill/03-slurm/reference.log0000664000175000017500000000012615264420307024645 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] 1/stop -triggered off ['1/t1'] cylc-flow-8.7.0/tests/functional/job-kill/03-slurm/flow.cylc0000664000175000017500000000064315264420307024033 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] expected task failures = 1/t1 [scheduling] [[graph]] R1 = t1:start=>stop [runtime] [[t1]] script = sleep 120 platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[[directives]]] --time=03:00 [[stop]] script=""" cylc kill "$CYLC_WORKFLOW_ID//*/t1" cylc stop "$CYLC_WORKFLOW_ID" """ cylc-flow-8.7.0/tests/functional/job-kill/04-pbs/0000775000175000017500000000000015264420307021630 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-kill/04-pbs/reference.log0000664000175000017500000000012615264420307024270 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] 1/stop -triggered off ['1/t1'] cylc-flow-8.7.0/tests/functional/job-kill/04-pbs/flow.cylc0000664000175000017500000000110515264420307023450 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] expected task failures = 1/t1 [scheduling] [[graph]] R1 = t1:start=>stop [runtime] [[t1]] script = sleep 120 {% if "CYLC_TEST_PLATFORM" in environ and environ["CYLC_TEST_PLATFORM"] %} platform == {{ environ['CYLC_TEST_PLATFORM'] }} {% endif %} [[[job]]] execution time limit = PT2M [[[directives]]] -l select=1:ncpus=1:mem=15mb [[stop]] script=""" cylc kill "$CYLC_WORKFLOW_ID//*/t1" cylc stop "$CYLC_WORKFLOW_ID" """ cylc-flow-8.7.0/tests/functional/job-kill/00-local/0000775000175000017500000000000015264420307022132 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-kill/00-local/reference.log0000664000175000017500000000040415264420307024571 0ustar alastairalastairInitial point: 1 Final point: 1 1/t2 -triggered off [] 1/t1 -triggered off [] 1/stop1 -triggered off ['1/t1', '1/t2'] 1/t4 -triggered off ['1/stop1'] 1/t3 -triggered off ['1/stop1'] 1/stop2 -triggered off ['1/t3', '1/t4'] 1/shutdown -triggered off ['1/stop2'] cylc-flow-8.7.0/tests/functional/job-kill/00-local/flow.cylc0000664000175000017500000000145215264420307023757 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] expected task failures = 1/t1, 1/t2, 1/t3, 1/t4, 1/stop2 [scheduling] [[graph]] R1 = """ t1:start & t2:start => stop1 => t3:start & t4:start => stop2? stop2:fail? => shutdown """ [runtime] [[T]] script = sleep 120 & echo $! >file; wait [[t1, t2, t3, t4]] inherit= T [[stop1]] script = """ # Kill 1/t1 and 1/t2 explicitly. cylc kill $CYLC_WORKFLOW_ID// //1/t1 //1/t2 || true """ [[stop2]] script = """ # Kill 1/t3, 1/t4, and myself! implicitly (kill all active tasks). cylc kill "$CYLC_WORKFLOW_ID//*" || true sleep 30 """ [[shutdown]] script = "cylc stop $CYLC_WORKFLOW_ID" cylc-flow-8.7.0/tests/functional/job-kill/04-pbs.t0000777000175000017500000000000015264420307024712 202-loadleveler.tustar alastairalastaircylc-flow-8.7.0/tests/functional/job-kill/02-loadleveler.t0000775000175000017500000000231515264420307023531 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test killing of jobs submitted to loadleveler, slurm, pbs... # TODO Check this test on a dockerized system or VM. JOB_RUNNER="${0##*\/??-}" export REQUIRE_PLATFORM="runner:${JOB_RUNNER%%.t} comms:tcp" . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 reftest purge exit cylc-flow-8.7.0/tests/functional/job-kill/02-loadleveler/0000775000175000017500000000000015264420307023340 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-kill/02-loadleveler/reference.log0000664000175000017500000000012615264420307026000 0ustar alastairalastairInitial point: 1 Final point: 1 t11/ -triggered off [] 1/stop -triggered off ['t11/'] cylc-flow-8.7.0/tests/functional/job-kill/02-loadleveler/flow.cylc0000664000175000017500000000122515264420307025163 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] expected task failures = 1/t1 [scheduling] [[graph]] R1=t1:start=>stop [runtime] [[t1]] script=sleep 120 {% if "CYLC_TEST_PLATFORM" in environ and environ["CYLC_TEST_PLATFORM"] %} platform = {{ environ["CYLC_TEST_PLATFORM"] }} {% endif %} [[[directives]]] class=serial job_type=serial notification=never resources=ConsumableCpus(1) ConsumableMemory(64mb) wall_clock_limit=180,120 [[stop]] script=""" cylc kill "$CYLC_WORKFLOW_ID//*/t1" cylc stop "$CYLC_WORKFLOW_ID" """ cylc-flow-8.7.0/tests/functional/job-kill/00-local.t0000775000175000017500000000404015264420307022320 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test kill local jobs. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 10 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-ps" for DIR in "${WORKFLOW_RUN_DIR}"/work/*/t*; do run_fail "${TEST_NAME}.$(basename "$DIR")" ps "$(cat "${DIR}/file")" done N=0 for FILE in "${WORKFLOW_RUN_DIR}"/log/job/*/t*/01/job.status; do run_fail "${TEST_NAME}-status-$((++N))" \ ps "$(awk -F= '$1 == "CYLC_JOB_PID" {print $2}' "$FILE")" done #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/job-kill/03-slurm.t0000777000175000017500000000000015264420307025267 202-loadleveler.tustar alastairalastaircylc-flow-8.7.0/tests/functional/job-kill/test_header0000777000175000017500000000000015264420307027140 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/job-kill/01-remote/0000775000175000017500000000000015264420307022334 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-kill/01-remote/reference.log0000664000175000017500000000016515264420307024777 0ustar alastairalastairInitial point: 1 Final point: 1 1/t2 -triggered off [] 1/t1 -triggered off [] 1/stop -triggered off ['1/t1', '1/t2'] cylc-flow-8.7.0/tests/functional/job-kill/01-remote/flow.cylc0000664000175000017500000000101515264420307024154 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] expected task failures = 1/t1, 1/t2 [scheduling] [[graph]] R1 = """ t1:start=>stop t2:start=>stop """ [runtime] [[T]] script=sleep 120 & echo $! >file; wait platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[t1]] inherit=T [[t2]] inherit=T [[stop]] script=""" cylc kill "$CYLC_WORKFLOW_ID//" '//1/t1' '//1/t2' || true cylc stop $CYLC_WORKFLOW_ID """ cylc-flow-8.7.0/tests/functional/job-kill/01-remote.t0000775000175000017500000000356115264420307022531 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test killing of jobs on a remote host. export REQUIRE_PLATFORM='loc:remote comms:tcp' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-ps run_fail "${TEST_NAME}" \ ssh -oBatchMode=yes -oConnectTimeout=5 -n "${CYLC_TEST_HOST}" \ "bash -c 'ps \$(cat cylc-run/${WORKFLOW_NAME}/work/*/t*/file)'" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/0000775000175000017500000000000015264420307024037 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-combination-scripts/08-vr-against-src.t0000664000175000017500000000576715264420307027330 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test `cylc vr` (Validate Reinstall restart) # Tests that VR doesn't modify the source directory for Cylc play. # See https://github.com/cylc/cylc-flow/issues/6209 . "$(dirname "$0")/test_header" set_test_number 9 # Setup (Run VIP, check that the play step fails in the correct way): WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" cp "${TEST_SOURCE_DIR}/vr_workflow_fail_on_play/flow.cylc" . # Run Cylc VIP run_fail "setup (vip)" \ cylc vip --debug \ --workflow-name "${WORKFLOW_NAME}" \ --no-run-name validate1="$(grep -Po "WARNING - vip:(.*)" "setup (vip).stderr" | awk -F ':' '{print $2}')" play1="$(grep -Po "WARNING - play:(.*)" "setup (vip).stderr" | awk -F ':' '{print $2}')" # Change the workflow to make the reinstall happen: echo "" >> flow.cylc # Run Cylc VR: TEST_NAME="${TEST_NAME_BASE}" run_fail "${TEST_NAME}" cylc vr "${WORKFLOW_NAME}" validate2="$(grep -Po "WARNING - vr:(.*)" "${TEST_NAME_BASE}.stderr" | awk -F ':' '{print $2}')" play2="$(grep -Po "WARNING - play:(.*)" "${TEST_NAME_BASE}.stderr" | awk -F ':' '{print $2}')" # Test that the correct source directory is opened at different # stages of Cylc VIP & VR TEST_NAME="outputs-created" if [[ -n $validate1 && -n $validate2 && -n $play1 && -n $play2 ]]; then ok "${TEST_NAME}" else fail "${TEST_NAME}" fi TEST_NAME="vip validate and play operate on different folders" if [[ $validate1 != "${play1}" ]]; then ok "${TEST_NAME}" else fail "${TEST_NAME}" fi TEST_NAME="vr & vip validate operate on the same folder" if [[ $validate1 == "${validate2}" ]]; then ok "${TEST_NAME}" else fail "${TEST_NAME}" fi TEST_NAME="vr validate and play operate on different folders" if [[ $validate2 != "${play2}" ]]; then ok "${TEST_NAME}" else fail "${TEST_NAME}" fi TEST_NAME="vip play loads from a cylc-run subdir" if [[ "${play2}" =~ cylc-run ]]; then ok "${TEST_NAME}" else fail "${TEST_NAME}" fi TEST_NAME="vr play loads from a cylc-run subdir" if [[ "${play2}" =~ cylc-run ]]; then ok "${TEST_NAME}" else fail "${TEST_NAME}" fi # Clean Up: run_ok "${TEST_NAME_BASE}-stop cylc stop ${WORKFLOW_NAME} --now --now" purge exit 0 cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/06-vip-named-run0000777000175000017500000000000015264420307027620 200-vipustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-combination-scripts/01-vr-reload.t0000664000175000017500000000414315264420307026337 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test `cylc vr` (Validate Reinstall) # In this case the target workflow is running so cylc reload is run. . "$(dirname "$0")/test_header" set_test_number 7 # Setup (Must be a running workflow, note the unusual absence of --no-detach) WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" cp "${TEST_SOURCE_DIR}/vr_workflow/flow.cylc" . run_ok "setup (vip)" \ cylc vip --debug \ --workflow-name "${WORKFLOW_NAME}" \ --no-run-name export WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}" poll_workflow_running # It validates and reloads: run_ok "${TEST_NAME_BASE}-runs" cylc vr "${WORKFLOW_NAME}" # Grep for VR reporting revalidation, reinstallation and reloading grep "\$" "${TEST_NAME_BASE}-runs.stdout" > VIPOUT.txt named_grep_ok "${TEST_NAME_BASE}-it-validated" "$ cylc validate --against-source" "VIPOUT.txt" named_grep_ok "${TEST_NAME_BASE}-it-installed" "$ cylc reinstall" "VIPOUT.txt" named_grep_ok "${TEST_NAME_BASE}-it-reloaded" "$ cylc reload" "VIPOUT.txt" cylc play "${WORKFLOW_NAME}" named_grep_ok "${TEST_NAME_BASE}-it-logged-reload" \ "Reloading the workflow definition" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" # Clean Up. run_ok "${TEST_NAME_BASE}-stop" cylc stop "${WORKFLOW_NAME}" --now --now purge exit 0 cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/04-vr-fail-validate.t0000664000175000017500000000350615264420307027600 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test `cylc vr` (Validate Reinstall restart) # Changes to the source cause VR to bail on validation. . "$(dirname "$0")/test_header" set_test_number 5 # Setup WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" cp "${TEST_SOURCE_DIR}/vr_workflow/flow.cylc" . run_ok "setup (vip)" \ cylc vip --debug \ --workflow-name "${WORKFLOW_NAME}" \ --no-run-name # Change source workflow and run vr: # Cut the runtime section out of the source flow. head -n 5 > tmp < flow.cylc cat tmp > flow.cylc TEST_NAME="${TEST_NAME_BASE}" run_fail "${TEST_NAME}" cylc vr "${WORKFLOW_NAME}" # Grep for reporting of revalidation, reinstallation, reloading and playing: named_grep_ok "${TEST_NAME_BASE}-it-tried" \ "$ cylc validate --against-source" "${TEST_NAME}.stdout" named_grep_ok "${TEST_NAME_BASE}-it-failed" \ "WorkflowConfigError" "${TEST_NAME}.stderr" # Clean Up: run_ok "${TEST_NAME_BASE}-stop" cylc stop "${WORKFLOW_NAME}" --now --now purge exit 0 cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/07-vr-remote.t0000664000175000017500000000341715264420307026375 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test `cylc vr` (Validate Reinstall restart) # Simple test that it works for a workflow reinvoked on a remote host export REQUIRE_PLATFORM='loc:remote runner:background fs:shared' . "$(dirname "$0")/test_header" set_test_number 3 create_test_global_config '' """ [scheduler] [[run hosts]] available = ${CYLC_TEST_HOST} """ # Setup WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" cp "${TEST_SOURCE_DIR}/vr_workflow_stop/flow.cylc" . run_ok "${TEST_NAME_BASE}-install" \ cylc vip \ --workflow-name "${WORKFLOW_NAME}" \ --no-detach # It validates and restarts: TEST_NAME="${TEST_NAME_BASE}-reinvoke" run_ok "${TEST_NAME}" cylc vr "${WORKFLOW_NAME}" --no-detach # Simply check that we have the expected scheduler logs: ls "${RUN_DIR}/${WORKFLOW_NAME}/runN/log/scheduler" > logdir.txt cmp_ok logdir.txt <<__HERE__ 01-start-01.log 02-restart-02.log log __HERE__ # Clean Up. purge cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/00-vip/0000775000175000017500000000000015264420307025052 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-combination-scripts/00-vip/reference.log0000664000175000017500000000005715264420307027515 0ustar alastairalastair13000101T0000Z/foo -triggered off [] in flow 1 cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/00-vip/flow.cylc0000664000175000017500000000017015264420307026673 0ustar alastairalastair[scheduler] allow implicit tasks = True [scheduling] initial cycle point = 1500 [[graph]] R1 = foo cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/09-vr-icp-now.t0000664000175000017500000000267315264420307026463 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Ensure that validate step of `cylc vr` does not set the --icp option for the # restart step, as this would cause an InputError. # See https://github.com/cylc/cylc-flow/issues/6262 . "$(dirname "$0")/test_header" set_test_number 2 WORKFLOW_ID=$(workflow_id) cp -r "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/flow.cylc" . run_ok "${TEST_NAME_BASE}-vip" cylc vip . \ --workflow-name "${WORKFLOW_ID}" \ --no-detach \ --no-run-name \ --mode simulation echo "# Some Comment" >> flow.cylc run_ok "${TEST_NAME_BASE}-vr" \ cylc vr "${WORKFLOW_ID}" --no-detach --mode simulation cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/06-vip-named-run.t0000664000175000017500000000327215264420307027135 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test `cylc vip` (Validate Install Play) . "$(dirname "$0")/test_header" set_test_number 5 WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" cp -r "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/flow.cylc" . cp -r "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/reference.log" . run_ok "${TEST_NAME_BASE}-from-path" \ cylc vip --no-detach --debug \ --workflow-name "${WORKFLOW_NAME}" \ --initial-cycle-point=1300 \ --run-name sardine \ --reference-test grep_ok "13000101T0000Z" "${TEST_NAME_BASE}-from-path.stdout" grep "\$" "${TEST_NAME_BASE}-from-path.stdout" > VIPOUT.txt named_grep_ok "${TEST_NAME_BASE}-it-validated" "$ cylc validate" "VIPOUT.txt" named_grep_ok "${TEST_NAME_BASE}-it-installed" "$ cylc install" "VIPOUT.txt" named_grep_ok "${TEST_NAME_BASE}-it-played" "$ cylc play" "VIPOUT.txt" purge exit 0 cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/00-vip.t0000664000175000017500000000326715264420307025247 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test `cylc vip` (Validate Install Play) . "$(dirname "$0")/test_header" set_test_number 5 WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" cp -r "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/flow.cylc" . cp -r "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/reference.log" . run_ok "${TEST_NAME_BASE}-from-path" \ cylc vip --no-detach --debug \ --workflow-name "${WORKFLOW_NAME}" \ --initial-cycle-point=1300 \ --no-run-name \ --reference-test \ grep_ok "13000101T0000Z" "${TEST_NAME_BASE}-from-path.stdout" grep "\$" "${TEST_NAME_BASE}-from-path.stdout" > VIPOUT.txt named_grep_ok "${TEST_NAME_BASE}-it-validated" "$ cylc validate" "VIPOUT.txt" named_grep_ok "${TEST_NAME_BASE}-it-installed" "$ cylc install" "VIPOUT.txt" named_grep_ok "${TEST_NAME_BASE}-it-played" "$ cylc play" "VIPOUT.txt" purge exit 0 cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/05-vr-fail-is-running.t0000664000175000017500000000403515264420307030077 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test `cylc vr` (Validate Reinstall) # In this case the target workflow is in an ambiguous state: We cannot tell # Whether it's running, paused or stopped. Cylc VR should validate before # reinstall: . "$(dirname "$0")/test_header" set_test_number 4 create_test_global_config "" """ [scheduler] [[main loop]] plugins = reset bad hosts """ # Setup WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" cp "${TEST_SOURCE_DIR}/vr_workflow/flow.cylc" . run_ok "setup (vip)" \ cylc vip --debug \ --workflow-name "${WORKFLOW_NAME}" \ --no-run-name # Get the workflow into an unreachable state CONTACTFILE="${RUN_DIR}/${WORKFLOW_NAME}/.service/contact" cp "$CONTACTFILE" "${CONTACTFILE}.old" poll test -e "${CONTACTFILE}" sed -i 's@CYLC_WORKFLOW_HOST=.*@CYLC_WORKFLOW_HOST=elephantshrew@' "${CONTACTFILE}" # It can't figure out whether the workflow is running: # Change source workflow and run vr: run_fail "${TEST_NAME_BASE}-runs" cylc vr "${WORKFLOW_NAME}" grep_ok "elephantshrew." "${TEST_NAME_BASE}-runs.stderr" # Clean Up: mv "${CONTACTFILE}.old" "$CONTACTFILE" run_ok "${TEST_NAME_BASE}-stop" cylc stop "${WORKFLOW_NAME}" --now --now purge exit 0 cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/vr_workflow_stop/0000775000175000017500000000000015264420307027465 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-combination-scripts/vr_workflow_stop/flow.cylc0000664000175000017500000000030615264420307031307 0ustar alastairalastair[scheduler] allow implicit tasks = true [[events]] startup handlers = cylc stop --now --now %(workflow)s [scheduling] initial cycle point = 1500 [[graph]] P1Y = foo cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/02-vr-restart.t0000664000175000017500000000354115264420307026557 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test `cylc vr` (Validate Reinstall restart) # In this case the target workflow is stopped so cylc play is run. . "$(dirname "$0")/test_header" set_test_number 6 # Setup WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" cp "${TEST_SOURCE_DIR}/vr_workflow/flow.cylc" . run_ok "setup (install)" \ cylc install \ --workflow-name "${WORKFLOW_NAME}" export WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}" # It validates and restarts: # Run VR run_ok "${TEST_NAME_BASE}-runs" cylc vr "${WORKFLOW_NAME}" # Grep for vr reporting revalidation, reinstallation and playing the workflow. grep "\$" "${TEST_NAME_BASE}-runs.stdout" > VIPOUT.txt named_grep_ok "${TEST_NAME_BASE}-it-revalidated" "$ cylc validate --against-source" "VIPOUT.txt" named_grep_ok "${TEST_NAME_BASE}-it-installed" "$ cylc reinstall" "VIPOUT.txt" named_grep_ok "${TEST_NAME_BASE}-it-played" "cylc play" "VIPOUT.txt" # Clean Up. run_ok "${TEST_NAME_BASE}-stop" cylc stop "${WORKFLOW_NAME}" --now --now purge exit 0 cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/09-vr-icp-now/0000775000175000017500000000000015264420307026266 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-combination-scripts/09-vr-icp-now/flow.cylc0000664000175000017500000000037015264420307030111 0ustar alastairalastair[scheduler] [[events]] restart timeout = PT0S [scheduling] initial cycle point = 2020 final cycle point = 2020 [[graph]] P1Y = foo [runtime] [[foo]] [[[simulation]]] default run length = PT0S cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/10-CYLC_WORKFLOW_SRC_DIR.t0000664000175000017500000000410415264420307027732 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test `CYLC_WORKFLOW_SRC_DIR.t` is set . "$(dirname "$0")/test_header" set_test_number 6 WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" cat > flow.cylc <<__HERE__ #!jinja2 # TEST: {{ CYLC_WORKFLOW_SRC_DIR }} [scheduler] allow implicit tasks = true [scheduling] [[graph]] R1 = foo __HERE__ run_ok "${TEST_NAME_BASE}-view" cylc view -p . named_grep_ok "${TEST_NAME_BASE}-src-path-in-view-p" "TEST: $PWD" \ "${TEST_NAME_BASE}-view.stdout" # It starts playing: run_ok "${TEST_NAME_BASE}-vip" \ cylc vip \ --pause \ --no-run-name \ --workflow-name "${WORKFLOW_NAME}" # It can get CYLC_WORKFLOW_SRC_DIR named_grep_ok "${TEST_NAME_BASE}-src-path-available" \ "TEST: $PWD" "${RUN_DIR}/${WORKFLOW_NAME}/log/config/flow-processed.cylc" # It can be updated with Cylc VR echo "[meta]" >> flow.cylc run_ok "${TEST_NAME_BASE}-vr" \ cylc vr "${WORKFLOW_NAME}" poll_grep "meta" "${RUN_DIR}/${WORKFLOW_NAME}/log/config/flow-processed.cylc" # It can get CYLC_WORKFLOW_SRC_DIR named_grep_ok "${TEST_NAME_BASE}-src-path-available" \ "TEST: $PWD" "${RUN_DIR}/${WORKFLOW_NAME}/log/config/flow-processed.cylc" cylc stop "${WORKFLOW_NAME}" purge "${WORKFLOW_NAME}" cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/test_header0000777000175000017500000000000015264420307032354 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-combination-scripts/vr_workflow/0000775000175000017500000000000015264420307026420 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-combination-scripts/vr_workflow/flow.cylc0000664000175000017500000000021615264420307030242 0ustar alastairalastair[scheduling] initial cycle point = 1500 [[graph]] P1Y = foo [runtime] [[foo]] script = cylc pause ${WORKFLOW_ID} cylc-flow-8.7.0/tests/functional/cylc-combination-scripts/vr_workflow_fail_on_play/0000775000175000017500000000000015264420307031134 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-combination-scripts/vr_workflow_fail_on_play/flow.cylc0000664000175000017500000000056315264420307032763 0ustar alastairalastair#!jinja2 {% from "sys" import argv %} {% from "cylc.flow" import LOG %} {% from "pathlib" import Path %} {% if argv[1] == "play" %} this = should cause cylc play to fail {% endif %} {% set SPATH = Path.cwd().__str__() %} {% do LOG.warning(argv[1] + ":" + SPATH) %} [scheduling] initial cycle point = 1500 [[graph]] P1Y = foo [runtime] [[foo]] cylc-flow-8.7.0/tests/functional/optional-outputs/0000775000175000017500000000000015264420307022466 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/optional-outputs/00-stall-on-partial.t0000664000175000017500000000264515264420307026262 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check that partially satisfied prerequisites count toward runahead limit. . "$(dirname "$0")/test_header" set_test_number 5 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --no-detach --reference-test --debug "${WORKFLOW_NAME}" LOG="${WORKFLOW_RUN_DIR}/log/scheduler/log" grep_ok "Partially satisfied prerequisites" "${LOG}" grep_ok "1/bar is waiting on \['1/foo:y'\]" "${LOG}" grep_ok "Workflow stalled" "${LOG}" purge exit cylc-flow-8.7.0/tests/functional/optional-outputs/01-stall-on-incomplete/0000775000175000017500000000000015264420307026572 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/optional-outputs/01-stall-on-incomplete/reference.log0000664000175000017500000000007315264420307031233 0ustar alastairalastairInitial point: 1 Final point: None 1/foo -triggered off [] cylc-flow-8.7.0/tests/functional/optional-outputs/01-stall-on-incomplete/flow.cylc0000664000175000017500000000060115264420307030412 0ustar alastairalastair# Should stall at runhead limit with incomplete foo [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] cycling mode = integer initial cycle point = 1 runahead limit = P0 [[graph]] P1 = "foo:x & foo:y => bar" [runtime] [[foo]] script = "cylc message x" [[[outputs]]] x = x y = y [[bar]] cylc-flow-8.7.0/tests/functional/optional-outputs/01-stall-on-incomplete.t0000664000175000017500000000264515264420307026766 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check that incomplete tasks count toward runahead limit. . "$(dirname "$0")/test_header" set_test_number 5 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --no-detach --reference-test --debug "${WORKFLOW_NAME}" LOG="${WORKFLOW_RUN_DIR}/log/scheduler/log" grep_ok "Incomplete tasks" "${LOG}" grep_ok '1/foo did not complete the required outputs:\n(.*\n){3}.*y\n' "${LOG}" -Pizo grep_ok "Workflow stalled" "${LOG}" purge exit cylc-flow-8.7.0/tests/functional/optional-outputs/02-no-stall-on-optional/0000775000175000017500000000000015264420307026673 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/optional-outputs/02-no-stall-on-optional/reference.log0000664000175000017500000000012015264420307031325 0ustar alastairalastairInitial point: 1 Final point: 2 1/foo -triggered off [] 2/foo -triggered off [] cylc-flow-8.7.0/tests/functional/optional-outputs/02-no-stall-on-optional/flow.cylc0000664000175000017500000000062715264420307030523 0ustar alastairalastair# Should not stall at runhead limit with incomplete foo [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] cycling mode = integer initial cycle point = 1 final cycle point = 2 runahead limit = P0 [[graph]] P1 = "foo:y? => bar" [runtime] [[foo]] script = "cylc message x" [[[outputs]]] x = x y = y [[bar]] cylc-flow-8.7.0/tests/functional/optional-outputs/08-finish-fail.t0000664000175000017500000000353515264420307025277 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------- # Test handling of failed tasks in finish triggers. . "$(dirname "$0")/test_header" set_test_number 3 # When a task with a finish trigger fails: # No runahead stall and no stall at final cycle point, because finish # triggers imply success is optional (i.e. no incomplete tasks created). init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CYLC__' [scheduler] [[events]] stall timeout = PT0S abort on stall timeout = True [scheduling] cycling mode = integer initial cycle point = 1 final cycle point = 2 runahead limit = P0 [[dependencies]] [[[P1]]] graph = "foo:finish" [runtime] [[foo]] script = false __FLOW_CYLC__ # Validate with a deprecation message TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" grep_ok "graph items were automatically upgraded" "${TEST_NAME}.stderr" # No stall expected. workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --no-detach --debug "${WORKFLOW_NAME}" purge cylc-flow-8.7.0/tests/functional/optional-outputs/test_header0000777000175000017500000000000015264420307031003 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/optional-outputs/02-no-stall-on-optional.t0000664000175000017500000000177215264420307027067 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check missing optional outputs do not cause a stall at the runahead limit. . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/optional-outputs/00-stall-on-partial/0000775000175000017500000000000015264420307026066 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/optional-outputs/00-stall-on-partial/reference.log0000664000175000017500000000007315264420307030527 0ustar alastairalastairInitial point: 1 Final point: None 1/foo -triggered off [] cylc-flow-8.7.0/tests/functional/optional-outputs/00-stall-on-partial/flow.cylc0000664000175000017500000000061315264420307027711 0ustar alastairalastair# Should stall at runhead limit with partially satisfied bar [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] cycling mode = integer initial cycle point = 1 runahead limit = P0 [[graph]] P1 = "foo:x & foo:y? => bar" [runtime] [[foo]] script = "cylc message x" [[[outputs]]] x = x y = y [[bar]] cylc-flow-8.7.0/tests/functional/cylc-lint/0000775000175000017500000000000015264420307021016 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-lint/01.lint-toml.t0000664000175000017500000000605015264420307023342 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test linting with a toml file present. . "$(dirname "$0")/test_header" set_test_number 12 # Set Up: rm etc/global.cylc LINE_LEN_NO=$(python -c "from cylc.flow.scripts.lint import LINE_LEN_NO; print(LINE_LEN_NO)") cat > flow.cylc <<__HERE__ # This is definitely not an OK flow.cylc file. \t[scheduler] [cylc] [[dependencies]] [runtime] [[foo]] inherit = hello [[[job]]] something\t __HERE__ mkdir sites cat > sites/niwa.cylc <<__HERE__ blatantly = not valid __HERE__ # Control tests TEST_NAME='it-lints-without-toml-file' run_fail "${TEST_NAME}" cylc lint TESTOUT="${TEST_NAME}.stdout" named_grep_ok "${TEST_NAME_BASE}-returns-error-code" "S004" "${TESTOUT}" named_grep_ok "${TEST_NAME_BASE}-returns-error-from-subdirectory" "niwa.cylc" "${TESTOUT}" named_grep_ok "${TEST_NAME_BASE}-returns-728-upgrade-code" "^\[U" "${TESTOUT}" # Add a pyproject.toml file cat > pyproject.toml <<__HERE__ [tool.cylc.lint] # Check against these rules rulesets = [ "style" ] # do not check for these errors ignore = [ "S004" ] # do not lint files matching # these globs: exclude = [ "sites/*.cylc", ] __HERE__ # Test that results are different: TEST_NAME='it-lints-with-toml-file' run_fail "${TEST_NAME}" cylc lint TESTOUT="${TEST_NAME}.stdout" grep_fail "S004" "${TESTOUT}" grep_fail "niwa.cylc" "${TESTOUT}" grep_fail "^\[U" "${TESTOUT}" # Add a max line length to the pyproject.toml. echo "" >> pyproject.toml echo "max-line-length = 4" >> pyproject.toml cat > flow.cylc <<__HERE__ script = """ How long a line is too long a line """ __HERE__ TEST_NAME="${TEST_NAME_BASE}-it-fails-if-max-line-length-set" run_fail "${TEST_NAME}" cylc lint named_grep_ok "${TEST_NAME}-line-too-long-message" \ "\[${LINE_LEN_NO}\] flow.cylc:2: line > 4 characters." \ "${TEST_NAME}.stdout" TEST_NAME='it-does-not-fail-if-max-line-length-set-but-ignored' cat > pyproject.toml <<__HERE__ [tool.cylc.lint] # Check against these rules rulesets = [ "style" ] # do not check for these errors ignore = [ "${LINE_LEN_NO}" ] exclude = [ "sites/*.cylc", ] max-line-length = 1 __HERE__ run_ok "${TEST_NAME}" cylc lint grep_ok "rules and found no issues" "${TEST_NAME}.stdout" cylc-flow-8.7.0/tests/functional/cylc-lint/00.lint.t0000664000175000017500000000603615264420307022374 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test workflow installation . "$(dirname "$0")/test_header" set_test_number 20 cat > flow.cylc <<__HERE__ # This is definitely not an OK flow.cylc file. [cylc] [[parameters]] __HERE__ rm etc/global.cylc TEST_NAME="${TEST_NAME_BASE}.vanilla" run_fail "${TEST_NAME}" cylc lint . named_grep_ok "${TEST_NAME_BASE}-check-for-error-code" "S004" "${TEST_NAME}.stdout" TEST_NAME="${TEST_NAME_BASE}.pick-a-ruleset" run_fail "${TEST_NAME}" cylc lint . -r 728 named_grep_ok "${TEST_NAME_BASE}-check-for-error-code" "U998" "${TEST_NAME}.stdout" TEST_NAME="${TEST_NAME_BASE}.inplace" run_fail "${TEST_NAME}" cylc lint . -i named_grep_ok "${TEST_NAME_BASE}-check-for-error-code-in-file" "U998" flow.cylc rm flow.cylc cat > suite.rc <<__HERE__ # This is definitely not an OK flow.cylc file. {{FOO}} __HERE__ TEST_NAME="${TEST_NAME_BASE}.pick-a-ruleset-728" run_fail "${TEST_NAME}" cylc lint . -r 728 named_grep_ok "${TEST_NAME_BASE}-do-not-upgrade-check-if-compat-mode" "Lint after renaming" "${TEST_NAME}.stderr" TEST_NAME="${TEST_NAME_BASE}.pick-a-ruleset-728-exit-zero" run_ok "${TEST_NAME}" cylc lint . -r 728 --exit-zero TEST_NAME="${TEST_NAME_BASE}.pick-a-ruleset-all" run_fail "${TEST_NAME}" cylc lint . -r all TEST_NAME="${TEST_NAME_BASE}.exit-zero" run_ok "${TEST_NAME}" cylc lint --exit-zero . rm suite.rc cat > flow.cylc <<__HERE__ # This one is fine [scheduler] __HERE__ TEST_NAME="${TEST_NAME_BASE}.zero-issues" run_ok "${TEST_NAME}" cylc lint . named_grep_ok "${TEST_NAME_BASE}-message-on-no-errors" "found no issues" "${TEST_NAME}.stdout" # It returns an error message if you attempt to lint a non-existent location TEST_NAME="it-fails-if-not-target" run_fail "${TEST_NAME}" cylc lint "a-$(uuidgen)" grep_ok "Workflow ID not found" "${TEST_NAME}.stderr" # It returns a reference in reference mode TEST_NAME="${TEST_NAME_BASE}-it-returns-a-reference" run_ok "${TEST_NAME}" cylc lint --list-codes named_grep_ok "${TEST_NAME}-contains-style-codes" "^S001:" "${TEST_NAME}.stdout" TEST_NAME="it-returns-a-reference-style" run_ok "${TEST_NAME}" cylc lint --list-codes -r 'style' named_grep_ok "${TEST_NAME}-contains-style-codes" "^S001:" "${TEST_NAME}.stdout" grep_fail "^U" "${TEST_NAME}.stdout" rm flow.cylc cylc-flow-8.7.0/tests/functional/cylc-lint/test_header0000777000175000017500000000000015264420307027333 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/0000775000175000017500000000000015264420307021376 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/11-remote-retrieve.t0000775000175000017500000000375515264420307025135 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc cat-log" for remote tasks with auto-retrieval. export REQUIRE_PLATFORM='loc:remote fs:indep' . "$(dirname "$0")/test_header" set_test_number 7 create_test_global_config "" " [platforms] [[${CYLC_TEST_PLATFORM}]] retrieve job logs = True" install_workflow "${TEST_NAME_BASE}" remote-simple TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" # Local job.out should exist (retrieved). LOCAL_JOB_OUT=$(cylc cat-log -f a -m d "${WORKFLOW_NAME}//1/a-task")/job.out exists_ok "${LOCAL_JOB_OUT}" # Distinguish local from remote job.out perl -pi -e 's/fox/FOX/' "${LOCAL_JOB_OUT}" # Cat the remote one. TEST_NAME=${TEST_NAME_BASE}-out-rem run_ok "${TEST_NAME}" cylc cat-log --force-remote -f o \ "${WORKFLOW_NAME}//1/a-task" grep_ok '^the quick brown fox$' "${TEST_NAME}.stdout" # Cat the local one. TEST_NAME=${TEST_NAME_BASE}-out-loc run_ok "${TEST_NAME}" cylc cat-log -f o "${WORKFLOW_NAME}//1/a-task" grep_ok '^the quick brown FOX$' "${TEST_NAME}.stdout" purge exit cylc-flow-8.7.0/tests/functional/cylc-cat-log/02-remote-custom-runtime-viewer-pbs.t0000775000175000017500000000247715264420307030364 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc cat-log" for viewing PBS runtime STDOUT/STDERR by a custom command export REQUIRE_PLATFORM='runner:pbs' . "$(dirname "$0")/test_header" OUT_VIEWER="$(cylc config -d -i "[platforms][$CYLC_TEST_PLATFORM]out viewer")" ERR_VIEWER="$(cylc config -d -i "[platforms][$CYLC_TEST_PLATFORM]err viewer")" if [[ -z "${ERR_VIEWER}" || -z "${OUT_VIEWER}" ]]; then skip_all 'remote viewers not configured for this platform' fi set_test_number 2 reftest purge exit cylc-flow-8.7.0/tests/functional/cylc-cat-log/09-cat-running.t0000775000175000017500000000212215264420307024236 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc cat-log" of currently-running local and remote jobs. export REQUIRE_PLATFORM='loc:remote comms:tcp' . "$(dirname "$0")/test_header" set_test_number 2 export REFTEST_OPTS="--abort-if-any-task-fails" reftest purge exit cylc-flow-8.7.0/tests/functional/cylc-cat-log/13-remote-out-err-tailer.t0000664000175000017500000000717515264420307026162 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc cat-log" with custom out/err tailers export REQUIRE_PLATFORM='loc:remote runner:background fs:indep comms:tcp' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 12 #------------------------------------------------------------------------------- # run the workflow TEST_NAME="${TEST_NAME_BASE}-validate" install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play -N "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- # change the platform the task ran on to the remote platform sqlite3 "${HOME}/cylc-run/${WORKFLOW_NAME}/log/db" " UPDATE task_jobs SET platform_name = '${CYLC_TEST_PLATFORM}', run_status = null WHERE name = 'foo' AND cycle = '1' ;" #------------------------------------------------------------------------------- # test cylc cat-log --mode=list-dir will not list job.out / err # (no tailer / viewer configured) create_test_global_config "" " [platforms] [[$CYLC_TEST_PLATFORM]] out tailer = err tailer = out viewer = err viewer = " TEST_NAME="${TEST_NAME_BASE}-list-dir-no-tailers" # NOTE: command will fail due to missing remote directory (this tests remote # error code is preserved) run_fail "${TEST_NAME}" cylc cat-log "${WORKFLOW_NAME}//1/foo" -m 'list-dir' # the job.out and job.err filees grep_fail "job.out" "${TEST_NAME}.stdout" grep_fail "job.err" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- # test cylc cat-log --mode=list-dir lists the tailed files # (both tailer and viewer configured) create_test_global_config "" " [platforms] [[$CYLC_TEST_PLATFORM]] out tailer = echo OUT err tailer = echo ERR out viewer = echo OUT err viewer = echo ERR " # test cylc cat-log --mode=list-dir lists the tailed files TEST_NAME="${TEST_NAME_BASE}-list-dir-with-tailers" # NOTE: command will fail due to missing remote directory (this tests remote # error code is preserved) run_fail "${TEST_NAME}" cylc cat-log "${WORKFLOW_NAME}//1/foo" -m 'list-dir' # the job.out and job.err filees grep_ok "job.out" "${TEST_NAME}.stdout" grep_ok "job.err" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- # test cylc cat-log runs the custom tailers TEST_NAME="${TEST_NAME_BASE}-cat-out" run_ok "${TEST_NAME}" cylc cat-log "${WORKFLOW_NAME}//1/foo" -f o -m t grep_ok "OUT" "${TEST_NAME}.stdout" run_ok "${TEST_NAME}" cylc cat-log "${WORKFLOW_NAME}//1/foo" -f e -m t grep_ok "ERR" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/cylc-cat-log/00-local/0000775000175000017500000000000015264420307022705 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/00-local/reference.log0000664000175000017500000000014215264420307025343 0ustar alastairalastair1/submit-failed -triggered off [] in flow 1 1/a-task -triggered off ['1/submit-failed'] in flow 1 cylc-flow-8.7.0/tests/functional/cylc-cat-log/00-local/flow.cylc0000664000175000017500000000175615264420307024541 0ustar alastairalastair[scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT3M [scheduling] [[graph]] R1 = submit-failed:submit-failed? => a-task [runtime] [[a-task]] script = """ # Write to task stdout log echo "the quick brown fox" # Write to task stderr log echo "jumped over the lazy dog" >&2 # Write to a custom log file echo "drugs, money & whitespace" > "${CYLC_TASK_LOG_ROOT} custom.log" # Generate a warning message in the workflow log. cylc message -p WARNING 'marmite and squashed bananas' # Remove the submit-failed task. cylc remove "${CYLC_WORKFLOW_NAME}//1/submit-failed" """ [[submit-failed]] # This causes a submission failure due to Bash syntax check. # In this case the job file isn't even written remotely. script = if cylc-flow-8.7.0/tests/functional/cylc-cat-log/09-cat-running/0000775000175000017500000000000015264420307024051 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/09-cat-running/reference.log0000664000175000017500000000032115264420307026506 0ustar alastairalastairInitial point: 1 Final point: 1 1/local-task -triggered off [] 1/remote-task -triggered off [] 1/cat-log -triggered off ['1/local-task', '1/remote-task'] 1/fin -triggered off ['1/local-task', '1/remote-task'] cylc-flow-8.7.0/tests/functional/cylc-cat-log/09-cat-running/flow.cylc0000664000175000017500000000322715264420307025700 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT3M [scheduling] [[graph]] R1 = """ # start "local-task" and "remote-task", then run cat-log local-task:echo & remote-task:echo => cat-log # add a downstream task to ensure "local-task" and "remote-task" # succeed (for the reference test) local-task & remote-task => fin """ [runtime] [[ECHO]] script = """ cylc__job__wait_cylc_message_started echo rubbish echo garbage >&2 cylc message -- 'echo done' # wait up to PT1M for the cat-log task to succeed # (the workflow will shut down if cat-log fails) sleep 60 # fail if the task was not orphaned by this point false """ [[[outputs]]] echo = "echo done" [[local-task]] inherit = ECHO [[remote-task]] platform = {{ environ['CYLC_TEST_PLATFORM'] }} inherit = ECHO [[cat-log]] script = """ for TASK in '1/local-task' '1/remote-task'; do cylc cat-log --debug -f o \ "${CYLC_WORKFLOW_ID}//${TASK}" \ | grep 'rubbish' cylc cat-log --debug -f e \ "${CYLC_WORKFLOW_ID}//${TASK}" \ | grep 'garbage' # orphan the sleep command cylc set "${CYLC_WORKFLOW_ID}//${TASK}" done """ [[fin]] run mode = skip cylc-flow-8.7.0/tests/functional/cylc-cat-log/15-remote_when_no_local_log.t0000775000175000017500000000523315264420307027037 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc cat-log" for a specific circumstance that caused cat-log to # not work properly. This tests simulates small window of time where a # job has finished but the logs have not yet been retrieved. In this # situation cat-log should remote log retrieval export REQUIRE_PLATFORM='loc:remote fs:indep' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 7 create_test_global_config "" " [platforms] [[${CYLC_TEST_PLATFORM}]] retrieve job logs = True retrieve job log expected files = job " install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" \ cylc play --debug --no-detach \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- # Confirm job.out is present locally (logs were retrieved). LOCAL_JOB_DIR=$(cylc cat-log -f a -m d "${WORKFLOW_NAME}//1/a-task") exists_ok "${LOCAL_JOB_DIR}/job.out" # Confirm job file was removed locally by b-task. exists_fail "${LOCAL_JOB_DIR}/job" # remote TEST_NAME=${TEST_NAME_BASE}-no_log_remote # Even though job.out is present, not all expected files are present, # so it should treat retrieval as incomplete and go remote: run_ok "$TEST_NAME" cylc cat-log --debug -f job.out "${WORKFLOW_NAME}//1/a-task" grep_ok "Not all logs present, getting job log remotely" "${TEST_NAME}.stderr" # remote TEST_NAME=${TEST_NAME_BASE}-no_log_remote_list_dir run_ok "$TEST_NAME" cylc cat-log --debug --mode=list-dir "${WORKFLOW_NAME}//1/a-task" grep_ok "Not all logs present, getting job log remotely" "${TEST_NAME}.stderr" # Clean up the task host. purge cylc-flow-8.7.0/tests/functional/cylc-cat-log/04-local-tail/0000775000175000017500000000000015264420307023640 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/04-local-tail/flow.cylc0000664000175000017500000000047415264420307025470 0ustar alastairalastair[scheduler] [[events]] abort on stall timeout = True stall timeout = PT2M [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = """ for I in $(seq 1 100); do echo "from $CYLC_TASK_NAME $I" sleep 1 done """ cylc-flow-8.7.0/tests/functional/cylc-cat-log/04-local-tail/bin/0000775000175000017500000000000015264420307024410 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/04-local-tail/bin/my-tailer.sh0000775000175000017500000000032415264420307026651 0ustar alastairalastair#!/usr/bin/env bash # Modify 'tail' output to prove that cylc used the custom tailer. # Exit immediately, for the test (i.e. don't 'tail -F') FILE="$1" tail -n +1 "${FILE}" | awk '{print "HELLO", $0; fflush() }' cylc-flow-8.7.0/tests/functional/cylc-cat-log/12-delete-kill.t0000664000175000017500000000370215264420307024200 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc cat-log" exits when the log file is deleted # or when tail is killed. . "$(dirname "$0")/test_header" set_test_number 2 # Get PID of tail cmd given the parent cat-log PPID get_tail_pid() { pgrep -P "$1" tail } init_workflow "${TEST_NAME_BASE}" << __EOF__ # whatever __EOF__ log_file="${WORKFLOW_RUN_DIR}/log/foo.log" echo "Hello, Mr. Thompson" > "$log_file" TEST_NAME="${TEST_NAME_BASE}-delete" cylc cat-log --mode=tail "$WORKFLOW_NAME" -f foo.log 2>&1 & cat_log_pid="$!" # Wait for tail to start poll get_tail_pid "$cat_log_pid" # We should be able to delete the log file run_ok "$TEST_NAME" rm "$log_file" # cat-log should exit (but exit code does not seem to be consistent across systems) poll_pid_done "$cat_log_pid" echo "Hello, Mr. Thompson" > "$log_file" TEST_NAME="${TEST_NAME_BASE}-kill" cylc cat-log --mode=tail "$WORKFLOW_NAME" -f foo.log 2>&1 & cat_log_pid="$!" # Wait for tail to start poll get_tail_pid "$cat_log_pid" kill "$(get_tail_pid "$cat_log_pid")" # cat-log should exit non-zero poll_pid_done "$cat_log_pid" run_fail "${TEST_NAME}_ret" wait "$cat_log_pid" purge cylc-flow-8.7.0/tests/functional/cylc-cat-log/06-log-rotation.t0000775000175000017500000000301415264420307024425 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Tests that cylc cat-log correctly handles log rotation. . "$(dirname "$0")/test_header" set_test_number 1 init_workflow "${TEST_NAME_BASE}" '/dev/null' # Populate its cylc-run dir with empty log files. LOG_DIR="$HOME/cylc-run/$WORKFLOW_NAME/log/scheduler" mkdir -p "${LOG_DIR}" touch -t '201001011200.00' "${LOG_DIR}/01-start-01.log" touch -t '201001011200.01' "${LOG_DIR}/02-start-01.log" touch -t '201001011200.02' "${LOG_DIR}/03-restart-02.log" # Test log rotation. for I in {0..2}; do basename "$(cylc cat-log "${WORKFLOW_NAME}" -m p -r "${I}")" done >'result' cmp_ok 'result' <<'__CMP__' 03-restart-02.log 02-start-01.log 01-start-01.log __CMP__ purge exit cylc-flow-8.7.0/tests/functional/cylc-cat-log/10-remote-no-retrieve.t0000775000175000017500000000333515264420307025540 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc cat-log" for remote tasks with no auto-retrieval. export REQUIRE_PLATFORM='loc:remote fs:indep' . "$(dirname "$0")/test_header" set_test_number 5 create_test_global_config "" " [platforms] [[${CYLC_TEST_PLATFORM}]] retrieve job logs = False" install_workflow "${TEST_NAME_BASE}" remote-simple TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" # Local job.out should not exist (not retrieved). LOCAL_JOB_DIR=$(cylc cat-log -f a -m d "${WORKFLOW_NAME}//1/a-task") exists_fail "${LOCAL_JOB_DIR}/job.out" # Cat the remote one. TEST_NAME=${TEST_NAME_BASE}-task-out run_ok "${TEST_NAME}" cylc cat-log -f o "${WORKFLOW_NAME}//1/a-task" grep_ok '^the quick brown fox$' "${TEST_NAME}.stdout" purge exit cylc-flow-8.7.0/tests/functional/cylc-cat-log/02-remote-custom-runtime-viewer-pbs/0000775000175000017500000000000015264420307030162 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/02-remote-custom-runtime-viewer-pbs/reference.log0000664000175000017500000000014015264420307032616 0ustar alastairalastairInitial point: 1 Final point: 1 1/a-task -triggered off [] 1/b-task -triggered off ['1/a-task'] cylc-flow-8.7.0/tests/functional/cylc-cat-log/02-remote-custom-runtime-viewer-pbs/flow.cylc0000664000175000017500000000116015264420307032003 0ustar alastairalastair#!Jinja2 [scheduling] [[graph]] R1 = a-task:echo => b-task [runtime] [[a-task]] script = """ echo rubbish echo garbage >&2 cylc message 'echo done' sleep 60 """ platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[[outputs]]] echo = "echo done" [[b-task]] script = """ sleep 10 # wait for buffer to flush? cylc cat-log --debug -f o "${CYLC_WORKFLOW_ID}//1/a-task" | grep 'rubbish' cylc cat-log --debug -f e "${CYLC_WORKFLOW_ID}//1/a-task" | grep 'garbage' """ cylc-flow-8.7.0/tests/functional/cylc-cat-log/04-local-tail.t0000775000175000017500000000370415264420307024034 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc cat-log" with a custom tail /command . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 4 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" create_test_global_config "" " [platforms] [[localhost]] tail command template = $PWD/bin/my-tailer.sh %(filename)s " #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" run_ok "wait-for-task-foo-start" \ cylc workflow-state "${WORKFLOW_NAME}//1/foo:started" --interval=1 --triggers TEST_NAME=${TEST_NAME_BASE}-cat-log cylc cat-log "${WORKFLOW_NAME}//1/foo" -f o -m t > "${TEST_NAME}.out" grep_ok "HELLO from foo 1" "${TEST_NAME}.out" #------------------------------------------------------------------------------- cylc stop --kill --max-polls=20 --interval=1 "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/cylc-cat-log/00-local.t0000775000175000017500000001722415264420307023103 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc cat-log" on the workflow host. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 43 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --no-detach "${WORKFLOW_NAME}" --reference-test #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-workflow-log-log run_ok "${TEST_NAME}" cylc cat-log "${WORKFLOW_NAME}" contains_ok "${TEST_NAME}.stdout" "${WORKFLOW_RUN_DIR}/log/scheduler/log" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-workflow-log-ok LOG_DIR="$(dirname "$(cylc cat-log -m p "${WORKFLOW_NAME}")")" echo "This is file 02-restart-02.log" > "${LOG_DIR}/02-restart-02.log" echo "This is file 03-restart-02.log" > "${LOG_DIR}/03-restart-02.log" # it should accept file paths relative to the scheduler log directory run_ok "${TEST_NAME}" cylc cat-log -f scheduler/03-restart-02.log "${WORKFLOW_NAME}" contains_ok "${TEST_NAME}.stdout" - << __END__ This is file 03-restart-02.log __END__ # it should pick the latest scheduler log file if no rotation number is provided run_ok "${TEST_NAME}" cylc cat-log --file s "${WORKFLOW_NAME}" contains_ok "${TEST_NAME}.stdout" - << __END__ This is file 03-restart-02.log __END__ # it should apply rotation number to scheduler log files run_ok "${TEST_NAME}" cylc cat-log -f s -r 1 "${WORKFLOW_NAME}" contains_ok "${TEST_NAME}.stdout" - << __END__ This is file 02-restart-02.log __END__ # it should list scheduler log files run_ok "${TEST_NAME}" cylc cat-log -m l "${WORKFLOW_NAME}" cmp_ok "${TEST_NAME}.stdout" - << __END__ config/01-start-01.cylc config/flow-processed.cylc install/01-install.log scheduler/01-start-01.log scheduler/02-restart-02.log scheduler/03-restart-02.log scheduler/reftest.log __END__ #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-task-out run_ok "${TEST_NAME}" cylc cat-log -f o "${WORKFLOW_NAME}//1/a-task" grep_ok '^the quick brown fox$' "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-task-job run_ok "${TEST_NAME}" cylc cat-log -f j "${WORKFLOW_NAME}//1/a-task" contains_ok "${TEST_NAME}.stdout" - << __END__ # SCRIPT: # Write to task stdout log echo "the quick brown fox" # Write to task stderr log echo "jumped over the lazy dog" >&2 # Write to a custom log file echo "drugs, money & whitespace" > "\${CYLC_TASK_LOG_ROOT} custom.log" # Generate a warning message in the workflow log. cylc message -p WARNING 'marmite and squashed bananas' __END__ #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-task-err run_ok "${TEST_NAME}" cylc cat-log -f e "${WORKFLOW_NAME}//1/a-task" grep_ok "jumped over the lazy dog" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-task-status run_ok "${TEST_NAME}" cylc cat-log -f s "${WORKFLOW_NAME}//1/a-task" grep_ok "CYLC_JOB_RUNNER_NAME=background" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-task-activity run_ok "${TEST_NAME}" cylc cat-log -f a "${WORKFLOW_NAME}//1/a-task" grep_ok '\[jobs-submit ret_code\] 0' "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-task-custom run_ok "${TEST_NAME}" cylc cat-log -f 'job custom.log' "${WORKFLOW_NAME}//1/a-task" grep_ok "drugs, money & whitespace" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-task-list-local-NN run_ok "${TEST_NAME}" cylc cat-log -f a -m l "${WORKFLOW_NAME}//1/a-task" contains_ok "${TEST_NAME}.stdout" <<__END__ job job-activity.log job custom.log job.err job.out job.status __END__ #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-task-list-local-01 run_ok "${TEST_NAME}" cylc cat-log -f a -m l -s 1 "${WORKFLOW_NAME}//1/a-task" contains_ok "${TEST_NAME}.stdout" <<__END__ job job-activity.log job custom.log job.err job.out job.status __END__ #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-task-list-local-02 run_fail cylc cat-log -f j -m l -s 2 "${WORKFLOW_NAME}//1/a-task" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-task-log-dir-NN run_ok "${TEST_NAME}" cylc cat-log -f j -m d "${WORKFLOW_NAME}//1/a-task" grep_ok "${WORKFLOW_NAME}/log/job/1/a-task/01$" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-task-log-dir-01 run_ok "${TEST_NAME}" cylc cat-log -f j -m d -s 1 "${WORKFLOW_NAME}//1/a-task" grep_ok "${WORKFLOW_NAME}/log/job/1/a-task/01$" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-task-job-path run_ok "${TEST_NAME}" cylc cat-log -f j -m p "${WORKFLOW_NAME}//1/a-task" grep_ok "${WORKFLOW_NAME}/log/job/1/a-task/01/job$" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- # it shouldn't let you modify the file path to access other resources # use the dedicated options TEST_NAME=${TEST_NAME_BASE}-un-norm-path run_fail "${TEST_NAME}" cylc cat-log -f j/../02/j "${WORKFLOW_NAME}//1/a-task" grep_ok 'InputError' "${TEST_NAME}.stderr" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-prepend-path run_ok "${TEST_NAME}-get-path" cylc cat-log -m p "${WORKFLOW_NAME}//1/a-task" run_ok "${TEST_NAME}" cylc cat-log --prepend-path "${WORKFLOW_NAME}//1/a-task" grep_ok "$(cat "#.*${TEST_NAME}-get-path.stdout")" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-submit-failed run_ok "${TEST_NAME}" cylc cat-log -m l "${WORKFLOW_NAME}//1/submit-failed" contains_ok "${TEST_NAME}.stdout" <<__END__ job.tmp job-activity.log __END__ #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-list-no-install-dir rm -r "${WORKFLOW_RUN_DIR}/log/install" run_ok "${TEST_NAME}-get-path" cylc cat-log -m l "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/cylc-cat-log/13-remote-out-err-tailer/0000775000175000017500000000000015264420307025763 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/13-remote-out-err-tailer/flow.cylc0000664000175000017500000000125515264420307027611 0ustar alastairalastair[scheduler] [[events]] abort on stall timeout = True stall timeout = PT2M [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = """ # wait for the started message to be received cylc__job__poll_grep_workflow_log -E 'foo.*running' # remove the out/err files rm "${CYLC_TASK_LOG_DIR}/job.out" rm "${CYLC_TASK_LOG_DIR}/job.err" # stop the workflow, orphaning this job cylc stop --now --now "${CYLC_WORKFLOW_ID}" 2>/dev/null >/dev/null # suppress any subsequent messages rm "${CYLC_WORKFLOW_RUN_DIR}/.service/contact" """ cylc-flow-8.7.0/tests/functional/cylc-cat-log/16-remote-tail-end/0000775000175000017500000000000015264420307024610 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/16-remote-tail-end/flow.cylc0000664000175000017500000000101515264420307026430 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = """ # Wait for "INFO - started" to be written to job.out: cylc__job__wait_cylc_message_started for I in $(seq 1 100); do echo "from $CYLC_TASK_NAME $I" done sleep 100 """ platform = {{environ['CYLC_TEST_PLATFORM'] | default("localhost")}} cylc-flow-8.7.0/tests/functional/cylc-cat-log/16-remote-tail-end/bin/0000775000175000017500000000000015264420307025360 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/16-remote-tail-end/bin/my-tailer.sh0000775000175000017500000000102215264420307027615 0ustar alastairalastair#!/usr/bin/env bash # Custom tailer that honours the %(lines)s substitution so the test can prove # that "cylc cat-log" passes the correct line count for each view mode: # * tail-from-start mode -> cylc substitutes "+1" -> whole file # * tail-end mode -> cylc substitutes "N" -> last N lines # Prefix output with HELLO to prove cylc used this custom tailer. # Exit immediately (i.e. don't follow) so the test does not block. LINES="$1" FILE="$2" tail -n "${LINES}" "${FILE}" | awk '{print "HELLO", $0; fflush() }' cylc-flow-8.7.0/tests/functional/cylc-cat-log/05-remote-tail/0000775000175000017500000000000015264420307024042 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/05-remote-tail/flow.cylc0000664000175000017500000000052615264420307025670 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] abort on stall timeout = True stall timeout = PT2M [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = """ for I in $(seq 1 100); do echo "from $CYLC_TASK_NAME $I" sleep 1 done""" platform={{environ['CYLC_TEST_PLATFORM'] | default("localhost")}} cylc-flow-8.7.0/tests/functional/cylc-cat-log/05-remote-tail/bin/0000775000175000017500000000000015264420307024612 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/05-remote-tail/bin/my-tailer.sh0000775000175000017500000000032415264420307027053 0ustar alastairalastair#!/usr/bin/env bash # Modify 'tail' output to prove that cylc used the custom tailer. # Exit immediately, for the test (i.e. don't 'tail -F') FILE="$1" tail -n +1 "${FILE}" | awk '{print "HELLO", $0; fflush() }' cylc-flow-8.7.0/tests/functional/cylc-cat-log/remote-simple/0000775000175000017500000000000015264420307024160 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/remote-simple/flow.cylc0000664000175000017500000000041715264420307026005 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] abort on stall timeout = True stall timeout = PT1M [scheduling] [[graph]] R1 = a-task [runtime] [[a-task]] script = echo "the quick brown fox" platform = {{ environ['CYLC_TEST_PLATFORM'] }} cylc-flow-8.7.0/tests/functional/cylc-cat-log/14-auto-mode.t0000664000175000017500000000552015264420307023701 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc cat-log" using auto mode with a custom tail /command . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 10 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" create_test_global_config "" " [platforms] [[localhost]] tail command template = $PWD/bin/my-tailer.sh %(filename)s " #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" cylc workflow-state "${WORKFLOW_NAME}//1/foo:started" --interval=1 --messages # Test that the custom tail command is used when the task is live TEST_NAME=${TEST_NAME_BASE}-cat-log-auto-tail run_ok "$TEST_NAME" cylc cat-log "${WORKFLOW_NAME}//1/foo" -f o -m a grep_ok "HELLO from foo 1" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- cylc stop --kill --max-polls=20 --interval=1 "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- # Test that the custom tail command is not used when the task is not live # (i.e., auto mode should not use the custom tail command) TEST_NAME=${TEST_NAME_BASE}-cat-log-auto-cat run_ok "$TEST_NAME" cylc cat-log "${WORKFLOW_NAME}//1/foo" -f o -m a grep_fail "HELLO from foo 1" "${TEST_NAME}.stdout" grep_ok "from foo 1" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- # Test that the custom tail command is not used when workflow logs are requested # (i.e., auto mode should not use the custom tail command) TEST_NAME=${TEST_NAME_BASE}-cat-log-auto-workflow # Use the workflow name without the task ID to get the workflow logs run_ok "$TEST_NAME" cylc cat-log "${WORKFLOW_NAME}" -m a grep_fail "HELLO" "${TEST_NAME}.stdout" grep_ok "Workflow:.*/f.*/cylc-cat-log/14-auto-mode" "${TEST_NAME}.stdout" purge cylc-flow-8.7.0/tests/functional/cylc-cat-log/16-remote-tail-end.t0000775000175000017500000001117615264420307025006 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc cat-log" tail view modes with a custom remote tail command that # honours the %(lines)s substitution: # * tail-from-start (-m t) -> cylc sends "+1" -> whole file # * tail-end (-m te) -> cylc sends "N" -> last N lines only # Also checks that a non-positive --tail-lines value is rejected. export REQUIRE_PLATFORM='loc:remote fs:indep comms:tcp runner:background' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 7 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" set -eu SSH='ssh -oBatchMode=yes -oConnectTimeout=5' SCP='scp -oBatchMode=yes -oConnectTimeout=5' $SSH -n "${CYLC_TEST_HOST}" "mkdir -p cylc-run/.bin" # shellcheck disable=SC2016 create_test_global_config "" " [platforms] [[$CYLC_TEST_PLATFORM]] tail command template = \$HOME/cylc-run/.bin/my-tailer.sh %(lines)s %(filename)s retrieve job logs = False " #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- $SCP "${PWD}/bin/my-tailer.sh" \ "${CYLC_TEST_HOST}:cylc-run/.bin/my-tailer.sh" #------------------------------------------------------------------------------- # Run detached. workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- poll_grep_workflow_log -E '1/foo/01:preparing.* => submitted' # Wait until enough output exists on the job host that "tail-end" with a small # line count cannot possibly include the first line (makes the test # deterministic regardless of how fast the job host is writing output). # shellcheck disable=SC2086 poll $SSH -n "${CYLC_TEST_HOST}" \ "grep -qs -- 'from foo 10' 'cylc-run/${WORKFLOW_NAME}/log/job/1/foo/NN/job.out'" #------------------------------------------------------------------------------- # cylc cat-log tail modes follow the file, so each invocation needs to be # killed; the custom tailer exits by itself but we add a timeout for safety. # # Scenario 1: tail-from-start (-m t) shows the whole file (cylc sends "+1"), # so the very first line of output is present. TEST_NAME="${TEST_NAME_BASE}-tail-from-start" timeout -s 'INT' 15 \ cylc cat-log "${WORKFLOW_NAME}//1/foo" -f 'o' -m 't' --force-remote \ >"${TEST_NAME}.out" 2>"${TEST_NAME}.err" || true grep_ok "HELLO from foo 1$" "${TEST_NAME}.out" #------------------------------------------------------------------------------- # Scenario 2: tail-end (-m te) with --tail-lines 3 shows only the last 3 lines # (cylc sends "3"): exactly 3 lines of output, and NOT the first line. TEST_NAME="${TEST_NAME_BASE}-tail-end-3" timeout -s 'INT' 15 \ cylc cat-log "${WORKFLOW_NAME}//1/foo" -f 'o' -m 'te' --tail-lines 3 \ --force-remote >"${TEST_NAME}.out" 2>"${TEST_NAME}.err" || true count_ok "HELLO from foo" "${TEST_NAME}.out" 3 grep_fail "HELLO from foo 1$" "${TEST_NAME}.out" #------------------------------------------------------------------------------- # Scenario 3: tail-end (-m te) with --tail-lines 1 shows just the last line. TEST_NAME="${TEST_NAME_BASE}-tail-end-1" timeout -s 'INT' 15 \ cylc cat-log "${WORKFLOW_NAME}//1/foo" -f 'o' -m 'te' --tail-lines 1 \ --force-remote >"${TEST_NAME}.out" 2>"${TEST_NAME}.err" || true count_ok "HELLO from foo" "${TEST_NAME}.out" 1 #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-stop run_ok "${TEST_NAME}" cylc stop --kill --max-polls=20 --interval=1 "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- $SSH -n "${CYLC_TEST_HOST}" "rm -rf cylc-run/.bin/" purge exit cylc-flow-8.7.0/tests/functional/cylc-cat-log/test_header0000777000175000017500000000000015264420307027713 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/01-remote/0000775000175000017500000000000015264420307023107 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/01-remote/flow.cylc0000664000175000017500000000103015264420307024724 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] [[graph]] R1 = a-task [runtime] [[a-task]] script = """ # Write to task stdout log echo "the quick brown fox" # Write to task stderr log echo "jumped over the lazy dog" >&2 # Write to a custom log file echo "drugs and money" > ${CYLC_TASK_LOG_ROOT}.custom-log """ platform = {{ environ['CYLC_TEST_PLATFORM'] }} cylc-flow-8.7.0/tests/functional/cylc-cat-log/05-remote-tail.t0000775000175000017500000000546115264420307024240 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc cat-log" with a custom remote tail command. export REQUIRE_PLATFORM='loc:remote comms:tcp runner:background' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 4 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" set -eu SSH='ssh -oBatchMode=yes -oConnectTimeout=5' SCP='scp -oBatchMode=yes -oConnectTimeout=5' $SSH -n "${CYLC_TEST_HOST}" "mkdir -p cylc-run/.bin" # shellcheck disable=SC2016 create_test_global_config "" " [platforms] [[$CYLC_TEST_PLATFORM]] tail command template = \$HOME/cylc-run/.bin/my-tailer.sh %(filename)s" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- $SCP "${PWD}/bin/my-tailer.sh" \ "${CYLC_TEST_HOST}:cylc-run/.bin/my-tailer.sh" #------------------------------------------------------------------------------- # Run detached. workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- poll_grep_workflow_log -E '1/foo/01:preparing.* => submitted' # cylc cat-log -m 't' tail-follows a file, so needs to be killed. # Send interrupt signal to tail command after 15 seconds. TEST_NAME="${TEST_NAME_BASE}-cat-log" timeout -s 'INT' 15 \ cylc cat-log "${WORKFLOW_NAME}//1/foo" -f 'o' -m 't' --force-remote \ >"${TEST_NAME}.out" 2>"${TEST_NAME}.err" || true grep_ok "HELLO from foo 1" "${TEST_NAME}.out" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-stop run_ok "${TEST_NAME}" cylc stop --kill --max-polls=20 --interval=1 "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- $SSH -n "${CYLC_TEST_HOST}" "rm -rf cylc-run/.bin/" purge exit cylc-flow-8.7.0/tests/functional/cylc-cat-log/14-auto-mode/0000775000175000017500000000000015264420307023512 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/14-auto-mode/flow.cylc0000664000175000017500000000047415264420307025342 0ustar alastairalastair[scheduler] [[events]] abort on stall timeout = True stall timeout = PT2M [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = """ for I in $(seq 1 100); do echo "from $CYLC_TASK_NAME $I" sleep 1 done """ cylc-flow-8.7.0/tests/functional/cylc-cat-log/14-auto-mode/bin/0000775000175000017500000000000015264420307024262 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/14-auto-mode/bin/my-tailer.sh0000775000175000017500000000032415264420307026523 0ustar alastairalastair#!/usr/bin/env bash # Modify 'tail' output to prove that cylc used the custom tailer. # Exit immediately, for the test (i.e. don't 'tail -F') FILE="$1" tail -n +1 "${FILE}" | awk '{print "HELLO", $0; fflush() }' cylc-flow-8.7.0/tests/functional/cylc-cat-log/01-remote.t0000775000175000017500000001232715264420307023304 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc cat-log" for remote tasks. export REQUIRE_PLATFORM='loc:remote fs:indep' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 27 create_test_global_config "" " [platforms] [[${CYLC_TEST_PLATFORM}]] retrieve job logs = False" install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-task-out run_ok "$TEST_NAME" cylc cat-log -f o "${WORKFLOW_NAME}//1/a-task" grep_ok '^the quick brown fox$' "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-task-job run_ok "$TEST_NAME" cylc cat-log -f j "${WORKFLOW_NAME}//1/a-task" contains_ok "${TEST_NAME}.stdout" - << __END__ # SCRIPT: # Write to task stdout log echo "the quick brown fox" # Write to task stderr log echo "jumped over the lazy dog" >&2 # Write to a custom log file echo "drugs and money" > \${CYLC_TASK_LOG_ROOT}.custom-log __END__ #------------------------------------------------------------------------------- # remote TEST_NAME=${TEST_NAME_BASE}-task-err run_ok "$TEST_NAME" cylc cat-log -f e "${WORKFLOW_NAME}//1/a-task" grep_ok "jumped over the lazy dog" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- # remote TEST_NAME=${TEST_NAME_BASE}-task-status run_ok "$TEST_NAME" cylc cat-log -f s "${WORKFLOW_NAME}//1/a-task" grep_ok "CYLC_JOB_RUNNER_NAME=$CYLC_TEST_JOB_RUNNER" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- # local TEST_NAME=${TEST_NAME_BASE}-task-activity run_ok "$TEST_NAME" cylc cat-log -f a "${WORKFLOW_NAME}//1/a-task" grep_ok '\[jobs-submit ret_code\] 0' "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- # remote TEST_NAME=${TEST_NAME_BASE}-task-custom run_ok "$TEST_NAME" cylc cat-log -f 'job.custom-log' "${WORKFLOW_NAME}//1/a-task" grep_ok "drugs and money" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- # local TEST_NAME=${TEST_NAME_BASE}-task-list-local-NN run_ok "$TEST_NAME" cylc cat-log -f a -m l "${WORKFLOW_NAME}//1/a-task" cmp_ok "${TEST_NAME}.stdout" <<__END__ job job-activity.log __END__ #------------------------------------------------------------------------------- # local TEST_NAME=${TEST_NAME_BASE}-task-list-local-01 run_ok "$TEST_NAME" cylc cat-log -f a -m l -s 1 "${WORKFLOW_NAME}//1/a-task" cmp_ok "${TEST_NAME}.stdout" <<__END__ job job-activity.log __END__ #------------------------------------------------------------------------------- # remote TEST_NAME=${TEST_NAME_BASE}-task-list-remote-NN run_ok "$TEST_NAME" cylc cat-log -m l "${WORKFLOW_NAME}//1/a-task" cmp_ok "${TEST_NAME}.stdout" <<__END__ job job-activity.log job.custom-log job.err job.out job.status job.xtrace __END__ #------------------------------------------------------------------------------- # remote TEST_NAME=${TEST_NAME_BASE}-task-log-dir-NN run_ok "$TEST_NAME" cylc cat-log -f j -m d "${WORKFLOW_NAME}//1/a-task" grep_ok "${WORKFLOW_NAME}/log/job/1/a-task/01$" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- # remote TEST_NAME=${TEST_NAME_BASE}-task-log-dir-01 run_ok "$TEST_NAME" cylc cat-log -m d -f j -s 1 "${WORKFLOW_NAME}//1/a-task" grep_ok "${WORKFLOW_NAME}/log/job/1/a-task/01$" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- # remote TEST_NAME=${TEST_NAME_BASE}-task-job-path run_ok "$TEST_NAME" cylc cat-log -m p -f j "${WORKFLOW_NAME}//1/a-task" grep_ok "${WORKFLOW_NAME}/log/job/1/a-task/01/job$" "${TEST_NAME}.stdout" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-un-norm-path run_fail "${TEST_NAME}" cylc cat-log "${WORKFLOW_NAME}//1/a-task" \ --remote-arg=j/../02/j \ --remote-arg=cat \ --remote-arg='tail -f' grep_ok 'InputError' "${TEST_NAME}.stderr" #------------------------------------------------------------------------------- # Clean up the task host. purge cylc-flow-8.7.0/tests/functional/cylc-cat-log/15-remote_when_no_local_log/0000775000175000017500000000000015264420307026644 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-cat-log/15-remote_when_no_local_log/flow.cylc0000664000175000017500000000112715264420307030470 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] abort on stall timeout = True stall timeout = PT1M [scheduling] [[graph]] R1 = a-task => b-task [runtime] [[a-task]] script = """ # Write to task stdout log echo "Master, Master! Where are those dreams that I've been after?" """ platform = {{CYLC_TEST_PLATFORM | default("localhost")}} [[b-task]] script = """ cylc__job__poll_grep_workflow_log 'TaskJobLogsRetrieveContext.*ret_code.*0' rm ${CYLC_WORKFLOW_RUN_DIR}/log/job/1/a-task/NN/job """ cylc-flow-8.7.0/tests/functional/cyclepoint/0000775000175000017500000000000015264420307021271 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclepoint/02-template.t0000775000175000017500000000452315264420307023517 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # basic cylc cyclepoint --template option tests . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 12 #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-extract run_ok "${TEST_NAME}.strf" cylc cyclepoint --template foo-%Y-%m-%d-%H.nc 20140809T12 cmp_ok "${TEST_NAME}.strf.stdout" - << __OUT__ foo-2014-08-09-12.nc __OUT__ #------------------------------------------------------------------------------- run_ok "${TEST_NAME}.yr" cylc cyclepoint --template CCYY 20140808T1200 cmp_ok "${TEST_NAME}.yr.stdout" - << __OUT__ 2014 __OUT__ #------------------------------------------------------------------------------- run_ok "${TEST_NAME}.month" cylc cyclepoint --template CC 20140809T1200 cmp_ok "${TEST_NAME}.month.stdout" - << __OUT__ 20 __OUT__ #------------------------------------------------------------------------------- run_ok "${TEST_NAME}.month" cylc cyclepoint --template MM 20140809T1200 cmp_ok "${TEST_NAME}.month.stdout" - << __OUT__ 08 __OUT__ #------------------------------------------------------------------------------- run_ok "${TEST_NAME}.day" cylc cyclepoint --template DD 20140809T1200 cmp_ok "${TEST_NAME}.day.stdout" - << __OUT__ 09 __OUT__ #------------------------------------------------------------------------------- run_ok "${TEST_NAME}.hour" cylc cyclepoint --template hh 20140809T1200 cmp_ok "${TEST_NAME}.hour.stdout" - << __OUT__ 12 __OUT__ cylc-flow-8.7.0/tests/functional/cyclepoint/00-time.t0000664000175000017500000001153715264420307022640 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # basic jinja2 expansion test . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 45 #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-use-env-var export CYLC_TASK_CYCLE_POINT=20100102T0300 run_ok "${TEST_NAME}.check-env" cylc cycle-point run_ok "${TEST_NAME}.year-only" cylc cycle-point --print-year cmp_ok "${TEST_NAME}.year-only.stdout" - << __OUT__ 2010 __OUT__ run_ok "${TEST_NAME}.month-only" cylc cycle-point --print-month cmp_ok "${TEST_NAME}.month-only.stdout" - << __OUT__ 01 __OUT__ run_ok "${TEST_NAME}.day-only" cylc cycle-point --print-day cmp_ok "${TEST_NAME}.day-only.stdout" - << __OUT__ 02 __OUT__ run_ok "${TEST_NAME}.hour-only" cylc cycle-point --print-hour cmp_ok "${TEST_NAME}.hour-only.stdout" - << __OUT__ 03 __OUT__ #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-offset-env-var run_ok "${TEST_NAME}.year" cylc cycle-point --offset-years=10 cmp_ok "${TEST_NAME}.year.stdout" - << __OUT__ 20200102T0300 __OUT__ run_ok "${TEST_NAME}.year-neg" cylc cycle-point --offset-years=-11 cmp_ok "${TEST_NAME}.year-neg.stdout" - << __OUT__ 19990102T0300 __OUT__ run_ok "${TEST_NAME}.month" cylc cycle-point --offset-months=2 cmp_ok "${TEST_NAME}.month.stdout" - << __OUT__ 20100302T0300 __OUT__ run_ok "${TEST_NAME}.month-neg" cylc cycle-point --offset-months=-1 cmp_ok "${TEST_NAME}.month-neg.stdout" - << __OUT__ 20091202T0300 __OUT__ run_ok "${TEST_NAME}.day" cylc cycle-point --offset-days=10 cmp_ok "${TEST_NAME}.day.stdout" - << __OUT__ 20100112T0300 __OUT__ run_ok "${TEST_NAME}.day-neg" cylc cycle-point --offset-days=-2 cmp_ok "${TEST_NAME}.day-neg.stdout" - << __OUT__ 20091231T0300 __OUT__ run_ok "${TEST_NAME}.hour" cylc cycle-point --offset-hours=10 cmp_ok "${TEST_NAME}.hour.stdout" - << __OUT__ 20100102T1300 __OUT__ run_ok "${TEST_NAME}.hour-neg" cylc cycle-point --offset-hours=-3 cmp_ok "${TEST_NAME}.hour-neg.stdout" - << __OUT__ 20100102T0000 __OUT__ #------------------------------------------------------------------------------- #Test with a supplied cycle time # N.B. this also checks environment variable being by CLI options TEST_NAME="${TEST_NAME_BASE}-print-supplied-ctime" run_ok "${TEST_NAME}.full" cylc cycle-point '2011-01-01' cmp_ok "${TEST_NAME}.full.stdout" - <<<'2011-01-01' run_ok "${TEST_NAME}-offset-week" cylc cycle-point --offset=P1W '20160301T06Z' cmp_ok "${TEST_NAME}-offset-week.stdout" - <<<'20160308T06Z' #------------------------------------------------------------------------------- unset CYLC_TASK_CYCLE_POINT # Test --equal option TEST_NAME="${TEST_NAME_BASE}-equal" run_ok "${TEST_NAME}-true" cylc cycle-point 2000 --equal 2000 run_fail "${TEST_NAME}-true" cylc cycle-point 2000 --equal 2001 run_fail "${TEST_NAME}-invalid" cylc cycle-point 2000 --equal x # Test --template option TEST_NAME="${TEST_NAME_BASE}-template" run_ok "${TEST_NAME}-pass" cylc cycle-point 2010-08 \ --offset-years=2 --template=foo-CCYY-MM.nc cmp_ok "${TEST_NAME}-pass.stdout" <<< 'foo-2012-08.nc' # invalid arg combo run_fail "${TEST_NAME}-fail" cylc cycle-point 2000 --template=x --print-year #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-fail" # no cycle point run_fail "${TEST_NAME}-1" cylc cycle-point # invalid cycle point run_fail "${TEST_NAME}-2" cylc cycle-point x # too many cycle points run_fail "${TEST_NAME}-3" cylc cycle-point 2000 2000 # invalid offsets run_ok "${TEST_NAME}-5" cylc cycle-point 2000 --offset-hours=1 # VALID run_fail "${TEST_NAME}-6" cylc cycle-point 2000 --offset-hours=x # INVALID run_fail "${TEST_NAME}-7" cylc cycle-point 2000 --offset-days=x run_fail "${TEST_NAME}-8" cylc cycle-point 2000 --offset-months=x run_fail "${TEST_NAME}-9" cylc cycle-point 2000 --offset-years=x # invalid ISO offset run_ok "${TEST_NAME}-10" cylc cycle-point 2000 --offset=P1Y # VALID run_fail "${TEST_NAME}-11" cylc cycle-point 2000 --offset=PT1Y # INVALID cylc-flow-8.7.0/tests/functional/cyclepoint/test_header0000777000175000017500000000000015264420307027606 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/0000775000175000017500000000000015264420307022063 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/04-submit-num.t0000775000175000017500000000303715264420307024577 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test usage of CYLC_TASK_SUBMIT_NUMBER. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/job-submission/09-activity-log-host-bad-submit/0000775000175000017500000000000015264420307027724 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/09-activity-log-host-bad-submit/reference.log0000664000175000017500000000026015264420307032363 0ustar alastairalastairInitial point: 19990101T0000Z Final point: 19990101T0000Z 19990101T0000Z/bad-submitter -triggered off [] 19990101T0000Z/grepper -triggered off ['19990101T0000Z/bad-submitter'] cylc-flow-8.7.0/tests/functional/job-submission/09-activity-log-host-bad-submit/flow.cylc0000664000175000017500000000140015264420307031542 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [[events]] expected task failures = 19990101T0000Z/bad-submitter [scheduling] initial cycle point=1999 final cycle point=1999 [[graph]] P1Y = bad-submitter:submit-failed? => grepper [runtime] [[root]] [[bad-submitter]] script = true platform = {{ CYLC_TEST_PLATFORM }} [[grepper]] script = """ set -x # Test that the original command is printed A_LOG="$(dirname "$0")/../../bad-submitter/01/job-activity.log" grep '\[jobs-submit cmd\] ssh .* {{CYLC_TEST_HOST}} .*cylc jobs-submit.*' \ "${A_LOG}" # Stop the workflow cleanly cylc stop "${CYLC_WORKFLOW_ID}" """ cylc-flow-8.7.0/tests/functional/job-submission/17-remote-localtime/0000775000175000017500000000000015264420307025552 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/17-remote-localtime/reference.log0000664000175000017500000000006715264420307030216 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/job-submission/17-remote-localtime/flow.cylc0000664000175000017500000000037615264420307027403 0ustar alastairalastair#!jinja2 [scheduler] UTC mode = False [scheduling] [[graph]] R1=t1 [runtime] [[t1]] script = test -z "${TZ:-}" platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[[job]]] execution time limit = PT1M cylc-flow-8.7.0/tests/functional/job-submission/06-garbage.t0000775000175000017500000000273615264420307024076 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test job submission, poll and kill with a garbage command. . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" if [[ -n "${PYTHONPATH:-}" ]]; then export PYTHONPATH="${PWD}/lib:${PYTHONPATH}" else export PYTHONPATH="${PWD}/lib" fi create_test_global_config ' [platforms] [[bad]] ' run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge exit ././@LongLink0000644000000000000000000000015100000000000011600 Lustar rootrootcylc-flow-8.7.0/tests/functional/job-submission/14-tidy-submits-of-prev-run-remote-host-with-shared-fs.tcylc-flow-8.7.0/tests/functional/job-submission/14-tidy-submits-of-prev-run-remote-host-with-shared-0000775000175000017500000000365315264420307033613 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test tidy of submits of previous runs. export REQUIRE_PLATFORM='loc:remote fs:shared' . "$(dirname "$0")/test_header" set_test_number 7 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" LOGD1="$RUN_DIR/${WORKFLOW_NAME}/log/job/1/t1/01" LOGD2="$RUN_DIR/${WORKFLOW_NAME}/log/job/1/t1/02" exists_ok "${LOGD1}" exists_ok "${LOGD2}" sed -i 's/script =.*$/script = true/' "${WORKFLOW_RUN_DIR}/flow.cylc" sed -i -n '1,/triggered off/p' "${WORKFLOW_RUN_DIR}/reference.log" delete_db workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" exists_ok "${LOGD1}" exists_fail "${LOGD2}" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/job-submission/01-job-nn-localhost/0000775000175000017500000000000015264420307025452 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/01-job-nn-localhost/reference.log0000664000175000017500000000007015264420307030110 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] cylc-flow-8.7.0/tests/functional/job-submission/01-job-nn-localhost/flow.cylc0000664000175000017500000000114215264420307027273 0ustar alastairalastair#!jinja2 [scheduler] UTC mode = True # Ignore DST [scheduling] [[graph]] R1 = foo [runtime] {% if CYLC_TEST_PLATFORM is defined %} [[root]] platform = {{ CYLC_TEST_PLATFORM }} {% endif %} [[foo]] script = """ JOB_LOG_DIR="$(dirname "${CYLC_TASK_LOG_DIR}")" NN_VALUE="$(readlink "${JOB_LOG_DIR}/NN")" # bash 4.2.0 bug: ((VAR == VAL)) does not trigger 'set -e': test "${CYLC_TASK_SUBMIT_NUMBER}" -eq "100" test "${NN_VALUE}" -eq "100" """ [[[job]]] execution retry delays = 99*PT0S cylc-flow-8.7.0/tests/functional/job-submission/01-job-nn-localhost/db.sqlite30000664000175000017500000000643015264420307027350 0ustar alastairalastairPRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE broadcast_events(time TEXT, change TEXT, point TEXT, namespace TEXT, key TEXT, value TEXT); CREATE TABLE broadcast_states(point TEXT, namespace TEXT, key TEXT, value TEXT, PRIMARY KEY(point, namespace, key)); CREATE TABLE inheritance(namespace TEXT, inheritance TEXT, PRIMARY KEY(namespace)); INSERT INTO inheritance VALUES('root','["root"]'); INSERT INTO inheritance VALUES('foo','["foo", "root"]'); CREATE TABLE workflow_params(key TEXT, value TEXT, PRIMARY KEY(key)); INSERT INTO workflow_params VALUES('cylc_version', '8.0.0'); INSERT INTO workflow_params VALUES('uuid_str', 'Something'); INSERT INTO workflow_params VALUES('UTC_mode', 1); CREATE TABLE workflow_template_vars(key TEXT, value TEXT, PRIMARY KEY(key)); CREATE TABLE task_action_timers(cycle TEXT, name TEXT, ctx_key TEXT, ctx TEXT, delays TEXT, num INTEGER, delay TEXT, timeout TEXT, PRIMARY KEY(cycle, name, ctx_key)); INSERT INTO task_action_timers VALUES('1','foo','"poll_timer"','["tuple", [[99, "running"]]]','[]',0,NULL,NULL); INSERT INTO task_action_timers VALUES('1','foo','["try_timers", "submit-retrying"]','null','[]',0,NULL,NULL); INSERT INTO task_action_timers VALUES('1','foo','["try_timers", "retrying"]','null','[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]',99,'0.0','1560508824.17287'); CREATE TABLE task_events(name TEXT, cycle TEXT, time TEXT, submit_num INTEGER, event TEXT, message TEXT); CREATE TABLE task_jobs(cycle TEXT, name TEXT, submit_num INTEGER, is_manual_submit INTEGER, try_num INTEGER, time_submit TEXT, time_submit_exit TEXT, submit_status INTEGER, time_run TEXT, time_run_exit TEXT, run_signal TEXT, run_status INTEGER, platform_name TEXT, job_runner_name TEXT, job_id TEXT, PRIMARY KEY(cycle, name, submit_num)); CREATE TABLE task_late_flags(cycle TEXT, name TEXT, value INTEGER, PRIMARY KEY(cycle, name)); CREATE TABLE task_outputs(cycle TEXT, name TEXT, flow_nums TEXT, outputs TEXT, PRIMARY KEY(cycle, name, flow_nums)); CREATE TABLE task_pool(cycle TEXT, name TEXT, flow_nums TEXT, status TEXT, is_held INTEGER, PRIMARY KEY(cycle, name, flow_nums)); INSERT INTO task_pool VALUES('1','foo','["1"]','waiting', 0); CREATE TABLE task_states(name TEXT, cycle TEXT, flow_nums TEXT, time_created TEXT, time_updated TEXT, submit_num INTEGER, status TEXT, flow_wait INTEGER, PRIMARY KEY(name, cycle, flow_nums)); INSERT INTO task_states VALUES('foo','1','["1"]', '2019-06-14T11:30:16+01:00','2019-06-14T11:40:24+01:00',99,'waiting','0'); CREATE TABLE task_prerequisites(cycle TEXT, name TEXT, flow_nums TEXT, prereq_name TEXT, prereq_cycle TEXT, prereq_output TEXT, satisfied TEXT, PRIMARY KEY(cycle, name, flow_nums, prereq_name, prereq_cycle, prereq_output)); CREATE TABLE task_timeout_timers(cycle TEXT, name TEXT, timeout REAL, PRIMARY KEY(cycle, name)); CREATE TABLE xtriggers(signature TEXT, results TEXT, PRIMARY KEY(signature)); COMMIT; cylc-flow-8.7.0/tests/functional/job-submission/10-at-shell/0000775000175000017500000000000015264420307024012 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/10-at-shell/reference.log0000664000175000017500000000007015264420307026450 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] cylc-flow-8.7.0/tests/functional/job-submission/10-at-shell/flow.cylc0000664000175000017500000000023215264420307025632 0ustar alastairalastair#!Jinja2 [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = true platform = {{ environ['CYLC_TEST_PLATFORM'] }} cylc-flow-8.7.0/tests/functional/job-submission/06-garbage/0000775000175000017500000000000015264420307023676 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/06-garbage/lib/0000775000175000017500000000000015264420307024444 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/06-garbage/lib/bad.py0000664000175000017500000000200615264420307025542 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Garbage batch system/job runner, for testing.""" class BadSubmitHandler: """Garbage batch system/job runner, for testing.""" SUBMIT_CMD_TMPL = "bad-bad-bad-submit '%(job)s'" JOB_RUNNER_HANDLER = BadSubmitHandler() cylc-flow-8.7.0/tests/functional/job-submission/06-garbage/reference.log0000664000175000017500000000012415264420307026334 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] 1/t2 -triggered off ['1/t1'] cylc-flow-8.7.0/tests/functional/job-submission/06-garbage/flow.cylc0000664000175000017500000000106615264420307025524 0ustar alastairalastair[scheduler] [[events]] expected task failures = 1/t1 [scheduling] [[graph]] R1 = t1:submit-fail? => t2 [runtime] [[t1]] script = true platform = bad [[t2]] script = """ grep -q -F \ 'platform: bad - Could not connect to bad' \ "${CYLC_WORKFLOW_LOG_DIR}/log" grep -q -F \ 'remote-init will retry if another host is available' \ "${CYLC_WORKFLOW_LOG_DIR}/log" cylc shutdown "${CYLC_WORKFLOW_ID}" """ cylc-flow-8.7.0/tests/functional/job-submission/16-timeout.t0000775000175000017500000000457015264420307024173 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that job submission kill on timeout results in a failed job submission. export REQUIRE_PLATFORM='runner:at comms:tcp' . "$(dirname "$0")/test_header" set_test_number 4 create_test_global_config "" " [scheduler] process pool timeout = PT10S [platforms] [[$CYLC_TEST_PLATFORM]] job runner command template = sleep 30 " install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-workflow-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" # egrep -m is stop matching after matches # -A is number of lines of context after match cylc cat-log "${WORKFLOW_NAME}" \ | grep -E -m 1 -A 2 "ERROR - \[jobs-submit cmd\]" \ | sed -e 's/^.* \(ERROR\)/\1/' > log WORKFLOW_LOG_DIR=$(cylc cat-log -m p "${WORKFLOW_NAME}" | sed 's/01-start-01\.//') JOB_LOG_DIR="${WORKFLOW_LOG_DIR%scheduler/log}" JOB_LOG_DIR="${JOB_LOG_DIR/$HOME/\$HOME}" DEFAULT_PATHS='--path=/bin --path=/usr/bin --path=/usr/local/bin --path=/sbin --path=/usr/sbin --path=/usr/local/sbin' cmp_ok log <<__END__ ERROR - [jobs-submit cmd] cylc jobs-submit --debug ${DEFAULT_PATHS} -- '${JOB_LOG_DIR}job' 1/foo/01 [jobs-submit ret_code] -9 [jobs-submit err] killed on timeout (PT10S) __END__ cylc workflow-state --old-format "${WORKFLOW_NAME}" > workflow-state.log # make sure foo submit failed and the stopper ran contains_ok workflow-state.log << __END__ stopper, 1, succeeded foo, 1, submit-failed __END__ purge cylc-flow-8.7.0/tests/functional/job-submission/04-submit-num/0000775000175000017500000000000015264420307024404 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/04-submit-num/flow.cylc0000664000175000017500000000141715264420307026232 0ustar alastairalastair[scheduler] [[events]] abort on stall timeout = True stall timeout = PT30S [scheduling] [[graph]] R1 = """ foo:fail? => bar foo? & bar => baz """ [runtime] [[foo]] script = """ echo "${CYLC_TASK_SUBMIT_NUMBER}" \ >>"${CYLC_WORKFLOW_RUN_DIR}/foo-submits.txt" # bash 4.2.0 bug: ((VAR == VAL)) does not trigger 'set -e': test "${CYLC_TASK_SUBMIT_NUMBER}" -gt "${CYLC_TASK_TRY_NUMBER}" """ [[[job]]] execution retry delays=2*PT0S [[bar]] script = cylc trigger "${CYLC_WORKFLOW_ID}//1/foo" [[baz]] script = """ printf "%d\n" {1..4} | cmp - "${CYLC_WORKFLOW_RUN_DIR}/foo-submits.txt" """ cylc-flow-8.7.0/tests/functional/job-submission/19-platform_select/0000775000175000017500000000000015264420307025475 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/19-platform_select/reference.log0000664000175000017500000000062215264420307030136 0ustar alastairalastair1/platform_no_subshell -triggered off [] in flow 1 1/host_no_subshell -triggered off [] in flow 1 1/platform_subshell -triggered off [] in flow 1 1/host_subshell -triggered off [] in flow 1 1/host_subshell_backticks -triggered off [] in flow 1 1/localhost_subshell -triggered off [] in flow 1 1/platform_subshell_suffix -triggered off [] in flow 1 1/platform_subshell_empty -triggered off [] in flow 1 cylc-flow-8.7.0/tests/functional/job-submission/19-platform_select/flow.cylc0000664000175000017500000000256315264420307027326 0ustar alastairalastair[meta] purpose = """ Test that subshells are handled for platform and host configs. Tasks of the form .*no_subshell act as control runs. """ [scheduler] UTC mode = True [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] [[dependencies]] R1 = """ host_no_subshell localhost_subshell platform_subshell:submit-fail? platform_no_subshell:submit-fail? platform_subshell_empty:submit-fail? platform_subshell_suffix:submit-fail? host_subshell:submit-fail? host_subshell_backticks:submit-fail? """ [runtime] [[root]] script = true [[platform_no_subshell]] platform = improbable platform name [[host_no_subshell]] [[[remote]]] host = localhost [[platform_subshell]] platform = $(echo "improbable platform name") [[platform_subshell_empty]] platform = $(echo "") [[platform_subshell_suffix]] platform = prefix-$( echo middle )-suffix [[host_subshell]] [[[remote]]] host = $(echo "improbable host name") [[host_subshell_backticks]] [[[remote]]] host = `echo "improbable host name"` [[localhost_subshell]] [[[remote]]] host = $(echo "localhost4.localdomain4") cylc-flow-8.7.0/tests/functional/job-submission/17-remote-localtime.t0000775000175000017500000000203215264420307025737 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that workflow does not set remote job TZ when in local time. export REQUIRE_PLATFORM='loc:remote' . "$(dirname "$0")/test_header" set_test_number 2 reftest purge exit cylc-flow-8.7.0/tests/functional/job-submission/15-garbage-platform-command-2/0000775000175000017500000000000015264420307027273 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/15-garbage-platform-command-2/reference.log0000664000175000017500000000012015264420307031725 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/foo -triggered off [] cylc-flow-8.7.0/tests/functional/job-submission/15-garbage-platform-command-2/flow.cylc0000664000175000017500000000055015264420307031116 0ustar alastairalastair[scheduler] [[events]] inactivity timeout = PT1M abort on inactivity timeout = True stall timeout = PT20S abort on stall timeout = True [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = true platform = $(my-host-select) [[[job]]] submission retry delays = PT5S cylc-flow-8.7.0/tests/functional/job-submission/15-garbage-platform-command-2/bin/0000775000175000017500000000000015264420307030043 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/15-garbage-platform-command-2/bin/my-host-select0000775000175000017500000000016115264420307032644 0ustar alastairalastair#!/usr/bin/env bash TXT="${CYLC_WORKFLOW_RUN_DIR}/my-host-select.txt" cat "${TXT}" || echo 'localhost' >"${TXT}" cylc-flow-8.7.0/tests/functional/job-submission/20-check-chunking.t0000775000175000017500000000310515264420307025352 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that a job containing more than 100 tasks will split into batches. . "$(dirname "$0")/test_header" set_test_number 3 create_test_global_config '' " [scheduler] process pool size = 1 [platforms] [[$CYLC_TEST_PLATFORM]] max batch submit size = 2 " install_workflow run_ok "${TEST_NAME_BASE}-validate" cylc validate \ -s "CYLC_TEST_PLATFORM='$CYLC_TEST_PLATFORM'" \ "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play \ -s "CYLC_TEST_PLATFORM='$CYLC_TEST_PLATFORM'" \ --debug \ --no-detach \ --reference-test \ "${WORKFLOW_NAME}" grep_ok \ "# will invoke in batches, sizes=\[2, 2, 1\]" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" # tidy up purge exit cylc-flow-8.7.0/tests/functional/job-submission/09-activity-log-host-bad-submit.t0000775000175000017500000000323115264420307030113 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test bad job submission, activity log has original command and some stderr # with the host name written. export REQUIRE_PLATFORM='runner:at loc:remote' . "$(dirname "$0")/test_header" set_test_number 2 create_test_global_config '' " [platforms] [[${CYLC_TEST_PLATFORM}]] job runner = at job runner command template = at non " install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" \ -s "CYLC_TEST_HOST='${CYLC_TEST_HOST}'" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test \ -s "CYLC_TEST_HOST='${CYLC_TEST_HOST}'" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/job-submission/21-late-sub-message/0000775000175000017500000000000015264420307025441 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/21-late-sub-message/lib/0000775000175000017500000000000015264420307026207 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/21-late-sub-message/lib/python/0000775000175000017500000000000015264420307027530 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/21-late-sub-message/lib/python/delayed_background.py0000664000175000017500000000221015264420307033703 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from time import sleep from cylc.flow.job_runner_handlers.background import BgCommandHandler class DelayedBgCommandHandler(BgCommandHandler): @classmethod def submit(cls, job_file_path, submit_opts): result = super().submit(job_file_path, submit_opts) sleep(10) return result JOB_RUNNER_HANDLER = DelayedBgCommandHandler() cylc-flow-8.7.0/tests/functional/job-submission/21-late-sub-message/flow.cylc0000664000175000017500000000026015264420307027262 0ustar alastairalastair# Test delayed job submission. [scheduling] [[graph]] R1 = foo => bar [runtime] [[bar]] [[foo]] platform = wobblygibblets script = sleep 20 cylc-flow-8.7.0/tests/functional/job-submission/15-garbage-platform-command-2.t0000775000175000017500000000226615264420307027471 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test recovery of a failed host select command. . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/job-submission/02-job-nn-remote-host.t0000775000175000017500000000262415264420307026125 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test remote host job log NN link correctness. export REQUIRE_PLATFORM='loc:remote' . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" mkdir -p "${WORKFLOW_RUN_DIR}/.service/" sqlite3 "${WORKFLOW_RUN_DIR}/.service/db" <'db.sqlite3' workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play --upgrade --reference-test --debug --no-detach \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/job-submission/07-multi.t0000775000175000017500000000736015264420307023637 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test job submission, multiple jobs per host. export REQUIRE_PLATFORM='loc:remote' . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" \ "${WORKFLOW_NAME}" RUN_DIR="$RUN_DIR/${WORKFLOW_NAME}" LOG="${RUN_DIR}/log/scheduler/log" sed -n 's/^.*\(cylc jobs-submit\)/\1/p' "${LOG}" | sort -u >'edited-workflow-log' PATHOPTS="--path=/bin --path=/usr/bin --path=/usr/local/bin --path=/sbin --path=/usr/sbin --path=/usr/local/sbin" sort >'edited-workflow-log-ref' <<__LOG__ cylc jobs-submit --debug --utc-mode $PATHOPTS -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 20200101T0000Z/t0/01 20200101T0000Z/t1/01 20200101T0000Z/t2/01 20200101T0000Z/t3/01 cylc jobs-submit --debug --utc-mode $PATHOPTS -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 20210101T0000Z/t0/01 20210101T0000Z/t1/01 20210101T0000Z/t2/01 20210101T0000Z/t3/01 cylc jobs-submit --debug --utc-mode $PATHOPTS -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 20220101T0000Z/t0/01 20220101T0000Z/t1/01 20220101T0000Z/t2/01 20220101T0000Z/t3/01 cylc jobs-submit --debug --utc-mode $PATHOPTS -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 20230101T0000Z/t0/01 20230101T0000Z/t1/01 20230101T0000Z/t2/01 20230101T0000Z/t3/01 cylc jobs-submit --debug --utc-mode $PATHOPTS -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 20240101T0000Z/t0/01 20240101T0000Z/t1/01 20240101T0000Z/t2/01 20240101T0000Z/t3/01 cylc jobs-submit --debug --utc-mode $PATHOPTS -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 20250101T0000Z/t0/01 20250101T0000Z/t1/01 20250101T0000Z/t2/01 20250101T0000Z/t3/01 cylc jobs-submit --debug --utc-mode --remote-mode $PATHOPTS -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 20200101T0000Z/t4/01 20200101T0000Z/t5/01 20200101T0000Z/t6/01 cylc jobs-submit --debug --utc-mode --remote-mode $PATHOPTS -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 20210101T0000Z/t4/01 20210101T0000Z/t5/01 20210101T0000Z/t6/01 cylc jobs-submit --debug --utc-mode --remote-mode $PATHOPTS -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 20220101T0000Z/t4/01 20220101T0000Z/t5/01 20220101T0000Z/t6/01 cylc jobs-submit --debug --utc-mode --remote-mode $PATHOPTS -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 20230101T0000Z/t4/01 20230101T0000Z/t5/01 20230101T0000Z/t6/01 cylc jobs-submit --debug --utc-mode --remote-mode $PATHOPTS -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 20240101T0000Z/t4/01 20240101T0000Z/t5/01 20240101T0000Z/t6/01 cylc jobs-submit --debug --utc-mode --remote-mode $PATHOPTS -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 20250101T0000Z/t4/01 20250101T0000Z/t5/01 20250101T0000Z/t6/01 __LOG__ cmp_ok 'edited-workflow-log' 'edited-workflow-log-ref' purge exit cylc-flow-8.7.0/tests/functional/job-submission/01-job-nn-localhost.t0000775000175000017500000000251115264420307025641 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test job log NN link correctness on reaching 100, localhost. . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" mkdir -p "${WORKFLOW_RUN_DIR}/.service/" sqlite3 "${WORKFLOW_RUN_DIR}/.service/db" <'db.sqlite3' workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play --reference-test --upgrade --debug --no-detach "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/job-submission/07-multi/0000775000175000017500000000000015264420307023441 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/07-multi/reference.log0000664000175000017500000001675415264420307026117 0ustar alastairalastairInitial point: 20200101T0000Z Final point: 20250101T0000Z 20200101T0000Z/t5 -triggered off ['20190101T0000Z/t0', '20190101T0000Z/t1', '20190101T0000Z/t2', '20190101T0000Z/t3', '20190101T0000Z/t4', '20190101T0000Z/t5', '20190101T0000Z/t6'] 20200101T0000Z/t1 -triggered off ['20190101T0000Z/t0', '20190101T0000Z/t1', '20190101T0000Z/t2', '20190101T0000Z/t3', '20190101T0000Z/t4', '20190101T0000Z/t5', '20190101T0000Z/t6'] 20200101T0000Z/t0 -triggered off ['20190101T0000Z/t0', '20190101T0000Z/t1', '20190101T0000Z/t2', '20190101T0000Z/t3', '20190101T0000Z/t4', '20190101T0000Z/t5', '20190101T0000Z/t6'] 20200101T0000Z/t2 -triggered off ['20190101T0000Z/t0', '20190101T0000Z/t1', '20190101T0000Z/t2', '20190101T0000Z/t3', '20190101T0000Z/t4', '20190101T0000Z/t5', '20190101T0000Z/t6'] 20200101T0000Z/t4 -triggered off ['20190101T0000Z/t0', '20190101T0000Z/t1', '20190101T0000Z/t2', '20190101T0000Z/t3', '20190101T0000Z/t4', '20190101T0000Z/t5', '20190101T0000Z/t6'] 20200101T0000Z/t3 -triggered off ['20190101T0000Z/t0', '20190101T0000Z/t1', '20190101T0000Z/t2', '20190101T0000Z/t3', '20190101T0000Z/t4', '20190101T0000Z/t5', '20190101T0000Z/t6'] 20200101T0000Z/t6 -triggered off ['20190101T0000Z/t0', '20190101T0000Z/t1', '20190101T0000Z/t2', '20190101T0000Z/t3', '20190101T0000Z/t4', '20190101T0000Z/t5', '20190101T0000Z/t6'] 20210101T0000Z/t5 -triggered off ['20200101T0000Z/t0', '20200101T0000Z/t1', '20200101T0000Z/t2', '20200101T0000Z/t3', '20200101T0000Z/t4', '20200101T0000Z/t5', '20200101T0000Z/t6'] 20210101T0000Z/t1 -triggered off ['20200101T0000Z/t0', '20200101T0000Z/t1', '20200101T0000Z/t2', '20200101T0000Z/t3', '20200101T0000Z/t4', '20200101T0000Z/t5', '20200101T0000Z/t6'] 20210101T0000Z/t0 -triggered off ['20200101T0000Z/t0', '20200101T0000Z/t1', '20200101T0000Z/t2', '20200101T0000Z/t3', '20200101T0000Z/t4', '20200101T0000Z/t5', '20200101T0000Z/t6'] 20210101T0000Z/t2 -triggered off ['20200101T0000Z/t0', '20200101T0000Z/t1', '20200101T0000Z/t2', '20200101T0000Z/t3', '20200101T0000Z/t4', '20200101T0000Z/t5', '20200101T0000Z/t6'] 20210101T0000Z/t4 -triggered off ['20200101T0000Z/t0', '20200101T0000Z/t1', '20200101T0000Z/t2', '20200101T0000Z/t3', '20200101T0000Z/t4', '20200101T0000Z/t5', '20200101T0000Z/t6'] 20210101T0000Z/t3 -triggered off ['20200101T0000Z/t0', '20200101T0000Z/t1', '20200101T0000Z/t2', '20200101T0000Z/t3', '20200101T0000Z/t4', '20200101T0000Z/t5', '20200101T0000Z/t6'] 20210101T0000Z/t6 -triggered off ['20200101T0000Z/t0', '20200101T0000Z/t1', '20200101T0000Z/t2', '20200101T0000Z/t3', '20200101T0000Z/t4', '20200101T0000Z/t5', '20200101T0000Z/t6'] 20220101T0000Z/t5 -triggered off ['20210101T0000Z/t0', '20210101T0000Z/t1', '20210101T0000Z/t2', '20210101T0000Z/t3', '20210101T0000Z/t4', '20210101T0000Z/t5', '20210101T0000Z/t6'] 20220101T0000Z/t2 -triggered off ['20210101T0000Z/t0', '20210101T0000Z/t1', '20210101T0000Z/t2', '20210101T0000Z/t3', '20210101T0000Z/t4', '20210101T0000Z/t5', '20210101T0000Z/t6'] 20220101T0000Z/t3 -triggered off ['20210101T0000Z/t0', '20210101T0000Z/t1', '20210101T0000Z/t2', '20210101T0000Z/t3', '20210101T0000Z/t4', '20210101T0000Z/t5', '20210101T0000Z/t6'] 20220101T0000Z/t0 -triggered off ['20210101T0000Z/t0', '20210101T0000Z/t1', '20210101T0000Z/t2', '20210101T0000Z/t3', '20210101T0000Z/t4', '20210101T0000Z/t5', '20210101T0000Z/t6'] 20220101T0000Z/t4 -triggered off ['20210101T0000Z/t0', '20210101T0000Z/t1', '20210101T0000Z/t2', '20210101T0000Z/t3', '20210101T0000Z/t4', '20210101T0000Z/t5', '20210101T0000Z/t6'] 20220101T0000Z/t1 -triggered off ['20210101T0000Z/t0', '20210101T0000Z/t1', '20210101T0000Z/t2', '20210101T0000Z/t3', '20210101T0000Z/t4', '20210101T0000Z/t5', '20210101T0000Z/t6'] 20220101T0000Z/t6 -triggered off ['20210101T0000Z/t0', '20210101T0000Z/t1', '20210101T0000Z/t2', '20210101T0000Z/t3', '20210101T0000Z/t4', '20210101T0000Z/t5', '20210101T0000Z/t6'] 20230101T0000Z/t2 -triggered off ['20220101T0000Z/t0', '20220101T0000Z/t1', '20220101T0000Z/t2', '20220101T0000Z/t3', '20220101T0000Z/t4', '20220101T0000Z/t5', '20220101T0000Z/t6'] 20230101T0000Z/t4 -triggered off ['20220101T0000Z/t0', '20220101T0000Z/t1', '20220101T0000Z/t2', '20220101T0000Z/t3', '20220101T0000Z/t4', '20220101T0000Z/t5', '20220101T0000Z/t6'] 20230101T0000Z/t5 -triggered off ['20220101T0000Z/t0', '20220101T0000Z/t1', '20220101T0000Z/t2', '20220101T0000Z/t3', '20220101T0000Z/t4', '20220101T0000Z/t5', '20220101T0000Z/t6'] 20230101T0000Z/t1 -triggered off ['20220101T0000Z/t0', '20220101T0000Z/t1', '20220101T0000Z/t2', '20220101T0000Z/t3', '20220101T0000Z/t4', '20220101T0000Z/t5', '20220101T0000Z/t6'] 20230101T0000Z/t6 -triggered off ['20220101T0000Z/t0', '20220101T0000Z/t1', '20220101T0000Z/t2', '20220101T0000Z/t3', '20220101T0000Z/t4', '20220101T0000Z/t5', '20220101T0000Z/t6'] 20230101T0000Z/t3 -triggered off ['20220101T0000Z/t0', '20220101T0000Z/t1', '20220101T0000Z/t2', '20220101T0000Z/t3', '20220101T0000Z/t4', '20220101T0000Z/t5', '20220101T0000Z/t6'] 20230101T0000Z/t0 -triggered off ['20220101T0000Z/t0', '20220101T0000Z/t1', '20220101T0000Z/t2', '20220101T0000Z/t3', '20220101T0000Z/t4', '20220101T0000Z/t5', '20220101T0000Z/t6'] 20240101T0000Z/t5 -triggered off ['20230101T0000Z/t0', '20230101T0000Z/t1', '20230101T0000Z/t2', '20230101T0000Z/t3', '20230101T0000Z/t4', '20230101T0000Z/t5', '20230101T0000Z/t6'] 20240101T0000Z/t0 -triggered off ['20230101T0000Z/t0', '20230101T0000Z/t1', '20230101T0000Z/t2', '20230101T0000Z/t3', '20230101T0000Z/t4', '20230101T0000Z/t5', '20230101T0000Z/t6'] 20240101T0000Z/t2 -triggered off ['20230101T0000Z/t0', '20230101T0000Z/t1', '20230101T0000Z/t2', '20230101T0000Z/t3', '20230101T0000Z/t4', '20230101T0000Z/t5', '20230101T0000Z/t6'] 20240101T0000Z/t3 -triggered off ['20230101T0000Z/t0', '20230101T0000Z/t1', '20230101T0000Z/t2', '20230101T0000Z/t3', '20230101T0000Z/t4', '20230101T0000Z/t5', '20230101T0000Z/t6'] 20240101T0000Z/t6 -triggered off ['20230101T0000Z/t0', '20230101T0000Z/t1', '20230101T0000Z/t2', '20230101T0000Z/t3', '20230101T0000Z/t4', '20230101T0000Z/t5', '20230101T0000Z/t6'] 20240101T0000Z/t1 -triggered off ['20230101T0000Z/t0', '20230101T0000Z/t1', '20230101T0000Z/t2', '20230101T0000Z/t3', '20230101T0000Z/t4', '20230101T0000Z/t5', '20230101T0000Z/t6'] 20240101T0000Z/t4 -triggered off ['20230101T0000Z/t0', '20230101T0000Z/t1', '20230101T0000Z/t2', '20230101T0000Z/t3', '20230101T0000Z/t4', '20230101T0000Z/t5', '20230101T0000Z/t6'] 20250101T0000Z/t4 -triggered off ['20240101T0000Z/t0', '20240101T0000Z/t1', '20240101T0000Z/t2', '20240101T0000Z/t3', '20240101T0000Z/t4', '20240101T0000Z/t5', '20240101T0000Z/t6'] 20250101T0000Z/t6 -triggered off ['20240101T0000Z/t0', '20240101T0000Z/t1', '20240101T0000Z/t2', '20240101T0000Z/t3', '20240101T0000Z/t4', '20240101T0000Z/t5', '20240101T0000Z/t6'] 20250101T0000Z/t2 -triggered off ['20240101T0000Z/t0', '20240101T0000Z/t1', '20240101T0000Z/t2', '20240101T0000Z/t3', '20240101T0000Z/t4', '20240101T0000Z/t5', '20240101T0000Z/t6'] 20250101T0000Z/t3 -triggered off ['20240101T0000Z/t0', '20240101T0000Z/t1', '20240101T0000Z/t2', '20240101T0000Z/t3', '20240101T0000Z/t4', '20240101T0000Z/t5', '20240101T0000Z/t6'] 20250101T0000Z/t1 -triggered off ['20240101T0000Z/t0', '20240101T0000Z/t1', '20240101T0000Z/t2', '20240101T0000Z/t3', '20240101T0000Z/t4', '20240101T0000Z/t5', '20240101T0000Z/t6'] 20250101T0000Z/t0 -triggered off ['20240101T0000Z/t0', '20240101T0000Z/t1', '20240101T0000Z/t2', '20240101T0000Z/t3', '20240101T0000Z/t4', '20240101T0000Z/t5', '20240101T0000Z/t6'] 20250101T0000Z/t5 -triggered off ['20240101T0000Z/t0', '20240101T0000Z/t1', '20240101T0000Z/t2', '20240101T0000Z/t3', '20240101T0000Z/t4', '20240101T0000Z/t5', '20240101T0000Z/t6'] cylc-flow-8.7.0/tests/functional/job-submission/07-multi/flow.cylc0000664000175000017500000000051315264420307025263 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [scheduling] initial cycle point = 2020 final cycle point = 2025 [[graph]] P1Y = T[-P1Y]:succeed-all => T [runtime] [[T]] script = true [[t0,t1,t2,t3]] inherit = T [[t4,t5,t6]] inherit = T platform = {{CYLC_TEST_PLATFORM}} cylc-flow-8.7.0/tests/functional/job-submission/20-check-chunking/0000775000175000017500000000000015264420307025163 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/20-check-chunking/reference.log0000664000175000017500000000035715264420307027631 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1_p3 -triggered off [] 1/t1_p4 -triggered off [] 1/t1_p2 -triggered off [] 1/t1_p5 -triggered off [] 1/t1_p1 -triggered off [] 1/fin -triggered off ['1/t1_p1', '1/t1_p2', '1/t1_p3', '1/t1_p4', '1/t1_p5'] cylc-flow-8.7.0/tests/functional/job-submission/20-check-chunking/flow.cylc0000664000175000017500000000053515264420307027011 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] abort on inactivity timeout = True abort on stall timeout = True stall timeout = PT0S inactivity timeout = PT10M [task parameters] p = 1..5 [scheduling] [[graph]] R1 = t1

=> fin [runtime] [[t1

]] platform = {{ CYLC_TEST_PLATFORM }} [[fin]] cylc-flow-8.7.0/tests/functional/job-submission/08-activity-log-host/0000775000175000017500000000000015264420307025676 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/08-activity-log-host/reference.log0000664000175000017500000000041515264420307030337 0ustar alastairalastairInitial point: 19990101T0000Z Final point: 19990101T0000Z 19990101T0000Z/sleeper -triggered off [] 19990101T0000Z/killer -triggered off ['19990101T0000Z/sleeper'] 19990101T0000Z/releaser -triggered off ['19990101T0000Z/killer'] 19990101T0000Z/sleeper -triggered off [] cylc-flow-8.7.0/tests/functional/job-submission/08-activity-log-host/flow.cylc0000664000175000017500000000115415264420307027522 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [scheduling] initial cycle point = 1999 final cycle point = 1999 [[graph]] P1Y = sleeper:start => killer => releaser [runtime] [[sleeper]] script = test "${CYLC_TASK_SUBMIT_NUMBER}" -eq 2 || sleep 60 platform = {{ CYLC_TEST_PLATFORM }} [[[job]]] execution retry delays = PT1S [[killer]] script = cylc kill "${CYLC_WORKFLOW_ID}//1999*/sleeper" [[releaser]] script = """ cylc__job__wait_cylc_message_started cylc release "${CYLC_WORKFLOW_ID}//1999*/sleeper" """ cylc-flow-8.7.0/tests/functional/job-submission/13-tidy-submits-of-prev-run-remote-host0000777000175000017500000000000015264420307036257 212-tidy-submits-of-prev-runustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/12-tidy-submits-of-prev-run.t0000775000175000017500000000333515264420307027312 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test tidy of submits of previous runs. . "$(dirname "$0")/test_header" set_test_number 7 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" LOGD1="$RUN_DIR/${WORKFLOW_NAME}/log/job/1/t1/01" LOGD2="$RUN_DIR/${WORKFLOW_NAME}/log/job/1/t1/02" exists_ok "${LOGD1}" exists_ok "${LOGD2}" sed -i 's/script =.*$/script = true/' "$RUN_DIR/$WORKFLOW_NAME/flow.cylc" sed -i -n '1,/triggered off/p' "$RUN_DIR/$WORKFLOW_NAME/reference.log" delete_db workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" exists_ok "${LOGD1}" exists_fail "${LOGD2}" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/job-submission/19-platform_select.t0000775000175000017500000000355015264420307025670 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test recovery of a failed host select command for a group of tasks. . "$(dirname "$0")/test_header" set_test_number 8 install_workflow "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" reftest_run logfile="${WORKFLOW_RUN_DIR}/log/scheduler/log" # Check that host = $(cmd) is correctly evaluated grep_ok \ "1/host_subshell/01:.* evaluated as 'improbable host name'" \ "${logfile}" grep_ok \ "1/localhost_subshell/01:.* evaluated as 'localhost'" \ "${logfile}" # Check that host = `cmd` is correctly evaluated grep_ok \ "1/host_subshell_backticks/01:.* evaluated as 'improbable host name'" \ "${logfile}" # Check that platform = $(cmd) correctly evaluated grep_ok \ "1/platform_subshell:.* evaluated as 'improbable platform name'" \ "${logfile}" grep_ok \ "1/platform_subshell_empty:.* evaluated as ''" \ "${logfile}" grep_ok \ "1/platform_subshell_suffix:.* evaluated as 'prefix-middle-suffix'" \ "${logfile}" # purge cylc-flow-8.7.0/tests/functional/job-submission/00-user/0000775000175000017500000000000015264420307023256 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/00-user/lib/0000775000175000017500000000000015264420307024024 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/00-user/lib/python/0000775000175000017500000000000015264420307025345 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/00-user/lib/python/my_background2.py0000664000175000017500000000171115264420307030625 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.job_runner_handlers.background import BgCommandHandler class MyBgCommandHandler(BgCommandHandler): pass JOB_RUNNER_HANDLER = MyBgCommandHandler() cylc-flow-8.7.0/tests/functional/job-submission/00-user/reference.log0000664000175000017500000000012015264420307025710 0ustar alastairalastairInitial point: 1 Final point: 1 1/bar -triggered off [] 1/foo -triggered off [] cylc-flow-8.7.0/tests/functional/job-submission/00-user/flow.cylc0000664000175000017500000000037415264420307025105 0ustar alastairalastair# Test job submission modules in lib/python/ and python/ (deprecated). [scheduling] [[graph]] R1 = foo & bar [runtime] [[root]] script = true # quick [[foo]] platform = testme [[bar]] platform = testme2 cylc-flow-8.7.0/tests/functional/job-submission/00-user/python/0000775000175000017500000000000015264420307024577 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/00-user/python/my_background.py0000664000175000017500000000171115264420307027775 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.job_runner_handlers.background import BgCommandHandler class MyBgCommandHandler(BgCommandHandler): pass JOB_RUNNER_HANDLER = MyBgCommandHandler() cylc-flow-8.7.0/tests/functional/job-submission/12-tidy-submits-of-prev-run/0000775000175000017500000000000015264420307027116 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/12-tidy-submits-of-prev-run/reference.log0000664000175000017500000000011615264420307031555 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] 1/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/job-submission/12-tidy-submits-of-prev-run/flow.cylc0000664000175000017500000000041415264420307030740 0ustar alastairalastair#!jinja2 [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = test "${CYLC_TASK_SUBMIT_NUMBER}" -eq 2 platform = {{ environ['CYLC_TEST_PLATFORM'] | default('localhost') }} [[[job]]] execution retry delays = P0Y cylc-flow-8.7.0/tests/functional/job-submission/13-tidy-submits-of-prev-run-remote-host.t0000775000175000017500000000474615264420307031566 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test tidy of submits of previous runs. export REQUIRE_PLATFORM='loc:remote' . "$(dirname "$0")/test_header" set_test_number 11 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" RLOGD1="cylc-run/${WORKFLOW_NAME}/log/job/1/t1/01" RLOGD2="cylc-run/${WORKFLOW_NAME}/log/job/1/t1/02" LOGD1="${RUN_DIR}/${WORKFLOW_NAME}/log/job/1/t1/01" LOGD2="${RUN_DIR}/${WORKFLOW_NAME}/log/job/1/t1/02" SSH='ssh -n -oBatchMode=yes -oConnectTimeout=5' # shellcheck disable=SC2086 run_ok "exists-rlogd1" ${SSH} "${CYLC_TEST_HOST}" test -e "${RLOGD1}" # shellcheck disable=SC2086 run_ok "exists-rlogd2" ${SSH} "${CYLC_TEST_HOST}" test -e "${RLOGD2}" exists_ok "${LOGD1}" exists_ok "${LOGD2}" sed -i 's/script =.*$/script = true/' "${RUN_DIR}/${WORKFLOW_NAME}/flow.cylc" sed -i -n '1,/triggered off/p' "${RUN_DIR}/${WORKFLOW_NAME}/reference.log" delete_db workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" # shellcheck disable=SC2086 run_ok "exists-rlogd1" ${SSH} "${CYLC_TEST_HOST}" test -e "${RLOGD1}" # shellcheck disable=SC2086 run_fail "not-exists-rlogd2" ${SSH} "${CYLC_TEST_HOST}" test -e "${RLOGD2}" exists_ok "${LOGD1}" exists_fail "${LOGD2}" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/job-submission/10-at-shell.t0000775000175000017500000000264015264420307024204 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test job submission via at, with SHELL set to tcsh export REQUIRE_PLATFORM='runner:at' . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" # By setting "SHELL=/bin/tcsh", "at" would run its command under "/bin/tcsh", # which would cause a failure of this test without the fix in #1749. workflow_run_ok "${TEST_NAME_BASE}-run" \ env 'SHELL=/bin/tcsh' cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/job-submission/02-job-nn-remote-host0000777000175000017500000000000015264420307031161 201-job-nn-localhostustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/11-garbage-platform-command.t0000775000175000017500000000305415264420307027322 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test job submission rundb task_jobs entry with garbage host command. . "$(dirname "$0")/test_header" set_test_number 3 create_test_global_config ' [platforms] [[badhost]] ' install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" sqlite3 \ "$RUN_DIR/${WORKFLOW_NAME}/log/db" \ "SELECT submit_num,submit_status FROM task_jobs WHERE name=='t1'" \ >'sqlite3.out' cmp_ok 'sqlite3.out' <<'__OUT__' 1|1 2|0 __OUT__ #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/job-submission/21-late-sub-message.t0000775000175000017500000000352115264420307025632 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test that a late job submitted message does not put the task back in the # submitted state (it is possible, though unlikely, for the job started # message to arrive first). # Uses a modified background job runner (defined in the workflow source # directory) that sleeps before returning after submitting the job. . "$(dirname "$0")/test_header" set_test_number 3 create_test_global_config "" " [platforms] [[wobblygibblets]] hosts = localhost job runner = delayed_background install target = localhost " install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" sqlite3 "$RUN_DIR/${WORKFLOW_NAME}/log/db" \ 'SELECT name, cycle, event from task_events;' >'sqlite3.out' cmp_ok 'sqlite3.out' <<'__OUT__' foo|1|submitted foo|1|started foo|1|succeeded bar|1|submitted bar|1|started bar|1|succeeded __OUT__ purge cylc-flow-8.7.0/tests/functional/job-submission/test_header0000777000175000017500000000000015264420307030400 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/08-activity-log-host.t0000775000175000017500000000260615264420307026072 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test job submission, activity log has remote host name export REQUIRE_PLATFORM='loc:remote comms:tcp' . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" \ "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/job-submission/03-job-nn-remote-host-with-shared-fs.t0000775000175000017500000000260015264420307030743 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test job log NN link correctness on reaching 100, remote (with shared fs). export REQUIRE_PLATFORM='loc:remote fs:shared' . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" mkdir "${WORKFLOW_RUN_DIR}/.service" sqlite3 "${WORKFLOW_RUN_DIR}/.service/db" <'db.sqlite3' workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play --reference-test --debug --no-detach --upgrade "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/job-submission/00-user.t0000775000175000017500000000232315264420307023446 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test user-defined job runner handlers can be used. . "$(dirname "$0")/test_header" set_test_number 2 create_test_global_config "" " [platforms] [[testme]] hosts = localhost job runner = my_background install target = localhost [[testme2]] hosts = localhost job runner = my_background2 install target = localhost " reftest exit cylc-flow-8.7.0/tests/functional/job-submission/18-clean-env/0000775000175000017500000000000015264420307024161 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/18-clean-env/flow.cylc0000664000175000017500000000055515264420307026011 0ustar alastairalastair[cylc] [[events]] inactivity timeout = PT30S abort on inactivity timeout = True abort on stall timeout = True stall timeout = PT0S [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = """ echo "BEEF ${BEEF:-undefined}" echo "CHEESE ${CHEESE:-undefined}" """ cylc-flow-8.7.0/tests/functional/job-submission/18-clean-env.t0000775000175000017500000000320015264420307024344 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that local jobs can be divorced from the scheduler environment. . "$(dirname "$0")/test_header" create_test_global_config "" " [platforms] [[localhost]] cylc path = $(dirname "$(command -v cylc)") clean job submission environment = True job submission environment pass-through = BEEF " set_test_number 4 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" # Export a variable and try to access from a task job. export BEEF=wellington export CHEESE=melted workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" cylc cat-log "${WORKFLOW_NAME}//1/foo" > job.out grep_ok "BEEF wellington" job.out grep_ok "CHEESE undefined" job.out purge exit ././@LongLink0000644000000000000000000000014700000000000011605 Lustar rootrootcylc-flow-8.7.0/tests/functional/job-submission/14-tidy-submits-of-prev-run-remote-host-with-shared-fscylc-flow-8.7.0/tests/functional/job-submission/14-tidy-submits-of-prev-run-remote-host-with-shared-0000777000175000017500000000000015264420307040552 212-tidy-submits-of-prev-runustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/03-job-nn-remote-host-with-shared-fs0000777000175000017500000000000015264420307034005 201-job-nn-localhostustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/11-garbage-platform-command/0000775000175000017500000000000015264420307027130 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/11-garbage-platform-command/reference.log0000664000175000017500000000015315264420307031570 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] 1/t2 -triggered off ['1/t1'] 1/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/job-submission/11-garbage-platform-command/flow.cylc0000664000175000017500000000063015264420307030752 0ustar alastairalastair[scheduler] [[events]] expected task failures = 1/t1 [scheduling] [[graph]] R1 = """t1:submit-fail? => t2""" [runtime] [[t1]] script = true platform = badhost [[t2]] script = """ cylc broadcast "${CYLC_WORKFLOW_ID}" \ -n 't1' -p '1' -s 'platform=localhost' cylc trigger "${CYLC_WORKFLOW_ID}//1/t1" """ cylc-flow-8.7.0/tests/functional/job-submission/16-timeout/0000775000175000017500000000000015264420307023775 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/job-submission/16-timeout/flow.cylc0000664000175000017500000000033615264420307025622 0ustar alastairalastair#!Jinja2 [scheduling] [[graph]] R1 = "foo:submit-fail? => stopper" [runtime] [[foo]] platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[stopper]] script = cylc stop "${CYLC_WORKFLOW_ID}" cylc-flow-8.7.0/tests/functional/inheritance/0000775000175000017500000000000015264420307021411 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/inheritance/01-circular.t0000664000175000017500000000326515264420307023626 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check circular inheritance fails validation with the correct error message. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-cmp" cmp_ok "${TEST_NAME_BASE}-validate.stderr" <<__ERR__ WorkflowConfigError: circular [runtime] inheritance? __ERR__ #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/inheritance/00-namespace-list.t0000775000175000017500000000377215264420307024734 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that members of namespace lists [[n1,n2,...]] are inserted into the # [runtime] ordered dict in the correct order. If just appended, they break # repeat-section override for the member. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-config cylc config -i runtime "${WORKFLOW_NAME}" > runtime.out cmp_ok runtime.out <<'__DONE__' [[root]] [[FAMILY]] [[m1]] inherit = FAMILY completion = succeeded [[[environment]]] FOO = foo [[m2]] inherit = FAMILY completion = succeeded [[[environment]]] FOO = bar [[m3]] inherit = FAMILY completion = succeeded [[[environment]]] FOO = foo __DONE__ #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/inheritance/02-bad-reinherit/0000775000175000017500000000000015264420307024345 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/inheritance/02-bad-reinherit/flow.cylc0000664000175000017500000000040415264420307026166 0ustar alastairalastair[meta] title = a workflow with a task that multiply inherits from the same family description = should fail validation [scheduling] [[graph]] R1 = foo [runtime] [[A]] [[B]] inherit = C, A [[C]] [[foo]] inherit = A, B cylc-flow-8.7.0/tests/functional/inheritance/02-bad-reinherit.t0000664000175000017500000000340315264420307024532 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check bad multi inheritance fails validation with the correct error message. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-cmp" cmp_ok "${TEST_NAME_BASE}-validate.stderr" <<__ERR__ WorkflowConfigError: ERROR: foo: bad runtime namespace inheritance hierarchy. See the cylc documentation on multiple inheritance. __ERR__ #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/inheritance/01-circular/0000775000175000017500000000000015264420307023433 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/inheritance/01-circular/flow.cylc0000664000175000017500000000040415264420307025254 0ustar alastairalastair[meta] title = a workflow with circular inheritance description = should fail validation [scheduling] [[graph]] R1 = foo [runtime] [[A]] inherit = B [[B]] inherit = C [[C]] inherit = B [[foo]] inherit = A cylc-flow-8.7.0/tests/functional/inheritance/00-namespace-list/0000775000175000017500000000000015264420307024533 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/inheritance/00-namespace-list/flow.cylc0000664000175000017500000000031415264420307026354 0ustar alastairalastair[scheduling] [[graph]] R1 = FAMILY [runtime] [[FAMILY]] [[m1,m2,m3]] inherit = FAMILY [[[environment]]] FOO = foo [[m2]] [[[environment]]] FOO = bar cylc-flow-8.7.0/tests/functional/inheritance/test_header0000777000175000017500000000000015264420307027726 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-diff/0000775000175000017500000000000015264420307020760 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-diff/04-icp-2.t0000775000175000017500000000470515264420307022311 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test for "cylc diff --icp=CYCLE_POINT", override. . "$(dirname "$0")/test_header" set_test_number 3 init_workflow "${TEST_NAME_BASE}-1" <<'__FLOW_CONFIG__' [scheduler] UTC mode = True [scheduling] [[graph]] R1 = foo => bar [runtime] [[foo, bar]] script = true __FLOW_CONFIG__ # shellcheck disable=SC2153 WORKFLOW_NAME1="${WORKFLOW_NAME}" init_workflow "${TEST_NAME_BASE}-2" <<'__FLOW_CONFIG__' [scheduler] UTC mode = True [scheduling] initial cycle point = 2016 [[graph]] R1 = food => barley [runtime] [[food, barley]] script = true __FLOW_CONFIG__ # shellcheck disable=SC2153 WORKFLOW_NAME2="${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}" \ cylc diff --icp=2020 "${WORKFLOW_NAME1}" "${WORKFLOW_NAME2}" cmp_ok "${TEST_NAME_BASE}.stdout" <<__OUT__ Parsing ${WORKFLOW_NAME1} (${RUN_DIR}/${WORKFLOW_NAME1}/flow.cylc) Parsing ${WORKFLOW_NAME2} (${RUN_DIR}/${WORKFLOW_NAME2}/flow.cylc) Workflow definitions ${WORKFLOW_NAME1} and ${WORKFLOW_NAME2} differ 2 items only in ${WORKFLOW_NAME1} (<) [runtime] [[foo]] < script = true < completion = succeeded [runtime] [[bar]] < script = true < completion = succeeded 2 items only in ${WORKFLOW_NAME2} (>) [runtime] [[food]] > script = true > completion = succeeded [runtime] [[barley]] > script = true > completion = succeeded 1 common items differ ${WORKFLOW_NAME1}(<) ${WORKFLOW_NAME2}(>) [scheduling] [[graph]] < R1 = foo => bar > R1 = food => barley __OUT__ cmp_ok "${TEST_NAME_BASE}.stderr" <'/dev/null' purge "${WORKFLOW_NAME1}" purge "${WORKFLOW_NAME2}" exit cylc-flow-8.7.0/tests/functional/cylc-diff/00-basic.t0000775000175000017500000000447715264420307022462 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Basic test for "cylc diff". . "$(dirname "$0")/test_header" set_test_number 3 init_workflow "${TEST_NAME_BASE}-1" <<'__FLOW_CONFIG__' [scheduling] [[graph]] R1 = foo => bar [runtime] [[foo, bar]] script = true __FLOW_CONFIG__ # shellcheck disable=SC2153 WORKFLOW_NAME1="${WORKFLOW_NAME}" init_workflow "${TEST_NAME_BASE}-2" <<'__FLOW_CONFIG__' [scheduling] [[graph]] R1 = food => barley [runtime] [[food, barley]] script = true __FLOW_CONFIG__ # shellcheck disable=SC2153 WORKFLOW_NAME2="${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}" cylc diff "${WORKFLOW_NAME1}" "${WORKFLOW_NAME2}" cmp_ok "${TEST_NAME_BASE}.stdout" <<__OUT__ Parsing ${WORKFLOW_NAME1} (${RUN_DIR}/${WORKFLOW_NAME1}/flow.cylc) Parsing ${WORKFLOW_NAME2} (${RUN_DIR}/${WORKFLOW_NAME2}/flow.cylc) Workflow definitions ${WORKFLOW_NAME1} and ${WORKFLOW_NAME2} differ 2 items only in ${WORKFLOW_NAME1} (<) [runtime] [[foo]] < script = true < completion = succeeded [runtime] [[bar]] < script = true < completion = succeeded 2 items only in ${WORKFLOW_NAME2} (>) [runtime] [[food]] > script = true > completion = succeeded [runtime] [[barley]] > script = true > completion = succeeded 1 common items differ ${WORKFLOW_NAME1}(<) ${WORKFLOW_NAME2}(>) [scheduling] [[graph]] < R1 = foo => bar > R1 = food => barley __OUT__ cmp_ok "${TEST_NAME_BASE}.stderr" <'/dev/null' purge "${WORKFLOW_NAME1}" purge "${WORKFLOW_NAME2}" exit cylc-flow-8.7.0/tests/functional/cylc-diff/01-same.t0000775000175000017500000000353715264420307022323 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test for "cylc diff" with 2 workflows pointing to same "flow.cylc". . "$(dirname "$0")/test_header" set_test_number 3 cat >'flow.cylc' <<'__FLOW_CONFIG__' [scheduling] [[graph]] R1 = foo => bar [runtime] [[foo, bar]] script = true __FLOW_CONFIG__ init_workflow "${TEST_NAME_BASE}-1" "${PWD}/flow.cylc" # shellcheck disable=SC2153 WORKFLOW_NAME1="${WORKFLOW_NAME}" # shellcheck disable=SC2153 WORKFLOW_NAME2="${WORKFLOW_NAME1%1}2" cylc install "${TEST_DIR}/${WORKFLOW_NAME1}" --workflow-name="${WORKFLOW_NAME2}" --no-run-name 2>'/dev/null' run_ok "${TEST_NAME_BASE}" cylc diff "${WORKFLOW_NAME1}" "${WORKFLOW_NAME2}" cmp_ok "${TEST_NAME_BASE}.stdout" <<__OUT__ Parsing ${WORKFLOW_NAME1} (${RUN_DIR}/${WORKFLOW_NAME1}/flow.cylc) Parsing ${WORKFLOW_NAME2} (${RUN_DIR}/${WORKFLOW_NAME2}/flow.cylc) Workflow definitions ${WORKFLOW_NAME1} and ${WORKFLOW_NAME2} are identical __OUT__ cmp_ok "${TEST_NAME_BASE}.stderr" <'/dev/null' purge "${WORKFLOW_NAME1}" purge "${WORKFLOW_NAME2}" exit cylc-flow-8.7.0/tests/functional/cylc-diff/03-icp.t0000775000175000017500000000463415264420307022152 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test for "cylc diff --icp=CYCLE_POINT". . "$(dirname "$0")/test_header" set_test_number 3 init_workflow "${TEST_NAME_BASE}-1" <<'__FLOW_CONFIG__' [scheduler] UTC mode = True [scheduling] [[graph]] R1 = foo => bar [runtime] [[foo, bar]] script = true __FLOW_CONFIG__ # shellcheck disable=SC2153 WORKFLOW_NAME1="${WORKFLOW_NAME}" init_workflow "${TEST_NAME_BASE}-2" <<'__FLOW_CONFIG__' [scheduler] UTC mode = True [scheduling] [[graph]] R1 = food => barley [runtime] [[food, barley]] script = true __FLOW_CONFIG__ # shellcheck disable=SC2153 WORKFLOW_NAME2="${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}" \ cylc diff --icp=2020 "${WORKFLOW_NAME1}" "${WORKFLOW_NAME2}" cmp_ok "${TEST_NAME_BASE}.stdout" <<__OUT__ Parsing ${WORKFLOW_NAME1} (${RUN_DIR}/${WORKFLOW_NAME1}/flow.cylc) Parsing ${WORKFLOW_NAME2} (${RUN_DIR}/${WORKFLOW_NAME2}/flow.cylc) Workflow definitions ${WORKFLOW_NAME1} and ${WORKFLOW_NAME2} differ 2 items only in ${WORKFLOW_NAME1} (<) [runtime] [[foo]] < script = true < completion = succeeded [runtime] [[bar]] < script = true < completion = succeeded 2 items only in ${WORKFLOW_NAME2} (>) [runtime] [[food]] > script = true > completion = succeeded [runtime] [[barley]] > script = true > completion = succeeded 1 common items differ ${WORKFLOW_NAME1}(<) ${WORKFLOW_NAME2}(>) [scheduling] [[graph]] < R1 = foo => bar > R1 = food => barley __OUT__ cmp_ok "${TEST_NAME_BASE}.stderr" <'/dev/null' purge "${WORKFLOW_NAME1}" purge "${WORKFLOW_NAME2}" exit cylc-flow-8.7.0/tests/functional/cylc-diff/test_header0000777000175000017500000000000015264420307027275 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-diff/02-identical.t0000775000175000017500000000355415264420307023332 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test for "cylc diff" with identical workflow definitions. . "$(dirname "$0")/test_header" set_test_number 3 init_workflow "${TEST_NAME_BASE}-1" <<'__FLOW_CONFIG__' [scheduling] [[graph]] R1 = foo => bar [runtime] [[foo, bar]] script = true __FLOW_CONFIG__ # shellcheck disable=SC2153 WORKFLOW_NAME1="${WORKFLOW_NAME}" init_workflow "${TEST_NAME_BASE}-2" <<'__FLOW_CONFIG__' [scheduling] [[graph]] R1 = foo => bar [runtime] [[foo, bar]] script = true __FLOW_CONFIG__ # shellcheck disable=SC2153 WORKFLOW_NAME2="${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}" cylc diff "${WORKFLOW_NAME1}" "${WORKFLOW_NAME2}" cmp_ok "${TEST_NAME_BASE}.stdout" <<__OUT__ Parsing ${WORKFLOW_NAME1} (${RUN_DIR}/${WORKFLOW_NAME1}/flow.cylc) Parsing ${WORKFLOW_NAME2} (${RUN_DIR}/${WORKFLOW_NAME2}/flow.cylc) Workflow definitions ${WORKFLOW_NAME1} and ${WORKFLOW_NAME2} are identical __OUT__ cmp_ok "${TEST_NAME_BASE}.stderr" <'/dev/null' purge "${WORKFLOW_NAME1}" purge "${WORKFLOW_NAME2}" exit cylc-flow-8.7.0/tests/functional/events/0000775000175000017500000000000015264420307020424 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/33-task-event-job-logs-retrieve-3.t0000775000175000017500000000403615264420307026620 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test remote job logs retrieval OK with only "job.out" on a succeeded task. export REQUIRE_PLATFORM='loc:remote fs:indep comms:tcp' . "$(dirname "$0")/test_header" set_test_number 5 create_test_global_config "" " [platforms] [[blackbriar]] hosts = ${CYLC_TEST_HOST} install target = ${CYLC_TEST_INSTALL_TARGET} retrieve job logs = True retrieve job logs retry delays = 2*PT5S " install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --reference-test -vv --no-detach "${WORKFLOW_NAME}" sed "/'job-logs-retrieve'/!d" \ "${WORKFLOW_RUN_DIR}/log/job/1/t1/01/job-activity.log" \ >'edited-activities.log' cmp_ok 'edited-activities.log' <<'__LOG__' [(('job-logs-retrieve', 'failed'), 1) ret_code] 1 [(('job-logs-retrieve', 'failed'), 1) err] File(s) not retrieved: job.err [(('job-logs-retrieve', 'failed'), 1) ret_code] 1 [(('job-logs-retrieve', 'failed'), 1) err] File(s) not retrieved: job.err __LOG__ exists_ok "${WORKFLOW_RUN_DIR}/log/job/1/t1/01/job.out" exists_fail "${WORKFLOW_RUN_DIR}/log/job/1/t1/01/job.err" purge exit cylc-flow-8.7.0/tests/functional/events/stall-timeout-ref/0000775000175000017500000000000015264420307024001 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/stall-timeout-ref/reference.log0000664000175000017500000000007015264420307026437 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] cylc-flow-8.7.0/tests/functional/events/stall-timeout-ref/flow.cylc0000664000175000017500000000053615264420307025630 0ustar alastairalastair[meta] description = This workflow is supposed to time out [scheduler] [[events]] abort on stall timeout = True stall timeout = PT1S expected task failures = 1/foo [scheduling] [[graph]] R1 = "foo" [runtime] [[foo]] script = false [[[simulation]]] fail cycle points = 1 cylc-flow-8.7.0/tests/functional/events/34-task-abort.t0000775000175000017500000000503515264420307023112 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test job abort-with-message and interaction with failed handler. . "$(dirname "$0")/test_header" set_test_number 6 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- # Check failed handler only call on last try. LOG="${WORKFLOW_RUN_DIR}/log/job/1/foo/NN/job-activity.log" grep "event-handler" "${LOG}" > 'edited-job-activity.log' cmp_ok 'edited-job-activity.log' <<'__LOG__' [(('event-handler-00', 'failed'), 2) cmd] echo "!!!FAILED!!!" failed 1/foo 2 '"ERROR: rust never sleeps"' [(('event-handler-00', 'failed'), 2) ret_code] 0 [(('event-handler-00', 'failed'), 2) out] !!!FAILED!!! failed 1/foo 2 "ERROR: rust never sleeps" __LOG__ #------------------------------------------------------------------------------- # Check job stdout stops at the abort call. LOG="${WORKFLOW_RUN_DIR}/log/job/1/foo/NN/job.out" # ...before abort grep_ok 'ONE' "${LOG}" # ...after abort grep_fail 'TWO' "${LOG}" #------------------------------------------------------------------------------- # Check only one CYLC_JOB_EXIT message written. JOB_STATUS="${WORKFLOW_RUN_DIR}/log/job/1/foo/NN/job.status" run_ok "${TEST_NAME_BASE}-message-count" \ test "$(grep -c '^CYLC_JOB_EXIT=' "$JOB_STATUS")" -eq '1' #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/events/00-workflow/0000775000175000017500000000000015264420307022513 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/00-workflow/flow.cylc0000664000175000017500000000103715264420307024337 0ustar alastairalastair# test some workflow event handler hooks [scheduler] [[events]] startup handlers = echo HELLO STARTUP workflow timeout = PT2S workflow timeout handlers = echo HELLO TIMEOUT abort on workflow timeout = False inactivity timeout = PT3S inactivity timeout handlers = echo HELLO INACTIVITY abort on inactivity timeout = False shutdown handlers = echo HELLO SHUTDOWN [scheduling] [[graph]] R1 = "foo" [runtime] [[foo]] platform = localhost script = sleep 10 cylc-flow-8.7.0/tests/functional/events/19-workflow-event-mail-globalcfg0000777000175000017500000000000015264420307032502 218-workflow-event-mailustar alastairalastaircylc-flow-8.7.0/tests/functional/events/30-task-event-mail-2/0000775000175000017500000000000015264420307024004 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/30-task-event-mail-2/flow.cylc0000664000175000017500000000075415264420307025635 0ustar alastairalastair#!jinja2 [meta] title = Task Event Mail [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S [[mail]] footer = see: http://localhost/stuff/%(owner)s/%(workflow)s/ task event batch interval = PT8S [scheduling] [[graph]] R1 = t1? & t2? & t3? & t4? & t5? [runtime] [[t1, t2, t3, t4, t5]] script = false execution retry delays = PT15S [[[events]]] mail events = failed, retry cylc-flow-8.7.0/tests/functional/events/10-task-event-job-logs-retrieve.t0000775000175000017500000000454615264420307026461 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test remote job logs retrieval, requires compatible version of cylc on remote # job host. export REQUIRE_PLATFORM='loc:remote' . "$(dirname "$0")/test_header" set_test_number 4 OPT_SET= create_test_global_config "" " [platforms] [[${CYLC_TEST_PLATFORM}]] retrieve job logs = True retrieve job logs retry delays = PT5S " OPT_SET='-s GLOBALCFG=True' install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # shellcheck disable=SC2086 run_ok "${TEST_NAME_BASE}-validate" \ cylc validate ${OPT_SET} \ -s "PLATFORM='${CYLC_TEST_PLATFORM}'" "${WORKFLOW_NAME}" # shellcheck disable=SC2086 workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach ${OPT_SET} \ -s "PLATFORM='${CYLC_TEST_PLATFORM}'" "${WORKFLOW_NAME}" sed "/'job-logs-retrieve'/!d" \ "${WORKFLOW_RUN_DIR}/log/job/1/t1/"{01,02,03}"/job-activity.log" \ >'edited-activities.log' cmp_ok 'edited-activities.log' <<'__LOG__' [(('job-logs-retrieve', 'retry'), 1) ret_code] 0 [(('job-logs-retrieve', 'retry'), 2) ret_code] 0 [(('job-logs-retrieve', 'succeeded'), 3) ret_code] 0 __LOG__ grep -F 'will run after' "${WORKFLOW_RUN_DIR}/log/scheduler/log" \ | cut -d' ' -f 4-12 | sort >"edited-log" cmp_ok 'edited-log' <<'__LOG__' 1/t1/01 handler:job-logs-retrieve for task event:retry will run after PT5S 1/t1/02 handler:job-logs-retrieve for task event:retry will run after PT5S 1/t1/03 handler:job-logs-retrieve for task event:succeeded will run after PT5S __LOG__ purge exit cylc-flow-8.7.0/tests/functional/events/timeout/0000775000175000017500000000000015264420307022112 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/timeout/flow.cylc0000664000175000017500000000043215264420307023734 0ustar alastairalastair[meta] description = This workflow is supposed to time out [scheduler] [[events]] workflow timeout = PT6S abort on workflow timeout = True [scheduling] [[graph]] R1 = "foo" [runtime] [[foo]] script = "cylc pause $CYLC_WORKFLOW_ID" cylc-flow-8.7.0/tests/functional/events/03-stall-timeout.t0000775000175000017500000000343515264420307023644 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Validate and run the workflow stall timeout workflow . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 4 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" stall-timeout #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_fail "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" grep_ok "WARNING - stall timer timed out after PT6S" "${TEST_NAME}.stderr" grep_ok 'Workflow shutting down - "abort on stall timeout" is set' \ "${TEST_NAME}.stderr" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/events/18-workflow-event-mail.t0000775000175000017500000000371115264420307024755 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test event mail. . "$(dirname "$0")/test_header" if ! command -v mail 2>'/dev/null'; then skip_all '"mail" command not available' fi set_test_number 3 mock_smtpd_init OPT_SET= if [[ "${TEST_NAME_BASE}" == *-globalcfg ]]; then create_test_global_config "" " [scheduler] [[events]] mail events = startup, shutdown [[mail]] footer = see: http://localhost/stuff/%(owner)s/%(workflow)s/ smtp = ${TEST_SMTPD_HOST}" OPT_SET='-s GLOBALCFG=True' else create_test_global_config " [scheduler] [[mail]] smtp = ${TEST_SMTPD_HOST} " fi install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # shellcheck disable=SC2086 run_ok "${TEST_NAME_BASE}-validate" \ cylc validate ${OPT_SET} "${WORKFLOW_NAME}" # shellcheck disable=SC2086 workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}" contains_ok "${TEST_SMTPD_LOG}" <<__LOG__ event: startup message: workflow starting event: shutdown message: AUTOMATIC see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/ __LOG__ purge mock_smtpd_kill exit cylc-flow-8.7.0/tests/functional/events/18-workflow-event-mail/0000775000175000017500000000000015264420307024563 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/18-workflow-event-mail/reference.log0000664000175000017500000000006715264420307027227 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/events/18-workflow-event-mail/flow.cylc0000664000175000017500000000100715264420307026404 0ustar alastairalastair#!jinja2 [meta] title=Workflow Event Mail [scheduler] {% if GLOBALCFG is not defined %} [[mail]] footer = see: http://localhost/stuff/%(owner)s/%(workflow)s/ [[events]] mail events = startup, shutdown {% endif %}{# not GLOBALCFG is not defined #} [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT3M [scheduling] [[graph]] R1=t1 [runtime] [[t1]] script=true cylc-flow-8.7.0/tests/functional/events/42-late-then-restart.t0000775000175000017500000000317515264420307024410 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test late event handler with restart. Event should be emitted once. . "$(dirname "$0")/test_header" set_test_number 5 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}-run" cylc play --debug --no-detach "${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}-restart" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" # Check that the workflow has emitted a single late event. grep -c 'WARNING.*late (late-time=.*)' \ <(cat "${WORKFLOW_RUN_DIR}/log/scheduler/"*.log) \ >'grep-log.out' cmp_ok 'grep-log.out' <<<'1' grep -c 'late (late-time=.*)' \ "${WORKFLOW_RUN_DIR}/log/scheduler/my-handler.out" \ > 'grep-my-handler.out' cmp_ok 'grep-my-handler.out' <<<'1' purge exit cylc-flow-8.7.0/tests/functional/events/04-stall-timeout-ref-live.t0000775000175000017500000000350715264420307025354 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Validate and run the workflow reference test live stall timeout workflow . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" 'stall-timeout-ref' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" RUN_MODE="$(basename "$0" | sed "s/.*-ref-\(.*\).t/\1/g")" workflow_run_fail "${TEST_NAME}" \ cylc play --reference-test --mode="${RUN_MODE}" --debug --no-detach \ "${WORKFLOW_NAME}" grep_ok "WARNING - stall timer timed out after PT1S" "${TEST_NAME}.stderr" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/events/11-cycle-task-event-job-logs-retrieve.t0000775000175000017500000000373315264420307027554 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test remote job logs retrieval, requires compatible version of cylc on remote # job host. export REQUIRE_PLATFORM='loc:remote fs:indep' . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" TEST_PLATFORM_CFG=$(cylc config -i "[platforms][$CYLC_TEST_PLATFORM]") create_test_global_config '' " [platforms] [[_retrieve, _no_retrieve]] $TEST_PLATFORM_CFG [[_retrieve]] retrieve job logs = True [[_no_retrieve]] retrieve job logs = False " run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" # There are 2 remote tasks. One with "retrieve job logs = True", one without. # Only y should have job.err and job.out retrieved. sed "/'job-logs-retrieve'/!d" \ "${WORKFLOW_RUN_DIR}/log/job/1/"{y,n}'/'{01,02}'/job-activity.log' \ >'edited-activities.log' cmp_ok 'edited-activities.log' <<__LOG__ [(('job-logs-retrieve', 'retry'), 1) ret_code] 0 [(('job-logs-retrieve', 'succeeded'), 2) ret_code] 0 __LOG__ purge cylc-flow-8.7.0/tests/functional/events/13-task-event-mail-globalcfg0000777000175000017500000000000015264420307030654 209-task-event-mailustar alastairalastaircylc-flow-8.7.0/tests/functional/events/51-task-event-job-logs-retrieve-4.t0000664000175000017500000000534215264420307026617 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- export REQUIRE_PLATFORM="loc:remote fs:indep comms:?(tcp|ssh)" . "$(dirname "$0")/test_header" set_test_number 3 #------------------------------------------------------------------------------- # It should retry job log retrieval even if all hosts are not contactable. #------------------------------------------------------------------------------- init_workflow "${TEST_NAME_BASE}-1" <<__FLOW_CONFIG__ [scheduling] [[graph]] R1 = """ remote """ [runtime] [[remote]] # script = sleep 1 platform = ${CYLC_TEST_PLATFORM} __FLOW_CONFIG__ # configure job retries on the test platform create_test_global_config '' " [platforms] [[${CYLC_TEST_PLATFORM}]] retrieve job logs = True retrieve job logs retry delays = 3*PT1S retrieve job logs command = fido " # * redirect retrieval attempts to a file where we can inspect them later # * make it look like retrieval failed due to network issues (255 ret code) JOB_LOG_RETR_CMD="${WORKFLOW_RUN_DIR}/bin/fido" RETRIEVAL_ATTEMPT_LOG="${WORKFLOW_RUN_DIR}/retrieval-attempt-log" mkdir "${WORKFLOW_RUN_DIR}/bin" cat > "${WORKFLOW_RUN_DIR}/bin/fido" <<__HERE__ #!/usr/bin/env bash echo "$@" >> "${RETRIEVAL_ATTEMPT_LOG}" exit 255 __HERE__ chmod +x "${JOB_LOG_RETR_CMD}" workflow_run_ok "${TEST_NAME_BASE}-play" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" # it should try retrieval three times # Note: it should reset bad_hosts to allow retries to happen TEST_NAME="${TEST_NAME_BASE}-retrieve-attempts" # shellcheck disable=SC2002 # (cat'ting into pipe to avoid having to sed out the filename) if [[ $(cat "${RETRIEVAL_ATTEMPT_LOG}" | wc -l) -eq 3 ]]; then ok "${TEST_NAME}" else fail "${TEST_NAME}" fi # then fail once the retries have been exhausted grep_workflow_log_ok "${TEST_NAME_BASE}-retrieve-fail" \ 'job-logs-retrieve for task event:succeeded failed' purge cylc-flow-8.7.0/tests/functional/events/33-task-event-job-logs-retrieve-3/0000775000175000017500000000000015264420307026425 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/33-task-event-job-logs-retrieve-3/reference.log0000664000175000017500000000006715264420307031071 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/events/33-task-event-job-logs-retrieve-3/flow.cylc0000664000175000017500000000055315264420307030253 0ustar alastairalastair#!jinja2 [meta] title=Task Event Job Log Retrieve 1 [scheduler] [[events]] abort on stall timeout = True stall timeout = PT1M expected task failures = 1/t1 [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = false err-script = rm -f "${CYLC_TASK_LOG_ROOT}.err" platform = blackbriar cylc-flow-8.7.0/tests/functional/events/48-workflow-aborted.t0000775000175000017500000000257715264420307024350 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test workflow abort event handler . "$(dirname "$0")/test_header" set_test_number 4 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" cylc cat-log "${WORKFLOW_NAME}" >'log' grep_ok 'CRITICAL - Workflow shutting down' 'log' cmp_ok "${WORKFLOW_RUN_DIR}/handler.out" <<'__OUT__' abort contact file unexpectedly modified __OUT__ purge cylc-flow-8.7.0/tests/functional/events/10-task-event-job-logs-retrieve/0000775000175000017500000000000015264420307026260 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/10-task-event-job-logs-retrieve/reference.log0000664000175000017500000000014515264420307030721 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] 1/t1 -triggered off [] 1/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/events/10-task-event-job-logs-retrieve/flow.cylc0000664000175000017500000000041615264420307030104 0ustar alastairalastair#!jinja2 [meta] title=Task Event Job Log Retrieve [scheduling] [[graph]] R1=t1 [runtime] [[t1]] script=test "${CYLC_TASK_TRY_NUMBER}" -eq 3 platform = {{ PLATFORM }} [[[job]]] execution retry delays=PT0S, 2*PT1S cylc-flow-8.7.0/tests/functional/events/27-workflow-stalled-dump-prereq-fam.t0000775000175000017500000000351515264420307027346 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test workflow event handler, dump unmet prereqs on stall . "$(dirname "$0")/test_header" set_test_number 9 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" grep_ok '"abort on stall timeout" is set' "${TEST_NAME_BASE}-run.stderr" grep_ok "ERROR - Incomplete tasks:" "${TEST_NAME_BASE}-run.stderr" grep_ok "1/foo did not complete the required outputs:\n.*succeeded" \ "${TEST_NAME_BASE}-run.stderr" \ -Pizo grep_ok "WARNING - Partially satisfied prerequisites:" \ "${TEST_NAME_BASE}-run.stderr" grep_ok "1/f_1 is waiting on \['1/foo:succeeded'\]" \ "${TEST_NAME_BASE}-run.stderr" grep_ok "1/f_2 is waiting on \['1/foo:succeeded'\]" \ "${TEST_NAME_BASE}-run.stderr" grep_ok "1/f_3 is waiting on \['1/foo:succeeded'\]" \ "${TEST_NAME_BASE}-run.stderr" purge exit cylc-flow-8.7.0/tests/functional/events/46-task-output-as-event.t0000775000175000017500000000475215264420307025073 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that known task outputs can be used as events. . "$(dirname "$0")/test_header" set_test_number 4 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script=""" cylc message -- ${CYLC_WORKFLOW_ID} ${CYLC_TASK_JOB} \ 'rose' 'lily' 'iris' 'WARNING:poison ivy' """ [[[outputs]]] rose = rose lily = lily iris = iris [[[events]]] handler events = rose, lily, iris, warning, arsenic # (arsenic is an invalid event) handlers = echo %(message)s __FLOW_CONFIG__ TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "$TEST_NAME" cylc validate "${WORKFLOW_NAME}" dump_std "${TEST_NAME}" grep_ok 'WARNING - Invalid event name.*arsenic' "${TEST_NAME}.stderr" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" cylc cat-log "${WORKFLOW_NAME}" \ | sed -n -e 's/^.*\(\[(('"'"'event-handler-00'"'"'.*$\)/\1/p' | sort >'log' cmp_ok log <<__END__ [(('event-handler-00', 'iris'), 1) cmd] echo iris [(('event-handler-00', 'iris'), 1) out] iris [(('event-handler-00', 'iris'), 1) ret_code] 0 [(('event-handler-00', 'lily'), 1) cmd] echo lily [(('event-handler-00', 'lily'), 1) out] lily [(('event-handler-00', 'lily'), 1) ret_code] 0 [(('event-handler-00', 'rose'), 1) cmd] echo rose [(('event-handler-00', 'rose'), 1) out] rose [(('event-handler-00', 'rose'), 1) ret_code] 0 [(('event-handler-00', 'warning-1'), 1) cmd] echo 'poison ivy' [(('event-handler-00', 'warning-1'), 1) out] poison ivy [(('event-handler-00', 'warning-1'), 1) ret_code] 0 __END__ purge exit cylc-flow-8.7.0/tests/functional/events/35-task-event-handler-importance.t0000775000175000017500000000333115264420307026674 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test new runtime config item importance #2289 . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------ set_test_number 3 #------------------------------------------------------------------------------ install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" T1_ACTIVITY_LOG="${WORKFLOW_RUN_DIR}/log/job/1/t1/NN/job-activity.log" grep_ok \ "\\[(('event-handler-00', 'failed'), 1) out\\] NAME = t1 POINT = 1 IMPORTANCE = 3 COLOR = red WORKFLOW-PRIORITY = HIGH" \ "${T1_ACTIVITY_LOG}" #------------------------------------------------------------------------------ purge cylc-flow-8.7.0/tests/functional/events/20-workflow-event-handlers/0000775000175000017500000000000015264420307025432 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/20-workflow-event-handlers/reference.log0000664000175000017500000000006715264420307030076 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/events/20-workflow-event-handlers/flow.cylc0000664000175000017500000000103615264420307027255 0ustar alastairalastair#!jinja2 [meta] title = Workflow Event Mail URL = http://myworkflows.com/${CYLC_WORKFLOW_ID}.html workflow-priority = HIGH [scheduler] [[events]] {% if GLOBALCFG is not defined %} handlers = echo 'Your %(workflow)s workflow has a %(event)s event and URL %(workflow_url)s and workflow-priority as %(workflow-priority)s and workflow-UUID as %(uuid)s.' handler events = startup {% endif %}{# not GLOBALCFG is not defined #} [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = true cylc-flow-8.7.0/tests/functional/events/17-task-event-job-logs-retrieve-command0000777000175000017500000000000015264420307035400 210-task-event-job-logs-retrieveustar alastairalastaircylc-flow-8.7.0/tests/functional/events/45-task-event-handler-multi-warning.t0000775000175000017500000000513115264420307027331 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that simultaneous warnings from a task job are all handled by the # warning event handler. # https://github.com/cylc/cylc-flow/issues/2806 . "$(dirname "$0")/test_header" set_test_number 3 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = """ cylc message -s WARNING -- ${CYLC_WORKFLOW_ID} ${CYLC_TASK_JOB} "cat" cylc message -s WARNING -- ${CYLC_WORKFLOW_ID} ${CYLC_TASK_JOB} "dog" cylc message -s WARNING -- ${CYLC_WORKFLOW_ID} ${CYLC_TASK_JOB} "fish" cylc message -s WARNING -- ${CYLC_WORKFLOW_ID} ${CYLC_TASK_JOB} "guinea pig" """ [[[events]]] handler events = warning handlers = echo "HANDLED %(message)s" __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" cylc cat-log "${WORKFLOW_NAME}" \ | sed -n -e 's/^.*\(\[(('"'"'event-handler-00'"'"'.*$\)/\1/p' >'log' contains_ok log <<__END__ [(('event-handler-00', 'warning-1'), 1) cmd] echo "HANDLED cat" [(('event-handler-00', 'warning-1'), 1) ret_code] 0 [(('event-handler-00', 'warning-1'), 1) out] HANDLED cat [(('event-handler-00', 'warning-2'), 1) cmd] echo "HANDLED dog" [(('event-handler-00', 'warning-2'), 1) ret_code] 0 [(('event-handler-00', 'warning-2'), 1) out] HANDLED dog [(('event-handler-00', 'warning-3'), 1) cmd] echo "HANDLED fish" [(('event-handler-00', 'warning-3'), 1) ret_code] 0 [(('event-handler-00', 'warning-3'), 1) out] HANDLED fish [(('event-handler-00', 'warning-4'), 1) cmd] echo "HANDLED 'guinea pig'" [(('event-handler-00', 'warning-4'), 1) ret_code] 0 [(('event-handler-00', 'warning-4'), 1) out] HANDLED 'guinea pig' __END__ purge exit cylc-flow-8.7.0/tests/functional/events/37-workflow-event-bad-custom-template.t0000775000175000017500000000263115264420307027703 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test workflow event handler, flexible interface . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run1" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" LOG="${WORKFLOW_RUN_DIR}/log/scheduler/log" MESSAGE="('workflow-event-handler-00', 'startup') bad template: echo %(rubbish)s" run_ok "${TEST_NAME_BASE}-run1-log" grep -q -F "ERROR - ${MESSAGE}" "${LOG}" purge exit cylc-flow-8.7.0/tests/functional/events/23-workflow-stalled-handler/0000775000175000017500000000000015264420307025561 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/23-workflow-stalled-handler/reference.log0000664000175000017500000000016615264420307030225 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/bar -triggered off ['1/foo'] 1/baz -triggered off ['1/bar'] cylc-flow-8.7.0/tests/functional/events/23-workflow-stalled-handler/flow.cylc0000664000175000017500000000061415264420307027405 0ustar alastairalastair[scheduler] [[events]] stall handlers = "cylc set %(workflow)s//1/bar" stall timeout = PT0S abort on stall timeout = False expected task failures = 1/bar [scheduling] [[graph]] R1 = foo => bar => baz [runtime] [[foo]] script = true [[bar]] script = false [[baz]] script = cylc remove "$CYLC_WORKFLOW_ID//1/bar" cylc-flow-8.7.0/tests/functional/events/38-task-event-handler-custom/0000775000175000017500000000000015264420307025660 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/38-task-event-handler-custom/reference.log0000664000175000017500000000016615264420307030324 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/bar -triggered off ['1/foo'] 1/baz -triggered off ['1/foo'] cylc-flow-8.7.0/tests/functional/events/38-task-event-handler-custom/flow.cylc0000664000175000017500000000171115264420307027503 0ustar alastairalastair[scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = """ foo:a => bar foo:b => baz """ [runtime] [[root]] script = true [[[events]]] custom handlers = echo !!CUSTOM!! %(point)s/%(name)s %(fish)s %(message)s [[[meta]]] fish = trout [[foo]] script = """ cylc__job__wait_cylc_message_started # Output message for triggering, and custom event handler. cylc message -p CUSTOM "Data ready for barring" # Generic message, not for triggering or custom event handler. cylc message "Aren't the hydrangeas nice?" # Output message for triggering, not custom event handler. cylc message "Data ready for bazzing" """ [[[outputs]]] a = "Data ready for barring" b = "Data ready for bazzing" [[[meta]]] fish = fugu cylc-flow-8.7.0/tests/functional/events/32-task-event-job-logs-retrieve-2.t0000775000175000017500000000344415264420307026620 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test remote job logs retrieval OK with only "job.out" on a succeeded task. export REQUIRE_PLATFORM='loc:remote fs:indep comms:tcp' . "$(dirname "$0")/test_header" set_test_number 5 create_test_global_config "" " [platforms] [[treadstone]] hosts = ${CYLC_TEST_HOST} install target = ${CYLC_TEST_INSTALL_TARGET} retrieve job logs = True " install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test -vv --no-detach "${WORKFLOW_NAME}" sed "/'job-logs-retrieve'/!d" \ "${WORKFLOW_RUN_DIR}/log/job/1/t1/01/job-activity.log" \ >'edited-activities.log' cmp_ok 'edited-activities.log' <<'__LOG__' [(('job-logs-retrieve', 'succeeded'), 1) ret_code] 0 __LOG__ exists_ok "${WORKFLOW_RUN_DIR}/log/job/1/t1/01/job.out" exists_fail "${WORKFLOW_RUN_DIR}/log/job/1/t1/01/job.err" purge cylc-flow-8.7.0/tests/functional/events/41-late/0000775000175000017500000000000015264420307021573 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/41-late/flow.cylc0000664000175000017500000000044615264420307023422 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] initial cycle point = now [[graph]] R1 = t1 => t2 [runtime] [[t1]] script = sleep 61 [[t2]] script = true [[[events]]] late offset = PT1M late handlers = my-handler %(message)s cylc-flow-8.7.0/tests/functional/events/41-late/bin/0000775000175000017500000000000015264420307022343 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/41-late/bin/my-handler0000775000175000017500000000014315264420307024327 0ustar alastairalastair#!/usr/bin/env bash set -eu ME="$(basename "$0")" echo "$@" >>"${CYLC_WORKFLOW_LOG_DIR}/${ME}.out" cylc-flow-8.7.0/tests/functional/events/30-task-event-mail-2.t0000775000175000017500000000452215264420307024177 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test event mail. . "$(dirname "$0")/test_header" if ! command -v mail 2>'/dev/null'; then skip_all '"mail" command not available' fi set_test_number 15 mock_smtpd_init OPT_SET=() if [[ "${TEST_NAME_BASE}" == *-globalcfg ]]; then create_test_global_config "" " [scheduler] [[mail]] footer = see: http://localhost/stuff/%(owner)s/%(workflow)s/ smtp = ${TEST_SMTPD_HOST} [task events] mail events = failed, retry, succeeded " OPT_SET=(-s 'GLOBALCFG=True') else create_test_global_config " [scheduler] [[mail]] smtp = ${TEST_SMTPD_HOST} " fi install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${OPT_SET[@]}" "$WORKFLOW_NAME" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${OPT_SET[@]}" "$WORKFLOW_NAME" # 1 - retry for i in {1..5}; do run_ok "${TEST_NAME_BASE}-t${i}-01" grep -Pizo "job: 1/t${i}/01.*\n.*event: retry" "$TEST_SMTPD_LOG" done # 2 - fail for i in {1..5}; do run_ok "${TEST_NAME_BASE}-t${i}-02" grep -Pizo "job: 1/t${i}/02.*\n.*event: failed" "$TEST_SMTPD_LOG" done contains_ok "${TEST_SMTPD_LOG}" <<__LOG__ see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/ __LOG__ run_ok "${TEST_NAME_BASE}-grep-log" \ grep -qPizo "Subject: \[. tasks retry\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" run_ok "${TEST_NAME_BASE}-grep-log" \ grep -qPizo "Subject: \[. tasks failed\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" purge mock_smtpd_kill cylc-flow-8.7.0/tests/functional/events/24-workflow-timeout/0000775000175000017500000000000015264420307024205 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/24-workflow-timeout/flow.cylc0000664000175000017500000000074015264420307026031 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] workflow timeout = PT1S workflow timeout handlers = echo "That was quick!" abort on workflow timeout = {{ABORT}} [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = """ cylc__job__poll_grep_workflow_log "workflow timer timed out" if [[ "{{ABORT}}" == "True" ]]; then cylc__job__poll_grep_workflow_log "Workflow shutting down" fi """ cylc-flow-8.7.0/tests/functional/events/43-late-spawn/0000775000175000017500000000000015264420307022723 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/43-late-spawn/flow.cylc0000664000175000017500000000034715264420307024552 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] initial cycle point = 2015 final cycle point = 2016 [[graph]] P1Y = t1 [runtime] [[t1]] script = true [[[events]]] late offset = PT1M cylc-flow-8.7.0/tests/functional/events/48-workflow-aborted/0000775000175000017500000000000015264420307024145 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/48-workflow-aborted/reference.log0000664000175000017500000000007315264420307026606 0ustar alastairalastairInitial point: 1 Final point: 1 1/modify -triggered off [] cylc-flow-8.7.0/tests/functional/events/48-workflow-aborted/flow.cylc0000664000175000017500000000153515264420307025774 0ustar alastairalastair[scheduler] allow implicit tasks = True [[main loop]] [[[health check]]] interval = PT1S [[events]] abort handlers = echo %(event)s "%(message)s" >"${CYLC_WORKFLOW_RUN_DIR}/handler.out" # These settings make the test avoid hanging if something goes wrong (nothing to do with the actual abort test): abort on inactivity timeout = true inactivity timeout = PT30S abort on stall timeout = true stall timeout = PT0S [scheduling] [[graph]] R1 = modify => t2 [runtime] [[modify]] script = """ # Pause the workflow, so it does not shutdown cylc pause "${CYLC_WORKFLOW_ID}" # Cause abort by making health check fail (contact file modified) echo 'TIME=MONEY' >>"${CYLC_WORKFLOW_RUN_DIR}/.service/contact" """ cylc-flow-8.7.0/tests/functional/events/29-task-event-mail-1.t0000775000175000017500000000350615264420307024207 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test event mail, single task event . "$(dirname "$0")/test_header" if ! command -v mail 2>'/dev/null'; then skip_all '"mail" command not available' fi set_test_number 5 mock_smtpd_init create_test_global_config " [scheduler] [[mail]] smtp = ${TEST_SMTPD_HOST} " install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # shellcheck disable=SC2086 run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "$WORKFLOW_NAME" # shellcheck disable=SC2086 workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "$WORKFLOW_NAME" run_ok "${TEST_NAME_BASE}-grep-log-1" \ grep -Pizo "job: 1/t1/01.*\n.*event: retry.*\n.*" "${TEST_SMTPD_LOG}" run_ok "${TEST_NAME_BASE}-grep-log-2" grep \ "see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/" \ "${TEST_SMTPD_LOG}" run_ok "${TEST_NAME_BASE}-grep-log-2" \ grep -Pizo "Subject: \\[1/t1/01 retry\\].*(\n)?.*${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" purge mock_smtpd_kill exit cylc-flow-8.7.0/tests/functional/events/06-stall-timeout-ref-simulation.t0000775000175000017500000000351515264420307026602 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Validate and run the workflow reference test simulation stall timeout workflow . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" 'stall-timeout-ref' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" RUN_MODE="$(basename "$0" | sed "s/.*-ref-\(.*\).t/\1/g")" workflow_run_fail "${TEST_NAME}" \ cylc play --reference-test --mode="${RUN_MODE}" --debug --no-detach \ "${WORKFLOW_NAME}" grep_ok "WARNING - stall timer timed out after PT1S" "${TEST_NAME}.stderr" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/events/28-inactivity/0000775000175000017500000000000015264420307023036 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/28-inactivity/flow.cylc0000664000175000017500000000043015264420307024656 0ustar alastairalastair[scheduler] UTC mode = True [[events]] inactivity timeout = PT10S abort on inactivity timeout = True [scheduling] [[graph]] R1 = foo [runtime] [[foo]] init-script = cylc__job__disable_fail_signals ERR EXIT script = exit 1 cylc-flow-8.7.0/tests/functional/events/35-task-event-handler-importance/0000775000175000017500000000000015264420307026504 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/35-task-event-handler-importance/flow.cylc0000664000175000017500000000071315264420307030330 0ustar alastairalastair[meta] priority = HIGH [scheduling] [[graph]] R1 = t1:fail => dummy [runtime] [[dummy]] script = true [[t1]] script = false [[[meta]]] URL = http://example.com importance = 3 color = red [[[events]]] failed handlers = echo 'NAME =' %(name)s 'POINT =' %(point)s 'IMPORTANCE =' %(importance)s 'COLOR =' %(color)s 'WORKFLOW-PRIORITY =' %(workflow_priority)s cylc-flow-8.7.0/tests/functional/events/25-held-not-stalled/0000775000175000017500000000000015264420307024010 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/25-held-not-stalled/flow.cylc0000664000175000017500000000051215264420307025631 0ustar alastairalastair[scheduler] [[events]] abort on inactivity timeout = False abort on stall timeout = True stall timeout = PT0S inactivity timeout handlers = cylc release --all '%(workflow)s' inactivity timeout = PT5S [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = true cylc-flow-8.7.0/tests/functional/events/43-late-spawn.t0000775000175000017500000000246415264420307023121 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test late event, with spawned tasks that are also late . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}-run" cylc play --debug --no-detach "${WORKFLOW_NAME}" grep -c 'WARNING.*late (late-time=.*)' \ <"${WORKFLOW_RUN_DIR}/log/scheduler/log" >'grep-log.out' cmp_ok 'grep-log.out' <<<'2' purge exit cylc-flow-8.7.0/tests/functional/events/50-ref-test-fail/0000775000175000017500000000000015264420307023310 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/50-ref-test-fail/reference.log0000664000175000017500000000012415264420307025746 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] 1/t2 -triggered off ['1/t1'] cylc-flow-8.7.0/tests/functional/events/50-ref-test-fail/flow.cylc0000664000175000017500000000015215264420307025131 0ustar alastairalastair[scheduling] [[graph]] R1 = t1 => t2 => t3 [runtime] [[t1, t2, t3]] script = true cylc-flow-8.7.0/tests/functional/events/11-cycle-task-event-job-logs-retrieve/0000775000175000017500000000000015264420307027356 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/11-cycle-task-event-job-logs-retrieve/reference.log0000664000175000017500000000017015264420307032015 0ustar alastairalastairInitial point: 1 Final point: 1 1/y -triggered off [] 1/y -triggered off [] 1/n -triggered off [] 1/n -triggered off [] cylc-flow-8.7.0/tests/functional/events/11-cycle-task-event-job-logs-retrieve/flow.cylc0000664000175000017500000000045115264420307031201 0ustar alastairalastair[meta] title = Task Event Job Log Retrieve [scheduling] [[graph]] R1 = y & n [runtime] [[root]] script = test "${CYLC_TASK_TRY_NUMBER}" -eq 2 execution retry delays = PT0S, PT1S [[y]] platform = _retrieve [[n]] platform = _no_retrieve cylc-flow-8.7.0/tests/functional/events/17-task-event-job-logs-retrieve-command.t0000775000175000017500000000516615264420307030103 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test remote job logs retrieval custom command, requires compatible version of # cylc on remote job host. export REQUIRE_PLATFORM='loc:remote' . "$(dirname "$0")/test_header" set_test_number 3 create_test_global_config "" " [platforms] [[${CYLC_TEST_PLATFORM}]] retrieve job logs = True retrieve job logs command = my-rsync " OPT_SET='-s GLOBALCFG=True' install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" mkdir -p "${RUN_DIR}/${WORKFLOW_NAME}/bin" cat >"${RUN_DIR}/${WORKFLOW_NAME}/bin/my-rsync" <<'__BASH__' #!/usr/bin/env bash set -eu echo "$@" >>"${CYLC_WORKFLOW_LOG_DIR}/my-rsync.log" exec rsync -a "$@" __BASH__ chmod +x "${RUN_DIR}/${WORKFLOW_NAME}/bin/my-rsync" # shellcheck disable=SC2086 run_ok "${TEST_NAME_BASE}-validate" \ cylc validate ${OPT_SET} -s "PLATFORM='${CYLC_TEST_PLATFORM}'" "${WORKFLOW_NAME}" # shellcheck disable=SC2086 workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach ${OPT_SET} \ -s "PLATFORM='${CYLC_TEST_PLATFORM}'" "${WORKFLOW_NAME}" WORKFLOW_LOG_D="${RUN_DIR}/${WORKFLOW_NAME}/log" sed 's/^.* -v //' "${WORKFLOW_LOG_D}/scheduler/my-rsync.log" >'my-rsync.log.edited' sed -i -E 's/--max-size=[^ ]* //' 'my-rsync.log.edited' # strip "retrieve job logs max size" arg sort -u 'my-rsync.log.edited' # strip out duplicates (can result from PBS log file spooling) OPT_HEAD='--include=/1 --include=/1/t1' OPT_TAIL='--exclude=/**' ARGS="${CYLC_TEST_HOST}:cylc-run/${WORKFLOW_NAME}/log/job/ ${WORKFLOW_LOG_D}/job/" cmp_ok 'my-rsync.log.edited' <<__LOG__ ${OPT_HEAD} --include=/1/t1/01 --include=/1/t1/01/** ${OPT_TAIL} ${ARGS} ${OPT_HEAD} --include=/1/t1/02 --include=/1/t1/02/** ${OPT_TAIL} ${ARGS} ${OPT_HEAD} --include=/1/t1/03 --include=/1/t1/03/** ${OPT_TAIL} ${ARGS} __LOG__ purge exit cylc-flow-8.7.0/tests/functional/events/21-workflow-event-handlers-globalcfg0000777000175000017500000000000015264420307034220 220-workflow-event-handlersustar alastairalastaircylc-flow-8.7.0/tests/functional/events/26-workflow-stalled-dump-prereq/0000775000175000017500000000000015264420307026410 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/26-workflow-stalled-dump-prereq/reference.log0000664000175000017500000000021615264420307031050 0ustar alastairalastairInitial point: 20100101T0000Z Final point: None 20100101T0000Z/foo -triggered off [] 20100101T0000Z/bar -triggered off ['20100101T0000Z/foo'] cylc-flow-8.7.0/tests/functional/events/26-workflow-stalled-dump-prereq/flow.cylc0000664000175000017500000000077615264420307030245 0ustar alastairalastair[scheduler] UTC mode = True # Ignore DST [[events]] abort on stall timeout = True stall timeout = PT0S expected task failures = 20100101T0000Z/bar [scheduling] initial cycle point = 20100101T0000Z [[graph]] # will abort on stall with unhandled failed bar T00, T06, T12, T18 = foo[-PT6H] & bar[-PT6H] => foo => bar => qux T12 = qux[-PT6H] => baz [runtime] [[root]] script = true [[foo,baz, qux]] [[bar]] script = false cylc-flow-8.7.0/tests/functional/events/37-workflow-event-bad-custom-template/0000775000175000017500000000000015264420307027511 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/37-workflow-event-bad-custom-template/reference.log0000664000175000017500000000006715264420307032155 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/events/37-workflow-event-bad-custom-template/flow.cylc0000664000175000017500000000026315264420307031335 0ustar alastairalastair[scheduler] [[events]] handlers = echo %(rubbish)s handler events = startup [scheduling] [[graph]] R1=t1 [runtime] [[t1]] script=true cylc-flow-8.7.0/tests/functional/events/20-workflow-event-handlers.t0000775000175000017500000000420215264420307025620 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test workflow event handler, flexible interface . "$(dirname "$0")/test_header" set_test_number 4 OPT_SET= if [[ "${TEST_NAME_BASE}" == *-globalcfg ]]; then create_test_global_config "" " [scheduler] [[events]] handlers = echo 'Your %(workflow)s workflow has a %(event)s event and URL %(workflow_url)s and workflow-priority as %(workflow-priority)s and workflow-UUID as %(uuid)s.' handler events = startup" OPT_SET='-s GLOBALCFG=True' fi install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # shellcheck disable=SC2086 run_ok "${TEST_NAME_BASE}-validate" \ cylc validate ${OPT_SET} "${WORKFLOW_NAME}" # shellcheck disable=SC2086 workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}" LOGD="$RUN_DIR/${WORKFLOW_NAME}/log" WORKFLOW_UUID="$(sqlite3 "${LOGD}/db" "SELECT value FROM workflow_params WHERE key=='uuid_str'")" LOG_FILE="${LOGD}/scheduler/log" grep_ok "\\[('workflow-event-handler-00', 'startup') ret_code\\] 0" "${LOG_FILE}" grep_ok "\\[('workflow-event-handler-00', 'startup') out\\] Your ${WORKFLOW_NAME} workflow has a startup event and URL http://myworkflows.com/${WORKFLOW_NAME}.html and workflow-priority as HIGH and workflow-UUID as ${WORKFLOW_UUID}." "${LOG_FILE}" purge exit cylc-flow-8.7.0/tests/functional/events/32-task-event-job-logs-retrieve-2/0000775000175000017500000000000015264420307026423 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/32-task-event-job-logs-retrieve-2/reference.log0000664000175000017500000000006715264420307031067 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/events/32-task-event-job-logs-retrieve-2/flow.cylc0000664000175000017500000000030415264420307030243 0ustar alastairalastair[meta] title = Task Event Job Log Retrieve 1 [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = rm -f "${CYLC_TASK_LOG_ROOT}.err" platform = treadstone cylc-flow-8.7.0/tests/functional/events/stall-timeout/0000775000175000017500000000000015264420307023227 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/stall-timeout/flow.cylc0000664000175000017500000000037315264420307025055 0ustar alastairalastair[meta] description = This workflow is supposed to time out [scheduler] [[events]] stall timeout = PT6S abort on stall timeout = True [scheduling] [[graph]] R1 = "foo" [runtime] [[foo]] script = false cylc-flow-8.7.0/tests/functional/events/38-task-event-handler-custom.t0000775000175000017500000000304615264420307026053 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test custom severity event handling. . "$(dirname "$0")/test_header" set_test_number 6 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" FOO_ACTIVITY_LOG="${WORKFLOW_RUN_DIR}/log/job/1/foo/NN/job-activity.log" WORKFLOW_LOG="${WORKFLOW_RUN_DIR}/log/scheduler/log" grep_ok \ "\[(('event-handler-00', 'custom-1'), 1) out\] !!CUSTOM!! 1/foo fugu Data ready for barring" \ "${FOO_ACTIVITY_LOG}" grep_ok "1/foo.*Data ready for barring" "${WORKFLOW_LOG}" -E grep_ok "1/foo.*Data ready for bazzing" "${WORKFLOW_LOG}" -E grep_ok "1/foo.*Aren't the hydrangeas nice" "${WORKFLOW_LOG}" -E purge cylc-flow-8.7.0/tests/functional/events/29-task-event-mail-1/0000775000175000017500000000000015264420307024013 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/29-task-event-mail-1/reference.log0000664000175000017500000000011615264420307026452 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] 1/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/events/29-task-event-mail-1/flow.cylc0000664000175000017500000000056615264420307025645 0ustar alastairalastair#!jinja2 [meta] title=Task Event Mail [scheduler] [[mail]] footer = see: http://localhost/stuff/%(owner)s/%(workflow)s/ [scheduling] [[graph]] R1=t1 [runtime] [[t1]] script=test "${CYLC_TASK_TRY_NUMBER}" -eq 2 [[[job]]] execution retry delays = PT1S [[[events]]] mail events = failed, retry cylc-flow-8.7.0/tests/functional/events/27-workflow-stalled-dump-prereq-fam/0000775000175000017500000000000015264420307027152 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/27-workflow-stalled-dump-prereq-fam/reference.log0000664000175000017500000000012015264420307031604 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/goo -triggered off [] cylc-flow-8.7.0/tests/functional/events/27-workflow-stalled-dump-prereq-fam/flow.cylc0000664000175000017500000000070515264420307030777 0ustar alastairalastair[scheduler] UTC mode = True # Ignore DST allow implicit tasks = True [[events]] abort on stall timeout = true stall timeout = PT0S expected task failures = 1/foo [scheduling] [[graph]] # will abort on stall with unhandled failed foo R1 = """foo & goo => FAM FAM:succeed-any => bar""" [runtime] [[foo]] script = false [[FAM]] [[f_1, f_2, f_3]] inherit = FAM cylc-flow-8.7.0/tests/functional/events/28-inactivity.t0000775000175000017500000000240615264420307023230 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test workflow event handler, abort on inactivity . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" grep_ok "inactivity timer timed out" "${TEST_NAME_BASE}-run.stderr" purge exit cylc-flow-8.7.0/tests/functional/events/24-workflow-timeout.t0000775000175000017500000000302715264420307024377 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test workflow timeout event and handler . "$(dirname "$0")/test_header" set_test_number 5 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate \ -s "ABORT=False" "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach -s "ABORT=False" "${WORKFLOW_NAME}" grep_workflow_log_ok grep-1 "That was quick!" purge install_workflow "${TEST_NAME_BASE}-2" "${TEST_NAME_BASE}" workflow_run_fail "${TEST_NAME_BASE}-run-2" \ cylc play --debug --no-detach -s "ABORT=True" "${WORKFLOW_NAME}" grep_workflow_log_ok grep-2 'Workflow shutting down - "abort on workflow timeout" is set' purge cylc-flow-8.7.0/tests/functional/events/00-workflow.t0000775000175000017500000000256115264420307022707 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test workflow event handlers . "$(dirname "$0")/test_header" set_test_number 6 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" grep_workflow_log_ok grep-startup "HELLO STARTUP" grep_workflow_log_ok grep-timeout "HELLO TIMEOUT" grep_workflow_log_ok grep-inactivity "HELLO INACTIVITY" grep_workflow_log_ok grep-shutdown "HELLO SHUTDOWN" cylc-flow-8.7.0/tests/functional/events/13-task-event-mail-globalcfg.t0000777000175000017500000000000015264420307031360 209-task-event-mail.tustar alastairalastaircylc-flow-8.7.0/tests/functional/events/26-workflow-stalled-dump-prereq.t0000775000175000017500000000327715264420307026611 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test workflow event handler, dump unmet prereqs on stall . "$(dirname "$0")/test_header" set_test_number 7 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" grep_ok '"abort on stall timeout" is set' "${TEST_NAME_BASE}-run.stderr" grep_ok "ERROR - Incomplete tasks:" "${TEST_NAME_BASE}-run.stderr" grep_ok "20100101T0000Z/bar did not complete the required outputs:\n.*succeeded" \ "${TEST_NAME_BASE}-run.stderr" \ -Pizo grep_ok "WARNING - Partially satisfied prerequisites:" \ "${TEST_NAME_BASE}-run.stderr" grep_ok "20100101T0600Z/foo is waiting on \['20100101T0000Z/bar:succeeded'\]" \ "${TEST_NAME_BASE}-run.stderr" purge exit cylc-flow-8.7.0/tests/functional/events/47-long-output.t0000775000175000017500000000635215264420307023347 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that a long output from an event handler is not going to hang or die. . "$(dirname "$0")/test_header" set_test_number 10 create_test_global_config "" " [scheduler] process pool timeout = PT10S " # Long STDOUT output init_workflow "${TEST_NAME_BASE}" <<__FLOW_CONFIG__ [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = true [[[events]]] succeeded handlers = cat "${CYLC_REPO_DIR}/COPYING" "${CYLC_REPO_DIR}/COPYING" "${CYLC_REPO_DIR}/COPYING" && echo __FLOW_CONFIG__ cd "$WORKFLOW_RUN_DIR" || exit 1 cat >'reference.log' <<'__REFLOG__' Initial point: 1 Final point: 1 1/t1 -triggered off [] __REFLOG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" cylc cat-log "${WORKFLOW_NAME}" >'catlog' sed -n 's/^.*\(GNU GENERAL PUBLIC LICENSE\)/\1/p' 'catlog' >'log-1' contains_ok 'log-1' <<'__LOG__' GNU GENERAL PUBLIC LICENSE GNU GENERAL PUBLIC LICENSE GNU GENERAL PUBLIC LICENSE __LOG__ run_ok "log-event-handler-00-out" \ grep -qF "[(('event-handler-00', 'succeeded'), 1) out]" 'catlog' run_ok "log-event-handler-ret-code" \ grep -qF "[(('event-handler-00', 'succeeded'), 1) ret_code] 0" 'catlog' purge # REPEAT: Long STDERR output init_workflow "${TEST_NAME_BASE}" <<__FLOW_CONFIG__ [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = true [[[events]]] succeeded handlers = cat "${CYLC_REPO_DIR}/COPYING" "${CYLC_REPO_DIR}/COPYING" "${CYLC_REPO_DIR}/COPYING" >&2 && echo __FLOW_CONFIG__ cd "${WORKFLOW_RUN_DIR}" || exit 1 cat >'reference.log' <<'__REFLOG__' Initial point: 1 Final point: 1 1/t1 -triggered off [] __REFLOG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" cylc cat-log "${WORKFLOW_NAME}" >'catlog' sed -n 's/^.*\(GNU GENERAL PUBLIC LICENSE\)/\1/p' 'catlog' >'log-1' contains_ok 'log-1' <<'__LOG__' GNU GENERAL PUBLIC LICENSE GNU GENERAL PUBLIC LICENSE GNU GENERAL PUBLIC LICENSE __LOG__ run_ok "log-event-handler-00-err" \ grep -qF "[(('event-handler-00', 'succeeded'), 1) err]" 'catlog' run_ok "log-event-handler-00-ret-code" \ grep -qF "[(('event-handler-00', 'succeeded'), 1) ret_code] 0" 'catlog' purge exit cylc-flow-8.7.0/tests/functional/events/41-late.t0000775000175000017500000000245215264420307021766 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test late event handler . "$(dirname "$0")/test_header" set_test_number 4 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}-run" cylc play --debug --no-detach "${WORKFLOW_NAME}" grep_ok 'late (late-time=.*)' "${WORKFLOW_RUN_DIR}/log/scheduler/log" grep_ok 'late (late-time=.*)' "${WORKFLOW_RUN_DIR}/log/scheduler/my-handler.out" purge exit cylc-flow-8.7.0/tests/functional/events/01-timeout.t0000775000175000017500000000337615264420307022531 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test workflow timeout . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 4 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" timeout #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_fail "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" grep_ok "WARNING - workflow timer timed out after PT6S" "${TEST_NAME}.stderr" grep_ok 'Workflow shutting down - "abort on workflow timeout" is set' \ "${TEST_NAME}.stderr" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/events/test_header0000777000175000017500000000000015264420307026741 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/events/19-workflow-event-mail-globalcfg.t0000777000175000017500000000000015264420307033206 218-workflow-event-mail.tustar alastairalastaircylc-flow-8.7.0/tests/functional/events/02-multi.t0000775000175000017500000000430515264420307022167 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test list of multiple event handlers. . "$(dirname "$0")/test_header" set_test_number 3 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduler] [[events]] inactivity timeout = PT30S [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = true [[[events]]] started handlers = echo %(workflow)s, echo %(name)s, echo %(start_time)s __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" JOB_STFILE="${WORKFLOW_RUN_DIR}/log/job/1/t1/01/job.status" JOB_START_TIME="$(sed -n 's/^CYLC_JOB_INIT_TIME=//p' "${JOB_STFILE}")" cylc cat-log "${WORKFLOW_NAME}" \ | sed -n -e 's/^.*\(\[(('"'"'event-handler-0.'"'"'.*$\)/\1/p' | sort >'log' cmp_ok log <<__END__ [(('event-handler-00', 'started'), 1) cmd] echo ${WORKFLOW_NAME} [(('event-handler-00', 'started'), 1) out] ${WORKFLOW_NAME} [(('event-handler-00', 'started'), 1) ret_code] 0 [(('event-handler-01', 'started'), 1) cmd] echo t1 [(('event-handler-01', 'started'), 1) out] t1 [(('event-handler-01', 'started'), 1) ret_code] 0 [(('event-handler-02', 'started'), 1) cmd] echo ${JOB_START_TIME} [(('event-handler-02', 'started'), 1) out] ${JOB_START_TIME} [(('event-handler-02', 'started'), 1) ret_code] 0 __END__ purge exit cylc-flow-8.7.0/tests/functional/events/25-held-not-stalled.t0000775000175000017500000000243115264420307024200 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test abort on stall does not apply to a workflow started with --hold-after # See also tests/functional/pause-resume/02-paused-not-stalled.t . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --hold-after=0 --no-detach "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/events/21-workflow-event-handlers-globalcfg.t0000777000175000017500000000000015264420307034724 220-workflow-event-handlers.tustar alastairalastaircylc-flow-8.7.0/tests/functional/events/34-task-abort/0000775000175000017500000000000015264420307022717 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/34-task-abort/reference.log0000664000175000017500000000016315264420307025360 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/foo -triggered off [] 1/handled -triggered off ['1/foo'] cylc-flow-8.7.0/tests/functional/events/34-task-abort/flow.cylc0000664000175000017500000000075615264420307024552 0ustar alastairalastair[meta] title = "Test job abort with retries and failed handler" [scheduler] [[events]] expected task failures = 1/foo [scheduling] [[graph]] R1 = "foo:fail => handled" [runtime] [[foo]] script = """ echo ONE cylc__job_abort "ERROR: rust never sleeps" echo TWO""" execution retry delays = PT0S [[[events]]] failed handlers = echo "!!!FAILED!!!" %(event)s %(id)s %(submit_num)s %(message)s [[handled]] script = true cylc-flow-8.7.0/tests/functional/events/23-workflow-stalled-handler.t0000775000175000017500000000176115264420307025756 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test workflow event handler, flexible interface with stall handler . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/events/09-task-event-mail/0000775000175000017500000000000015264420307023653 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/09-task-event-mail/reference.log0000664000175000017500000000011615264420307026312 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] 1/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/events/09-task-event-mail/flow.cylc0000664000175000017500000000104115264420307025472 0ustar alastairalastair#!jinja2 [meta] title=Task Event Mail [scheduler] {% if GLOBALCFG is not defined %} [[mail]] footer = see: http://localhost/stuff/%(owner)s/%(workflow)s/ {% endif %}{# not GLOBALCFG is not defined #} [scheduling] [[graph]] R1=t1 [runtime] [[t1]] script=test "${CYLC_TASK_TRY_NUMBER}" -eq 2 [[[job]]] execution retry delays = PT5S {% if GLOBALCFG is not defined %} [[[events]]] mail events = failed, retry, succeeded {% endif %}{# not GLOBALCFG is not defined #} cylc-flow-8.7.0/tests/functional/events/09-task-event-mail.t0000775000175000017500000000442415264420307024047 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test event mail, 2 different task events . "$(dirname "$0")/test_header" if ! command -v mail 2>'/dev/null'; then skip_all '"mail" command not available' fi set_test_number 6 mock_smtpd_init OPT_SET= if [[ "${TEST_NAME_BASE}" == *-globalcfg ]]; then create_test_global_config "" " [scheduler] [[mail]] footer = see: http://localhost/stuff/%(owner)s/%(workflow)s/ smtp = ${TEST_SMTPD_HOST} [task events] mail events = failed, retry, succeeded " OPT_SET='-s GLOBALCFG=True' else create_test_global_config " [scheduler] [[mail]] smtp = ${TEST_SMTPD_HOST} " fi install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # shellcheck disable=SC2086 run_ok "${TEST_NAME_BASE}-validate" \ cylc validate ${OPT_SET} "${WORKFLOW_NAME}" # shellcheck disable=SC2086 workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}-grep-log-1" \ grep -Pizo 'job: 1/t1/01.*\n.*event: retry' "${TEST_SMTPD_LOG}" run_ok "${TEST_NAME_BASE}-grep-log-2" \ grep -Pizo 'job: 1/t1/02.*\n.*event: succeeded' "${TEST_SMTPD_LOG}" run_ok "${TEST_NAME_BASE}-grep-log-3" \ grep -Pizo "Subject: \[1/t1/01 retry\].*(\n)?.* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" run_ok "${TEST_NAME_BASE}-grep-log-4" \ grep -Pizo "Subject: \[1/t1/02 succeeded\].*(\n)?.* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" purge mock_smtpd_kill exit cylc-flow-8.7.0/tests/functional/events/42-late-then-restart/0000775000175000017500000000000015264420307024212 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/42-late-then-restart/flow.cylc0000664000175000017500000000051515264420307026036 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] initial cycle point = now [[graph]] R1 = t1 => t2 [runtime] [[t1]] script = cylc stop --now "${CYLC_WORKFLOW_ID}"; sleep 61 [[t2]] script = true [[[events]]] late offset = PT1M late handlers = my-handler %(message)s cylc-flow-8.7.0/tests/functional/events/42-late-then-restart/bin/0000775000175000017500000000000015264420307024762 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/events/42-late-then-restart/bin/my-handler0000775000175000017500000000014315264420307026746 0ustar alastairalastair#!/usr/bin/env bash set -eu ME="$(basename "$0")" echo "$@" >>"${CYLC_WORKFLOW_LOG_DIR}/${ME}.out" cylc-flow-8.7.0/tests/functional/ext-trigger/0000775000175000017500000000000015264420307021361 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/ext-trigger/02-cycle-point/0000775000175000017500000000000015264420307024026 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/ext-trigger/02-cycle-point/flow.cylc0000664000175000017500000000147615264420307025661 0ustar alastairalastair[meta] title = "Test for Github Issue 1893" description = """Cycle-point specific external triggering in a date-time cycling workflow. The workflow will time out and abort if the ext trigger fails.""" [scheduler] cycle point format = %Y [[events]] abort on stall timeout = True stall timeout = PT30S [scheduling] initial cycle point = 2020 final cycle point = 2020 [[special tasks]] external-trigger = ext("cheese on toast for $CYLC_TASK_CYCLE_POINT") [[graph]] P1Y = ext & trig [runtime] [[ext]] # Externally triggered task. script = echo $CYLC_EXT_TRIGGER_ID [[trig]] # Task to do the "external" triggering. script = cylc ext-trigger $CYLC_WORKFLOW_ID \ "cheese on toast for $CYLC_TASK_CYCLE_POINT" "blarghh!" cylc-flow-8.7.0/tests/functional/ext-trigger/02-cycle-point.t0000664000175000017500000000215515264420307024216 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test GitHub #1893 - cycle point specific external triggers. . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --no-detach "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/ext-trigger/00-satellite/0000775000175000017500000000000015264420307023564 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/ext-trigger/00-satellite/reference.log0000664000175000017500000000160615264420307026230 0ustar alastairalastairInitial point: 1 Final point: 5 1/prep -triggered off [] 1/satsim -triggered off ['1/prep'] 1/get_data -triggered off ['0/get_data', '1/prep'] 2/get_data -triggered off ['1/get_data'] 1/proc1 -triggered off ['1/get_data'] 1/proc2 -triggered off ['1/proc1'] 3/get_data -triggered off ['2/get_data'] 2/proc1 -triggered off ['2/get_data'] 4/get_data -triggered off ['3/get_data'] 1/products -triggered off ['1/proc2'] 3/proc1 -triggered off ['3/get_data'] 2/proc2 -triggered off ['2/proc1'] 5/get_data -triggered off ['4/get_data'] 2/products -triggered off ['2/proc2'] 3/proc2 -triggered off ['3/proc1'] 4/proc1 -triggered off ['4/get_data'] 4/proc2 -triggered off ['4/proc1'] 3/products -triggered off ['3/proc2'] 5/proc1 -triggered off ['5/get_data'] 4/products -triggered off ['4/proc2'] 5/proc2 -triggered off ['5/proc1'] 5/products -triggered off ['5/proc2'] 5/collate -triggered off ['5/products'] cylc-flow-8.7.0/tests/functional/ext-trigger/00-satellite/flow.cylc0000664000175000017500000001027715264420307025416 0ustar alastairalastair#!Jinja2 # TEST WORKFLOW ADAPTED FROM examples/satellite/ext-trigger/ [meta] title = Real time satellite data processing demo, variant 3 of 3 description = """ Successive cycle points retrieve and processes the next arbitrarily timed and labelled dataset, in parallel if the data comes in quickly. This variant of the workflow has initial get_data tasks with external triggers: they do not submit until triggered by an external system. """ # Note that the satellite simulator task here that supplies the external event # trigger happens to be a workflow task - i.e. it is not really "external" - but # this is only a convenience - an easy route to a self-contained example workflow. # you can monitor output processing with: # $ watch -n 1 \ # "find ~/cylc-run//share; find ~/cylc-run//work" {% set N_DATASETS = 5 %} # define shared directories (could use runtime namespaces for this) {% set DATA_IN_DIR = "$CYLC_WORKFLOW_SHARE_DIR/incoming" %} {% set PRODUCT_DIR = "$CYLC_WORKFLOW_SHARE_DIR/products" %} [scheduler] UTC mode = True [scheduling] cycling mode = integer initial cycle point = 1 final cycle point = {{N_DATASETS}} runahead limit = P4 [[special tasks]] external-trigger = get_data("new dataset ready for processing") [[graph]] R1 = prep => satsim & get_data P1 = """ # Processing chain for each dataset get_data => proc1 => proc2 => products # As one dataset is retrieved, start waiting on another. get_data[-P1] => get_data """ R1//{{N_DATASETS}} = products => collate # last cycle [runtime] [[prep]] script = rm -rf $CYLC_WORKFLOW_SHARE_DIR $CYLC_WORKFLOW_WORK_DIR [[[meta]]] title = clean the workflow output directories [[satsim]] pre-script = mkdir -p {{DATA_IN_DIR}} script = """ COUNT=0 while ((COUNT < {{N_DATASETS}})); do # sleep $((RANDOM % 20)) # Generate datasets very quickly to test parallel processing. DATA_ID=$(date +%s).$((RANDOM % 100)) DATA_FILE=dataset-${DATA_ID}.raw touch {{DATA_IN_DIR}}/$DATA_FILE ((COUNT += 1)) # (required to distinguish fast-arriving messages). # Trigger downstream processing in the workflow. cylc ext-trigger $CYLC_WORKFLOW_ID \ "new dataset ready for processing" $DATA_ID done """ [[[meta]]] title = simulate a satellite data feed description = """ Generates {{N_DATASETS}} arbitrarily labelled datasets very quickly, to show parallel processing streams. """ [[WORKDIR]] # Define a common cycle-point-specific work-directory for all # processing tasks so that they all work on the same dataset. work sub-directory = proc-$CYLC_TASK_CYCLE_POINT [[[environment]]] DATASET = dataset-$CYLC_EXT_TRIGGER_ID [[get_data]] inherit = WORKDIR script = mv {{DATA_IN_DIR}}/${DATASET}.raw $PWD [[[meta]]] title = retrieve next dataset description = just do it - we know it exists already [[proc1]] inherit = WORKDIR script = mv ${DATASET}.raw ${DATASET}.proc1 [[[meta]]] title = convert .raw dataset to .proc1 form [[proc2]] inherit = WORKDIR script = mv ${DATASET}.proc1 ${DATASET}.proc2 [[[meta]]] title = convert .proc1 dataset to .proc2 form [[products]] inherit = WORKDIR script = """ mkdir -p {{PRODUCT_DIR}} mv ${DATASET}.proc2 {{PRODUCT_DIR}}/${DATASET}.prod """ [[[meta]]] title = generate products from .proc2 processed dataset [[collate]] # Note you might want to use "cylc workflow-state" to check that # _all_ product tasks have finished before collating results. script = """ echo PRODUCTS: ls {{PRODUCT_DIR}} """ [[[meta]]] title = collate all products from the workflow run cylc-flow-8.7.0/tests/functional/ext-trigger/01-no-nudge.t0000664000175000017500000000253415264420307023504 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that external trigger events stimulate task processing even when nothing # else is happening in the workflow. # Note this test will probably become irrelevant once we go to entirely # event-driven scheduling. . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --no-detach "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/ext-trigger/01-no-nudge/0000775000175000017500000000000015264420307023313 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/ext-trigger/01-no-nudge/flow.cylc0000664000175000017500000000256115264420307025142 0ustar alastairalastair[meta] title = "Test for Github Issue 1543" description = """ External trigger events should stimulate task processing even when nothing else is happening in the workflow. Here, long-running task bar ext-triggers foo when nothing else is happening. If task processing occurs foo will submit and kill bar, allowing the workflow to shutdown. Otherwise, foo won't submit, bar will keep running, and the workflow will time out. """ [scheduler] [[events]] abort on stall timeout = True stall timeout = PT30S abort on inactivity timeout = True inactivity timeout = PT30S [scheduling] [[special tasks]] external-trigger = foo("drugs and money") [[graph]] # killed tasks are held to prevent retries; they have to be # released before they can be removed. R1 = """ foo & bar? bar:fail? => handler """ [runtime] [[foo]] script = """ cylc kill "$CYLC_WORKFLOW_ID//1/bar" cylc__job__poll_grep_workflow_log -E '1/bar.* \(internal\)failed' cylc release "$CYLC_WORKFLOW_ID//1/bar" """ [[bar]] script = """ sleep 5 cylc ext-trigger $CYLC_WORKFLOW_ID "drugs and money" 12345 sleep 60 """ [[handler]] script = true cylc-flow-8.7.0/tests/functional/ext-trigger/test_header0000777000175000017500000000000015264420307027676 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/ext-trigger/00-satellite.t0000664000175000017500000000174215264420307023755 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Validate and run the external trigger test workflow . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/cylc-trigger/0000775000175000017500000000000015264420307021513 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-trigger/06-already-active.t0000664000175000017500000000231315264420307025014 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test triggering an already-active task just generates a warning. . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" purge cylc-flow-8.7.0/tests/functional/cylc-trigger/07-kill-trigger.t0000664000175000017500000000241515264420307024522 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test killing a job, then re-triggering it whilst it is "held". # See https://github.com/cylc/cylc-flow/issues/6398 . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" purge cylc-flow-8.7.0/tests/functional/cylc-trigger/01-queued/0000775000175000017500000000000015264420307023221 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-trigger/01-queued/reference.log0000664000175000017500000000012715264420307025662 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/bar -triggered off ['1/foo'] cylc-flow-8.7.0/tests/functional/cylc-trigger/01-queued/flow.cylc0000664000175000017500000000065715264420307025054 0ustar alastairalastair[scheduling] [[queues]] [[[my_queue]]] limit = 1 members = METASYNTACTIC [[graph]] R1 = "foo:start => bar" [runtime] [[METASYNTACTIC]] [[foo]] inherit = METASYNTACTIC script = """ cylc__job__wait_cylc_message_started cylc trigger "$CYLC_WORKFLOW_ID//1/bar" """ [[bar]] inherit = METASYNTACTIC script = true cylc-flow-8.7.0/tests/functional/cylc-trigger/07-kill-trigger/0000775000175000017500000000000015264420307024333 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-trigger/07-kill-trigger/reference.log0000664000175000017500000000010015264420307026763 0ustar alastairalastair1/a -triggered off [] in flow 1 1/a -triggered off [] in flow 1 cylc-flow-8.7.0/tests/functional/cylc-trigger/07-kill-trigger/flow.cylc0000664000175000017500000000201215264420307026151 0ustar alastairalastair[scheduler] [[events]] inactivity timeout = PT1M expected task failures = "1/a" [scheduling] [[graph]] R1 = a [runtime] [[a]] script = """ if [[ $CYLC_TASK_SUBMIT_NUMBER == 1 ]]; then # wait for the scheduler to receive the started message, # then kill the job cylc__job__poll_grep_workflow_log -E '1/a.*running' cylc kill "${CYLC_WORKFLOW_ID}//1/a" # do not succeed immediately after issuing the kill command or the # workflow may shut down as complete before registering task failure # (This polling grep will never complete, but you know what I mean.) cylc__job__poll_grep_workflow_log -E '1/a.*failed' fi """ [[[events]]] # when the scheduler receives the failed message, trigger the task # to run again, it should run instantly failed handlers = cylc trigger "%(workflow)s//1/a" cylc-flow-8.7.0/tests/functional/cylc-trigger/00-compat/0000775000175000017500000000000015264420307023213 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-trigger/00-compat/reference.log0000664000175000017500000000012715264420307025654 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/bar -triggered off ['1/foo'] cylc-flow-8.7.0/tests/functional/cylc-trigger/00-compat/flow.cylc0000664000175000017500000000024615264420307025040 0ustar alastairalastair[scheduling] [[graph]] R1 = foo => bar [runtime] [[foo]] script = cylc trigger "${CYLC_WORKFLOW_ID}//^/bar" [[bar]] script = true cylc-flow-8.7.0/tests/functional/cylc-trigger/04-filter-names.t0000775000175000017500000000256315264420307024520 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test triggering matching names in a cycle. . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" # Ensure that 1/loser is only triggered once. JOB_LOG_DIR="$RUN_DIR/${WORKFLOW_NAME}/log/job" run_ok "${TEST_NAME_BASE}-loser-nn" \ test "$(readlink "${JOB_LOG_DIR}/1/loser/NN")" = '01' purge exit cylc-flow-8.7.0/tests/functional/cylc-trigger/02-filter-failed.t0000775000175000017500000000203515264420307024631 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test triggering multiple failed tasks in a cycle point. # (Pre SoD this matched the task pool by failed state). . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/cylc-trigger/01-queued.t0000777000175000017500000000000015264420307025266 200-compat.tustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-trigger/test_header0000777000175000017500000000000015264420307030030 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-trigger/00-compat.t0000775000175000017500000000172015264420307023403 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test basic usage of "cylc trigger" . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/cylc-trigger/06-already-active/0000775000175000017500000000000015264420307024630 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-trigger/06-already-active/flow.cylc0000664000175000017500000000125315264420307026454 0ustar alastairalastair# test triggering an already active task [scheduler] [[events]] inactivity timeout = PT1M abort on inactivity timeout = True [scheduling] [[graph]] R1 = "triggeree:start & triggerer" [runtime] [[triggerer]] script = """ cylc__job__poll_grep_workflow_log "1/triggeree.* => running" -E cylc trigger "$CYLC_WORKFLOW_ID//1/triggeree" cylc__job__poll_grep_workflow_log \ "Job already in process - ignoring trigger" -E """ [[triggeree]] script = """ cylc__job__poll_grep_workflow_log \ "Job already in process - ignoring trigger" -E """ cylc-flow-8.7.0/tests/functional/cylc-trigger/02-filter-failed/0000775000175000017500000000000015264420307024441 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-trigger/02-filter-failed/reference.log0000664000175000017500000000054615264420307027107 0ustar alastairalastairInitial point: 1 Final point: 1 1/fixable1 -triggered off [] 1/fixable2 -triggered off [] 1/fixable3 -triggered off [] 1/fixer -triggered off [] 1/fixable1 -triggered off [] 1/fixable2 -triggered off [] 1/fixable3 -triggered off [] 1/z1 -triggered off ['1/fixable1', '1/fixable2', '1/fixable3'] 1/z2 -triggered off ['1/fixable1', '1/fixable2', '1/fixable3'] cylc-flow-8.7.0/tests/functional/cylc-trigger/02-filter-failed/flow.cylc0000664000175000017500000000161415264420307026266 0ustar alastairalastair[scheduler] [[events]] expected task failures = 1/fixable1, 1/fixable2, 1/fixable3 [scheduling] [[graph]] # Unhandled failures stay around for retriggering by "fixer" R1 = """ fixer FIXABLES:succeed-all => Z """ [runtime] [[FIXABLES]] script = test "${CYLC_TASK_SUBMIT_NUMBER}" -eq 2 [[fixable1, fixable2, fixable3]] inherit = FIXABLES [[fixer]] script = """ cylc__job__wait_cylc_message_started cylc__job__poll_grep_workflow_log '\[1/fixable1/01:running\] failed/ERR' cylc__job__poll_grep_workflow_log '\[1/fixable2/01:running\] failed/ERR' cylc__job__poll_grep_workflow_log '\[1/fixable3/01:running\] failed/ERR' cylc trigger "${CYLC_WORKFLOW_ID}//1/fixable*:failed" """ [[Z]] script = true [[z1, z2]] inherit = Z cylc-flow-8.7.0/tests/functional/cylc-trigger/04-filter-names/0000775000175000017500000000000015264420307024322 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-trigger/04-filter-names/reference.log0000664000175000017500000000114615264420307026765 0ustar alastairalastairInitial point: 1 Final point: 1 1/fixable-1a -triggered off [] 1/fixable-1b -triggered off [] 1/fixable-2a -triggered off [] 1/fixable-2b -triggered off [] 1/fixable-3 -triggered off [] 1/loser -triggered off [] 1/fixer -triggered off [] 1/fixable-1a -triggered off [] 1/fixable-1b -triggered off [] 1/fixable-2a -triggered off [] 1/fixable-2b -triggered off [] 1/fixable-3 -triggered off [] 1/z1 -triggered off ['1/fixable-1a', '1/fixable-1b', '1/fixable-2a', '1/fixable-2b', '1/fixable-3', '1/loser'] 1/z2 -triggered off ['1/fixable-1a', '1/fixable-1b', '1/fixable-2a', '1/fixable-2b', '1/fixable-3', '1/loser'] cylc-flow-8.7.0/tests/functional/cylc-trigger/04-filter-names/flow.cylc0000664000175000017500000000247715264420307026157 0ustar alastairalastair[scheduler] [[events]] expected task failures = 1/fixable-1a, 1/fixable-1b, 1/fixable-2a, 1/fixable-2b, 1/fixable-3, 1/loser [scheduling] [[graph]] R1 = """ # Unhandled failures stay around for retriggering by "fixer" fixer FIXABLES:succeed-all & loser:fail => Z """ [runtime] [[FIXABLES]] script = test "${CYLC_TASK_SUBMIT_NUMBER}" -eq 2 [[FIXABLE-1, FIXABLE-2, FIXABLE-3]] inherit = FIXABLES [[fixable-1a, fixable-1b]] inherit = FIXABLE-1 [[fixable-2a, fixable-2b]] inherit = FIXABLE-2 [[fixable-3]] inherit = FIXABLE-3 [[fixer]] script = """ cylc__job__wait_cylc_message_started cylc__job__poll_grep_workflow_log -E '1/fixable-1a/01.* failed' cylc__job__poll_grep_workflow_log -E '1/fixable-1b/01.* failed' cylc__job__poll_grep_workflow_log -E '1/fixable-2a/01.* failed' cylc__job__poll_grep_workflow_log -E '1/fixable-2b/01.* failed' cylc__job__poll_grep_workflow_log -E '1/fixable-3/01.* failed' cylc trigger "${CYLC_WORKFLOW_ID}//" \ '//1/FIXABLE-1' '//1/fixable-2*' '//1/fixable-3' """ [[loser]] script = false [[Z]] script = true [[z1, z2]] inherit = Z cylc-flow-8.7.0/tests/functional/cyclers/0000775000175000017500000000000015264420307020564 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/rmany_reverse/0000775000175000017500000000000015264420307023445 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/rmany_reverse/reference.log0000664000175000017500000000067715264420307026120 0ustar alastairalastairInitial point: 20150221T0000Z Final point: 20150221T1800Z 20150221T0000Z/foo -triggered off [] 20150221T0600Z/foo -triggered off [] 20150221T1200Z/foo -triggered off [] 20150221T0000Z/nonstop -triggered off ['20150221T0000Z/foo'] 20150221T0600Z/nonstop -triggered off ['20150221T0600Z/foo'] 20150221T1200Z/nonstop -triggered off ['20150221T1200Z/foo'] 20150221T1800Z/foo -triggered off [] 20150221T1800Z/stop -triggered off ['20150221T1800Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/rmany_reverse/flow.cylc0000664000175000017500000000045615264420307025275 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 2015-02-21T00 final cycle point = 2015-02-21T18 [[graph]] PT6H = foo R1/P0D = foo => stop R20/PT6H/-PT6H = foo => nonstop [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/rmany_reverse/graph.plain.ref0000664000175000017500000000115615264420307026351 0ustar alastairalastairedge "20150221T0000Z/foo" "20150221T0000Z/nonstop" edge "20150221T0600Z/foo" "20150221T0600Z/nonstop" edge "20150221T1200Z/foo" "20150221T1200Z/nonstop" edge "20150221T1800Z/foo" "20150221T1800Z/stop" graph node "20150221T0000Z/foo" "foo\n20150221T0000Z" node "20150221T0000Z/nonstop" "nonstop\n20150221T0000Z" node "20150221T0600Z/foo" "foo\n20150221T0600Z" node "20150221T0600Z/nonstop" "nonstop\n20150221T0600Z" node "20150221T1200Z/foo" "foo\n20150221T1200Z" node "20150221T1200Z/nonstop" "nonstop\n20150221T1200Z" node "20150221T1800Z/foo" "foo\n20150221T1800Z" node "20150221T1800Z/stop" "stop\n20150221T1800Z" stop cylc-flow-8.7.0/tests/functional/cyclers/multidaily/0000775000175000017500000000000015264420307022741 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/multidaily/reference.log0000664000175000017500000000406715264420307025411 0ustar alastairalastairInitial point: 20001231T0100 Final point: 20010114 20001231T0100Z/baz -triggered off ['20001227T0100Z/baz'] 20001231T0100Z/foo -triggered off ['20001230T0100Z/foo'] 20001231T0100Z/qux -triggered off ['20001231T0100Z/baz'] 20010104T0100Z/baz -triggered off ['20001231T0100Z/baz'] 20001231T0100Z/bar -triggered off ['20001231T0100Z/foo'] 20010101T0100Z/foo -triggered off ['20001231T0100Z/foo'] 20010104T0100Z/qux -triggered off ['20010104T0100Z/baz'] 20010101T0100Z/bar -triggered off ['20010101T0100Z/foo'] 20010102T0100Z/foo -triggered off ['20010101T0100Z/foo'] 20010108T0100Z/baz -triggered off ['20010104T0100Z/baz'] 20010102T0100Z/bar -triggered off ['20010102T0100Z/foo'] 20010103T0100Z/foo -triggered off ['20010102T0100Z/foo'] 20010108T0100Z/qux -triggered off ['20010108T0100Z/baz'] 20010103T0100Z/bar -triggered off ['20010103T0100Z/foo'] 20010104T0100Z/foo -triggered off ['20010103T0100Z/foo'] 20010104T0100Z/bar -triggered off ['20010104T0100Z/foo'] 20010105T0100Z/foo -triggered off ['20010104T0100Z/foo'] 20010105T0100Z/bar -triggered off ['20010105T0100Z/foo'] 20010106T0100Z/foo -triggered off ['20010105T0100Z/foo'] 20010112T0100Z/baz -triggered off ['20010108T0100Z/baz'] 20010106T0100Z/bar -triggered off ['20010106T0100Z/foo'] 20010107T0100Z/foo -triggered off ['20010106T0100Z/foo'] 20010112T0100Z/qux -triggered off ['20010112T0100Z/baz'] 20010108T0100Z/foo -triggered off ['20010107T0100Z/foo'] 20010107T0100Z/bar -triggered off ['20010107T0100Z/foo'] 20010108T0100Z/bar -triggered off ['20010108T0100Z/foo'] 20010109T0100Z/foo -triggered off ['20010108T0100Z/foo'] 20010109T0100Z/bar -triggered off ['20010109T0100Z/foo'] 20010110T0100Z/foo -triggered off ['20010109T0100Z/foo'] 20010110T0100Z/bar -triggered off ['20010110T0100Z/foo'] 20010111T0100Z/foo -triggered off ['20010110T0100Z/foo'] 20010111T0100Z/bar -triggered off ['20010111T0100Z/foo'] 20010112T0100Z/foo -triggered off ['20010111T0100Z/foo'] 20010112T0100Z/bar -triggered off ['20010112T0100Z/foo'] 20010113T0100Z/foo -triggered off ['20010112T0100Z/foo'] 20010113T0100Z/bar -triggered off ['20010113T0100Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/multidaily/flow.cylc0000664000175000017500000000045315264420307024566 0ustar alastairalastair[scheduler] cycle point time zone = Z allow implicit tasks = True [scheduling] initial cycle point = 20001231T0100 final cycle point = 20010114 [[graph]] P1D = "foo[-P1D] => foo => bar" P4D = "baz[-P4D] => baz => qux" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/multidaily/graph.plain.ref0000664000175000017500000000641115264420307025644 0ustar alastairalastairedge "20001231T0100Z/baz" "20001231T0100Z/qux" edge "20001231T0100Z/baz" "20010104T0100Z/baz" edge "20001231T0100Z/foo" "20001231T0100Z/bar" edge "20001231T0100Z/foo" "20010101T0100Z/foo" edge "20010101T0100Z/foo" "20010101T0100Z/bar" edge "20010101T0100Z/foo" "20010102T0100Z/foo" edge "20010102T0100Z/foo" "20010102T0100Z/bar" edge "20010102T0100Z/foo" "20010103T0100Z/foo" edge "20010103T0100Z/foo" "20010103T0100Z/bar" edge "20010103T0100Z/foo" "20010104T0100Z/foo" edge "20010104T0100Z/baz" "20010104T0100Z/qux" edge "20010104T0100Z/baz" "20010108T0100Z/baz" edge "20010104T0100Z/foo" "20010104T0100Z/bar" edge "20010104T0100Z/foo" "20010105T0100Z/foo" edge "20010105T0100Z/foo" "20010105T0100Z/bar" edge "20010105T0100Z/foo" "20010106T0100Z/foo" edge "20010106T0100Z/foo" "20010106T0100Z/bar" edge "20010106T0100Z/foo" "20010107T0100Z/foo" edge "20010107T0100Z/foo" "20010107T0100Z/bar" edge "20010107T0100Z/foo" "20010108T0100Z/foo" edge "20010108T0100Z/baz" "20010108T0100Z/qux" edge "20010108T0100Z/baz" "20010112T0100Z/baz" edge "20010108T0100Z/foo" "20010108T0100Z/bar" edge "20010108T0100Z/foo" "20010109T0100Z/foo" edge "20010109T0100Z/foo" "20010109T0100Z/bar" edge "20010109T0100Z/foo" "20010110T0100Z/foo" edge "20010110T0100Z/foo" "20010110T0100Z/bar" edge "20010110T0100Z/foo" "20010111T0100Z/foo" edge "20010111T0100Z/foo" "20010111T0100Z/bar" edge "20010111T0100Z/foo" "20010112T0100Z/foo" edge "20010112T0100Z/baz" "20010112T0100Z/qux" edge "20010112T0100Z/foo" "20010112T0100Z/bar" edge "20010112T0100Z/foo" "20010113T0100Z/foo" edge "20010113T0100Z/foo" "20010113T0100Z/bar" graph node "20001231T0100Z/bar" "bar\n20001231T0100Z" node "20001231T0100Z/baz" "baz\n20001231T0100Z" node "20001231T0100Z/foo" "foo\n20001231T0100Z" node "20001231T0100Z/qux" "qux\n20001231T0100Z" node "20010101T0100Z/bar" "bar\n20010101T0100Z" node "20010101T0100Z/foo" "foo\n20010101T0100Z" node "20010102T0100Z/bar" "bar\n20010102T0100Z" node "20010102T0100Z/foo" "foo\n20010102T0100Z" node "20010103T0100Z/bar" "bar\n20010103T0100Z" node "20010103T0100Z/foo" "foo\n20010103T0100Z" node "20010104T0100Z/bar" "bar\n20010104T0100Z" node "20010104T0100Z/baz" "baz\n20010104T0100Z" node "20010104T0100Z/foo" "foo\n20010104T0100Z" node "20010104T0100Z/qux" "qux\n20010104T0100Z" node "20010105T0100Z/bar" "bar\n20010105T0100Z" node "20010105T0100Z/foo" "foo\n20010105T0100Z" node "20010106T0100Z/bar" "bar\n20010106T0100Z" node "20010106T0100Z/foo" "foo\n20010106T0100Z" node "20010107T0100Z/bar" "bar\n20010107T0100Z" node "20010107T0100Z/foo" "foo\n20010107T0100Z" node "20010108T0100Z/bar" "bar\n20010108T0100Z" node "20010108T0100Z/baz" "baz\n20010108T0100Z" node "20010108T0100Z/foo" "foo\n20010108T0100Z" node "20010108T0100Z/qux" "qux\n20010108T0100Z" node "20010109T0100Z/bar" "bar\n20010109T0100Z" node "20010109T0100Z/foo" "foo\n20010109T0100Z" node "20010110T0100Z/bar" "bar\n20010110T0100Z" node "20010110T0100Z/foo" "foo\n20010110T0100Z" node "20010111T0100Z/bar" "bar\n20010111T0100Z" node "20010111T0100Z/foo" "foo\n20010111T0100Z" node "20010112T0100Z/bar" "bar\n20010112T0100Z" node "20010112T0100Z/baz" "baz\n20010112T0100Z" node "20010112T0100Z/foo" "foo\n20010112T0100Z" node "20010112T0100Z/qux" "qux\n20010112T0100Z" node "20010113T0100Z/bar" "bar\n20010113T0100Z" node "20010113T0100Z/foo" "foo\n20010113T0100Z" stop cylc-flow-8.7.0/tests/functional/cyclers/20-multidaily_local.t0000775000175000017500000000425515264420307024530 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test intercycle dependencies, local time. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- CHOSEN_WORKFLOW="$(basename "$0" | sed "s/^.*-\(.*\)\.t/\1/g")" install_workflow "${TEST_NAME_BASE}" "${CHOSEN_WORKFLOW}" CURRENT_TZ_UTC_OFFSET="Z" sed -i "s/Z/$CURRENT_TZ_UTC_OFFSET/g" "${WORKFLOW_RUN_DIR}/reference.log" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-graph" graph_workflow "${WORKFLOW_NAME}" "${WORKFLOW_NAME}.graph.plain" \ 20001231T0100 20010114 sed "s/Z/$CURRENT_TZ_UTC_OFFSET/g" \ "$TEST_SOURCE_DIR/$CHOSEN_WORKFLOW/graph.plain.ref" > 'graph.plain.local.ref' cmp_ok "${WORKFLOW_NAME}.graph.plain" 'graph.plain.local.ref' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/cyclers/multiweekly/0000775000175000017500000000000015264420307023137 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/multiweekly/reference.log0000664000175000017500000000167615264420307025612 0ustar alastairalastairInitial point: 1000W011 Final point: 1000W064 09991230T0000Z/baz -triggered off ['09991209T0000Z/baz'] 09991230T0000Z/foo -triggered off ['09991223T0000Z/foo'] 09991230T0000Z/bar -triggered off ['09991230T0000Z/foo'] 09991230T0000Z/qux -triggered off ['09991230T0000Z/baz'] 10000120T0000Z/baz -triggered off ['09991230T0000Z/baz'] 10000106T0000Z/foo -triggered off ['09991230T0000Z/foo'] 10000106T0000Z/bar -triggered off ['10000106T0000Z/foo'] 10000120T0000Z/qux -triggered off ['10000120T0000Z/baz'] 10000113T0000Z/foo -triggered off ['10000106T0000Z/foo'] 10000113T0000Z/bar -triggered off ['10000113T0000Z/foo'] 10000120T0000Z/foo -triggered off ['10000113T0000Z/foo'] 10000120T0000Z/bar -triggered off ['10000120T0000Z/foo'] 10000127T0000Z/foo -triggered off ['10000120T0000Z/foo'] 10000127T0000Z/bar -triggered off ['10000127T0000Z/foo'] 10000203T0000Z/foo -triggered off ['10000127T0000Z/foo'] 10000203T0000Z/bar -triggered off ['10000203T0000Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/multiweekly/flow.cylc0000664000175000017500000000043415264420307024763 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 1000W011 final cycle point = 1000W064 [[graph]] P1W = "foo[-P1W] => foo => bar" P3W = "baz[-P3W] => baz => qux" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/multiweekly/graph.plain.ref0000664000175000017500000000263515264420307026046 0ustar alastairalastairedge "09991230T0000Z/baz" "09991230T0000Z/qux" edge "09991230T0000Z/baz" "10000120T0000Z/baz" edge "09991230T0000Z/foo" "09991230T0000Z/bar" edge "09991230T0000Z/foo" "10000106T0000Z/foo" edge "10000106T0000Z/foo" "10000106T0000Z/bar" edge "10000106T0000Z/foo" "10000113T0000Z/foo" edge "10000113T0000Z/foo" "10000113T0000Z/bar" edge "10000113T0000Z/foo" "10000120T0000Z/foo" edge "10000120T0000Z/baz" "10000120T0000Z/qux" edge "10000120T0000Z/foo" "10000120T0000Z/bar" edge "10000120T0000Z/foo" "10000127T0000Z/foo" edge "10000127T0000Z/foo" "10000127T0000Z/bar" edge "10000127T0000Z/foo" "10000203T0000Z/foo" edge "10000203T0000Z/foo" "10000203T0000Z/bar" graph node "09991230T0000Z/bar" "bar\n09991230T0000Z" node "09991230T0000Z/baz" "baz\n09991230T0000Z" node "09991230T0000Z/foo" "foo\n09991230T0000Z" node "09991230T0000Z/qux" "qux\n09991230T0000Z" node "10000106T0000Z/bar" "bar\n10000106T0000Z" node "10000106T0000Z/foo" "foo\n10000106T0000Z" node "10000113T0000Z/bar" "bar\n10000113T0000Z" node "10000113T0000Z/foo" "foo\n10000113T0000Z" node "10000120T0000Z/bar" "bar\n10000120T0000Z" node "10000120T0000Z/baz" "baz\n10000120T0000Z" node "10000120T0000Z/foo" "foo\n10000120T0000Z" node "10000120T0000Z/qux" "qux\n10000120T0000Z" node "10000127T0000Z/bar" "bar\n10000127T0000Z" node "10000127T0000Z/foo" "foo\n10000127T0000Z" node "10000203T0000Z/bar" "bar\n10000203T0000Z" node "10000203T0000Z/foo" "foo\n10000203T0000Z" stop cylc-flow-8.7.0/tests/functional/cyclers/r1_initial_immortal/0000775000175000017500000000000015264420307024523 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/r1_initial_immortal/reference.log0000664000175000017500000000233315264420307027165 0ustar alastairalastairInitial point: 20140101T0000Z Final point: None 20140101T0000Z/cold_foo -triggered off [] 20140101T0000Z/foo -triggered off ['20131231T1200Z/foo', '20140101T0000Z/cold_foo'] 20140101T1200Z/foo -triggered off ['20140101T0000Z/cold_foo', '20140101T0000Z/foo'] 20140102T0000Z/foo -triggered off ['20140101T0000Z/cold_foo', '20140101T1200Z/foo'] 20140102T1200Z/foo -triggered off ['20140101T0000Z/cold_foo', '20140102T0000Z/foo'] 20140103T0000Z/foo -triggered off ['20140101T0000Z/cold_foo', '20140102T1200Z/foo'] 20140103T1200Z/foo -triggered off ['20140101T0000Z/cold_foo', '20140103T0000Z/foo'] 20140105T0000Z/stop -triggered off [] 20140104T0000Z/foo -triggered off ['20140101T0000Z/cold_foo', '20140103T1200Z/foo'] 20140104T1200Z/foo -triggered off ['20140101T0000Z/cold_foo', '20140104T0000Z/foo'] 20140105T0000Z/foo -triggered off ['20140101T0000Z/cold_foo', '20140104T1200Z/foo', '20140105T0000Z/stop'] 20140105T1200Z/foo -triggered off ['20140101T0000Z/cold_foo', '20140105T0000Z/foo'] 20140106T0000Z/foo -triggered off ['20140101T0000Z/cold_foo', '20140105T1200Z/foo'] 20140106T1200Z/foo -triggered off ['20140101T0000Z/cold_foo', '20140106T0000Z/foo'] 20140107T0000Z/foo -triggered off ['20140101T0000Z/cold_foo', '20140106T1200Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/r1_initial_immortal/flow.cylc0000664000175000017500000000053615264420307026352 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20140101 [[graph]] R1 = "cold_foo" PT12H = "cold_foo[^] & foo[-PT12H] => foo" R1/+P4D = "stop => foo" [runtime] [[root]] script = true [[stop]] script = cylc stop "${CYLC_WORKFLOW_ID}//20140107" cylc-flow-8.7.0/tests/functional/cyclers/r1_initial_immortal/graph.plain.ref0000664000175000017500000000554115264420307027431 0ustar alastairalastairedge "20140101T0000Z/cold_foo" "20140101T0000Z/foo" edge "20140101T0000Z/cold_foo" "20140101T1200Z/foo" edge "20140101T0000Z/cold_foo" "20140102T0000Z/foo" edge "20140101T0000Z/cold_foo" "20140102T1200Z/foo" edge "20140101T0000Z/cold_foo" "20140103T0000Z/foo" edge "20140101T0000Z/cold_foo" "20140103T1200Z/foo" edge "20140101T0000Z/cold_foo" "20140104T0000Z/foo" edge "20140101T0000Z/cold_foo" "20140104T1200Z/foo" edge "20140101T0000Z/cold_foo" "20140105T0000Z/foo" edge "20140101T0000Z/cold_foo" "20140105T1200Z/foo" edge "20140101T0000Z/cold_foo" "20140106T0000Z/foo" edge "20140101T0000Z/cold_foo" "20140106T1200Z/foo" edge "20140101T0000Z/cold_foo" "20140107T0000Z/foo" edge "20140101T0000Z/cold_foo" "20140107T1200Z/foo" edge "20140101T0000Z/cold_foo" "20140108T0000Z/foo" edge "20140101T0000Z/cold_foo" "20140108T1200Z/foo" edge "20140101T0000Z/cold_foo" "20140109T0000Z/foo" edge "20140101T0000Z/cold_foo" "20140109T1200Z/foo" edge "20140101T0000Z/cold_foo" "20140110T0000Z/foo" edge "20140101T0000Z/foo" "20140101T1200Z/foo" edge "20140101T1200Z/foo" "20140102T0000Z/foo" edge "20140102T0000Z/foo" "20140102T1200Z/foo" edge "20140102T1200Z/foo" "20140103T0000Z/foo" edge "20140103T0000Z/foo" "20140103T1200Z/foo" edge "20140103T1200Z/foo" "20140104T0000Z/foo" edge "20140104T0000Z/foo" "20140104T1200Z/foo" edge "20140104T1200Z/foo" "20140105T0000Z/foo" edge "20140105T0000Z/foo" "20140105T1200Z/foo" edge "20140105T0000Z/stop" "20140105T0000Z/foo" edge "20140105T1200Z/foo" "20140106T0000Z/foo" edge "20140106T0000Z/foo" "20140106T1200Z/foo" edge "20140106T1200Z/foo" "20140107T0000Z/foo" edge "20140107T0000Z/foo" "20140107T1200Z/foo" edge "20140107T1200Z/foo" "20140108T0000Z/foo" edge "20140108T0000Z/foo" "20140108T1200Z/foo" edge "20140108T1200Z/foo" "20140109T0000Z/foo" edge "20140109T0000Z/foo" "20140109T1200Z/foo" edge "20140109T1200Z/foo" "20140110T0000Z/foo" graph node "20140101T0000Z/cold_foo" "cold_foo\n20140101T0000Z" node "20140101T0000Z/foo" "foo\n20140101T0000Z" node "20140101T1200Z/foo" "foo\n20140101T1200Z" node "20140102T0000Z/foo" "foo\n20140102T0000Z" node "20140102T1200Z/foo" "foo\n20140102T1200Z" node "20140103T0000Z/foo" "foo\n20140103T0000Z" node "20140103T1200Z/foo" "foo\n20140103T1200Z" node "20140104T0000Z/foo" "foo\n20140104T0000Z" node "20140104T1200Z/foo" "foo\n20140104T1200Z" node "20140105T0000Z/foo" "foo\n20140105T0000Z" node "20140105T0000Z/stop" "stop\n20140105T0000Z" node "20140105T1200Z/foo" "foo\n20140105T1200Z" node "20140106T0000Z/foo" "foo\n20140106T0000Z" node "20140106T1200Z/foo" "foo\n20140106T1200Z" node "20140107T0000Z/foo" "foo\n20140107T0000Z" node "20140107T1200Z/foo" "foo\n20140107T1200Z" node "20140108T0000Z/foo" "foo\n20140108T0000Z" node "20140108T1200Z/foo" "foo\n20140108T1200Z" node "20140109T0000Z/foo" "foo\n20140109T0000Z" node "20140109T1200Z/foo" "foo\n20140109T1200Z" node "20140110T0000Z/foo" "foo\n20140110T0000Z" stop cylc-flow-8.7.0/tests/functional/cyclers/multiyearly/0000775000175000017500000000000015264420307023144 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/multiyearly/reference.log0000664000175000017500000000276015264420307025612 0ustar alastairalastairInitial point: 1005 Final point: 1014 10050101T0000Z/baz -triggered off ['10010101T0000Z/baz'] 10050101T0000Z/foo -triggered off ['10040101T0000Z/foo'] 10050101T0000Z/qux -triggered off ['10050101T0000Z/baz'] 10090101T0000Z/baz -triggered off ['10050101T0000Z/baz'] 10050101T0000Z/bar -triggered off ['10050101T0000Z/foo'] 10060101T0000Z/foo -triggered off ['10050101T0000Z/foo'] 10060101T0000Z/bar -triggered off ['10060101T0000Z/foo'] 10070101T0000Z/foo -triggered off ['10060101T0000Z/foo'] 10090101T0000Z/qux -triggered off ['10090101T0000Z/baz'] 10070101T0000Z/bar -triggered off ['10070101T0000Z/foo'] 10080101T0000Z/foo -triggered off ['10070101T0000Z/foo'] 10130101T0000Z/baz -triggered off ['10090101T0000Z/baz'] 10080101T0000Z/bar -triggered off ['10080101T0000Z/foo'] 10090101T0000Z/foo -triggered off ['10080101T0000Z/foo'] 10130101T0000Z/qux -triggered off ['10130101T0000Z/baz'] 10090101T0000Z/bar -triggered off ['10090101T0000Z/foo'] 10100101T0000Z/foo -triggered off ['10090101T0000Z/foo'] 10100101T0000Z/bar -triggered off ['10100101T0000Z/foo'] 10110101T0000Z/foo -triggered off ['10100101T0000Z/foo'] 10110101T0000Z/bar -triggered off ['10110101T0000Z/foo'] 10120101T0000Z/foo -triggered off ['10110101T0000Z/foo'] 10120101T0000Z/bar -triggered off ['10120101T0000Z/foo'] 10130101T0000Z/foo -triggered off ['10120101T0000Z/foo'] 10130101T0000Z/bar -triggered off ['10130101T0000Z/foo'] 10140101T0000Z/foo -triggered off ['10130101T0000Z/foo'] 10140101T0000Z/bar -triggered off ['10140101T0000Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/multiyearly/flow.cylc0000664000175000017500000000042415264420307024767 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 1005 final cycle point = 1014 [[graph]] P1Y = "foo[-P1Y] => foo => bar" P4Y = "baz[-P4Y] => baz => qux" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/multiyearly/graph.plain.ref0000664000175000017500000000452315264420307026051 0ustar alastairalastairedge "10050101T0000Z/baz" "10050101T0000Z/qux" edge "10050101T0000Z/baz" "10090101T0000Z/baz" edge "10050101T0000Z/foo" "10050101T0000Z/bar" edge "10050101T0000Z/foo" "10060101T0000Z/foo" edge "10060101T0000Z/foo" "10060101T0000Z/bar" edge "10060101T0000Z/foo" "10070101T0000Z/foo" edge "10070101T0000Z/foo" "10070101T0000Z/bar" edge "10070101T0000Z/foo" "10080101T0000Z/foo" edge "10080101T0000Z/foo" "10080101T0000Z/bar" edge "10080101T0000Z/foo" "10090101T0000Z/foo" edge "10090101T0000Z/baz" "10090101T0000Z/qux" edge "10090101T0000Z/baz" "10130101T0000Z/baz" edge "10090101T0000Z/foo" "10090101T0000Z/bar" edge "10090101T0000Z/foo" "10100101T0000Z/foo" edge "10100101T0000Z/foo" "10100101T0000Z/bar" edge "10100101T0000Z/foo" "10110101T0000Z/foo" edge "10110101T0000Z/foo" "10110101T0000Z/bar" edge "10110101T0000Z/foo" "10120101T0000Z/foo" edge "10120101T0000Z/foo" "10120101T0000Z/bar" edge "10120101T0000Z/foo" "10130101T0000Z/foo" edge "10130101T0000Z/baz" "10130101T0000Z/qux" edge "10130101T0000Z/foo" "10130101T0000Z/bar" edge "10130101T0000Z/foo" "10140101T0000Z/foo" edge "10140101T0000Z/foo" "10140101T0000Z/bar" graph node "10050101T0000Z/bar" "bar\n10050101T0000Z" node "10050101T0000Z/baz" "baz\n10050101T0000Z" node "10050101T0000Z/foo" "foo\n10050101T0000Z" node "10050101T0000Z/qux" "qux\n10050101T0000Z" node "10060101T0000Z/bar" "bar\n10060101T0000Z" node "10060101T0000Z/foo" "foo\n10060101T0000Z" node "10070101T0000Z/bar" "bar\n10070101T0000Z" node "10070101T0000Z/foo" "foo\n10070101T0000Z" node "10080101T0000Z/bar" "bar\n10080101T0000Z" node "10080101T0000Z/foo" "foo\n10080101T0000Z" node "10090101T0000Z/bar" "bar\n10090101T0000Z" node "10090101T0000Z/baz" "baz\n10090101T0000Z" node "10090101T0000Z/foo" "foo\n10090101T0000Z" node "10090101T0000Z/qux" "qux\n10090101T0000Z" node "10100101T0000Z/bar" "bar\n10100101T0000Z" node "10100101T0000Z/foo" "foo\n10100101T0000Z" node "10110101T0000Z/bar" "bar\n10110101T0000Z" node "10110101T0000Z/foo" "foo\n10110101T0000Z" node "10120101T0000Z/bar" "bar\n10120101T0000Z" node "10120101T0000Z/foo" "foo\n10120101T0000Z" node "10130101T0000Z/bar" "bar\n10130101T0000Z" node "10130101T0000Z/baz" "baz\n10130101T0000Z" node "10130101T0000Z/foo" "foo\n10130101T0000Z" node "10130101T0000Z/qux" "qux\n10130101T0000Z" node "10140101T0000Z/bar" "bar\n10140101T0000Z" node "10140101T0000Z/foo" "foo\n10140101T0000Z" stop cylc-flow-8.7.0/tests/functional/cyclers/r1_initial_back_comp_standalone_line/0000775000175000017500000000000015264420307030034 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/r1_initial_back_comp_standalone_line/reference.log0000664000175000017500000000054015264420307032474 0ustar alastairalastairInitial point: 2014010100 Final point: 2014010400 2014010100/cold_foo -triggered off [] 2014010100/foo_midnight -triggered off ['2014010100/cold_foo'] 2014010200/foo_midnight -triggered off ['2014010100/foo_midnight'] 2014010300/foo_midnight -triggered off ['2014010200/foo_midnight'] 2014010400/foo_midnight -triggered off ['2014010300/foo_midnight'] cylc-flow-8.7.0/tests/functional/cyclers/r1_initial_back_comp_standalone_line/flow.cylc0000664000175000017500000000055115264420307031660 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20140101 final cycle point = 20140104 [[special tasks]] start-up = cold_foo [[graph]] 0 = """ cold_foo foo_midnight[T-24] & cold_foo => foo_midnight """ [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/r1_initial_back_comp_standalone_line/graph.plain.ref0000664000175000017500000000043715264420307032741 0ustar alastairalastairedge "2014010100/cold_foo" "2014010100/foo_midnight" edge "2014010100/foo_midnight" "2014010200/foo_midnight" graph node "2014010100/cold_foo" "cold_foo\n2014010100" node "2014010100/foo_midnight" "foo_midnight\n2014010100" node "2014010200/foo_midnight" "foo_midnight\n2014010200" stop cylc-flow-8.7.0/tests/functional/cyclers/03-multidaily.t0000775000175000017500000000205015264420307023346 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20001231T0100" # shellcheck disable=SC2034 FINALCP="20010114" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/exclusions_advanced/0000775000175000017500000000000015264420307024605 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/exclusions_advanced/reference.log0000664000175000017500000001535315264420307027255 0ustar alastairalastairInitial point: 20000101T0000Z Final point: 20000102T1200Z 20000101T0000Z/start -triggered off [] 20000101T0000Z/dibble -triggered off [] 20000101T0100Z/nip -triggered off [] 20000101T0000Z/qux -triggered off [] 20000101T0030Z/quux -triggered off [] 20000101T0100Z/wibble -triggered off [] 20000101T0000Z/foo -triggered off ['20000101T0000Z/start'] 20000101T0200Z/nip -triggered off [] 20000101T0115Z/bob -triggered off [] 20000101T0130Z/quux -triggered off [] 20000101T0200Z/wibble -triggered off [] 20000101T0115Z/toot -triggered off [] 20000101T0300Z/nip -triggered off [] 20000101T0230Z/quux -triggered off [] 20000101T0300Z/wibble -triggered off [] 20000101T0215Z/bob -triggered off [] 20000101T0230Z/dibble -triggered off [] 20000101T0215Z/toot -triggered off [] 20000101T0300Z/qux -triggered off [] 20000101T0300Z/baz -triggered off [] 20000101T0300Z/dibble -triggered off [] 20000101T0400Z/nip -triggered off [] 20000101T0400Z/wibble -triggered off [] 20000101T0330Z/quux -triggered off [] 20000101T0415Z/bob -triggered off [] 20000101T0415Z/toot -triggered off [] 20000101T0430Z/quux -triggered off [] 20000101T0500Z/nip -triggered off [] 20000101T0500Z/wibble -triggered off [] 20000101T0515Z/bob -triggered off [] 20000101T0515Z/toot -triggered off [] 20000101T0530Z/quux -triggered off [] 20000101T0600Z/dibble -triggered off [] 20000101T0600Z/qux -triggered off [] 20000101T0630Z/quux -triggered off [] 20000101T0715Z/bob -triggered off [] 20000101T0700Z/nip -triggered off [] 20000101T0715Z/toot -triggered off [] 20000101T0700Z/wibble -triggered off [] 20000101T0730Z/quux -triggered off [] 20000101T0815Z/bob -triggered off [] 20000101T0815Z/toot -triggered off [] 20000101T0800Z/wibble -triggered off [] 20000101T0800Z/nip -triggered off [] 20000101T0830Z/quux -triggered off [] 20000101T0900Z/wibble -triggered off [] 20000101T0900Z/baz -triggered off [] 20000101T0900Z/nip -triggered off [] 20000101T0900Z/qux -triggered off [] 20000101T0930Z/quux -triggered off [] 20000101T1000Z/nip -triggered off [] 20000101T1000Z/wibble -triggered off [] 20000101T1015Z/bob -triggered off [] 20000101T1015Z/toot -triggered off [] 20000101T1030Z/quux -triggered off [] 20000101T1100Z/wibble -triggered off [] 20000101T1100Z/nip -triggered off [] 20000101T1115Z/bob -triggered off [] 20000101T1115Z/toot -triggered off [] 20000101T1130Z/quux -triggered off [] 20000101T1200Z/qux -triggered off [] 20000101T1230Z/quux -triggered off [] 20000101T1300Z/nip -triggered off [] 20000101T1300Z/wibble -triggered off [] 20000101T1315Z/bob -triggered off [] 20000101T1315Z/toot -triggered off [] 20000101T1330Z/quux -triggered off [] 20000101T1400Z/wibble -triggered off [] 20000101T1400Z/nip -triggered off [] 20000101T1415Z/bob -triggered off [] 20000101T1415Z/toot -triggered off [] 20000101T1430Z/quux -triggered off [] 20000101T1500Z/nip -triggered off [] 20000101T1500Z/wibble -triggered off [] 20000101T1500Z/baz -triggered off [] 20000101T1500Z/dibble -triggered off [] 20000101T1500Z/qux -triggered off [] 20000101T1530Z/quux -triggered off [] 20000101T1600Z/nip -triggered off [] 20000101T1600Z/wibble -triggered off [] 20000101T1615Z/toot -triggered off [] 20000101T1615Z/bob -triggered off [] 20000101T1630Z/quux -triggered off [] 20000101T1700Z/nip -triggered off [] 20000101T1700Z/wibble -triggered off [] 20000101T1715Z/bob -triggered off [] 20000101T1715Z/toot -triggered off [] 20000101T1730Z/quux -triggered off [] 20000101T1800Z/dibble -triggered off [] 20000101T1800Z/qux -triggered off [] 20000101T1830Z/quux -triggered off [] 20000101T1900Z/nip -triggered off [] 20000101T1900Z/wibble -triggered off [] 20000101T1915Z/bob -triggered off [] 20000101T1915Z/toot -triggered off [] 20000101T1930Z/quux -triggered off [] 20000101T1945Z/dibble -triggered off [] 20000101T2000Z/wibble -triggered off [] 20000101T2000Z/nip -triggered off [] 20000101T2015Z/bob -triggered off [] 20000101T2015Z/toot -triggered off [] 20000101T2030Z/quux -triggered off [] 20000101T2100Z/wibble -triggered off [] 20000101T2100Z/qux -triggered off [] 20000101T2100Z/baz -triggered off [] 20000101T2100Z/dibble -triggered off [] 20000101T2100Z/nip -triggered off [] 20000101T2130Z/quux -triggered off [] 20000101T2200Z/nip -triggered off [] 20000101T2200Z/wibble -triggered off [] 20000101T2215Z/bob -triggered off [] 20000101T2215Z/toot -triggered off [] 20000101T2230Z/quux -triggered off [] 20000101T2300Z/nip -triggered off [] 20000101T2300Z/wibble -triggered off [] 20000101T2315Z/toot -triggered off [] 20000101T2315Z/bob -triggered off [] 20000101T2330Z/quux -triggered off [] 20000102T0000Z/qux -triggered off [] 20000102T0000Z/dibble -triggered off [] 20000102T0000Z/bar -triggered off [] 20000102T0030Z/quux -triggered off [] 20000102T0100Z/nip -triggered off [] 20000102T0100Z/wibble -triggered off [] 20000102T0115Z/bob -triggered off [] 20000102T0115Z/toot -triggered off [] 20000102T0130Z/quux -triggered off [] 20000102T0200Z/nip -triggered off [] 20000102T0200Z/wibble -triggered off [] 20000102T0215Z/toot -triggered off [] 20000102T0215Z/bob -triggered off [] 20000102T0230Z/dibble -triggered off [] 20000102T0230Z/quux -triggered off [] 20000102T0300Z/baz -triggered off [] 20000102T0300Z/qux -triggered off [] 20000102T0300Z/nip -triggered off [] 20000102T0300Z/wibble -triggered off [] 20000102T0300Z/dibble -triggered off [] 20000102T0330Z/quux -triggered off [] 20000102T0400Z/wibble -triggered off [] 20000102T0400Z/nip -triggered off [] 20000102T0415Z/bob -triggered off [] 20000102T0430Z/quux -triggered off [] 20000102T0415Z/toot -triggered off [] 20000102T0500Z/nip -triggered off [] 20000102T0500Z/wibble -triggered off [] 20000102T0515Z/toot -triggered off [] 20000102T0515Z/bob -triggered off [] 20000102T0530Z/quux -triggered off [] 20000102T0600Z/qux -triggered off [] 20000102T0600Z/dibble -triggered off [] 20000102T0630Z/quux -triggered off [] 20000102T0700Z/nip -triggered off [] 20000102T0700Z/wibble -triggered off [] 20000102T0715Z/bob -triggered off [] 20000102T0715Z/toot -triggered off [] 20000102T0800Z/wibble -triggered off [] 20000102T0800Z/nip -triggered off [] 20000102T0730Z/quux -triggered off [] 20000102T0815Z/toot -triggered off [] 20000102T0815Z/bob -triggered off [] 20000102T0900Z/nip -triggered off [] 20000102T0830Z/quux -triggered off [] 20000102T0900Z/baz -triggered off [] 20000102T0900Z/wibble -triggered off [] 20000102T0900Z/qux -triggered off [] 20000102T0930Z/quux -triggered off [] 20000102T1000Z/wibble -triggered off [] 20000102T1000Z/nip -triggered off [] 20000102T1015Z/toot -triggered off [] 20000102T1015Z/bob -triggered off [] 20000102T1030Z/quux -triggered off [] 20000102T1115Z/toot -triggered off [] 20000102T1115Z/bob -triggered off [] 20000102T1100Z/nip -triggered off [] 20000102T1200Z/pub -triggered off [] 20000102T1100Z/wibble -triggered off [] 20000102T1130Z/quux -triggered off [] 20000102T1200Z/qux -triggered off [] cylc-flow-8.7.0/tests/functional/cyclers/exclusions_advanced/flow.cylc0000664000175000017500000000210015264420307026421 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20000101T00Z final cycle point = 20000102T12Z [[graph]] R1/^ = start => foo # Don't run at the initial cycle point # Also test whitespace tolerance T00 ! ^ = bar # Run 3-hourly but not 6 hourly. PT3H!PT6H = baz # Run 4 hourly but not at 0800 from ICP PT3H!T08 = qux # Run hourly but not at 00:00, 06:00, 12:00, 18:00 T-00!(T00, T06, T12, T18) = nip # Run half-hourly but not on the hour from ICP PT30M!T-00 = quux # Run hourly on the hour, but not 6 hourly on the hour T-00!PT6H = wibble # Run hourly at 15 minutes past except every 3rd hour T-15!T-15/PT3H = bob # Run hourly at 15 minutes past except every 3rd hour # Implicit start point T-15!PT3H = toot # Stacked sequences T0230, PT3H! (T09, T12 ) , T1945 = dibble R1/$ = pub [runtime] [[root]] script = echo success cylc-flow-8.7.0/tests/functional/cyclers/exclusions_advanced/graph.plain.ref0000664000175000017500000002150415264420307027510 0ustar alastairalastairedge "20000101T0000Z/start" "20000101T0000Z/foo" graph node "20000101T0000Z/dibble" "dibble\n20000101T0000Z" node "20000101T0000Z/foo" "foo\n20000101T0000Z" node "20000101T0000Z/qux" "qux\n20000101T0000Z" node "20000101T0000Z/start" "start\n20000101T0000Z" node "20000101T0030Z/quux" "quux\n20000101T0030Z" node "20000101T0100Z/nip" "nip\n20000101T0100Z" node "20000101T0100Z/wibble" "wibble\n20000101T0100Z" node "20000101T0115Z/bob" "bob\n20000101T0115Z" node "20000101T0115Z/toot" "toot\n20000101T0115Z" node "20000101T0130Z/quux" "quux\n20000101T0130Z" node "20000101T0200Z/nip" "nip\n20000101T0200Z" node "20000101T0200Z/wibble" "wibble\n20000101T0200Z" node "20000101T0215Z/bob" "bob\n20000101T0215Z" node "20000101T0215Z/toot" "toot\n20000101T0215Z" node "20000101T0230Z/dibble" "dibble\n20000101T0230Z" node "20000101T0230Z/quux" "quux\n20000101T0230Z" node "20000101T0300Z/baz" "baz\n20000101T0300Z" node "20000101T0300Z/dibble" "dibble\n20000101T0300Z" node "20000101T0300Z/nip" "nip\n20000101T0300Z" node "20000101T0300Z/qux" "qux\n20000101T0300Z" node "20000101T0300Z/wibble" "wibble\n20000101T0300Z" node "20000101T0330Z/quux" "quux\n20000101T0330Z" node "20000101T0400Z/nip" "nip\n20000101T0400Z" node "20000101T0400Z/wibble" "wibble\n20000101T0400Z" node "20000101T0415Z/bob" "bob\n20000101T0415Z" node "20000101T0415Z/toot" "toot\n20000101T0415Z" node "20000101T0430Z/quux" "quux\n20000101T0430Z" node "20000101T0500Z/nip" "nip\n20000101T0500Z" node "20000101T0500Z/wibble" "wibble\n20000101T0500Z" node "20000101T0515Z/bob" "bob\n20000101T0515Z" node "20000101T0515Z/toot" "toot\n20000101T0515Z" node "20000101T0530Z/quux" "quux\n20000101T0530Z" node "20000101T0600Z/dibble" "dibble\n20000101T0600Z" node "20000101T0600Z/qux" "qux\n20000101T0600Z" node "20000101T0630Z/quux" "quux\n20000101T0630Z" node "20000101T0700Z/nip" "nip\n20000101T0700Z" node "20000101T0700Z/wibble" "wibble\n20000101T0700Z" node "20000101T0715Z/bob" "bob\n20000101T0715Z" node "20000101T0715Z/toot" "toot\n20000101T0715Z" node "20000101T0730Z/quux" "quux\n20000101T0730Z" node "20000101T0800Z/nip" "nip\n20000101T0800Z" node "20000101T0800Z/wibble" "wibble\n20000101T0800Z" node "20000101T0815Z/bob" "bob\n20000101T0815Z" node "20000101T0815Z/toot" "toot\n20000101T0815Z" node "20000101T0830Z/quux" "quux\n20000101T0830Z" node "20000101T0900Z/baz" "baz\n20000101T0900Z" node "20000101T0900Z/nip" "nip\n20000101T0900Z" node "20000101T0900Z/qux" "qux\n20000101T0900Z" node "20000101T0900Z/wibble" "wibble\n20000101T0900Z" node "20000101T0930Z/quux" "quux\n20000101T0930Z" node "20000101T1000Z/nip" "nip\n20000101T1000Z" node "20000101T1000Z/wibble" "wibble\n20000101T1000Z" node "20000101T1015Z/bob" "bob\n20000101T1015Z" node "20000101T1015Z/toot" "toot\n20000101T1015Z" node "20000101T1030Z/quux" "quux\n20000101T1030Z" node "20000101T1100Z/nip" "nip\n20000101T1100Z" node "20000101T1100Z/wibble" "wibble\n20000101T1100Z" node "20000101T1115Z/bob" "bob\n20000101T1115Z" node "20000101T1115Z/toot" "toot\n20000101T1115Z" node "20000101T1130Z/quux" "quux\n20000101T1130Z" node "20000101T1200Z/qux" "qux\n20000101T1200Z" node "20000101T1230Z/quux" "quux\n20000101T1230Z" node "20000101T1300Z/nip" "nip\n20000101T1300Z" node "20000101T1300Z/wibble" "wibble\n20000101T1300Z" node "20000101T1315Z/bob" "bob\n20000101T1315Z" node "20000101T1315Z/toot" "toot\n20000101T1315Z" node "20000101T1330Z/quux" "quux\n20000101T1330Z" node "20000101T1400Z/nip" "nip\n20000101T1400Z" node "20000101T1400Z/wibble" "wibble\n20000101T1400Z" node "20000101T1415Z/bob" "bob\n20000101T1415Z" node "20000101T1415Z/toot" "toot\n20000101T1415Z" node "20000101T1430Z/quux" "quux\n20000101T1430Z" node "20000101T1500Z/baz" "baz\n20000101T1500Z" node "20000101T1500Z/dibble" "dibble\n20000101T1500Z" node "20000101T1500Z/nip" "nip\n20000101T1500Z" node "20000101T1500Z/qux" "qux\n20000101T1500Z" node "20000101T1500Z/wibble" "wibble\n20000101T1500Z" node "20000101T1530Z/quux" "quux\n20000101T1530Z" node "20000101T1600Z/nip" "nip\n20000101T1600Z" node "20000101T1600Z/wibble" "wibble\n20000101T1600Z" node "20000101T1615Z/bob" "bob\n20000101T1615Z" node "20000101T1615Z/toot" "toot\n20000101T1615Z" node "20000101T1630Z/quux" "quux\n20000101T1630Z" node "20000101T1700Z/nip" "nip\n20000101T1700Z" node "20000101T1700Z/wibble" "wibble\n20000101T1700Z" node "20000101T1715Z/bob" "bob\n20000101T1715Z" node "20000101T1715Z/toot" "toot\n20000101T1715Z" node "20000101T1730Z/quux" "quux\n20000101T1730Z" node "20000101T1800Z/dibble" "dibble\n20000101T1800Z" node "20000101T1800Z/qux" "qux\n20000101T1800Z" node "20000101T1830Z/quux" "quux\n20000101T1830Z" node "20000101T1900Z/nip" "nip\n20000101T1900Z" node "20000101T1900Z/wibble" "wibble\n20000101T1900Z" node "20000101T1915Z/bob" "bob\n20000101T1915Z" node "20000101T1915Z/toot" "toot\n20000101T1915Z" node "20000101T1930Z/quux" "quux\n20000101T1930Z" node "20000101T1945Z/dibble" "dibble\n20000101T1945Z" node "20000101T2000Z/nip" "nip\n20000101T2000Z" node "20000101T2000Z/wibble" "wibble\n20000101T2000Z" node "20000101T2015Z/bob" "bob\n20000101T2015Z" node "20000101T2015Z/toot" "toot\n20000101T2015Z" node "20000101T2030Z/quux" "quux\n20000101T2030Z" node "20000101T2100Z/baz" "baz\n20000101T2100Z" node "20000101T2100Z/dibble" "dibble\n20000101T2100Z" node "20000101T2100Z/nip" "nip\n20000101T2100Z" node "20000101T2100Z/qux" "qux\n20000101T2100Z" node "20000101T2100Z/wibble" "wibble\n20000101T2100Z" node "20000101T2130Z/quux" "quux\n20000101T2130Z" node "20000101T2200Z/nip" "nip\n20000101T2200Z" node "20000101T2200Z/wibble" "wibble\n20000101T2200Z" node "20000101T2215Z/bob" "bob\n20000101T2215Z" node "20000101T2215Z/toot" "toot\n20000101T2215Z" node "20000101T2230Z/quux" "quux\n20000101T2230Z" node "20000101T2300Z/nip" "nip\n20000101T2300Z" node "20000101T2300Z/wibble" "wibble\n20000101T2300Z" node "20000101T2315Z/bob" "bob\n20000101T2315Z" node "20000101T2315Z/toot" "toot\n20000101T2315Z" node "20000101T2330Z/quux" "quux\n20000101T2330Z" node "20000102T0000Z/bar" "bar\n20000102T0000Z" node "20000102T0000Z/dibble" "dibble\n20000102T0000Z" node "20000102T0000Z/qux" "qux\n20000102T0000Z" node "20000102T0030Z/quux" "quux\n20000102T0030Z" node "20000102T0100Z/nip" "nip\n20000102T0100Z" node "20000102T0100Z/wibble" "wibble\n20000102T0100Z" node "20000102T0115Z/bob" "bob\n20000102T0115Z" node "20000102T0115Z/toot" "toot\n20000102T0115Z" node "20000102T0130Z/quux" "quux\n20000102T0130Z" node "20000102T0200Z/nip" "nip\n20000102T0200Z" node "20000102T0200Z/wibble" "wibble\n20000102T0200Z" node "20000102T0215Z/bob" "bob\n20000102T0215Z" node "20000102T0215Z/toot" "toot\n20000102T0215Z" node "20000102T0230Z/dibble" "dibble\n20000102T0230Z" node "20000102T0230Z/quux" "quux\n20000102T0230Z" node "20000102T0300Z/baz" "baz\n20000102T0300Z" node "20000102T0300Z/dibble" "dibble\n20000102T0300Z" node "20000102T0300Z/nip" "nip\n20000102T0300Z" node "20000102T0300Z/qux" "qux\n20000102T0300Z" node "20000102T0300Z/wibble" "wibble\n20000102T0300Z" node "20000102T0330Z/quux" "quux\n20000102T0330Z" node "20000102T0400Z/nip" "nip\n20000102T0400Z" node "20000102T0400Z/wibble" "wibble\n20000102T0400Z" node "20000102T0415Z/bob" "bob\n20000102T0415Z" node "20000102T0415Z/toot" "toot\n20000102T0415Z" node "20000102T0430Z/quux" "quux\n20000102T0430Z" node "20000102T0500Z/nip" "nip\n20000102T0500Z" node "20000102T0500Z/wibble" "wibble\n20000102T0500Z" node "20000102T0515Z/bob" "bob\n20000102T0515Z" node "20000102T0515Z/toot" "toot\n20000102T0515Z" node "20000102T0530Z/quux" "quux\n20000102T0530Z" node "20000102T0600Z/dibble" "dibble\n20000102T0600Z" node "20000102T0600Z/qux" "qux\n20000102T0600Z" node "20000102T0630Z/quux" "quux\n20000102T0630Z" node "20000102T0700Z/nip" "nip\n20000102T0700Z" node "20000102T0700Z/wibble" "wibble\n20000102T0700Z" node "20000102T0715Z/bob" "bob\n20000102T0715Z" node "20000102T0715Z/toot" "toot\n20000102T0715Z" node "20000102T0730Z/quux" "quux\n20000102T0730Z" node "20000102T0800Z/nip" "nip\n20000102T0800Z" node "20000102T0800Z/wibble" "wibble\n20000102T0800Z" node "20000102T0815Z/bob" "bob\n20000102T0815Z" node "20000102T0815Z/toot" "toot\n20000102T0815Z" node "20000102T0830Z/quux" "quux\n20000102T0830Z" node "20000102T0900Z/baz" "baz\n20000102T0900Z" node "20000102T0900Z/nip" "nip\n20000102T0900Z" node "20000102T0900Z/qux" "qux\n20000102T0900Z" node "20000102T0900Z/wibble" "wibble\n20000102T0900Z" node "20000102T0930Z/quux" "quux\n20000102T0930Z" node "20000102T1000Z/nip" "nip\n20000102T1000Z" node "20000102T1000Z/wibble" "wibble\n20000102T1000Z" node "20000102T1015Z/bob" "bob\n20000102T1015Z" node "20000102T1015Z/toot" "toot\n20000102T1015Z" node "20000102T1030Z/quux" "quux\n20000102T1030Z" node "20000102T1100Z/nip" "nip\n20000102T1100Z" node "20000102T1100Z/wibble" "wibble\n20000102T1100Z" node "20000102T1115Z/bob" "bob\n20000102T1115Z" node "20000102T1115Z/toot" "toot\n20000102T1115Z" node "20000102T1130Z/quux" "quux\n20000102T1130Z" node "20000102T1200Z/pub" "pub\n20000102T1200Z" node "20000102T1200Z/qux" "qux\n20000102T1200Z" stop cylc-flow-8.7.0/tests/functional/cyclers/09-offset_initial.t0000775000175000017500000000204615264420307024203 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20140101" # shellcheck disable=SC2034 FINALCP="20140102T12" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/18-r1_multi_start.t0000775000175000017500000000204715264420307024156 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="2014-01-01" # shellcheck disable=SC2034 FINALCP="2014-01-04" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/multihourly/0000775000175000017500000000000015264420307023161 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/multihourly/reference.log0000664000175000017500000000565615264420307025636 0ustar alastairalastairInitial point: 20000131T0100Z Final point: 20000202T0600+0600 20000131T1400+13/baz -triggered off ['20000131T0800+13/baz'] 20000131T1400+13/foo -triggered off ['20000131T1100+13/foo'] 20000131T1400+13/qux -triggered off ['20000131T1400+13/baz'] 20000131T2000+13/baz -triggered off ['20000131T1400+13/baz'] 20000131T1400+13/bar -triggered off ['20000131T1400+13/foo'] 20000131T1700+13/foo -triggered off ['20000131T1400+13/foo'] 20000131T2000+13/qux -triggered off ['20000131T2000+13/baz'] 20000201T0200+13/baz -triggered off ['20000131T2000+13/baz'] 20000131T1700+13/bar -triggered off ['20000131T1700+13/foo'] 20000131T2000+13/foo -triggered off ['20000131T1700+13/foo'] 20000201T0200+13/qux -triggered off ['20000201T0200+13/baz'] 20000201T0800+13/baz -triggered off ['20000201T0200+13/baz'] 20000131T2000+13/bar -triggered off ['20000131T2000+13/foo'] 20000131T2300+13/foo -triggered off ['20000131T2000+13/foo'] 20000201T0800+13/qux -triggered off ['20000201T0800+13/baz'] 20000131T2300+13/bar -triggered off ['20000131T2300+13/foo'] 20000201T0200+13/foo -triggered off ['20000131T2300+13/foo'] 20000201T1400+13/baz -triggered off ['20000201T0800+13/baz'] 20000201T0200+13/bar -triggered off ['20000201T0200+13/foo'] 20000201T0500+13/foo -triggered off ['20000201T0200+13/foo'] 20000201T1400+13/qux -triggered off ['20000201T1400+13/baz'] 20000201T0800+13/foo -triggered off ['20000201T0500+13/foo'] 20000201T0500+13/bar -triggered off ['20000201T0500+13/foo'] 20000201T1100+13/foo -triggered off ['20000201T0800+13/foo'] 20000201T0800+13/bar -triggered off ['20000201T0800+13/foo'] 20000201T2000+13/baz -triggered off ['20000201T1400+13/baz'] 20000201T1400+13/foo -triggered off ['20000201T1100+13/foo'] 20000201T1100+13/bar -triggered off ['20000201T1100+13/foo'] 20000201T2000+13/qux -triggered off ['20000201T2000+13/baz'] 20000201T1700+13/foo -triggered off ['20000201T1400+13/foo'] 20000201T1400+13/bar -triggered off ['20000201T1400+13/foo'] 20000202T0200+13/baz -triggered off ['20000201T2000+13/baz'] 20000201T2000+13/foo -triggered off ['20000201T1700+13/foo'] 20000201T1700+13/bar -triggered off ['20000201T1700+13/foo'] 20000202T0200+13/qux -triggered off ['20000202T0200+13/baz'] 20000202T0800+13/baz -triggered off ['20000202T0200+13/baz'] 20000201T2300+13/foo -triggered off ['20000201T2000+13/foo'] 20000201T2000+13/bar -triggered off ['20000201T2000+13/foo'] 20000202T0800+13/qux -triggered off ['20000202T0800+13/baz'] 20000201T2300+13/bar -triggered off ['20000201T2300+13/foo'] 20000202T0200+13/foo -triggered off ['20000201T2300+13/foo'] 20000202T0200+13/bar -triggered off ['20000202T0200+13/foo'] 20000202T0500+13/foo -triggered off ['20000202T0200+13/foo'] 20000202T0500+13/bar -triggered off ['20000202T0500+13/foo'] 20000202T0800+13/foo -triggered off ['20000202T0500+13/foo'] 20000202T0800+13/bar -triggered off ['20000202T0800+13/foo'] 20000202T1100+13/foo -triggered off ['20000202T0800+13/foo'] 20000202T1100+13/bar -triggered off ['20000202T1100+13/foo'] cylc-flow-8.7.0/tests/functional/cyclers/multihourly/flow.cylc0000664000175000017500000000047415264420307025011 0ustar alastairalastair[scheduler] cycle point time zone = +13 allow implicit tasks = True [scheduling] initial cycle point = 20000131T0100Z final cycle point = 20000202T0600+0600 [[graph]] PT3H = "foo[-PT3H] => foo => bar" PT6H = "baz[-PT6H] => baz => qux" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/multihourly/graph.plain.ref0000664000175000017500000001136515264420307026070 0ustar alastairalastairedge "20000131T1400+13/baz" "20000131T1400+13/qux" edge "20000131T1400+13/baz" "20000131T2000+13/baz" edge "20000131T1400+13/foo" "20000131T1400+13/bar" edge "20000131T1400+13/foo" "20000131T1700+13/foo" edge "20000131T1700+13/foo" "20000131T1700+13/bar" edge "20000131T1700+13/foo" "20000131T2000+13/foo" edge "20000131T2000+13/baz" "20000131T2000+13/qux" edge "20000131T2000+13/baz" "20000201T0200+13/baz" edge "20000131T2000+13/foo" "20000131T2000+13/bar" edge "20000131T2000+13/foo" "20000131T2300+13/foo" edge "20000131T2300+13/foo" "20000131T2300+13/bar" edge "20000131T2300+13/foo" "20000201T0200+13/foo" edge "20000201T0200+13/baz" "20000201T0200+13/qux" edge "20000201T0200+13/baz" "20000201T0800+13/baz" edge "20000201T0200+13/foo" "20000201T0200+13/bar" edge "20000201T0200+13/foo" "20000201T0500+13/foo" edge "20000201T0500+13/foo" "20000201T0500+13/bar" edge "20000201T0500+13/foo" "20000201T0800+13/foo" edge "20000201T0800+13/baz" "20000201T0800+13/qux" edge "20000201T0800+13/baz" "20000201T1400+13/baz" edge "20000201T0800+13/foo" "20000201T0800+13/bar" edge "20000201T0800+13/foo" "20000201T1100+13/foo" edge "20000201T1100+13/foo" "20000201T1100+13/bar" edge "20000201T1100+13/foo" "20000201T1400+13/foo" edge "20000201T1400+13/baz" "20000201T1400+13/qux" edge "20000201T1400+13/baz" "20000201T2000+13/baz" edge "20000201T1400+13/foo" "20000201T1400+13/bar" edge "20000201T1400+13/foo" "20000201T1700+13/foo" edge "20000201T1700+13/foo" "20000201T1700+13/bar" edge "20000201T1700+13/foo" "20000201T2000+13/foo" edge "20000201T2000+13/baz" "20000201T2000+13/qux" edge "20000201T2000+13/baz" "20000202T0200+13/baz" edge "20000201T2000+13/foo" "20000201T2000+13/bar" edge "20000201T2000+13/foo" "20000201T2300+13/foo" edge "20000201T2300+13/foo" "20000201T2300+13/bar" edge "20000201T2300+13/foo" "20000202T0200+13/foo" edge "20000202T0200+13/baz" "20000202T0200+13/qux" edge "20000202T0200+13/baz" "20000202T0800+13/baz" edge "20000202T0200+13/foo" "20000202T0200+13/bar" edge "20000202T0200+13/foo" "20000202T0500+13/foo" edge "20000202T0500+13/foo" "20000202T0500+13/bar" edge "20000202T0500+13/foo" "20000202T0800+13/foo" edge "20000202T0800+13/baz" "20000202T0800+13/qux" edge "20000202T0800+13/foo" "20000202T0800+13/bar" edge "20000202T0800+13/foo" "20000202T1100+13/foo" edge "20000202T1100+13/foo" "20000202T1100+13/bar" graph node "20000131T1400+13/bar" "bar\n20000131T1400+13" node "20000131T1400+13/baz" "baz\n20000131T1400+13" node "20000131T1400+13/foo" "foo\n20000131T1400+13" node "20000131T1400+13/qux" "qux\n20000131T1400+13" node "20000131T1700+13/bar" "bar\n20000131T1700+13" node "20000131T1700+13/foo" "foo\n20000131T1700+13" node "20000131T2000+13/bar" "bar\n20000131T2000+13" node "20000131T2000+13/baz" "baz\n20000131T2000+13" node "20000131T2000+13/foo" "foo\n20000131T2000+13" node "20000131T2000+13/qux" "qux\n20000131T2000+13" node "20000131T2300+13/bar" "bar\n20000131T2300+13" node "20000131T2300+13/foo" "foo\n20000131T2300+13" node "20000201T0200+13/bar" "bar\n20000201T0200+13" node "20000201T0200+13/baz" "baz\n20000201T0200+13" node "20000201T0200+13/foo" "foo\n20000201T0200+13" node "20000201T0200+13/qux" "qux\n20000201T0200+13" node "20000201T0500+13/bar" "bar\n20000201T0500+13" node "20000201T0500+13/foo" "foo\n20000201T0500+13" node "20000201T0800+13/bar" "bar\n20000201T0800+13" node "20000201T0800+13/baz" "baz\n20000201T0800+13" node "20000201T0800+13/foo" "foo\n20000201T0800+13" node "20000201T0800+13/qux" "qux\n20000201T0800+13" node "20000201T1100+13/bar" "bar\n20000201T1100+13" node "20000201T1100+13/foo" "foo\n20000201T1100+13" node "20000201T1400+13/bar" "bar\n20000201T1400+13" node "20000201T1400+13/baz" "baz\n20000201T1400+13" node "20000201T1400+13/foo" "foo\n20000201T1400+13" node "20000201T1400+13/qux" "qux\n20000201T1400+13" node "20000201T1700+13/bar" "bar\n20000201T1700+13" node "20000201T1700+13/foo" "foo\n20000201T1700+13" node "20000201T2000+13/bar" "bar\n20000201T2000+13" node "20000201T2000+13/baz" "baz\n20000201T2000+13" node "20000201T2000+13/foo" "foo\n20000201T2000+13" node "20000201T2000+13/qux" "qux\n20000201T2000+13" node "20000201T2300+13/bar" "bar\n20000201T2300+13" node "20000201T2300+13/foo" "foo\n20000201T2300+13" node "20000202T0200+13/bar" "bar\n20000202T0200+13" node "20000202T0200+13/baz" "baz\n20000202T0200+13" node "20000202T0200+13/foo" "foo\n20000202T0200+13" node "20000202T0200+13/qux" "qux\n20000202T0200+13" node "20000202T0500+13/bar" "bar\n20000202T0500+13" node "20000202T0500+13/foo" "foo\n20000202T0500+13" node "20000202T0800+13/bar" "bar\n20000202T0800+13" node "20000202T0800+13/baz" "baz\n20000202T0800+13" node "20000202T0800+13/foo" "foo\n20000202T0800+13" node "20000202T0800+13/qux" "qux\n20000202T0800+13" node "20000202T1100+13/bar" "bar\n20000202T1100+13" node "20000202T1100+13/foo" "foo\n20000202T1100+13" stop cylc-flow-8.7.0/tests/functional/cyclers/05-multimonthly.t0000775000175000017500000000203315264420307023741 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="1000" # shellcheck disable=SC2034 FINALCP="1001" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/17-yearly.t0000775000175000017500000000204715264420307022511 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="2005023T12" # shellcheck disable=SC2034 FINALCP="2008001T11" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/r5_initial/0000775000175000017500000000000015264420307022623 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/r5_initial/reference.log0000664000175000017500000000115115264420307025262 0ustar alastairalastairInitial point: 20140101 Final point: 20140102T12 20140101T0000Z/xyzzy -triggered off [] 20140101T0300Z/xyzzy -triggered off [] 20140101T0600Z/xyzzy -triggered off [] 20140101T0000Z/bar -triggered off ['20140101T0000Z/xyzzy'] 20140101T0300Z/bar -triggered off ['20140101T0300Z/xyzzy'] 20140101T0900Z/xyzzy -triggered off [] 20140101T0600Z/bar -triggered off ['20140101T0600Z/xyzzy'] 20140101T0900Z/bar -triggered off ['20140101T0900Z/xyzzy'] 20140101T1200Z/xyzzy -triggered off [] 20140101T1200Z/bar -triggered off ['20140101T1200Z/xyzzy'] 20140101T1500Z/xyzzy -triggered off [] 20140101T1800Z/xyzzy -triggered off [] cylc-flow-8.7.0/tests/functional/cyclers/r5_initial/flow.cylc0000664000175000017500000000054115264420307024446 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20140101 final cycle point = 20140101T18 [[graph]] # xyzzy should not spawn bar in the last two cycle points: PT3H = xyzzy # 7 cycles R5//PT3H = "xyzzy => bar" # 5 cycles [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/r5_initial/graph.plain.ref0000664000175000017500000000153415264420307025527 0ustar alastairalastairedge "20140101T0000Z/xyzzy" "20140101T0000Z/bar" edge "20140101T0300Z/xyzzy" "20140101T0300Z/bar" edge "20140101T0600Z/xyzzy" "20140101T0600Z/bar" edge "20140101T0900Z/xyzzy" "20140101T0900Z/bar" edge "20140101T1200Z/xyzzy" "20140101T1200Z/bar" graph node "20140101T0000Z/bar" "bar\n20140101T0000Z" node "20140101T0000Z/xyzzy" "xyzzy\n20140101T0000Z" node "20140101T0300Z/bar" "bar\n20140101T0300Z" node "20140101T0300Z/xyzzy" "xyzzy\n20140101T0300Z" node "20140101T0600Z/bar" "bar\n20140101T0600Z" node "20140101T0600Z/xyzzy" "xyzzy\n20140101T0600Z" node "20140101T0900Z/bar" "bar\n20140101T0900Z" node "20140101T0900Z/xyzzy" "xyzzy\n20140101T0900Z" node "20140101T1200Z/bar" "bar\n20140101T1200Z" node "20140101T1200Z/xyzzy" "xyzzy\n20140101T1200Z" node "20140101T1500Z/xyzzy" "xyzzy\n20140101T1500Z" node "20140101T1800Z/xyzzy" "xyzzy\n20140101T1800Z" stop cylc-flow-8.7.0/tests/functional/cyclers/r5_final/0000775000175000017500000000000015264420307022263 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/r5_final/reference.log0000664000175000017500000000103315264420307024721 0ustar alastairalastairInitial point: 20140101 Final point: 20140102T12 20140102T0000Z/xyzzy -triggered off [] 20140102T0300Z/xyzzy -triggered off [] 20140102T0600Z/xyzzy -triggered off [] 20140102T0000Z/bar -triggered off ['20140102T0000Z/xyzzy'] 20140102T0300Z/bar -triggered off ['20140102T0300Z/xyzzy'] 20140102T0600Z/bar -triggered off ['20140102T0600Z/xyzzy'] 20140102T0900Z/xyzzy -triggered off [] 20140102T1200Z/xyzzy -triggered off [] 20140102T0900Z/bar -triggered off ['20140102T0900Z/xyzzy'] 20140102T1200Z/bar -triggered off ['20140102T1200Z/xyzzy'] cylc-flow-8.7.0/tests/functional/cyclers/r5_final/flow.cylc0000664000175000017500000000036015264420307024105 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20140101 final cycle point = 20140102T12 [[graph]] R5/PT3H = "xyzzy => bar" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/r5_final/graph.plain.ref0000664000175000017500000000136415264420307025170 0ustar alastairalastairedge "20140102T0000Z/xyzzy" "20140102T0000Z/bar" edge "20140102T0300Z/xyzzy" "20140102T0300Z/bar" edge "20140102T0600Z/xyzzy" "20140102T0600Z/bar" edge "20140102T0900Z/xyzzy" "20140102T0900Z/bar" edge "20140102T1200Z/xyzzy" "20140102T1200Z/bar" graph node "20140102T0000Z/bar" "bar\n20140102T0000Z" node "20140102T0000Z/xyzzy" "xyzzy\n20140102T0000Z" node "20140102T0300Z/bar" "bar\n20140102T0300Z" node "20140102T0300Z/xyzzy" "xyzzy\n20140102T0300Z" node "20140102T0600Z/bar" "bar\n20140102T0600Z" node "20140102T0600Z/xyzzy" "xyzzy\n20140102T0600Z" node "20140102T0900Z/bar" "bar\n20140102T0900Z" node "20140102T0900Z/xyzzy" "xyzzy\n20140102T0900Z" node "20140102T1200Z/bar" "bar\n20140102T1200Z" node "20140102T1200Z/xyzzy" "xyzzy\n20140102T1200Z" stop cylc-flow-8.7.0/tests/functional/cyclers/multimonthly/0000775000175000017500000000000015264420307023331 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/multimonthly/reference.log0000664000175000017500000000350615264420307025776 0ustar alastairalastairInitial point: 1000 Final point: 1001 10000101T0000Z/baz -triggered off ['09990701T0000Z/baz'] 10000101T0000Z/foo -triggered off ['09991201T0000Z/foo'] 10000101T0000Z/bar -triggered off ['10000101T0000Z/foo'] 10000101T0000Z/qux -triggered off ['10000101T0000Z/baz'] 10000701T0000Z/baz -triggered off ['10000101T0000Z/baz'] 10000201T0000Z/foo -triggered off ['10000101T0000Z/foo'] 10000701T0000Z/qux -triggered off ['10000701T0000Z/baz'] 10000201T0000Z/bar -triggered off ['10000201T0000Z/foo'] 10000301T0000Z/foo -triggered off ['10000201T0000Z/foo'] 10000301T0000Z/bar -triggered off ['10000301T0000Z/foo'] 10000401T0000Z/foo -triggered off ['10000301T0000Z/foo'] 10000401T0000Z/bar -triggered off ['10000401T0000Z/foo'] 10000501T0000Z/foo -triggered off ['10000401T0000Z/foo'] 10010101T0000Z/baz -triggered off ['10000701T0000Z/baz'] 10000501T0000Z/bar -triggered off ['10000501T0000Z/foo'] 10000601T0000Z/foo -triggered off ['10000501T0000Z/foo'] 10010101T0000Z/qux -triggered off ['10010101T0000Z/baz'] 10000701T0000Z/foo -triggered off ['10000601T0000Z/foo'] 10000601T0000Z/bar -triggered off ['10000601T0000Z/foo'] 10000801T0000Z/foo -triggered off ['10000701T0000Z/foo'] 10000701T0000Z/bar -triggered off ['10000701T0000Z/foo'] 10000901T0000Z/foo -triggered off ['10000801T0000Z/foo'] 10000801T0000Z/bar -triggered off ['10000801T0000Z/foo'] 10001001T0000Z/foo -triggered off ['10000901T0000Z/foo'] 10000901T0000Z/bar -triggered off ['10000901T0000Z/foo'] 10001101T0000Z/foo -triggered off ['10001001T0000Z/foo'] 10001001T0000Z/bar -triggered off ['10001001T0000Z/foo'] 10001201T0000Z/foo -triggered off ['10001101T0000Z/foo'] 10001101T0000Z/bar -triggered off ['10001101T0000Z/foo'] 10010101T0000Z/foo -triggered off ['10001201T0000Z/foo'] 10001201T0000Z/bar -triggered off ['10001201T0000Z/foo'] 10010101T0000Z/bar -triggered off ['10010101T0000Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/multimonthly/flow.cylc0000664000175000017500000000042415264420307025154 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 1000 final cycle point = 1001 [[graph]] P1M = "foo[-P1M] => foo => bar" P6M = "baz[-P6M] => baz => qux" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/multimonthly/graph.plain.ref0000664000175000017500000000561515264420307026241 0ustar alastairalastairedge "10000101T0000Z/baz" "10000101T0000Z/qux" edge "10000101T0000Z/baz" "10000701T0000Z/baz" edge "10000101T0000Z/foo" "10000101T0000Z/bar" edge "10000101T0000Z/foo" "10000201T0000Z/foo" edge "10000201T0000Z/foo" "10000201T0000Z/bar" edge "10000201T0000Z/foo" "10000301T0000Z/foo" edge "10000301T0000Z/foo" "10000301T0000Z/bar" edge "10000301T0000Z/foo" "10000401T0000Z/foo" edge "10000401T0000Z/foo" "10000401T0000Z/bar" edge "10000401T0000Z/foo" "10000501T0000Z/foo" edge "10000501T0000Z/foo" "10000501T0000Z/bar" edge "10000501T0000Z/foo" "10000601T0000Z/foo" edge "10000601T0000Z/foo" "10000601T0000Z/bar" edge "10000601T0000Z/foo" "10000701T0000Z/foo" edge "10000701T0000Z/baz" "10000701T0000Z/qux" edge "10000701T0000Z/baz" "10010101T0000Z/baz" edge "10000701T0000Z/foo" "10000701T0000Z/bar" edge "10000701T0000Z/foo" "10000801T0000Z/foo" edge "10000801T0000Z/foo" "10000801T0000Z/bar" edge "10000801T0000Z/foo" "10000901T0000Z/foo" edge "10000901T0000Z/foo" "10000901T0000Z/bar" edge "10000901T0000Z/foo" "10001001T0000Z/foo" edge "10001001T0000Z/foo" "10001001T0000Z/bar" edge "10001001T0000Z/foo" "10001101T0000Z/foo" edge "10001101T0000Z/foo" "10001101T0000Z/bar" edge "10001101T0000Z/foo" "10001201T0000Z/foo" edge "10001201T0000Z/foo" "10001201T0000Z/bar" edge "10001201T0000Z/foo" "10010101T0000Z/foo" edge "10010101T0000Z/baz" "10010101T0000Z/qux" edge "10010101T0000Z/foo" "10010101T0000Z/bar" graph node "10000101T0000Z/bar" "bar\n10000101T0000Z" node "10000101T0000Z/baz" "baz\n10000101T0000Z" node "10000101T0000Z/foo" "foo\n10000101T0000Z" node "10000101T0000Z/qux" "qux\n10000101T0000Z" node "10000201T0000Z/bar" "bar\n10000201T0000Z" node "10000201T0000Z/foo" "foo\n10000201T0000Z" node "10000301T0000Z/bar" "bar\n10000301T0000Z" node "10000301T0000Z/foo" "foo\n10000301T0000Z" node "10000401T0000Z/bar" "bar\n10000401T0000Z" node "10000401T0000Z/foo" "foo\n10000401T0000Z" node "10000501T0000Z/bar" "bar\n10000501T0000Z" node "10000501T0000Z/foo" "foo\n10000501T0000Z" node "10000601T0000Z/bar" "bar\n10000601T0000Z" node "10000601T0000Z/foo" "foo\n10000601T0000Z" node "10000701T0000Z/bar" "bar\n10000701T0000Z" node "10000701T0000Z/baz" "baz\n10000701T0000Z" node "10000701T0000Z/foo" "foo\n10000701T0000Z" node "10000701T0000Z/qux" "qux\n10000701T0000Z" node "10000801T0000Z/bar" "bar\n10000801T0000Z" node "10000801T0000Z/foo" "foo\n10000801T0000Z" node "10000901T0000Z/bar" "bar\n10000901T0000Z" node "10000901T0000Z/foo" "foo\n10000901T0000Z" node "10001001T0000Z/bar" "bar\n10001001T0000Z" node "10001001T0000Z/foo" "foo\n10001001T0000Z" node "10001101T0000Z/bar" "bar\n10001101T0000Z" node "10001101T0000Z/foo" "foo\n10001101T0000Z" node "10001201T0000Z/bar" "bar\n10001201T0000Z" node "10001201T0000Z/foo" "foo\n10001201T0000Z" node "10010101T0000Z/bar" "bar\n10010101T0000Z" node "10010101T0000Z/baz" "baz\n10010101T0000Z" node "10010101T0000Z/foo" "foo\n10010101T0000Z" node "10010101T0000Z/qux" "qux\n10010101T0000Z" stop cylc-flow-8.7.0/tests/functional/cyclers/hourly/0000775000175000017500000000000015264420307022106 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/hourly/reference.log0000664000175000017500000000115515264420307024551 0ustar alastairalastairInitial point: 20000229T2000 Final point: 20000301 20000229T2000Z/foo -triggered off ['20000229T1900Z/foo'] 20000229T2000Z/bar -triggered off ['20000229T2000Z/foo'] 20000229T2100Z/foo -triggered off ['20000229T2000Z/foo'] 20000229T2100Z/bar -triggered off ['20000229T2100Z/foo'] 20000229T2200Z/foo -triggered off ['20000229T2100Z/foo'] 20000229T2200Z/bar -triggered off ['20000229T2200Z/foo'] 20000229T2300Z/foo -triggered off ['20000229T2200Z/foo'] 20000229T2300Z/bar -triggered off ['20000229T2300Z/foo'] 20000301T0000Z/foo -triggered off ['20000229T2300Z/foo'] 20000301T0000Z/bar -triggered off ['20000301T0000Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/hourly/flow.cylc0000664000175000017500000000037315264420307023734 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20000229T2000 final cycle point = 20000301 [[graph]] PT1H = "foo[-PT1H] => foo => bar" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/hourly/graph.plain.ref0000664000175000017500000000162215264420307025010 0ustar alastairalastairedge "20000229T2000Z/foo" "20000229T2000Z/bar" edge "20000229T2000Z/foo" "20000229T2100Z/foo" edge "20000229T2100Z/foo" "20000229T2100Z/bar" edge "20000229T2100Z/foo" "20000229T2200Z/foo" edge "20000229T2200Z/foo" "20000229T2200Z/bar" edge "20000229T2200Z/foo" "20000229T2300Z/foo" edge "20000229T2300Z/foo" "20000229T2300Z/bar" edge "20000229T2300Z/foo" "20000301T0000Z/foo" edge "20000301T0000Z/foo" "20000301T0000Z/bar" graph node "20000229T2000Z/bar" "bar\n20000229T2000Z" node "20000229T2000Z/foo" "foo\n20000229T2000Z" node "20000229T2100Z/bar" "bar\n20000229T2100Z" node "20000229T2100Z/foo" "foo\n20000229T2100Z" node "20000229T2200Z/bar" "bar\n20000229T2200Z" node "20000229T2200Z/foo" "foo\n20000229T2200Z" node "20000229T2300Z/bar" "bar\n20000229T2300Z" node "20000229T2300Z/foo" "foo\n20000229T2300Z" node "20000301T0000Z/bar" "bar\n20000301T0000Z" node "20000301T0000Z/foo" "foo\n20000301T0000Z" stop cylc-flow-8.7.0/tests/functional/cyclers/16-weekly.t0000775000175000017500000000205615264420307022503 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="2005W015T12" # shellcheck disable=SC2034 FINALCP="2005W065T11-0100" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/offset_final/0000775000175000017500000000000015264420307023223 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/offset_final/reference.log0000664000175000017500000000021715264420307025664 0ustar alastairalastairInitial point: 20140101 Final point: 20140102T12 20140101T1200Z/bar -triggered off [] 20140101T1200Z/baz -triggered off ['20140101T1200Z/bar'] cylc-flow-8.7.0/tests/functional/cyclers/offset_final/flow.cylc0000664000175000017500000000035715264420307025053 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20140101 final cycle point = 20140102T12 [[graph]] R1//-P1D = "bar => baz" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/offset_final/graph.plain.ref0000664000175000017500000000023215264420307026121 0ustar alastairalastairedge "20140101T1200Z/bar" "20140101T1200Z/baz" graph node "20140101T1200Z/bar" "bar\n20140101T1200Z" node "20140101T1200Z/baz" "baz\n20140101T1200Z" stop cylc-flow-8.7.0/tests/functional/cyclers/r1_middle/0000775000175000017500000000000015264420307022424 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/r1_middle/reference.log0000664000175000017500000000021715264420307025065 0ustar alastairalastairInitial point: 20140101 Final point: 20140102T12 20140101T0010Z/foo -triggered off [] 20140101T0010Z/bar -triggered off ['20140101T0010Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/r1_middle/flow.cylc0000664000175000017500000000036015264420307024246 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20140101 final cycle point = 20140102T12 [[graph]] R1/+PT10M = "foo => bar" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/r1_middle/graph.plain.ref0000664000175000017500000000023215264420307025322 0ustar alastairalastairedge "20140101T0010Z/foo" "20140101T0010Z/bar" graph node "20140101T0010Z/bar" "bar\n20140101T0010Z" node "20140101T0010Z/foo" "foo\n20140101T0010Z" stop cylc-flow-8.7.0/tests/functional/cyclers/weekly/0000775000175000017500000000000015264420307022064 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/weekly/reference.log0000664000175000017500000000134515264420307024530 0ustar alastairalastairInitial point: 2005W015T12 Final point: 2005W065T11-0100 20050107T1200Z/foo -triggered off ['20041231T1200Z/foo'] 20050107T1200Z/bar -triggered off ['20050107T1200Z/foo'] 20050114T1200Z/foo -triggered off ['20050107T1200Z/foo'] 20050114T1200Z/bar -triggered off ['20050114T1200Z/foo'] 20050121T1200Z/foo -triggered off ['20050114T1200Z/foo'] 20050121T1200Z/bar -triggered off ['20050121T1200Z/foo'] 20050128T1200Z/foo -triggered off ['20050121T1200Z/foo'] 20050128T1200Z/bar -triggered off ['20050128T1200Z/foo'] 20050204T1200Z/foo -triggered off ['20050128T1200Z/foo'] 20050204T1200Z/bar -triggered off ['20050204T1200Z/foo'] 20050211T1200Z/foo -triggered off ['20050204T1200Z/foo'] 20050211T1200Z/bar -triggered off ['20050211T1200Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/weekly/flow.cylc0000664000175000017500000000037715264420307023716 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 2005W015T12 final cycle point = 2005W065T11-0100 [[graph]] P1W = "foo[-P1W] => foo => bar" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/weekly/graph.plain.ref0000664000175000017500000000212015264420307024760 0ustar alastairalastairedge "20050107T1200Z/foo" "20050107T1200Z/bar" edge "20050107T1200Z/foo" "20050114T1200Z/foo" edge "20050114T1200Z/foo" "20050114T1200Z/bar" edge "20050114T1200Z/foo" "20050121T1200Z/foo" edge "20050121T1200Z/foo" "20050121T1200Z/bar" edge "20050121T1200Z/foo" "20050128T1200Z/foo" edge "20050128T1200Z/foo" "20050128T1200Z/bar" edge "20050128T1200Z/foo" "20050204T1200Z/foo" edge "20050204T1200Z/foo" "20050204T1200Z/bar" edge "20050204T1200Z/foo" "20050211T1200Z/foo" edge "20050211T1200Z/foo" "20050211T1200Z/bar" graph node "20050107T1200Z/bar" "bar\n20050107T1200Z" node "20050107T1200Z/foo" "foo\n20050107T1200Z" node "20050114T1200Z/bar" "bar\n20050114T1200Z" node "20050114T1200Z/foo" "foo\n20050114T1200Z" node "20050121T1200Z/bar" "bar\n20050121T1200Z" node "20050121T1200Z/foo" "foo\n20050121T1200Z" node "20050128T1200Z/bar" "bar\n20050128T1200Z" node "20050128T1200Z/foo" "foo\n20050128T1200Z" node "20050204T1200Z/bar" "bar\n20050204T1200Z" node "20050204T1200Z/foo" "foo\n20050204T1200Z" node "20050211T1200Z/bar" "bar\n20050211T1200Z" node "20050211T1200Z/foo" "foo\n20050211T1200Z" stop cylc-flow-8.7.0/tests/functional/cyclers/35-day_of_week.t0000775000175000017500000000204515264420307023456 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20100101T0000Z" # shellcheck disable=SC2034 FINALCP="+P3W" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/offset_initial/0000775000175000017500000000000015264420307023563 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/offset_initial/reference.log0000664000175000017500000000051315264420307026223 0ustar alastairalastairInitial point: 20140101 Final point: 20140102T12 20140102T0000Z/bar -triggered off [] 20140102T0600Z/bar -triggered off [] 20140102T0000Z/baz -triggered off ['20140102T0000Z/bar'] 20140102T1200Z/bar -triggered off [] 20140102T0600Z/baz -triggered off ['20140102T0600Z/bar'] 20140102T1200Z/baz -triggered off ['20140102T1200Z/bar'] cylc-flow-8.7.0/tests/functional/cyclers/offset_initial/flow.cylc0000664000175000017500000000036015264420307025405 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20140101 final cycle point = 20140102T12 [[graph]] +P1D/PT6H = "bar => baz" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/offset_initial/graph.plain.ref0000664000175000017500000000067015264420307026467 0ustar alastairalastairedge "20140102T0000Z/bar" "20140102T0000Z/baz" edge "20140102T0600Z/bar" "20140102T0600Z/baz" edge "20140102T1200Z/bar" "20140102T1200Z/baz" graph node "20140102T0000Z/bar" "bar\n20140102T0000Z" node "20140102T0000Z/baz" "baz\n20140102T0000Z" node "20140102T0600Z/bar" "bar\n20140102T0600Z" node "20140102T0600Z/baz" "baz\n20140102T0600Z" node "20140102T1200Z/bar" "bar\n20140102T1200Z" node "20140102T1200Z/baz" "baz\n20140102T1200Z" stop cylc-flow-8.7.0/tests/functional/cyclers/33-integer1.t0000775000175000017500000000203015264420307022710 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="1" # shellcheck disable=SC2034 FINALCP="+P15" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/40-integer_exclusions_advanced.t0000775000175000017500000000274515264420307026743 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test intercycle dependencies. . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" 'integer_exclusions_advanced' #------------------------------------------------------------------------------- run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" graph_workflow "${WORKFLOW_NAME}" "${WORKFLOW_NAME}.graph.plain" 1 "+P15" cmp_ok "${WORKFLOW_NAME}.graph.plain" 'graph.plain.ref' workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/cyclers/12-r1_middle.t0000775000175000017500000000204615264420307023036 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20140101" # shellcheck disable=SC2034 FINALCP="20140102T12" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/00-daily-find.out0000664000175000017500000000312415264420307023552 0ustar alastairalastairlog/job/20131231T2300Z/bar/01/job log/job/20131231T2300Z/bar/01/job.err log/job/20131231T2300Z/bar/01/job.out log/job/20131231T2300Z/bar/01/job.status log/job/20131231T2300Z/bar/01/job.xtrace log/job/20131231T2300Z/bar/01/job-activity.log log/job/20131231T2300Z/foo/01/job log/job/20131231T2300Z/foo/01/job.err log/job/20131231T2300Z/foo/01/job.out log/job/20131231T2300Z/foo/01/job.status log/job/20131231T2300Z/foo/01/job.xtrace log/job/20131231T2300Z/foo/01/job-activity.log log/job/20140101T2300Z/bar/01/job log/job/20140101T2300Z/bar/01/job.err log/job/20140101T2300Z/bar/01/job.out log/job/20140101T2300Z/bar/01/job.status log/job/20140101T2300Z/bar/01/job.xtrace log/job/20140101T2300Z/bar/01/job-activity.log log/job/20140101T2300Z/foo/01/job log/job/20140101T2300Z/foo/01/job.err log/job/20140101T2300Z/foo/01/job.out log/job/20140101T2300Z/foo/01/job.status log/job/20140101T2300Z/foo/01/job.xtrace log/job/20140101T2300Z/foo/01/job-activity.log log/job/20140102T2300Z/bar/01/job log/job/20140102T2300Z/bar/01/job.err log/job/20140102T2300Z/bar/01/job.out log/job/20140102T2300Z/bar/01/job.status log/job/20140102T2300Z/bar/01/job.xtrace log/job/20140102T2300Z/bar/01/job-activity.log log/job/20140102T2300Z/foo/01/job log/job/20140102T2300Z/foo/01/job.err log/job/20140102T2300Z/foo/01/job.out log/job/20140102T2300Z/foo/01/job.status log/job/20140102T2300Z/foo/01/job.xtrace log/job/20140102T2300Z/foo/01/job-activity.log work/20131231T2300Z/bar/typing work/20131231T2300Z/foo/typing work/20140101T2300Z/bar/typing work/20140101T2300Z/foo/typing work/20140102T2300Z/bar/typing work/20140102T2300Z/foo/typing cylc-flow-8.7.0/tests/functional/cyclers/29-r1_restricted.t0000775000175000017500000000205115264420307023754 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20130808T00" # shellcheck disable=SC2034 FINALCP="20130808T18" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/monthly_complex/0000775000175000017500000000000015264420307024005 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/monthly_complex/reference.log0000664000175000017500000000224515264420307026451 0ustar alastairalastair20000430T0100Z/flub -triggered off [] 20000406T0000Z/foo -triggered off [] 20000331T0100Z/flob -triggered off ['20000430T0100Z/flub'] 20000401T0100Z/flob -triggered off ['20000331T0100Z/flob', '20000430T0100Z/flub'] 20000402T0100Z/flob -triggered off ['20000401T0100Z/flob', '20000430T0100Z/flub'] 20000403T0100Z/flob -triggered off ['20000402T0100Z/flob', '20000430T0100Z/flub'] 20000506T0000Z/foo -triggered off [] 20000508T1200Z/bar -triggered off ['20000506T0000Z/foo'] 20000606T0000Z/foo -triggered off [] 20000605T1200Z/bar -triggered off ['20000606T0000Z/foo'] 20000706T0000Z/foo -triggered off [] 20000703T1200Z/bar -triggered off ['20000706T0000Z/foo'] 20000806T0000Z/foo -triggered off [] 20000731T1200Z/bar -triggered off ['20000806T0000Z/foo'] 20000906T0000Z/foo -triggered off [] 20000828T1200Z/bar -triggered off ['20000906T0000Z/foo'] 20001006T0000Z/foo -triggered off [] 20000925T1200Z/bar -triggered off ['20001006T0000Z/foo'] 20001106T0000Z/foo -triggered off [] 20001023T1200Z/bar -triggered off ['20001106T0000Z/foo'] 20001206T0000Z/foo -triggered off [] 20001120T1200Z/bar -triggered off ['20001206T0000Z/foo'] 20001218T1200Z/bar -triggered off ['20001206T0000Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/monthly_complex/flow.cylc0000664000175000017500000000055115264420307025631 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20000331T0100Z final cycle point = 2001 [[graph]] 20T-P2W/P1M = "foo" 30T06+P8DT6H/P3W = "foo[20T-P2W] => bar" R1/+P1M = "flub" R4//P1D = "flub[^+P1M] & flob[-P1D] => flob" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/subhourly/0000775000175000017500000000000015264420307022620 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/subhourly/reference.log0000664000175000017500000000100015264420307025250 0ustar alastairalastairInitial point: 20131231T2300 Final point: 20140101T0050 20131231T2300Z/foo -triggered off ['20131231T2230Z/foo'] 20131231T2300Z/bar -triggered off ['20131231T2300Z/foo'] 20131231T2330Z/foo -triggered off ['20131231T2300Z/foo'] 20131231T2330Z/bar -triggered off ['20131231T2330Z/foo'] 20140101T0000Z/foo -triggered off ['20131231T2330Z/foo'] 20140101T0000Z/bar -triggered off ['20140101T0000Z/foo'] 20140101T0030Z/foo -triggered off ['20140101T0000Z/foo'] 20140101T0030Z/bar -triggered off ['20140101T0030Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/subhourly/flow.cylc0000664000175000017500000000040215264420307024437 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20131231T2300 final cycle point = 20140101T0050 [[graph]] PT30M = "foo[-PT30M] => foo => bar" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/subhourly/graph.plain.ref0000664000175000017500000000132415264420307025521 0ustar alastairalastairedge "20131231T2300Z/foo" "20131231T2300Z/bar" edge "20131231T2300Z/foo" "20131231T2330Z/foo" edge "20131231T2330Z/foo" "20131231T2330Z/bar" edge "20131231T2330Z/foo" "20140101T0000Z/foo" edge "20140101T0000Z/foo" "20140101T0000Z/bar" edge "20140101T0000Z/foo" "20140101T0030Z/foo" edge "20140101T0030Z/foo" "20140101T0030Z/bar" graph node "20131231T2300Z/bar" "bar\n20131231T2300Z" node "20131231T2300Z/foo" "foo\n20131231T2300Z" node "20131231T2330Z/bar" "bar\n20131231T2330Z" node "20131231T2330Z/foo" "foo\n20131231T2330Z" node "20140101T0000Z/bar" "bar\n20140101T0000Z" node "20140101T0000Z/foo" "foo\n20140101T0000Z" node "20140101T0030Z/bar" "bar\n20140101T0030Z" node "20140101T0030Z/foo" "foo\n20140101T0030Z" stop cylc-flow-8.7.0/tests/functional/cyclers/28-missing-cycling-sequence.t0000775000175000017500000000341415264420307026112 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check that missing cycling task fails validation. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- init_workflow "${TEST_NAME_BASE}" << __FLOW__ [scheduling] initial cycle point = 20140808T00 [[graph]] P1D = foo[-P1D] => bar [runtime] [[foo,bar]] __FLOW__ #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-cmp" cmp_ok "${TEST_NAME_BASE}-validate.stderr" <<__ERR__ TaskDefError: No cycling sequences defined for foo __ERR__ #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/cyclers/14-r5_initial-integer.t0000775000175000017500000000302415264420307024667 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test triggering from unbounded to bound sequences - integer cycling. . "$(dirname "$0")/test_header" set_test_number 3 CHOSEN_WORKFLOW="$(basename "$0" | sed "s/^[0-9]*-\(.*\)\.t/\1/g")" install_workflow "${TEST_NAME_BASE}" "${CHOSEN_WORKFLOW}" TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-graph" graph_workflow "${WORKFLOW_NAME}" "${WORKFLOW_NAME}.graph.plain" 1 7 cmp_ok "${WORKFLOW_NAME}.graph.plain" "$TEST_SOURCE_DIR/$CHOSEN_WORKFLOW/graph.plain.ref" TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" purge cylc-flow-8.7.0/tests/functional/cyclers/10-r1_final.t0000775000175000017500000000204615264420307022667 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20140101" # shellcheck disable=SC2034 FINALCP="20140102T12" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/day_of_week/0000775000175000017500000000000015264420307023040 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/day_of_week/reference.log0000664000175000017500000000031615264420307025501 0ustar alastairalastairInitial point: 20100101T0000Z Final point: 20100125T0000Z 20100104T0000Z/foo -triggered off [] 20100111T0000Z/foo -triggered off [] 20100118T0000Z/foo -triggered off [] 20100125T0000Z/foo -triggered off [] cylc-flow-8.7.0/tests/functional/cyclers/day_of_week/flow.cylc0000664000175000017500000000032215264420307024660 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] initial cycle point = 20100101T0000Z final cycle point = 20100125T0000Z [[graph]] R5/W-1/P1W = foo [runtime] [[foo]] script = true cylc-flow-8.7.0/tests/functional/cyclers/day_of_week/graph.plain.ref0000664000175000017500000000023315264420307025737 0ustar alastairalastairgraph node "20100104T0000Z/foo" "foo\n20100104T0000Z" node "20100111T0000Z/foo" "foo\n20100111T0000Z" node "20100118T0000Z/foo" "foo\n20100118T0000Z" stop cylc-flow-8.7.0/tests/functional/cyclers/26-0000_rollunder.t0000775000175000017500000000265215264420307023653 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test intercycle dependencies. . "$(dirname "$0")/test_header" set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" 0000_rollunder #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" grep_ok "Cannot dump TimePoint year: -1 not in bounds 0 to 9999." \ "${TEST_NAME}.stderr" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/cyclers/daily_final/0000775000175000017500000000000015264420307023037 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/daily_final/reference.log0000664000175000017500000000022015264420307025472 0ustar alastairalastairInitial point: 20140101T0000Z Final point: 20140102T1200Z 20140101T0000Z/daily_foo -triggered off [] 20140102T0000Z/daily_foo -triggered off [] cylc-flow-8.7.0/tests/functional/cyclers/daily_final/flow.cylc0000664000175000017500000000032115264420307024656 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] initial cycle point = 20140101 final cycle point = 20140102T12 [[graph]] R//T00 = "daily_foo" [runtime] [[daily_foo]] script = true cylc-flow-8.7.0/tests/functional/cyclers/daily_final/graph.plain.ref0000664000175000017500000000020415264420307025734 0ustar alastairalastairgraph node "20140101T0000Z/daily_foo" "daily_foo\n20140101T0000Z" node "20140102T0000Z/daily_foo" "daily_foo\n20140102T0000Z" stop cylc-flow-8.7.0/tests/functional/cyclers/07-multiyearly.t0000775000175000017500000000203315264420307023556 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="1004" # shellcheck disable=SC2034 FINALCP="1014" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/02-monthly.t0000775000175000017500000000204515264420307022666 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20000131T0100Z" # shellcheck disable=SC2034 FINALCP="2001" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/multidaily_local/0000775000175000017500000000000015264420307024113 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/multidaily_local/reference.log0000664000175000017500000000406715264420307026563 0ustar alastairalastairInitial point: 20001231T0100 Final point: 20010114 20001231T0100Z/baz -triggered off ['20001227T0100Z/baz'] 20001231T0100Z/foo -triggered off ['20001230T0100Z/foo'] 20001231T0100Z/qux -triggered off ['20001231T0100Z/baz'] 20010104T0100Z/baz -triggered off ['20001231T0100Z/baz'] 20001231T0100Z/bar -triggered off ['20001231T0100Z/foo'] 20010101T0100Z/foo -triggered off ['20001231T0100Z/foo'] 20010104T0100Z/qux -triggered off ['20010104T0100Z/baz'] 20010101T0100Z/bar -triggered off ['20010101T0100Z/foo'] 20010102T0100Z/foo -triggered off ['20010101T0100Z/foo'] 20010108T0100Z/baz -triggered off ['20010104T0100Z/baz'] 20010102T0100Z/bar -triggered off ['20010102T0100Z/foo'] 20010103T0100Z/foo -triggered off ['20010102T0100Z/foo'] 20010108T0100Z/qux -triggered off ['20010108T0100Z/baz'] 20010103T0100Z/bar -triggered off ['20010103T0100Z/foo'] 20010104T0100Z/foo -triggered off ['20010103T0100Z/foo'] 20010104T0100Z/bar -triggered off ['20010104T0100Z/foo'] 20010105T0100Z/foo -triggered off ['20010104T0100Z/foo'] 20010105T0100Z/bar -triggered off ['20010105T0100Z/foo'] 20010106T0100Z/foo -triggered off ['20010105T0100Z/foo'] 20010112T0100Z/baz -triggered off ['20010108T0100Z/baz'] 20010106T0100Z/bar -triggered off ['20010106T0100Z/foo'] 20010107T0100Z/foo -triggered off ['20010106T0100Z/foo'] 20010112T0100Z/qux -triggered off ['20010112T0100Z/baz'] 20010108T0100Z/foo -triggered off ['20010107T0100Z/foo'] 20010107T0100Z/bar -triggered off ['20010107T0100Z/foo'] 20010108T0100Z/bar -triggered off ['20010108T0100Z/foo'] 20010109T0100Z/foo -triggered off ['20010108T0100Z/foo'] 20010109T0100Z/bar -triggered off ['20010109T0100Z/foo'] 20010110T0100Z/foo -triggered off ['20010109T0100Z/foo'] 20010110T0100Z/bar -triggered off ['20010110T0100Z/foo'] 20010111T0100Z/foo -triggered off ['20010110T0100Z/foo'] 20010111T0100Z/bar -triggered off ['20010111T0100Z/foo'] 20010112T0100Z/foo -triggered off ['20010111T0100Z/foo'] 20010112T0100Z/bar -triggered off ['20010112T0100Z/foo'] 20010113T0100Z/foo -triggered off ['20010112T0100Z/foo'] 20010113T0100Z/bar -triggered off ['20010113T0100Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/multidaily_local/flow.cylc0000664000175000017500000000041515264420307025736 0ustar alastairalastair[scheduler] allow implicit tasks = True [scheduling] initial cycle point = 20001231T0100 final cycle point = 20010114 [[graph]] P1D = "foo[-P1D] => foo => bar" P4D = "baz[-P4D] => baz => qux" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/multidaily_local/graph.plain.ref0000664000175000017500000000641115264420307027016 0ustar alastairalastairedge "20001231T0100Z/baz" "20001231T0100Z/qux" edge "20001231T0100Z/baz" "20010104T0100Z/baz" edge "20001231T0100Z/foo" "20001231T0100Z/bar" edge "20001231T0100Z/foo" "20010101T0100Z/foo" edge "20010101T0100Z/foo" "20010101T0100Z/bar" edge "20010101T0100Z/foo" "20010102T0100Z/foo" edge "20010102T0100Z/foo" "20010102T0100Z/bar" edge "20010102T0100Z/foo" "20010103T0100Z/foo" edge "20010103T0100Z/foo" "20010103T0100Z/bar" edge "20010103T0100Z/foo" "20010104T0100Z/foo" edge "20010104T0100Z/baz" "20010104T0100Z/qux" edge "20010104T0100Z/baz" "20010108T0100Z/baz" edge "20010104T0100Z/foo" "20010104T0100Z/bar" edge "20010104T0100Z/foo" "20010105T0100Z/foo" edge "20010105T0100Z/foo" "20010105T0100Z/bar" edge "20010105T0100Z/foo" "20010106T0100Z/foo" edge "20010106T0100Z/foo" "20010106T0100Z/bar" edge "20010106T0100Z/foo" "20010107T0100Z/foo" edge "20010107T0100Z/foo" "20010107T0100Z/bar" edge "20010107T0100Z/foo" "20010108T0100Z/foo" edge "20010108T0100Z/baz" "20010108T0100Z/qux" edge "20010108T0100Z/baz" "20010112T0100Z/baz" edge "20010108T0100Z/foo" "20010108T0100Z/bar" edge "20010108T0100Z/foo" "20010109T0100Z/foo" edge "20010109T0100Z/foo" "20010109T0100Z/bar" edge "20010109T0100Z/foo" "20010110T0100Z/foo" edge "20010110T0100Z/foo" "20010110T0100Z/bar" edge "20010110T0100Z/foo" "20010111T0100Z/foo" edge "20010111T0100Z/foo" "20010111T0100Z/bar" edge "20010111T0100Z/foo" "20010112T0100Z/foo" edge "20010112T0100Z/baz" "20010112T0100Z/qux" edge "20010112T0100Z/foo" "20010112T0100Z/bar" edge "20010112T0100Z/foo" "20010113T0100Z/foo" edge "20010113T0100Z/foo" "20010113T0100Z/bar" graph node "20001231T0100Z/bar" "bar\n20001231T0100Z" node "20001231T0100Z/baz" "baz\n20001231T0100Z" node "20001231T0100Z/foo" "foo\n20001231T0100Z" node "20001231T0100Z/qux" "qux\n20001231T0100Z" node "20010101T0100Z/bar" "bar\n20010101T0100Z" node "20010101T0100Z/foo" "foo\n20010101T0100Z" node "20010102T0100Z/bar" "bar\n20010102T0100Z" node "20010102T0100Z/foo" "foo\n20010102T0100Z" node "20010103T0100Z/bar" "bar\n20010103T0100Z" node "20010103T0100Z/foo" "foo\n20010103T0100Z" node "20010104T0100Z/bar" "bar\n20010104T0100Z" node "20010104T0100Z/baz" "baz\n20010104T0100Z" node "20010104T0100Z/foo" "foo\n20010104T0100Z" node "20010104T0100Z/qux" "qux\n20010104T0100Z" node "20010105T0100Z/bar" "bar\n20010105T0100Z" node "20010105T0100Z/foo" "foo\n20010105T0100Z" node "20010106T0100Z/bar" "bar\n20010106T0100Z" node "20010106T0100Z/foo" "foo\n20010106T0100Z" node "20010107T0100Z/bar" "bar\n20010107T0100Z" node "20010107T0100Z/foo" "foo\n20010107T0100Z" node "20010108T0100Z/bar" "bar\n20010108T0100Z" node "20010108T0100Z/baz" "baz\n20010108T0100Z" node "20010108T0100Z/foo" "foo\n20010108T0100Z" node "20010108T0100Z/qux" "qux\n20010108T0100Z" node "20010109T0100Z/bar" "bar\n20010109T0100Z" node "20010109T0100Z/foo" "foo\n20010109T0100Z" node "20010110T0100Z/bar" "bar\n20010110T0100Z" node "20010110T0100Z/foo" "foo\n20010110T0100Z" node "20010111T0100Z/bar" "bar\n20010111T0100Z" node "20010111T0100Z/foo" "foo\n20010111T0100Z" node "20010112T0100Z/bar" "bar\n20010112T0100Z" node "20010112T0100Z/baz" "baz\n20010112T0100Z" node "20010112T0100Z/foo" "foo\n20010112T0100Z" node "20010112T0100Z/qux" "qux\n20010112T0100Z" node "20010113T0100Z/bar" "bar\n20010113T0100Z" node "20010113T0100Z/foo" "foo\n20010113T0100Z" stop cylc-flow-8.7.0/tests/functional/cyclers/00-daily.t0000775000175000017500000000205515264420307022275 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20131231T2300" # shellcheck disable=SC2034 FINALCP="20140103T0000" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/integer_exclusions_advanced/0000775000175000017500000000000015264420307026322 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/integer_exclusions_advanced/reference.log0000664000175000017500000000201115264420307030755 0ustar alastairalastairInitial point: 1 Final point: 16 2/woo -triggered off [] 2/qux -triggered off [] 1/bar -triggered off [] 1/foo -triggered off [] 3/cthulhu -triggered off [] 4/qux -triggered off [] 5/cthulhu -triggered off [] 3/bar -triggered off [] 4/woo -triggered off [] 4/foo -triggered off [] 5/foo -triggered off [] 6/woo -triggered off [] 9/cthulhu -triggered off [] 10/qux -triggered off [] 5/bar -triggered off [] 8/woo -triggered off [] 11/cthulhu -triggered off [] 12/qux -triggered off [] 6/foo -triggered off [] 7/bar -triggered off [] 10/woo -triggered off [] 8/foo -triggered off [] 9/bar -triggered off [] 12/woo -triggered off [] 11/bar -triggered off [] 9/foo -triggered off [] 15/cthulhu -triggered off [] 13/bar -triggered off [] 10/foo -triggered off [] 14/woo -triggered off [] 14/qux -triggered off [] 16/woo -triggered off [] 15/bar -triggered off [] 11/foo -triggered off [] 16/qux -triggered off [] 12/foo -triggered off [] 13/foo -triggered off [] 14/foo -triggered off [] 15/foo -triggered off [] 16/foo -triggered off [] cylc-flow-8.7.0/tests/functional/cyclers/integer_exclusions_advanced/flow.cylc0000664000175000017500000000107615264420307030151 0ustar alastairalastair[scheduler] allow implicit tasks = True [scheduling] initial cycle point = 1 final cycle point = +P15 # = 16 runahead limit = P11 cycling mode = integer [[graph]] R/P1!(2,3,7) = foo P1 ! P2 = woo P1 ! +P1/P2 = bar P1 !(P2,6,8) = qux R/1/P2!P3 = cthulhu [runtime] [[foo]] script = """ cylc__job__wait_cylc_message_started cylc message -- "${CYLC_WORKFLOW_ID}" "${CYLC_TASK_JOB}" 'the cheese is ready' """ [[[outputs]]] out1 = the cheese is ready cylc-flow-8.7.0/tests/functional/cyclers/integer_exclusions_advanced/graph.plain.ref0000664000175000017500000000171315264420307031225 0ustar alastairalastairgraph node "1/bar" "bar\n1" node "3/bar" "bar\n3" node "5/bar" "bar\n5" node "7/bar" "bar\n7" node "9/bar" "bar\n9" node "11/bar" "bar\n11" node "13/bar" "bar\n13" node "15/bar" "bar\n15" node "3/cthulhu" "cthulhu\n3" node "5/cthulhu" "cthulhu\n5" node "9/cthulhu" "cthulhu\n9" node "11/cthulhu" "cthulhu\n11" node "15/cthulhu" "cthulhu\n15" node "1/foo" "foo\n1" node "4/foo" "foo\n4" node "5/foo" "foo\n5" node "6/foo" "foo\n6" node "8/foo" "foo\n8" node "9/foo" "foo\n9" node "10/foo" "foo\n10" node "11/foo" "foo\n11" node "12/foo" "foo\n12" node "13/foo" "foo\n13" node "14/foo" "foo\n14" node "15/foo" "foo\n15" node "16/foo" "foo\n16" node "2/qux" "qux\n2" node "4/qux" "qux\n4" node "10/qux" "qux\n10" node "12/qux" "qux\n12" node "14/qux" "qux\n14" node "16/qux" "qux\n16" node "2/woo" "woo\n2" node "4/woo" "woo\n4" node "6/woo" "woo\n6" node "8/woo" "woo\n8" node "10/woo" "woo\n10" node "12/woo" "woo\n12" node "14/woo" "woo\n14" node "16/woo" "woo\n16" stop cylc-flow-8.7.0/tests/functional/cyclers/monthly/0000775000175000017500000000000015264420307022256 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/monthly/reference.log0000664000175000017500000000260715264420307024724 0ustar alastairalastairInitial point: 20000131T0100 Final point: 2001 20000131T0100Z/foo -triggered off ['19991231T0100Z/foo'] 20000229T0100Z/foo -triggered off ['20000129T0100Z/foo'] 20000131T0100Z/bar -triggered off ['20000131T0100Z/foo'] 20000329T0100Z/foo -triggered off ['20000229T0100Z/foo'] 20000229T0100Z/bar -triggered off ['20000229T0100Z/foo'] 20000429T0100Z/foo -triggered off ['20000329T0100Z/foo'] 20000329T0100Z/bar -triggered off ['20000329T0100Z/foo'] 20000529T0100Z/foo -triggered off ['20000429T0100Z/foo'] 20000429T0100Z/bar -triggered off ['20000429T0100Z/foo'] 20000629T0100Z/foo -triggered off ['20000529T0100Z/foo'] 20000529T0100Z/bar -triggered off ['20000529T0100Z/foo'] 20000729T0100Z/foo -triggered off ['20000629T0100Z/foo'] 20000629T0100Z/bar -triggered off ['20000629T0100Z/foo'] 20000829T0100Z/foo -triggered off ['20000729T0100Z/foo'] 20000729T0100Z/bar -triggered off ['20000729T0100Z/foo'] 20000929T0100Z/foo -triggered off ['20000829T0100Z/foo'] 20000829T0100Z/bar -triggered off ['20000829T0100Z/foo'] 20001029T0100Z/foo -triggered off ['20000929T0100Z/foo'] 20000929T0100Z/bar -triggered off ['20000929T0100Z/foo'] 20001129T0100Z/foo -triggered off ['20001029T0100Z/foo'] 20001029T0100Z/bar -triggered off ['20001029T0100Z/foo'] 20001229T0100Z/foo -triggered off ['20001129T0100Z/foo'] 20001129T0100Z/bar -triggered off ['20001129T0100Z/foo'] 20001229T0100Z/bar -triggered off ['20001229T0100Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/monthly/flow.cylc0000664000175000017500000000036615264420307024106 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20000131T0100Z final cycle point = 2001 [[graph]] P1M = "foo[-P1M] => foo => bar" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/monthly/graph.plain.ref0000664000175000017500000000422515264420307025162 0ustar alastairalastairedge "20000131T0100Z/foo" "20000131T0100Z/bar" edge "20000229T0100Z/foo" "20000229T0100Z/bar" edge "20000229T0100Z/foo" "20000329T0100Z/foo" edge "20000329T0100Z/foo" "20000329T0100Z/bar" edge "20000329T0100Z/foo" "20000429T0100Z/foo" edge "20000429T0100Z/foo" "20000429T0100Z/bar" edge "20000429T0100Z/foo" "20000529T0100Z/foo" edge "20000529T0100Z/foo" "20000529T0100Z/bar" edge "20000529T0100Z/foo" "20000629T0100Z/foo" edge "20000629T0100Z/foo" "20000629T0100Z/bar" edge "20000629T0100Z/foo" "20000729T0100Z/foo" edge "20000729T0100Z/foo" "20000729T0100Z/bar" edge "20000729T0100Z/foo" "20000829T0100Z/foo" edge "20000829T0100Z/foo" "20000829T0100Z/bar" edge "20000829T0100Z/foo" "20000929T0100Z/foo" edge "20000929T0100Z/foo" "20000929T0100Z/bar" edge "20000929T0100Z/foo" "20001029T0100Z/foo" edge "20001029T0100Z/foo" "20001029T0100Z/bar" edge "20001029T0100Z/foo" "20001129T0100Z/foo" edge "20001129T0100Z/foo" "20001129T0100Z/bar" edge "20001129T0100Z/foo" "20001229T0100Z/foo" edge "20001229T0100Z/foo" "20001229T0100Z/bar" graph node "20000131T0100Z/bar" "bar\n20000131T0100Z" node "20000131T0100Z/foo" "foo\n20000131T0100Z" node "20000229T0100Z/bar" "bar\n20000229T0100Z" node "20000229T0100Z/foo" "foo\n20000229T0100Z" node "20000329T0100Z/bar" "bar\n20000329T0100Z" node "20000329T0100Z/foo" "foo\n20000329T0100Z" node "20000429T0100Z/bar" "bar\n20000429T0100Z" node "20000429T0100Z/foo" "foo\n20000429T0100Z" node "20000529T0100Z/bar" "bar\n20000529T0100Z" node "20000529T0100Z/foo" "foo\n20000529T0100Z" node "20000629T0100Z/bar" "bar\n20000629T0100Z" node "20000629T0100Z/foo" "foo\n20000629T0100Z" node "20000729T0100Z/bar" "bar\n20000729T0100Z" node "20000729T0100Z/foo" "foo\n20000729T0100Z" node "20000829T0100Z/bar" "bar\n20000829T0100Z" node "20000829T0100Z/foo" "foo\n20000829T0100Z" node "20000929T0100Z/bar" "bar\n20000929T0100Z" node "20000929T0100Z/foo" "foo\n20000929T0100Z" node "20001029T0100Z/bar" "bar\n20001029T0100Z" node "20001029T0100Z/foo" "foo\n20001029T0100Z" node "20001129T0100Z/bar" "bar\n20001129T0100Z" node "20001129T0100Z/foo" "foo\n20001129T0100Z" node "20001229T0100Z/bar" "bar\n20001229T0100Z" node "20001229T0100Z/foo" "foo\n20001229T0100Z" stop cylc-flow-8.7.0/tests/functional/cyclers/48-icp-cutoff.t0000775000175000017500000000260615264420307023250 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test ICP cutoff bug. . "$(dirname "$0")/test_header" set_test_number 1 init_workflow "${TEST_NAME_BASE}" <<'__WORKFLOW__' [scheduler] cycle point time zone = Z [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] initial cycle point = 20171101T0000Z [[graph]] R1 = foo R1/T00 = foo[^] => qux R1/T06 = foo[^] => qux [runtime] [[foo, qux]] script = true __WORKFLOW__ run_ok "${TEST_NAME_BASE}" cylc play "${WORKFLOW_NAME}" --no-detach purge exit cylc-flow-8.7.0/tests/functional/cyclers/08-offset_final.t0000775000175000017500000000204615264420307023642 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20140101" # shellcheck disable=SC2034 FINALCP="20140102T12" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/0000_rollunder/0000775000175000017500000000000015264420307023231 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/0000_rollunder/flow.cylc0000664000175000017500000000032215264420307025051 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] initial cycle point = 00000101T00 final cycle point = 00000101T00 [[graph]] P1D = "foo[-P1Y] => foo" [runtime] [[foo]] script = true cylc-flow-8.7.0/tests/functional/cyclers/11-r1_initial.t0000775000175000017500000000204615264420307023230 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20140101" # shellcheck disable=SC2034 FINALCP="20140102T12" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/39-exclusions_advanced.t0000775000175000017500000000471715264420307025237 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test intercycle dependencies. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- if [[ -f "$TEST_SOURCE_DIR/${TEST_NAME_BASE}-find.out" ]]; then set_test_number 4 else set_test_number 3 fi #------------------------------------------------------------------------------- CHOSEN_WORKFLOW="$(basename "$0" | sed "s/^.*-\(.*\)\.t/\1/g")" install_workflow "${TEST_NAME_BASE}" "${CHOSEN_WORKFLOW}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-graph" graph_workflow "${WORKFLOW_NAME}" "${WORKFLOW_NAME}.graph.plain" \ "20000101T00Z" "20010102T12Z" cmp_ok "${WORKFLOW_NAME}.graph.plain" "$TEST_SOURCE_DIR/$CHOSEN_WORKFLOW/graph.plain.ref" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- if [[ -f "$TEST_SOURCE_DIR/${TEST_NAME_BASE}-find.out" ]]; then TEST_NAME="${TEST_NAME_BASE}-find" WORKFLOW_DIR="$RUN_DIR/${WORKFLOW_NAME}" (cd "${WORKFLOW_DIR}" && find 'log/job' 'work' -type f) | sort -V >"${TEST_NAME}" cmp_ok "${TEST_NAME}" "$TEST_SOURCE_DIR/${TEST_NAME_BASE}-find.out" fi #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/cyclers/test_header0000777000175000017500000000000015264420307027101 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/r1_multi_start/0000775000175000017500000000000015264420307023535 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/r1_multi_start/reference.log0000664000175000017500000000120715264420307026176 0ustar alastairalastairInitial point: 2014-01-01 Final point: 2014-01-04 20140101T0000Z/cold_foo -triggered off [] 20140101T0000Z/foo_midnight -triggered off ['20131231T0000Z/foo_midnight', '20140101T0000Z/cold_foo'] 20140101T0600Z/foo_dawn -triggered off ['20131231T0600Z/foo_dawn', '20140101T0000Z/cold_foo'] 20140102T0000Z/foo_midnight -triggered off ['20140101T0000Z/foo_midnight'] 20140102T0600Z/foo_dawn -triggered off ['20140101T0600Z/foo_dawn'] 20140103T0000Z/foo_midnight -triggered off ['20140102T0000Z/foo_midnight'] 20140103T0600Z/foo_dawn -triggered off ['20140102T0600Z/foo_dawn'] 20140104T0000Z/foo_midnight -triggered off ['20140103T0000Z/foo_midnight'] cylc-flow-8.7.0/tests/functional/cyclers/r1_multi_start/flow.cylc0000664000175000017500000000064015264420307025360 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 2014-01-01 final cycle point = 2014-01-04 [[graph]] R1 = "cold_foo" R1/T00 = "cold_foo[^] => foo_midnight" R1/T06 = "cold_foo[^] => foo_dawn" T00 = "foo_midnight[-P1D] => foo_midnight" T06 = "foo_dawn[-P1D] => foo_dawn" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/r1_multi_start/graph.plain.ref0000664000175000017500000000164615264420307026445 0ustar alastairalastairedge "20140101T0000Z/cold_foo" "20140101T0000Z/foo_midnight" edge "20140101T0000Z/cold_foo" "20140101T0600Z/foo_dawn" edge "20140101T0000Z/foo_midnight" "20140102T0000Z/foo_midnight" edge "20140101T0600Z/foo_dawn" "20140102T0600Z/foo_dawn" edge "20140102T0000Z/foo_midnight" "20140103T0000Z/foo_midnight" edge "20140102T0600Z/foo_dawn" "20140103T0600Z/foo_dawn" edge "20140103T0000Z/foo_midnight" "20140104T0000Z/foo_midnight" graph node "20140101T0000Z/cold_foo" "cold_foo\n20140101T0000Z" node "20140101T0000Z/foo_midnight" "foo_midnight\n20140101T0000Z" node "20140101T0600Z/foo_dawn" "foo_dawn\n20140101T0600Z" node "20140102T0000Z/foo_midnight" "foo_midnight\n20140102T0000Z" node "20140102T0600Z/foo_dawn" "foo_dawn\n20140102T0600Z" node "20140103T0000Z/foo_midnight" "foo_midnight\n20140103T0000Z" node "20140103T0600Z/foo_dawn" "foo_dawn\n20140103T0600Z" node "20140104T0000Z/foo_midnight" "foo_midnight\n20140104T0000Z" stop cylc-flow-8.7.0/tests/functional/cyclers/r5_initial-integer/0000775000175000017500000000000015264420307024256 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/r5_initial-integer/reference.log0000664000175000017500000000057315264420307026724 0ustar alastairalastairInitial point: 1 Final point: 7 1/xyzzy -triggered off [] 2/xyzzy -triggered off [] 3/xyzzy -triggered off [] 1/bar -triggered off ['1/xyzzy'] 2/bar -triggered off ['2/xyzzy'] 4/xyzzy -triggered off [] 3/bar -triggered off ['3/xyzzy'] 4/bar -triggered off ['4/xyzzy'] 5/xyzzy -triggered off [] 5/bar -triggered off ['5/xyzzy'] 6/xyzzy -triggered off [] 7/xyzzy -triggered off [] cylc-flow-8.7.0/tests/functional/cyclers/r5_initial-integer/flow.cylc0000664000175000017500000000052315264420307026101 0ustar alastairalastair[scheduler] allow implicit tasks = True [scheduling] cycling mode = integer initial cycle point = 1 final cycle point = 7 [[graph]] # xyzzy should not spawn bar in the last two cycle points: P1 = xyzzy # 7 cycles R5//P1 = "xyzzy => bar" # 5 cycles [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/r5_initial-integer/graph.plain.ref0000664000175000017500000000064215264420307027161 0ustar alastairalastairedge "1/xyzzy" "1/bar" edge "2/xyzzy" "2/bar" edge "3/xyzzy" "3/bar" edge "4/xyzzy" "4/bar" edge "5/xyzzy" "5/bar" graph node "1/bar" "bar\n1" node "2/bar" "bar\n2" node "3/bar" "bar\n3" node "4/bar" "bar\n4" node "5/bar" "bar\n5" node "1/xyzzy" "xyzzy\n1" node "2/xyzzy" "xyzzy\n2" node "3/xyzzy" "xyzzy\n3" node "4/xyzzy" "xyzzy\n4" node "5/xyzzy" "xyzzy\n5" node "6/xyzzy" "xyzzy\n6" node "7/xyzzy" "xyzzy\n7" stop cylc-flow-8.7.0/tests/functional/cyclers/daily/0000775000175000017500000000000015264420307021666 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/daily/reference.log0000664000175000017500000000061615264420307024332 0ustar alastairalastairInitial point: 20131231T2300 Final point: 20140103T0000 20131231T2300Z/foo -triggered off ['20131230T2300Z/foo'] 20131231T2300Z/bar -triggered off ['20131231T2300Z/foo'] 20140101T2300Z/foo -triggered off ['20131231T2300Z/foo'] 20140101T2300Z/bar -triggered off ['20140101T2300Z/foo'] 20140102T2300Z/foo -triggered off ['20140101T2300Z/foo'] 20140102T2300Z/bar -triggered off ['20140102T2300Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/daily/flow.cylc0000664000175000017500000000040615264420307023511 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20131231T2300 final cycle point = 20140103T0000 [[graph]] P1D = "foo[-P1D] => foo => bar" [runtime] [[root]] script = touch typing cylc-flow-8.7.0/tests/functional/cyclers/daily/graph.plain.ref0000664000175000017500000000102615264420307024566 0ustar alastairalastairedge "20131231T2300Z/foo" "20131231T2300Z/bar" edge "20131231T2300Z/foo" "20140101T2300Z/foo" edge "20140101T2300Z/foo" "20140101T2300Z/bar" edge "20140101T2300Z/foo" "20140102T2300Z/foo" edge "20140102T2300Z/foo" "20140102T2300Z/bar" graph node "20131231T2300Z/bar" "bar\n20131231T2300Z" node "20131231T2300Z/foo" "foo\n20131231T2300Z" node "20140101T2300Z/bar" "bar\n20140101T2300Z" node "20140101T2300Z/foo" "foo\n20140101T2300Z" node "20140102T2300Z/bar" "bar\n20140102T2300Z" node "20140102T2300Z/foo" "foo\n20140102T2300Z" stop cylc-flow-8.7.0/tests/functional/cyclers/37-exclusions.t0000775000175000017500000000471615264420307023407 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test intercycle dependencies. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- if [[ -f "$TEST_SOURCE_DIR/${TEST_NAME_BASE}-find.out" ]]; then set_test_number 4 else set_test_number 3 fi #------------------------------------------------------------------------------- CHOSEN_WORKFLOW="$(basename "$0" | sed "s/^.*-\(.*\)\.t/\1/g")" install_workflow "${TEST_NAME_BASE}" "${CHOSEN_WORKFLOW}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-graph" graph_workflow "${WORKFLOW_NAME}" "${WORKFLOW_NAME}.graph.plain" \ "20000101T00Z" "20000102T12Z" cmp_ok "${WORKFLOW_NAME}.graph.plain" "$TEST_SOURCE_DIR/$CHOSEN_WORKFLOW/graph.plain.ref" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- if [[ -f "$TEST_SOURCE_DIR/${TEST_NAME_BASE}-find.out" ]]; then TEST_NAME="${TEST_NAME_BASE}-find" WORKFLOW_DIR="$RUN_DIR/${WORKFLOW_NAME}" (cd "$WORKFLOW_DIR" && find 'log/job' 'work' -type f) | sort -V >"${TEST_NAME}" cmp_ok "${TEST_NAME}" "$TEST_SOURCE_DIR/${TEST_NAME_BASE}-find.out" fi #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/cyclers/r1_final/0000775000175000017500000000000015264420307022257 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/r1_final/reference.log0000664000175000017500000000022515264420307024717 0ustar alastairalastairInitial point: 20140101 Final point: 20140102T12 20140102T1200Z/foo -triggered off [] 20140102T1200Z/final_foo -triggered off ['20140102T1200Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/r1_final/flow.cylc0000664000175000017500000000036315264420307024104 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20140101 final cycle point = 20140102T12 [[graph]] R1/P0D = "foo => final_foo" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/r1_final/graph.plain.ref0000664000175000017500000000025415264420307025161 0ustar alastairalastairedge "20140102T1200Z/foo" "20140102T1200Z/final_foo" graph node "20140102T1200Z/final_foo" "final_foo\n20140102T1200Z" node "20140102T1200Z/foo" "foo\n20140102T1200Z" stop cylc-flow-8.7.0/tests/functional/cyclers/15-subhourly.t0000775000175000017500000000205515264420307023235 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20131231T2300" # shellcheck disable=SC2034 FINALCP="20140101T0050" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/32-rmany_reverse.t0000775000175000017500000000205515264420307024061 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="2015-02-21T00" # shellcheck disable=SC2034 FINALCP="2015-02-21T18" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/yearly/0000775000175000017500000000000015264420307022071 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/yearly/reference.log0000664000175000017500000000061015264420307024527 0ustar alastairalastairInitial point: 2005023T12 Final point: 2008001T11 20050123T1200Z/foo -triggered off ['20040123T1200Z/foo'] 20050123T1200Z/bar -triggered off ['20050123T1200Z/foo'] 20060123T1200Z/foo -triggered off ['20050123T1200Z/foo'] 20060123T1200Z/bar -triggered off ['20060123T1200Z/foo'] 20070123T1200Z/foo -triggered off ['20060123T1200Z/foo'] 20070123T1200Z/bar -triggered off ['20070123T1200Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/yearly/flow.cylc0000664000175000017500000000037015264420307023714 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 2005023T12 final cycle point = 2008001T11 [[graph]] P1Y = "foo[-P1Y] => foo => bar" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/yearly/graph.plain.ref0000664000175000017500000000102615264420307024771 0ustar alastairalastairedge "20050123T1200Z/foo" "20050123T1200Z/bar" edge "20050123T1200Z/foo" "20060123T1200Z/foo" edge "20060123T1200Z/foo" "20060123T1200Z/bar" edge "20060123T1200Z/foo" "20070123T1200Z/foo" edge "20070123T1200Z/foo" "20070123T1200Z/bar" graph node "20050123T1200Z/bar" "bar\n20050123T1200Z" node "20050123T1200Z/foo" "foo\n20050123T1200Z" node "20060123T1200Z/bar" "bar\n20060123T1200Z" node "20060123T1200Z/foo" "foo\n20060123T1200Z" node "20070123T1200Z/bar" "bar\n20070123T1200Z" node "20070123T1200Z/foo" "foo\n20070123T1200Z" stop cylc-flow-8.7.0/tests/functional/cyclers/06-multiweekly.t0000775000175000017500000000204315264420307023551 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="1000W011" # shellcheck disable=SC2034 FINALCP="1000W064" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/rnone_reverse/0000775000175000017500000000000015264420307023440 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/rnone_reverse/reference.log0000664000175000017500000000067715264420307026113 0ustar alastairalastairInitial point: 20150221T0000Z Final point: 20150221T1800Z 20150221T0000Z/foo -triggered off [] 20150221T0600Z/foo -triggered off [] 20150221T1200Z/foo -triggered off [] 20150221T0000Z/nonstop -triggered off ['20150221T0000Z/foo'] 20150221T0600Z/nonstop -triggered off ['20150221T0600Z/foo'] 20150221T1200Z/nonstop -triggered off ['20150221T1200Z/foo'] 20150221T1800Z/foo -triggered off [] 20150221T1800Z/stop -triggered off ['20150221T1800Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/rnone_reverse/flow.cylc0000664000175000017500000000045415264420307025266 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 2015-02-21T00 final cycle point = 2015-02-21T18 [[graph]] PT6H = foo R1/P0D = foo => stop R/PT6H/-PT6H = foo => nonstop [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/rnone_reverse/graph.plain.ref0000664000175000017500000000115615264420307026344 0ustar alastairalastairedge "20150221T0000Z/foo" "20150221T0000Z/nonstop" edge "20150221T0600Z/foo" "20150221T0600Z/nonstop" edge "20150221T1200Z/foo" "20150221T1200Z/nonstop" edge "20150221T1800Z/foo" "20150221T1800Z/stop" graph node "20150221T0000Z/foo" "foo\n20150221T0000Z" node "20150221T0000Z/nonstop" "nonstop\n20150221T0000Z" node "20150221T0600Z/foo" "foo\n20150221T0600Z" node "20150221T0600Z/nonstop" "nonstop\n20150221T0600Z" node "20150221T1200Z/foo" "foo\n20150221T1200Z" node "20150221T1200Z/nonstop" "nonstop\n20150221T1200Z" node "20150221T1800Z/foo" "foo\n20150221T1800Z" node "20150221T1800Z/stop" "stop\n20150221T1800Z" stop cylc-flow-8.7.0/tests/functional/cyclers/common0000775000175000017500000000357715264420307022016 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . . "$(dirname "$0")/test_header" if [[ -f "$TEST_SOURCE_DIR/${TEST_NAME_BASE}-find.out" ]]; then set_test_number 4 else set_test_number 3 fi CHOSEN_WORKFLOW="$(basename "$0" | sed "s/^.*-\(.*\)\.t/\1/g")" install_workflow "${TEST_NAME_BASE}" "${CHOSEN_WORKFLOW}" TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-graph" graph_workflow "${WORKFLOW_NAME}" "${WORKFLOW_NAME}.graph.plain" \ -- "${INITIALCP}" "${FINALCP}" cmp_ok "${WORKFLOW_NAME}.graph.plain" "$TEST_SOURCE_DIR/$CHOSEN_WORKFLOW/graph.plain.ref" TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" if [[ -f "$TEST_SOURCE_DIR/${TEST_NAME_BASE}-find.out" ]]; then TEST_NAME="${TEST_NAME_BASE}-find" WORKFLOW_RUN_DIR="$RUN_DIR/${WORKFLOW_NAME}" { (cd "${WORKFLOW_RUN_DIR}" && find 'log/job' -type f) (cd "${WORKFLOW_RUN_DIR}" && find 'work' -type f) } | sort -V >"${TEST_NAME}" cmp_ok "${TEST_NAME}" "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}-find.out" fi purge cylc-flow-8.7.0/tests/functional/cyclers/01-hourly.t0000775000175000017500000000205015264420307022511 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20000229T2000" # shellcheck disable=SC2034 FINALCP="20000301" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/25-aeon.t0000775000175000017500000000210315264420307022116 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="-13800000000-02-29T05:30" # shellcheck disable=SC2034 FINALCP="+05400000000-12-31T23:59" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/aeon/0000775000175000017500000000000015264420307021506 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/aeon/reference.log0000664000175000017500000000242115264420307024146 0ustar alastairalastairInitial point: -138000000000229T0530Z Final point: +054000000001231T2359Z -138000000000229T0530Z/big_bang -triggered off [] -138000000000229T0530Z/count_the_aeons -triggered off ['-138000000000229T0530Z/big_bang'] -128000000000229T0530Z/count_the_aeons -triggered off [] -118000000000229T0530Z/count_the_aeons -triggered off [] -108000000000229T0530Z/count_the_aeons -triggered off [] -098000000000229T0530Z/count_the_aeons -triggered off [] -088000000000229T0530Z/count_the_aeons -triggered off [] -078000000000229T0530Z/count_the_aeons -triggered off [] -068000000000229T0530Z/count_the_aeons -triggered off [] -058000000000229T0530Z/count_the_aeons -triggered off [] -048000000000229T0530Z/count_the_aeons -triggered off [] -038000000000229T0530Z/count_the_aeons -triggered off [] -028000000000229T0530Z/count_the_aeons -triggered off [] -018000000000229T0530Z/count_the_aeons -triggered off [] -008000000000229T0530Z/count_the_aeons -triggered off [] +002000000000229T0530Z/count_the_aeons -triggered off [] +012000000000229T0530Z/count_the_aeons -triggered off [] +022000000000229T0530Z/count_the_aeons -triggered off [] +032000000000229T0530Z/count_the_aeons -triggered off [] +042000000000229T0530Z/count_the_aeons -triggered off [] +052000000000229T0530Z/count_the_aeons -triggered off [] cylc-flow-8.7.0/tests/functional/cyclers/aeon/flow.cylc0000664000175000017500000000064515264420307023336 0ustar alastairalastair[scheduler] cycle point num expanded year digits = 7 UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = -13800000000-02-29T05:30 # Big Bang final cycle point = +05400000000-12-31T23:59 # Sun leaves main sequence [[graph]] R1 = "big_bang => count_the_aeons" +P1000000000Y/P1000000000Y = "count_the_aeons" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/aeon/graph.plain.ref0000664000175000017500000000360515264420307024413 0ustar alastairalastairedge "-138000000000229T0530Z/big_bang" "-138000000000229T0530Z/count_the_aeons" graph node "+002000000000229T0530Z/count_the_aeons" "count_the_aeons\n+002000000000229T0530Z" node "+012000000000229T0530Z/count_the_aeons" "count_the_aeons\n+012000000000229T0530Z" node "+022000000000229T0530Z/count_the_aeons" "count_the_aeons\n+022000000000229T0530Z" node "+032000000000229T0530Z/count_the_aeons" "count_the_aeons\n+032000000000229T0530Z" node "+042000000000229T0530Z/count_the_aeons" "count_the_aeons\n+042000000000229T0530Z" node "+052000000000229T0530Z/count_the_aeons" "count_the_aeons\n+052000000000229T0530Z" node "-008000000000229T0530Z/count_the_aeons" "count_the_aeons\n-008000000000229T0530Z" node "-018000000000229T0530Z/count_the_aeons" "count_the_aeons\n-018000000000229T0530Z" node "-028000000000229T0530Z/count_the_aeons" "count_the_aeons\n-028000000000229T0530Z" node "-038000000000229T0530Z/count_the_aeons" "count_the_aeons\n-038000000000229T0530Z" node "-048000000000229T0530Z/count_the_aeons" "count_the_aeons\n-048000000000229T0530Z" node "-058000000000229T0530Z/count_the_aeons" "count_the_aeons\n-058000000000229T0530Z" node "-068000000000229T0530Z/count_the_aeons" "count_the_aeons\n-068000000000229T0530Z" node "-078000000000229T0530Z/count_the_aeons" "count_the_aeons\n-078000000000229T0530Z" node "-088000000000229T0530Z/count_the_aeons" "count_the_aeons\n-088000000000229T0530Z" node "-098000000000229T0530Z/count_the_aeons" "count_the_aeons\n-098000000000229T0530Z" node "-108000000000229T0530Z/count_the_aeons" "count_the_aeons\n-108000000000229T0530Z" node "-118000000000229T0530Z/count_the_aeons" "count_the_aeons\n-118000000000229T0530Z" node "-128000000000229T0530Z/count_the_aeons" "count_the_aeons\n-128000000000229T0530Z" node "-138000000000229T0530Z/big_bang" "big_bang\n-138000000000229T0530Z" node "-138000000000229T0530Z/count_the_aeons" "count_the_aeons\n-138000000000229T0530Z" stop cylc-flow-8.7.0/tests/functional/cyclers/14-r5_initial.t0000775000175000017500000000204615264420307023237 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20140101" # shellcheck disable=SC2034 FINALCP="20140101T18" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/34-r1_initial_immortal.t0000775000175000017500000000205115264420307025135 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20140101" # shellcheck disable=SC2034 FINALCP="20140110T0000Z" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/exclusions/0000775000175000017500000000000015264420307022760 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/exclusions/reference.log0000664000175000017500000000165415264420307025427 0ustar alastairalastairInitial point: 20000101T0000Z Final point: 20000102T1200Z 20000101T0000Z/quux -triggered off [] 20000101T0000Z/start -triggered off [] 20000101T0000Z/baz -triggered off [] 20000101T0000Z/qux -triggered off [] 20000101T0000Z/nip -triggered off [] 20000101T1200Z/baz -triggered off [] 20000101T1200Z/nip -triggered off [] 20000101T1000Z/quux -triggered off [] 20000101T0000Z/foo -triggered off ['20000101T0000Z/start'] 20000101T1500Z/quux -triggered off [] 20000101T1800Z/nip -triggered off [] 20000101T2000Z/quux -triggered off [] 20000102T0000Z/nip -triggered off [] 20000102T0000Z/qux -triggered off [] 20000102T0000Z/bar -triggered off [] 20000102T0000Z/baz -triggered off [] 20000102T0100Z/quux -triggered off [] 20000102T0600Z/nip -triggered off [] 20000102T0600Z/quux -triggered off [] 20000102T1200Z/pub -triggered off [] 20000102T1200Z/qux -triggered off [] 20000102T1200Z/nip -triggered off [] 20000102T1100Z/quux -triggered off [] cylc-flow-8.7.0/tests/functional/cyclers/exclusions/flow.cylc0000664000175000017500000000131115264420307024577 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20000101T00Z final cycle point = 20000102T12Z [[graph]] R1 = start => foo # Don't run at the initial cycle point # Also test whitespace tolerance T00 ! ^ = bar # Don't run at the final cycle point PT12H!$ = baz # Don't run at a specific datetime PT12H!20000101T1200Z = qux # Don't run at a datetime with an offset R/PT6H!20000101T05Z+PT1H = nip # Don't run a multiple specific datetimes PT5H!(20000101T05Z,20000102T05Z) = quux R1/$ = pub [runtime] [[root]] script = echo success cylc-flow-8.7.0/tests/functional/cyclers/exclusions/graph.plain.ref0000664000175000017500000000223615264420307025664 0ustar alastairalastairedge "20000101T0000Z/start" "20000101T0000Z/foo" graph node "20000101T0000Z/baz" "baz\n20000101T0000Z" node "20000101T0000Z/foo" "foo\n20000101T0000Z" node "20000101T0000Z/nip" "nip\n20000101T0000Z" node "20000101T0000Z/quux" "quux\n20000101T0000Z" node "20000101T0000Z/qux" "qux\n20000101T0000Z" node "20000101T0000Z/start" "start\n20000101T0000Z" node "20000101T1000Z/quux" "quux\n20000101T1000Z" node "20000101T1200Z/baz" "baz\n20000101T1200Z" node "20000101T1200Z/nip" "nip\n20000101T1200Z" node "20000101T1500Z/quux" "quux\n20000101T1500Z" node "20000101T1800Z/nip" "nip\n20000101T1800Z" node "20000101T2000Z/quux" "quux\n20000101T2000Z" node "20000102T0000Z/bar" "bar\n20000102T0000Z" node "20000102T0000Z/baz" "baz\n20000102T0000Z" node "20000102T0000Z/nip" "nip\n20000102T0000Z" node "20000102T0000Z/qux" "qux\n20000102T0000Z" node "20000102T0100Z/quux" "quux\n20000102T0100Z" node "20000102T0600Z/nip" "nip\n20000102T0600Z" node "20000102T0600Z/quux" "quux\n20000102T0600Z" node "20000102T1100Z/quux" "quux\n20000102T1100Z" node "20000102T1200Z/nip" "nip\n20000102T1200Z" node "20000102T1200Z/pub" "pub\n20000102T1200Z" node "20000102T1200Z/qux" "qux\n20000102T1200Z" stop cylc-flow-8.7.0/tests/functional/cyclers/r1_initial/0000775000175000017500000000000015264420307022617 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/r1_initial/reference.log0000664000175000017500000000053215264420307025260 0ustar alastairalastairInitial point: 20140101 Final point: 20140102T12 20140101T0000Z/cold_foo -triggered off [] 20140101T0000Z/foo -triggered off ['20131231T1200Z/foo', '20140101T0000Z/cold_foo'] 20140101T1200Z/foo -triggered off ['20140101T0000Z/foo'] 20140102T0000Z/foo -triggered off ['20140101T1200Z/foo'] 20140102T1200Z/foo -triggered off ['20140102T0000Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/r1_initial/flow.cylc0000664000175000017500000000042315264420307024441 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20140101 final cycle point = 20140102T12 [[graph]] R1 = "cold_foo => foo" PT12H = "foo[-PT12H] => foo" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/r1_initial/graph.plain.ref0000664000175000017500000000070615264420307025523 0ustar alastairalastairedge "20140101T0000Z/cold_foo" "20140101T0000Z/foo" edge "20140101T0000Z/foo" "20140101T1200Z/foo" edge "20140101T1200Z/foo" "20140102T0000Z/foo" edge "20140102T0000Z/foo" "20140102T1200Z/foo" graph node "20140101T0000Z/cold_foo" "cold_foo\n20140101T0000Z" node "20140101T0000Z/foo" "foo\n20140101T0000Z" node "20140101T1200Z/foo" "foo\n20140101T1200Z" node "20140102T0000Z/foo" "foo\n20140102T0000Z" node "20140102T1200Z/foo" "foo\n20140102T1200Z" stop cylc-flow-8.7.0/tests/functional/cyclers/31-rnone_reverse.t0000775000175000017500000000205515264420307024053 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="2015-02-21T00" # shellcheck disable=SC2034 FINALCP="2015-02-21T18" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/04-multihourly.t0000775000175000017500000000206315264420307023573 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20000131T0100Z" # shellcheck disable=SC2034 FINALCP="20000202T0600+0600" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cyclers/integer1/0000775000175000017500000000000015264420307022302 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/integer1/reference.log0000664000175000017500000000305615264420307024747 0ustar alastairalastair10/bar -triggered off ['10/foo', '11/woo'] 13/bar -triggered off ['13/foo', '14/woo'] 1/bar -triggered off ['1/foo', '2/woo'] 4/bar -triggered off ['4/foo', '5/woo'] 7/bar -triggered off ['7/foo', '8/woo'] 16/baz -triggered off [] Final point: 16 10/foo -triggered off ['10/seq', '11/woo', '7/foo', '9/woo'] 13/foo -triggered off ['10/foo', '12/woo', '13/seq', '14/woo'] 1/foo -triggered off ['-2/foo', '0/woo', '1/seq', '2/woo'] 4/foo -triggered off ['1/foo', '3/woo', '4/seq', '5/woo'] 7/foo -triggered off ['4/foo', '6/woo', '7/seq', '8/woo'] Initial point: 1 1/nang -triggered off [] 12/ning -triggered off [] 16/ning -triggered off [] 4/ning -triggered off [] 2/nong -triggered off [] 8/nong -triggered off [] 10/on_toast -triggered off ['7/foo'] 13/on_toast -triggered off ['10/foo'] 16/on_toast -triggered off ['13/foo'] 1/on_toast -triggered off ['-2/foo'] 4/on_toast -triggered off ['1/foo'] 7/on_toast -triggered off ['4/foo'] 16/quux -triggered off [] 8/quux -triggered off [] 13/qux -triggered off ['10/foo'] 7/qux -triggered off ['4/foo'] 10/seq -triggered off ['7/seq'] 13/seq -triggered off ['10/seq'] 16/seq -triggered off ['13/seq'] 1/seq -triggered off [] 4/seq -triggered off ['1/seq'] 7/seq -triggered off ['4/seq'] 1/wibble -triggered off [] 7/wobble -triggered off ['1/wibble'] 11/woo -triggered off [] 12/woo -triggered off [] 14/woo -triggered off [] 15/woo -triggered off [] 2/woo -triggered off [] 3/woo -triggered off [] 5/woo -triggered off [] 6/woo -triggered off [] 8/woo -triggered off [] 9/woo -triggered off [] 16/wubble -triggered off ['7/wobble'] cylc-flow-8.7.0/tests/functional/cyclers/integer1/flow.cylc0000664000175000017500000000230615264420307024126 0ustar alastairalastair[scheduler] allow implicit tasks = True [scheduling] initial cycle point = 1 final cycle point = +P15 # = 16 runahead limit = P11 cycling mode = integer [[special tasks]] sequential = seq [[graph]] R1 = wibble R1/+P6 = wibble[^] => wobble R1/P0 = wobble[^+P6] => wubble P3 = """ seq => foo # sequential task foo[-P3] => foo # prev instance trigger foo => bar # plain trigger woo[+P1] => foo # prev cycle woo[+P1] => bar woo[-P1] => foo # next cycle foo[-P3]:out1 => on_toast # message outputs """ +P1/P3 = woo R/+P2/P3 = woo R/7/P6 = foo[-P3] => qux # every second cycle R1/$ = baz R/P4!8 = ning R/P4!(4,12) = quux # Multiple exclusion points R1/^ = nang R/+P1/P6!14 = nong [runtime] [[root]] script = true [[foo]] script = """ cylc__job__wait_cylc_message_started cylc message -- "${CYLC_WORKFLOW_ID}" "${CYLC_TASK_JOB}" "the cheese is ready" """ [[[outputs]]] out1 = "the cheese is ready" cylc-flow-8.7.0/tests/functional/cyclers/integer1/graph.plain.ref0000664000175000017500000000424315264420307025206 0ustar alastairalastairedge "1/foo" "1/bar" edge "1/foo" "4/foo" edge "1/foo" "4/on_toast" edge "4/foo" "4/bar" edge "4/foo" "7/foo" edge "4/foo" "7/on_toast" edge "4/foo" "7/qux" edge "7/foo" "7/bar" edge "7/foo" "10/foo" edge "7/foo" "10/on_toast" edge "10/foo" "10/bar" edge "10/foo" "13/foo" edge "10/foo" "13/on_toast" edge "10/foo" "13/qux" edge "13/foo" "13/bar" edge "13/foo" "16/foo" edge "13/foo" "16/on_toast" edge "16/foo" "16/bar" edge "1/seq" "1/foo" edge "4/seq" "4/foo" edge "7/seq" "7/foo" edge "10/seq" "10/foo" edge "13/seq" "13/foo" edge "16/seq" "16/foo" edge "1/wibble" "7/wobble" edge "7/wobble" "16/wubble" edge "2/woo" "1/bar" edge "2/woo" "1/foo" edge "3/woo" "4/foo" edge "5/woo" "4/bar" edge "5/woo" "4/foo" edge "6/woo" "7/foo" edge "8/woo" "7/bar" edge "8/woo" "7/foo" edge "9/woo" "10/foo" edge "11/woo" "10/bar" edge "11/woo" "10/foo" edge "12/woo" "13/foo" edge "14/woo" "13/bar" edge "14/woo" "13/foo" edge "15/woo" "16/foo" edge "17/woo" "16/bar" edge "17/woo" "16/foo" graph node "1/bar" "bar\n1" node "4/bar" "bar\n4" node "7/bar" "bar\n7" node "10/bar" "bar\n10" node "13/bar" "bar\n13" node "16/bar" "bar\n16" node "16/baz" "baz\n16" node "1/foo" "foo\n1" node "4/foo" "foo\n4" node "7/foo" "foo\n7" node "10/foo" "foo\n10" node "13/foo" "foo\n13" node "16/foo" "foo\n16" node "1/nang" "nang\n1" node "4/ning" "ning\n4" node "12/ning" "ning\n12" node "16/ning" "ning\n16" node "2/nong" "nong\n2" node "8/nong" "nong\n8" node "1/on_toast" "on_toast\n1" node "4/on_toast" "on_toast\n4" node "7/on_toast" "on_toast\n7" node "10/on_toast" "on_toast\n10" node "13/on_toast" "on_toast\n13" node "16/on_toast" "on_toast\n16" node "8/quux" "quux\n8" node "16/quux" "quux\n16" node "7/qux" "qux\n7" node "13/qux" "qux\n13" node "1/seq" "seq\n1" node "4/seq" "seq\n4" node "7/seq" "seq\n7" node "10/seq" "seq\n10" node "13/seq" "seq\n13" node "16/seq" "seq\n16" node "1/wibble" "wibble\n1" node "7/wobble" "wobble\n7" node "2/woo" "woo\n2" node "3/woo" "woo\n3" node "5/woo" "woo\n5" node "6/woo" "woo\n6" node "8/woo" "woo\n8" node "9/woo" "woo\n9" node "11/woo" "woo\n11" node "12/woo" "woo\n12" node "14/woo" "woo\n14" node "15/woo" "woo\n15" node "17/woo" "woo\n17" node "16/wubble" "wubble\n16" stop cylc-flow-8.7.0/tests/functional/cyclers/r1_restricted/0000775000175000017500000000000015264420307023336 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cyclers/r1_restricted/reference.log0000664000175000017500000000077215264420307026005 0ustar alastairalastairInitial point: 20130808T0000Z Final point: 20130808T1800Z 20130808T0000Z/setup_foo -triggered off [] 20130808T0000Z/foo -triggered off ['20130808T0000Z/setup_foo'] 20130808T0600Z/foo -triggered off ['20130808T0000Z/foo'] 20130808T1200Z/foo -triggered off ['20130808T0600Z/foo'] 20130808T0600Z/bar -triggered off ['20130808T0600Z/foo'] 20130808T1200Z/bar -triggered off ['20130808T1200Z/foo'] 20130808T1800Z/foo -triggered off ['20130808T1200Z/foo'] 20130808T1800Z/bar -triggered off ['20130808T1800Z/foo'] cylc-flow-8.7.0/tests/functional/cyclers/r1_restricted/flow.cylc0000664000175000017500000000044215264420307025161 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20130808T00 final cycle point = 20130808T18 [[graph]] R1 = "setup_foo => foo" +PT6H/PT6H = "foo[-PT6H] => foo => bar" [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/cyclers/r1_restricted/graph.plain.ref0000664000175000017500000000134615264420307026243 0ustar alastairalastairedge "20130808T0000Z/foo" "20130808T0600Z/foo" edge "20130808T0000Z/setup_foo" "20130808T0000Z/foo" edge "20130808T0600Z/foo" "20130808T0600Z/bar" edge "20130808T0600Z/foo" "20130808T1200Z/foo" edge "20130808T1200Z/foo" "20130808T1200Z/bar" edge "20130808T1200Z/foo" "20130808T1800Z/foo" edge "20130808T1800Z/foo" "20130808T1800Z/bar" graph node "20130808T0000Z/foo" "foo\n20130808T0000Z" node "20130808T0000Z/setup_foo" "setup_foo\n20130808T0000Z" node "20130808T0600Z/bar" "bar\n20130808T0600Z" node "20130808T0600Z/foo" "foo\n20130808T0600Z" node "20130808T1200Z/bar" "bar\n20130808T1200Z" node "20130808T1200Z/foo" "foo\n20130808T1200Z" node "20130808T1800Z/bar" "bar\n20130808T1800Z" node "20130808T1800Z/foo" "foo\n20130808T1800Z" stop cylc-flow-8.7.0/tests/functional/cyclers/13-r5_final.t0000775000175000017500000000204615264420307022676 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # variables used in the sourced common script below. # shellcheck disable=SC2034 INITIALCP="20140101" # shellcheck disable=SC2034 FINALCP="20140102T12" source "$(dirname "$0")"/common cylc-flow-8.7.0/tests/functional/cylc-kill/0000775000175000017500000000000015264420307021003 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-kill/04-handlers/0000775000175000017500000000000015264420307023024 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-kill/04-handlers/reference.log0000664000175000017500000000023615264420307025466 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off [] 1/killer -triggered off ['1/a', '1/b'] 1/end -triggered off ['1/a', '1/b', '1/c'] cylc-flow-8.7.0/tests/functional/cylc-kill/04-handlers/flow.cylc0000664000175000017500000000177115264420307024655 0ustar alastairalastair[scheduler] allow implicit tasks = True [[events]] expected task failures = 1/a, 1/b, 1/c stall timeout = PT0S abort on stall timeout = True [scheduling] [[graph]] R1 = """ a:started => killer a:failed => end b:submitted? => killer b:submit-failed? => end c:submit-failed? => end c:submitted? => nope """ [runtime] [[a, b, c]] [[[events]]] failed handlers = echo %(id)s submission failed handlers = echo %(id)s [[a]] script = sleep 40 [[b]] platform = old_street [[c]] platform = $(sleep 40; echo localhost) [[killer]] script = """ cylc kill "$CYLC_WORKFLOW_ID//1/a" "$CYLC_WORKFLOW_ID//1/b" cylc__job__poll_grep_workflow_log -E '1\/c.* => preparing' cylc kill "$CYLC_WORKFLOW_ID//1/c" """ [[end]] script = cylc stop "$CYLC_WORKFLOW_ID" --now --now cylc-flow-8.7.0/tests/functional/cylc-kill/00-multi-hosts-compat.t0000775000175000017500000000347515264420307025172 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test kill multiple jobs on localhost and a remote host export REQUIRE_PLATFORM='loc:remote comms:tcp' . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" RUN_DIR="$RUN_DIR/${WORKFLOW_NAME}" LOG="${RUN_DIR}/log/scheduler/log" sed -n 's/^.*\(cylc jobs-kill\)/\1/p' "${LOG}" | sort -u >'edited-workflow-log' sort >'edited-workflow-log-ref' <<__LOG__ cylc jobs-kill --debug -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 1/remote-1/01 1/remote-2/01 cylc jobs-kill --debug -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 1/local-1/01 1/local-2/01 1/local-3/01 __LOG__ cmp_ok 'edited-workflow-log' 'edited-workflow-log-ref' purge exit cylc-flow-8.7.0/tests/functional/cylc-kill/02-submitted.t0000775000175000017500000000171115264420307023412 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test kill running jobs only . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/cylc-kill/04-handlers.t0000664000175000017500000000317315264420307023215 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test event handlers when killing running/submitted/preparing tasks. # Any downstream tasks that depend on the `:submit-fail`/`:fail` outputs # SHOULD run. # Handlers for the `submission failed`/`failed` events SHOULD run. export REQUIRE_PLATFORM='runner:at' . "$(dirname "$0")/test_header" set_test_number 5 # Create platform that ensures job will be in submitted state for long enough create_test_global_config '' " [platforms] [[old_street]] job runner = at job runner command template = at now + 5 minutes hosts = localhost install target = localhost " install_and_validate reftest_run grep_workflow_log_ok "grep-a" "[(('event-handler-00', 'failed'), 1) out] 1/a" -F for task in b c; do grep_workflow_log_ok "grep-${task}" \ "[(('event-handler-00', 'submission failed'), 1) out] 1/${task}" -F done purge cylc-flow-8.7.0/tests/functional/cylc-kill/05-retries.t0000664000175000017500000000250515264420307023071 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test retries when killing running/submitted/preparing tasks. # As killing tasks puts them in the held state, retries should NOT go ahead. export REQUIRE_PLATFORM='runner:at' . "$(dirname "$0")/test_header" set_test_number 2 # Create platform that ensures job will be in submitted state for long enough create_test_global_config '' " [platforms] [[old_street]] job runner = at job runner command template = at now + 5 minutes hosts = localhost install target = localhost " install_and_validate reftest_run purge cylc-flow-8.7.0/tests/functional/cylc-kill/01-multi-hosts.t0000775000175000017500000000347615264420307023713 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test kill multiple jobs on localhost and a remote host export REQUIRE_PLATFORM='loc:remote comms:tcp' . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" RUN_DIR="$RUN_DIR/${WORKFLOW_NAME}" LOG="${RUN_DIR}/log/scheduler/log" sed -n 's/^.*\(cylc jobs-kill\)/\1/p' "${LOG}" | sort -u >'edited-workflow-log' sort >'edited-workflow-log-ref' <<__LOG__ cylc jobs-kill --debug -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 1/remote-1/01 1/remote-2/01 cylc jobs-kill --debug -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 1/local-1/01 1/local-2/01 1/local-3/01 __LOG__ cmp_ok 'edited-workflow-log' 'edited-workflow-log-ref' purge exit cylc-flow-8.7.0/tests/functional/cylc-kill/test_header0000777000175000017500000000000015264420307027320 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-kill/01-multi-hosts/0000775000175000017500000000000015264420307023511 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-kill/01-multi-hosts/reference.log0000664000175000017500000000053015264420307026150 0ustar alastairalastairInitial point: 1 Final point: 1 1/remote-1 -triggered off [] 1/local-1 -triggered off ['1/remote-1', '1/remote-2'] 1/local-2 -triggered off ['1/remote-1', '1/remote-2'] 1/remote-2 -triggered off [] 1/local-3 -triggered off ['1/remote-1', '1/remote-2'] 1/killer -triggered off ['1/local-1', '1/local-2', '1/local-3', '1/remote-1', '1/remote-2'] cylc-flow-8.7.0/tests/functional/cylc-kill/01-multi-hosts/flow.cylc0000664000175000017500000000145715264420307025343 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [[events]] expected task failures = 1/local-1, 1/local-2, 1/local-3, 1/remote-1, 1/remote-2 [scheduling] [[graph]] R1 = """ # wait for the remote tasks to start before triggering the # local ones in order to factor out remote-init time remote-1:start & remote-2:start => local-1 & local-2 & local-3 KILLABLE:start-all => killer """ [runtime] [[KILLABLE]] script = sleep 60 [[local-1, local-2, local-3]] inherit = KILLABLE [[remote-1, remote-2]] inherit = KILLABLE platform = {{ CYLC_TEST_PLATFORM }} [[killer]] script = """ cylc kill "${CYLC_WORKFLOW_ID}//1/KILLABLE" cylc stop "${CYLC_WORKFLOW_ID}" """ cylc-flow-8.7.0/tests/functional/cylc-kill/02-submitted/0000775000175000017500000000000015264420307023222 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-kill/02-submitted/reference.log0000664000175000017500000000042015264420307025657 0ustar alastairalastairInitial point: 1 Final point: 1 1/killable-1 -triggered off [] 1/killable-2 -triggered off [] 1/killable-3 -triggered off [] 1/killer -triggered off ['1/killable-1', '1/killable-2', '1/killable-3'] 1/stopper -triggered off ['1/killable-1', '1/killable-2', '1/killable-3'] cylc-flow-8.7.0/tests/functional/cylc-kill/02-submitted/flow.cylc0000664000175000017500000000200715264420307025044 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [[events]] expected task failures = 1/killable-1, 1/killable-2, 1/killable-3 [scheduling] [[graph]] R1 = """ KILLABLE:submit-all? => killer KILLABLE:submit-fail-all? => stopper """ [runtime] [[KILLABLE]] init-script=""" echo "CYLC_JOB_PID=$$" >>"$0.status" sleep 60 """ script=true [[killable-1, killable-2, killable-3]] inherit=KILLABLE [[killer]] script=""" cylc__job__wait_cylc_message_started cylc__job__poll_grep_workflow_log -F '1/killable-1 -triggered' cylc__job__poll_grep_workflow_log -F '1/killable-2 -triggered' cylc__job__poll_grep_workflow_log -F '1/killable-3 -triggered' # (Avoid killing myself if my started message hasn't arrived yet:) cylc kill "${CYLC_WORKFLOW_ID}//*/killable*:submitted" """ [[stopper]] script=cylc stop "${CYLC_WORKFLOW_ID}" cylc-flow-8.7.0/tests/functional/cylc-kill/00-multi-hosts-compat/0000775000175000017500000000000015264420307024771 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-kill/00-multi-hosts-compat/reference.log0000664000175000017500000000053015264420307027430 0ustar alastairalastairInitial point: 1 Final point: 1 1/remote-1 -triggered off [] 1/local-1 -triggered off ['1/remote-1', '1/remote-2'] 1/local-2 -triggered off ['1/remote-1', '1/remote-2'] 1/remote-2 -triggered off [] 1/local-3 -triggered off ['1/remote-1', '1/remote-2'] 1/killer -triggered off ['1/local-1', '1/local-2', '1/local-3', '1/remote-1', '1/remote-2'] cylc-flow-8.7.0/tests/functional/cylc-kill/00-multi-hosts-compat/flow.cylc0000664000175000017500000000145615264420307026622 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [[events]] expected task failures = 1/local-1, 1/local-2, 1/local-3, 1/remote-1, 1/remote-2 [scheduling] [[graph]] R1 = """ # wait for the remote tasks to start before triggering the # local ones in order to factor out remote-init time remote-1:start & remote-2:start => local-1 & local-2 & local-3 KILLABLE:start-all => killer """ [runtime] [[KILLABLE]] script = sleep 60 [[local-1, local-2, local-3]] inherit = KILLABLE [[remote-1, remote-2]] inherit = KILLABLE platform = {{CYLC_TEST_PLATFORM}} [[killer]] script = """ cylc kill "${CYLC_WORKFLOW_ID}//1/KILLABLE" cylc stop "${CYLC_WORKFLOW_ID}" """ cylc-flow-8.7.0/tests/functional/cylc-kill/05-retries/0000775000175000017500000000000015264420307022702 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-kill/05-retries/reference.log0000664000175000017500000000022515264420307025342 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off [] 1/killer -triggered off ['1/a', '1/b'] 1/end -triggered off ['1/killer'] cylc-flow-8.7.0/tests/functional/cylc-kill/05-retries/flow.cylc0000664000175000017500000000214115264420307024523 0ustar alastairalastair[scheduler] allow implicit tasks = True [[events]] expected task failures = 1/a, 1/b, 1/c stall timeout = PT0S abort on stall timeout = True [scheduling] [[graph]] R1 = """ a:started & b:submitted? => killer => end # Should not get these outputs: a:failed? | b:submit-failed? | c:submit-failed? => nope """ [runtime] [[a]] script = sleep 40 execution retry delays = PT0S [[b]] platform = old_street submission retry delays = PT0S [[c]] platform = $(sleep 40; echo localhost) submission retry delays = PT0S [[killer]] script = """ cylc kill "$CYLC_WORKFLOW_ID//1/a" "$CYLC_WORKFLOW_ID//1/b" cylc__job__poll_grep_workflow_log -E '1\/c.* => preparing' cylc kill "$CYLC_WORKFLOW_ID//1/c" """ [[end]] script = """ for task in a b c; do cylc__job__poll_grep_workflow_log -E "1\/${task}.* retrying" done cylc stop "$CYLC_WORKFLOW_ID" --now --now """ cylc-flow-8.7.0/tests/functional/cylc-show/0000775000175000017500000000000015264420307021030 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-show/05-complex/0000775000175000017500000000000015264420307022721 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-show/05-complex/flow.cylc0000664000175000017500000000107415264420307024546 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [scheduling] initial cycle point = 2000 final cycle point = +P1D runahead limit = P1 [[graph]] T00 = a & b & (c | (d & e)) & f[-P1D] => f [runtime] [[root]] script = true [[a, b, c, d, e]] [[f]] script = """ # show myself. sleep 4 cylc show "${CYLC_WORKFLOW_ID}//${CYLC_TASK_CYCLE_POINT}/f" >>{{ TEST_OUTPUT_PATH }} cylc show --list-prereqs "${CYLC_WORKFLOW_ID}//${CYLC_TASK_CYCLE_POINT}/f" >>{{ TEST_OUTPUT_PATH }} """ cylc-flow-8.7.0/tests/functional/cylc-show/05-complex.t0000664000175000017500000000701215264420307023106 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test cylc show for a basic task. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" TEST_SHOW_OUTPUT_PATH="$PWD/${TEST_NAME_BASE}-show.stdout" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate \ --set="TEST_OUTPUT_PATH='$TEST_SHOW_OUTPUT_PATH'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" run_ok "${TEST_NAME}" cylc play \ --no-detach --set="TEST_OUTPUT_PATH='$TEST_SHOW_OUTPUT_PATH'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-show" contains_ok "${TEST_SHOW_OUTPUT_PATH}" << '__OUT__' title: (not given) description: (not given) URL: (not given) state: running prerequisites: ('⨯': not satisfied) ✓ 1 & 2 & (3 | (4 & 5)) & 0 ✓ 0 = 19991231T0000Z/f succeeded ✓ 1 = 20000101T0000Z/a succeeded ✓ 2 = 20000101T0000Z/b succeeded ✓ 3 = 20000101T0000Z/c succeeded ✓ 4 = 20000101T0000Z/d succeeded ✓ 5 = 20000101T0000Z/e succeeded outputs: ('⨯': not completed) ⨯ 20000101T0000Z/f expired ✓ 20000101T0000Z/f submitted ⨯ 20000101T0000Z/f submit-failed ✓ 20000101T0000Z/f started ⨯ 20000101T0000Z/f succeeded ⨯ 20000101T0000Z/f failed output completion: incomplete ⨯ ┆ succeeded 19991231T0000Z/f succeeded 20000101T0000Z/a succeeded 20000101T0000Z/b succeeded 20000101T0000Z/c succeeded 20000101T0000Z/d succeeded 20000101T0000Z/e succeeded title: (not given) description: (not given) URL: (not given) state: running prerequisites: ('⨯': not satisfied) ✓ 1 & 2 & (3 | (4 & 5)) & 0 ✓ 0 = 20000101T0000Z/f succeeded ✓ 1 = 20000102T0000Z/a succeeded ✓ 2 = 20000102T0000Z/b succeeded ✓ 3 = 20000102T0000Z/c succeeded ✓ 4 = 20000102T0000Z/d succeeded ✓ 5 = 20000102T0000Z/e succeeded outputs: ('⨯': not completed) ⨯ 20000102T0000Z/f expired ✓ 20000102T0000Z/f submitted ⨯ 20000102T0000Z/f submit-failed ✓ 20000102T0000Z/f started ⨯ 20000102T0000Z/f succeeded ⨯ 20000102T0000Z/f failed output completion: incomplete ⨯ ┆ succeeded 20000101T0000Z/f succeeded 20000102T0000Z/a succeeded 20000102T0000Z/b succeeded 20000102T0000Z/c succeeded 20000102T0000Z/d succeeded 20000102T0000Z/e succeeded __OUT__ #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/cylc-show/06-past-present-future.t0000664000175000017500000000413315264420307025376 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test cylc-show prerequisites for past, future, and present tasks. . "$(dirname "$0")/test_header" set_test_number 6 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" # First run: task c shuts the scheduler down then fails. TEST_NAME="${TEST_NAME_BASE}-run-1" workflow_run_ok "${TEST_NAME}" cylc play --no-detach --debug "${WORKFLOW_NAME}" # Restart: task kick-c triggers off of c's failure, and re-triggers c. # Then, c runs `cylc show` on tasks b, c, and d. TEST_NAME="${TEST_NAME_BASE}-run-2" workflow_run_ok "${TEST_NAME}" cylc play --no-detach --debug "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-show.past" contains_ok "$WORKFLOW_RUN_DIR/show-b.txt" <<__END__ state: succeeded prerequisites: (n/a for past tasks) __END__ # Note trigger command satisfies off-flow prerequisites. TEST_NAME="${TEST_NAME_BASE}-show.present" contains_ok "${WORKFLOW_RUN_DIR}/show-c.txt" <<__END__ state: running prerequisites: ('⨯': not satisfied) ✓ 1/b succeeded __END__ TEST_NAME="${TEST_NAME_BASE}-show.future" contains_ok "${WORKFLOW_RUN_DIR}/show-d.txt" <<__END__ state: waiting prerequisites: ('⨯': not satisfied) ⨯ 1/c succeeded __END__ purge cylc-flow-8.7.0/tests/functional/cylc-show/test_header0000777000175000017500000000000015264420307027345 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-show/clock-triggered-non-utc-mode/0000775000175000017500000000000015264420307026400 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-show/clock-triggered-non-utc-mode/reference-untz.log0000664000175000017500000000056315264420307032043 0ustar alastairalastairRun mode: live Initial point: 20140808T0900$TZ_OFFSET_BASIC Final point: 20140808T0900$TZ_OFFSET_BASIC Cold Start 20140808T0900$TZ_OFFSET_BASIC 20140808T0900$TZ_OFFSET_BASIC/woo -triggered off [] 20140808T0900$TZ_OFFSET_BASIC/foo -triggered off ['20140808T0900$TZ_OFFSET_BASIC/woo'] 20140808T0900$TZ_OFFSET_BASIC/show -triggered off ['20140808T0900$TZ_OFFSET_BASIC/woo'] cylc-flow-8.7.0/tests/functional/cylc-show/clock-triggered-non-utc-mode/flow.cylc0000664000175000017500000000074615264420307030232 0ustar alastairalastair#!jinja2 [scheduler] cycle point time zone = {{ TZ_OFFSET_BASIC }} [scheduling] initial cycle point = 20140808T09 final cycle point = 20140808T09 [[special tasks]] clock-trigger = foo(PT5M) [[graph]] PT1H = "woo => foo & show" [runtime] [[woo]] script = true [[foo]] script = sleep 10 [[show]] script = """ sleep 4 cylc show "$CYLC_WORKFLOW_ID//20140808T0900{{ TZ_OFFSET_BASIC }}/foo" >{{ TEST_SHOW_OUTPUT_PATH }} """ cylc-flow-8.7.0/tests/functional/cylc-show/06-past-present-future/0000775000175000017500000000000015264420307025210 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-show/06-past-present-future/flow.cylc0000664000175000017500000000166715264420307027045 0ustar alastairalastair[scheduler] allow implicit tasks = True [[events]] inactivity timeout = PT1M abort on inactivity timeout = True [scheduling] [[graph]] R1 = """ a => b => c? => d c:fail? => kick-c """ [runtime] # First run: c stops the scheduler then fails. # On restart, kick-c retriggers c to run 'cylc show'. [[kick-c]] script = cylc trigger "$CYLC_WORKFLOW_ID//1/c" [[c]] script = """ if ((CYLC_TASK_SUBMIT_NUMBER == 1)); then cylc stop --now --max-polls=10 --interval=1 $CYLC_WORKFLOW_ID false else # Allow time for c submission => running sleep 2 cylc show "$CYLC_WORKFLOW_ID//1/b" >> $CYLC_WORKFLOW_RUN_DIR/show-b.txt cylc show "$CYLC_WORKFLOW_ID//1/c" >> $CYLC_WORKFLOW_RUN_DIR/show-c.txt cylc show "$CYLC_WORKFLOW_ID//1/d" >> $CYLC_WORKFLOW_RUN_DIR/show-d.txt fi """ cylc-flow-8.7.0/tests/functional/cylc-reinstall/0000775000175000017500000000000015264420307022045 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-reinstall/00-simple.t0000664000175000017500000000441415264420307023743 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test workflow re-installation . "$(dirname "$0")/test_header" set_test_number 8 # Test basic cylc reinstall, named run given TEST_NAME="${TEST_NAME_BASE}-basic-named-run" make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 run_ok "${TEST_NAME}" cylc install cmp_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE} __OUT__ run_ok "basic-reinstall" cylc reinstall "${RND_WORKFLOW_NAME}/run1" REINSTALL_LOG="$(find "${RND_WORKFLOW_RUNDIR}/run1/log/install" -type f -name '*reinstall.log')" grep_ok "REINSTALLED ${RND_WORKFLOW_NAME}/run1 from ${RND_WORKFLOW_SOURCE}" "${REINSTALL_LOG}" popd || exit 1 purge_rnd_workflow #------------------------------------------------------------------------------ # Test install/reinstall executed from elsewhere in filesystem TEST_NAME="${TEST_NAME_BASE}-named-flow" make_rnd_workflow pushd "${TMPDIR}" || exit 1 run_ok "${TEST_NAME}-install" cylc install "${RND_WORKFLOW_SOURCE}" --workflow-name="${RND_WORKFLOW_NAME}" cmp_ok "${TEST_NAME}-install.stdout" <<__OUT__ INSTALLED ${RND_WORKFLOW_NAME}/run1 from ${RND_WORKFLOW_SOURCE} __OUT__ run_ok "${TEST_NAME}-reinstall" cylc reinstall "${RND_WORKFLOW_NAME}/run1" cmp_ok "${TEST_NAME}-reinstall.stdout" <<__OUT__ REINSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE} Successfully reinstalled. Done __OUT__ popd || exit 1 purge_rnd_workflow rm -rf "${RUN_DIR:?}/${RND_WORKFLOW_NAME}/" exit cylc-flow-8.7.0/tests/functional/cylc-reinstall/01-file-transfer.t0000664000175000017500000000673315264420307025222 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test workflow re-installation file transfer . "$(dirname "$0")/test_header" if ! command -v 'tree' >'/dev/null'; then skip_all '"tree" command not available' fi set_test_number 13 # Need to override any symlink dirs set in global.cylc: create_test_global_config "" " [install] [[symlink dirs]] [[[localhost]]] run = log = work = share = share/cycle = " # Test cylc install copies files to run dir successfully. TEST_NAME="${TEST_NAME_BASE}-basic" make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 mkdir .git .svn dir1 dir2-be-removed touch .git/file1 .svn/file1 dir1/file1 dir2-be-removed/file1 file1 file2 echo '01-*' > .cylcignore # filter out test files run_ok "${TEST_NAME}" cylc install tree_excludes='*.log|01-file-transfer*|rose-suite*.conf|opt' tree -a -v -I "${tree_excludes}" --charset=ascii --noreport "${RND_WORKFLOW_RUNDIR}/run1" > '01-file-transfer-basic-tree.out' cmp_ok '01-file-transfer-basic-tree.out' <<__OUT__ ${RND_WORKFLOW_RUNDIR}/run1 |-- dir1 | \`-- file1 |-- dir2-be-removed | \`-- file1 |-- file1 |-- file2 |-- flow.cylc \`-- log \`-- install __OUT__ contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE} __OUT__ run_ok "${TEST_NAME}" cylc install mkdir new_dir touch new_dir/new_file1 new_dir/new_file2 rm -rf dir2-be-removed file2 run_ok "${TEST_NAME}-reinstall" cylc reinstall "${RND_WORKFLOW_NAME}/run2" REINSTALL_LOG="$(find "${RND_WORKFLOW_RUNDIR}/run2/log/install" -type f -name '*reinstall.log')" grep_ok '(del\.|\*deleting) dir2-be-removed/file1' "$REINSTALL_LOG" -E grep_ok '(del\.|\*deleting) file2' "$REINSTALL_LOG" -E grep_ok '(send|recv) new_dir/$' "$REINSTALL_LOG" -E grep_ok '(send|recv) new_dir/new_file1' "$REINSTALL_LOG" -E grep_ok '(send|recv) new_dir/new_file2' "$REINSTALL_LOG" -E tree -a -v -I "${tree_excludes}" --charset=ascii --noreport "${RND_WORKFLOW_RUNDIR}/run2" > 'after-reinstall-run2-tree.out' cmp_ok 'after-reinstall-run2-tree.out' <<__OUT__ ${RND_WORKFLOW_RUNDIR}/run2 |-- dir1 | \`-- file1 |-- file1 |-- flow.cylc |-- log | \`-- install \`-- new_dir |-- new_file1 \`-- new_file2 __OUT__ contains_ok "${TEST_NAME}-reinstall.stdout" <<__OUT__ REINSTALLED $RND_WORKFLOW_NAME/run2 from ${RND_WORKFLOW_SOURCE} __OUT__ # Test cylc reinstall affects only named run, i.e. run1 should be unaffected in this case tree -a -v -I "${tree_excludes}" --charset=ascii --noreport "${RND_WORKFLOW_RUNDIR}/run1" > 'after-reinstall-run1-tree.out' cmp_ok 'after-reinstall-run1-tree.out' '01-file-transfer-basic-tree.out' popd || exit 1 purge_rnd_workflow exit cylc-flow-8.7.0/tests/functional/cylc-reinstall/03-external-files.t0000664000175000017500000000322215264420307025373 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test "cylc reinstall" handling of external (non source) files in the run dir. . "$(dirname "$0")/test_header" set_test_number 6 make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 # 1. Should delete an external file. # Install source files. run_ok "install-normal" cylc install # Install an "external" file. EXT_FILE=$HOME/cylc-run/${RND_WORKFLOW_NAME}/run1/external touch "${EXT_FILE}" run_ok "reinstall-normal" cylc reinstall "${RND_WORKFLOW_NAME}" exists_fail "${EXT_FILE}" # 2. Unless it is listed in .cylcignore. basename "${EXT_FILE}" > .cylcignore run_ok "install-cylcignore" cylc install EXT_FILE=$HOME/cylc-run/${RND_WORKFLOW_NAME}/run2/external touch "${EXT_FILE}" run_ok "reinstall-cylcignore" cylc reinstall "${RND_WORKFLOW_NAME}" exists_ok "${EXT_FILE}" purge_rnd_workflow cylc-flow-8.7.0/tests/functional/cylc-reinstall/02-failures.t0000664000175000017500000001040415264420307024262 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test workflow reinstallation expected failures . "$(dirname "$0")/test_header" set_test_number 23 # Test fail no workflow run dir TEST_NAME="${TEST_NAME_BASE}-reinstall-no-run-dir" make_rnd_workflow run_ok "${TEST_NAME}-install" cylc install "${RND_WORKFLOW_SOURCE}" --workflow-name="${RND_WORKFLOW_NAME}" --no-run-name rm -rf "${RND_WORKFLOW_RUNDIR}" run_fail "${TEST_NAME}-reinstall" cylc reinstall "${RND_WORKFLOW_NAME}" cmp_ok "${TEST_NAME}-reinstall.stderr" <<__ERR__ InputError: Workflow ID not found: ${RND_WORKFLOW_RUNDIR#*/cylc-run/} (Directory not found: ${RND_WORKFLOW_RUNDIR}) __ERR__ purge_rnd_workflow # Test fail no workflow source dir TEST_NAME="${TEST_NAME_BASE}-reinstall-no-source-dir" make_rnd_workflow run_ok "${TEST_NAME}-install" cylc install "${RND_WORKFLOW_SOURCE}" --workflow-name="${RND_WORKFLOW_NAME}" --no-run-name rm -rf "${RND_WORKFLOW_SOURCE}" run_fail "${TEST_NAME}-reinstall" cylc reinstall "${RND_WORKFLOW_NAME}" cmp_ok "${TEST_NAME}-reinstall.stderr" <<__ERR__ WorkflowFilesError: Workflow source dir is not accessible: "${RND_WORKFLOW_SOURCE}". Restore the source or modify the "${RND_WORKFLOW_RUNDIR}/_cylc-install/source" symlink to continue. __ERR__ purge_rnd_workflow # Test fail no flow.cylc or suite.rc file TEST_NAME="${TEST_NAME_BASE}-no-flow-file" make_rnd_workflow run_ok "${TEST_NAME}-install" cylc install "${RND_WORKFLOW_SOURCE}" --workflow-name="${RND_WORKFLOW_NAME}" --no-run-name rm -f "${RND_WORKFLOW_SOURCE}/flow.cylc" run_fail "${TEST_NAME}" cylc reinstall "${RND_WORKFLOW_NAME}" grep_ok 'No flow.cylc file in' "${TEST_NAME}.stderr" purge_rnd_workflow # Test fail both flow.cylc and suite.rc file TEST_NAME="${TEST_NAME_BASE}-both-flow-and-suite-file" make_rnd_workflow run_ok "${TEST_NAME}-install" cylc install "${RND_WORKFLOW_SOURCE}" --workflow-name="${RND_WORKFLOW_NAME}" --no-run-name touch "${RND_WORKFLOW_SOURCE}/suite.rc" run_fail "${TEST_NAME}" cylc reinstall "${RND_WORKFLOW_NAME}" cmp_ok "${TEST_NAME}.stderr" <<__ERR__ WorkflowFilesError: Both flow.cylc and suite.rc are present in ${RND_WORKFLOW_SOURCE}. \ Please remove suite.rc. For more information visit: \ https://cylc.github.io/cylc-doc/stable/html/7-to-8/major-changes/compatibility-mode.html __ERR__ purge_rnd_workflow # Test source dir can not contain '_cylc-install, log, share, work' dirs for cylc reinstall for DIR in 'work' 'share' 'log' '_cylc-install'; do TEST_NAME="${TEST_NAME_BASE}-${DIR}-forbidden-in-source" make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 cylc install --no-run-name --workflow-name="${RND_WORKFLOW_NAME}" mkdir ${DIR} run_fail "${TEST_NAME}" cylc reinstall "${RND_WORKFLOW_NAME}" cmp_ok "${TEST_NAME}.stderr" <<__ERR__ WorkflowFilesError: ${RND_WORKFLOW_NAME} installation failed - ${DIR} exists in source directory. __ERR__ purge_rnd_workflow popd || exit 1 done # Test cylc reinstall raises error when no source dir. TEST_NAME="${TEST_NAME_BASE}-reinstall-no-source-raises-error" make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 run_ok "${TEST_NAME}-install" cylc install --no-run-name --workflow-name="${RND_WORKFLOW_NAME}" pushd "${RND_WORKFLOW_RUNDIR}" || exit 1 rm -rf "_cylc-install" run_fail "${TEST_NAME}-reinstall" cylc reinstall "$RND_WORKFLOW_NAME" cmp_ok "${TEST_NAME}-reinstall.stderr" <<__ERR__ WorkflowFilesError: "${RND_WORKFLOW_NAME}" was not installed with cylc install. __ERR__ popd || exit 1 popd || exit 1 purge_rnd_workflow exit cylc-flow-8.7.0/tests/functional/cylc-reinstall/test_header0000777000175000017500000000000015264420307030362 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/lib/0000775000175000017500000000000015264420307017666 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/lib/bash/0000775000175000017500000000000015264420307020603 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/lib/bash/test_header0000664000175000017500000011013315264420307023014 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # NAME # test_header # # SYNOPSIS # . $CYLC_REPO_DIR/tests/lib/bash/test_header # # DESCRIPTION # Interface for constructing tests under a TAP harness (the "prove" # command). # # FUNCTIONS # set_test_number N # echo a total number of tests for TAP to read. # ok TEST_NAME # echo a TAP OK message for TEST_NAME. # fail TEST_NAME # echo a TAP fail message for TEST_NAME. If $CYLC_TEST_DEBUG is set, # cat $TEST_NAME.stderr to stderr. # run_ok TEST_NAME COMMAND ... # Run COMMAND with any following options/arguments and store stdout # and stderr in TEST_NAME.stdout and TEST_NAME.stderr. # This is expected to have a return code of 0, which ok's the test. # run_fail TEST_NAME COMMAND ... # Run COMMAND with any following options/arguments and store stdout # and stderr in TEST_NAME.stdout and TEST_NAME.stderr. # This is expected to have a non-zero return code, which ok's the test. # run_graphql_ok TEST_NAME WORKFLOW JSON # Send a graphql query to the workflow, extract the result from # ${TEST_NAME}.stdout # cmp_ok FILE_TEST FILE_CONTROL # Compare FILE_TEST with a file or stdin given by FILE_CONTROL (stdin # if FILE_CONTROL is "-" or missing). By default, it uses "diff -u" to # compare files. However, if an alternate command such as "xxdiff -D" # is desirable (e.g. for debugging), # "export CYLC_TEST_DIFF_CMD=xxdiff -D". # cmp_ok_re FILE_TEST FILE_CONTROL # Match a file against a regex file line by line. # Reads in the test and control files line by line and checks the test # line re.match'es the control line. # cmp_json TEST_NAME FILE_TEST [FILE_CONTROL] # Alternative implementation to cmp_json_ok, compare two JSON files # and display any differences as a unified(ish) diff: # * Works with nested dictionaries. # * Impervious to dictionary order. # contains_ok FILE_TEST [FILE_CONTROL] # Make sure that each line in FILE_TEST is present in FILE_CONTROL # (stdin if FILE_CONTROL is "-" or missing). # grep_ok PATTERN FILE [$OPTS] # Run "grep [$OPTS] -q -e PATTERN FILE". # grep_workflow_log_ok TEST_NAME PATTERN [$OPTS] # Run "grep [$OPTS] -s -e PATTERN ". # named_grep_ok NAME PATTERN FILE [$OPTS] # Run grep_ok with a custom test name. # OPTS: put grep options like '-E' (extended regex) at end of line. # grep_fail PATTERN FILE [$OPTS] # Run "grep [$OPTS] -q -e PATTERN FILE", expect no match. # count_ok PATTERN FILE COUNT # Test that PATTERN occurs in exactly COUNT lines of FILE. # exists_ok FILE # Test that FILE exists # exists_fail FILE # Test that FILE does not exist # init_workflow TEST_NAME [SOURCE] [[RUN_NUMS]] # Create a workflow from SOURCE's "flow.cylc" called: # "cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR##*tests/}/${TEST_NAME}" # Provides WORKFLOW_NAME and WORKFLOW_RUN_DIR variables. # RUN_NUMS (defaults to false): If false run cylc install --no-rundir # install_workflow TEST_NAME SOURCE [RUN_NUMS] # Same as init_workflow, but SOURCE must be a directory containing a # "flow.cylc" file. # log_scan TEST_NAME FILE ATTEMPTS DELAY PATTERN... # Monitor FILE polling every DELAY seconds for maximum of ATTEMPTS # tries grepping for each PATTERN in turn. Tests will only pass if the # PATTERNs appear in FILE in the correct order. Runs one test per # pattern, each prefixed by TEST_NAME. # set LOG_SCAN_GREP_OPTS in the environment, e.g. "-E" for "grep -E" # make_rnd_workflow # Create a randomly-named workflow source directory. # mock_smtpd_init # Start a mock SMTP server daemon for testing. Write host:port setting # to the variable TEST_SMTPD_HOST. Write pid of daemon to # TEST_SMTPD_PID. Write log to TEST_SMTPD_LOG. # mock_smtpd_kill # Kill the mock SMTP server daemon process. # poll COMMAND # Run COMMAND in 1 second intervals for a minute until COMMAND returns # a zero value, or exit 1 (abort test) with a timeout message. # poll_grep ... # Shorthand for 'poll grep -s ...'. # poll_grep_workflow_log ... # Shorthand for 'poll_grep ... "${WORKFLOW_RUN_DIR}/log/scheduler/log"' # poll_pid_done PID # Poll until process with PID is done. # poll_workflow_running # Shorthand for 'poll test -e "${WORKFLOW_RUN_DIR}/.service/contact"' # poll_workflow_stopped # Shorthand for 'poll "!" test -e "${WORKFLOW_RUN_DIR}/.service/contact"'. # Note that you might want to call poll_workflow_running before this as # it may return before contact file is created in the first place. # get_workflow_uuid # Extract the UUID from the contact file or echo '' if not present. # poll_workflow_restart [TIMEOUT [UUID]] # Wait for the workflow to restart. # Provide the UUID from the current run if workflow is not # guaranteed to be running. # purge [WORKFLOW_NAME [PLATFORM]] # Remove a workflow from the filesystem. # Defaults to removing $WORKFLOW_NAME on $CYLC_TEST_PLATFORM # purge_rnd_workflow # Remove a workflow source dir and rundir. # Typically used with make_rnd_workflow # skip N SKIP_REASON # echo "ok $((++T)) # skip SKIP_REASON" N times. # skip_all SKIP_REASON # echo "1..0 # SKIP SKIP_REASON" and exit. # ssh_install_cylc HOST # Install cylc on a remote host. # reftest [TEST_NAME [SOURCE]] # Install a reference workflow using `install_workflow`, run a validation # test on the workflow and run the reference workflow with `workflow_run_ok`. # Expect 2 OK tests. # install_and_validate # The first part of reftest, to allow separate use. # Expect 1 OK test. # reftest_run # The guts of reftest, to allow separate use. # Expect 1 OK test. # create_test_global_config [PRE [POST]] # Create a new global config file $PWD/etc from global-tests.cylc # with PRE and POST pre- and ap-pended. # PRE and POST are strings. # The global config starts with #!Jinja2 in case Jinja2 is used. # localhost_fqdn # Get the FQDN of the current host using the same mechanism Cylc uses. # get_fqdn [TARGET] # SSH to TARGET and return `hostname -f`. # dump_std TEST_NAME # Dump stdout and stderr of TEST_NAME to the test log dir. # delete_db [WORKFLOW_RUN_DIR] # Delete private/public workflow databases for a cold start. # # VARIABLES # LOG_INDENT # indentation width of secondary log lines # # TODO: document other variables used by test scripts # #------------------------------------------------------------------------------- set -eu shopt -s extglob # For indented secondary log lines; must match the width in cylc.flow.loggingutil export LOG_INDENT=" " FAILURES=0 SIGNALS="EXIT INT" TEST_DIR= TEST_RHOST_CYLC_DIR= FINALLY() { for S in ${SIGNALS}; do trap '' "${S}" done if [[ -n "${TEST_DIR}" ]]; then cd ~ rm -rf "${TEST_DIR}" fi if [[ -n "${TEST_RHOST_CYLC_DIR}" ]]; then # TEST_RHOST_CYLC_DIR is a local variable of this script # shellcheck disable=SC2029 ssh -oBatchMode=yes -oConnectTimeout=5 "${TEST_RHOST_CYLC_DIR%%:*}" \ "rm -fr ${TEST_RHOST_CYLC_DIR#*:}" fi if [[ -n "${TEST_SMTPD_PID:-}" ]]; then kill "${TEST_SMTPD_PID}" fi if ((FAILURES > 0)); then echo -e "\n stdout and stderr stored in: ${TEST_LOG_DIR}" >&2 if "${WORKFLOW_RUN_FAILS}"; then echo -e " workflow logs can be found under: ${WORKFLOW_LOG_DIR}/" >&2 fi fi } for S in ${SIGNALS}; do # Signal as argument to FINALLY # shellcheck disable=SC2064 trap "FINALLY ${S}" "${S}" done TEST_NUMBER=0 if command -v lsof >/dev/null; then HAS_LSOF=true else HAS_LSOF=false fi set_test_number() { echo "1..$1" } ok() { echo "ok $((++TEST_NUMBER)) - $*" } fail() { ((++FAILURES)) echo "not ok $((++TEST_NUMBER)) - $*" if [[ -n "${CYLC_TEST_DEBUG:-}" ]]; then echo >'/dev/tty' echo "${TEST_NAME_BASE} ${TEST_NAME}" >'/dev/tty' cat "${TEST_NAME}.stderr" >'/dev/tty' fi } dump_std() { local TEST_NAME="$1" mkdir -p "${TEST_LOG_DIR}" # directory may not exist if run fails very early if [[ -s "${TEST_NAME}.stdout" ]]; then cp -p "${TEST_NAME}.stdout" "${TEST_LOG_DIR}/${TEST_NAME}.stdout" fi cp -p "${TEST_NAME}.stderr" "${TEST_LOG_DIR}/${TEST_NAME}.stderr" } run_ok() { local TEST_NAME="$1" shift 1 if ! "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then fail "${TEST_NAME}" dump_std "${TEST_NAME}" return fi ok "${TEST_NAME}" } run_fail() { local TEST_NAME="$1" shift 1 if "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then fail "${TEST_NAME}" dump_std "${TEST_NAME}" return fi ok "${TEST_NAME}" } run_graphql_ok () { # this is a thin wrapper to `cylc client` which strips newlines from $2 to # make it valid JSON local TEST_NAME="$1" local WORKFLOW="$2" local JSON="$3" # shellcheck disable=SC2086 run_ok "${TEST_NAME}" cylc client \ "${WORKFLOW}" \ 'graphql' \ < <(echo ${JSON}) } workflow_run_ok() { local TEST_NAME="$1" shift 1 if "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then ok "${TEST_NAME}" return fi fail "${TEST_NAME}" WORKFLOW_RUN_FAILS=true WORKFLOW_LOG_DIR="${WORKFLOW_RUN_DIR}/log/scheduler" mkdir -p "${WORKFLOW_LOG_DIR}" # directory may not exist if run fails very early dump_std "${TEST_NAME}" } workflow_run_fail() { local TEST_NAME="$1" shift 1 if ! "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then ok "${TEST_NAME}" return fi fail "${TEST_NAME}" WORKFLOW_RUN_FAILS=true WORKFLOW_LOG_DIR="${WORKFLOW_RUN_DIR}/log/scheduler" mkdir -p "${WORKFLOW_LOG_DIR}" # directory may not exist if run fails very early dump_std "${TEST_NAME}" } cmp_ok() { local FILE_TEST="$1" local FILE_CONTROL="${2:--}" local TEST_NAME TEST_NAME="$(basename "${FILE_TEST}")-cmp-ok" local DIFF_CMD=${CYLC_TEST_DIFF_CMD:-'diff -u'} if ${DIFF_CMD} "${FILE_CONTROL}" "${FILE_TEST}" >"${TEST_NAME}.stderr" 2>&1 then ok "${TEST_NAME}" return else cat "${TEST_NAME}.stderr" >&2 fi dump_std "$TEST_NAME" fail "${TEST_NAME}" } cmp_ok_re() { local FILE_TEST="$1" local FILE_CONTROL="${2:--}" local TEST_NAME TEST_NAME="$(basename "${FILE_TEST}")-cmp-ok" if python3 -c ' import re import sys def compare(test_file, control_file): for ind, (test_line, control_line) in enumerate( zip(test_file, control_file) ): if test_line is None: sys.stderr.write("Test file is missing lines") sys.exit(1) if control_line is None: sys.stderr.write("Test file has extra lines") sys.exit(1) if not re.match(control_line, test_line): sys.stderr.write( f"Line {ind+1}: \"{test_line}\" does not match /{control_line}/" .replace("\n", "") ) sys.exit(1) test_filename, control_filename = sys.argv[1:3] with open(test_filename, "r") as test_file: if control_filename == "-": compare(test_file, sys.stdin) else: with open(control_filename, "r") as control_file: compare(test_file, control_file) ' "${FILE_TEST}" "${FILE_CONTROL}"; then ok "${TEST_NAME}" else ( echo -e '\n# Test file:' cat "${FILE_TEST}" ) >&2 fail "${TEST_NAME}" fi } cmp_json() { local TEST_NAME="$1" local FILE_TEST="$2" local FILE_CONTROL="${3:--}" if python3 "$CYLC_REPO_DIR/tests/f/lib/python/diffr.py" \ "$(cat "$FILE_CONTROL")" "$(cat "$FILE_TEST")" \ >"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr" then ok "${TEST_NAME}" return else cat "${TEST_NAME}.stderr" >&2 fi dump_std "$TEST_NAME" fail "${TEST_NAME}" } contains_ok() { local FILE_TEST="$1" local FILE_CONTROL="${2:--}" local TEST_NAME TEST_NAME="$(basename "${FILE_TEST}")-contains-ok" LANG=C comm -13 <(LANG=C sort "${FILE_TEST}") <(LANG=C sort "${FILE_CONTROL}") \ 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr" if [[ -s "${TEST_NAME}.stdout" ]]; then mkdir -p "${TEST_LOG_DIR}" { echo 'Missing lines:' cat "${TEST_NAME}.stdout" echo 'Test File:' cat "${FILE_TEST}" } >>"${TEST_NAME}.stderr" cp -p "${TEST_NAME}.stderr" "${TEST_LOG_DIR}/${TEST_NAME}.stderr" fail "${TEST_NAME}" return fi ok "${TEST_NAME}" } count_ok() { local BRE="$1" local FILE="$2" local COUNT="$3" local TEST_NAME NEW_COUNT TEST_NAME="$(basename "${FILE}")-count-ok" NEW_COUNT=$(grep -c "${BRE}" "${FILE}") if (( NEW_COUNT == COUNT )); then ok "${TEST_NAME}" return fi mkdir -p "${TEST_LOG_DIR}" echo "Found ${NEW_COUNT} (not ${COUNT}) of ${BRE} in ${FILE}" \ >"${TEST_LOG_DIR}/${TEST_NAME}.stderr" fail "${TEST_NAME}" } grep_ok() { # (Put extra grep options like '-E' at end of the command line) local BRE="$1" local FILE="$2" shift 2 OPTS="$*" named_grep_ok "$(basename "${FILE}")-grep-ok" "$BRE" "$FILE" "$OPTS" } grep_workflow_log_ok() { local TEST_NAME="$1" local PATTERN="$2" shift 2 OPTS="$*" local LOG_FILE="${WORKFLOW_RUN_DIR}/log/scheduler/log" # shellcheck disable=SC2086 if grep ${OPTS} -s -e "$PATTERN" "$LOG_FILE"; then ok "${TEST_NAME}" return fi mkdir -p "${TEST_LOG_DIR}" { cat <<__ERR__ Can't find: =========== ${PATTERN} =========== in: =========== __ERR__ cat "$LOG_FILE" } >"${TEST_LOG_DIR}/${TEST_NAME}.stderr" fail "${TEST_NAME}" } named_grep_ok() { # (Put extra grep options like '-E' at end of the command line) local TEST_NAME="$1" local BRE="$2" local FILE="$3" shift 3 local OPTS="$*" # shellcheck disable=SC2086 if grep ${OPTS} -q -e "${BRE}" "${FILE}"; then ok "${TEST_NAME}" return fi mkdir -p "${TEST_LOG_DIR}" { cat <<__ERR__ Can't find: =========== ${BRE} =========== in: =========== __ERR__ cat "${FILE}" } >"${TEST_LOG_DIR}/${TEST_NAME}.stderr" fail "${TEST_NAME}" } grep_fail() { local BRE="$1" local FILE="$2" shift 2 local OPTS="$*" local TEST_NAME TEST_NAME="$(basename "${FILE}")-grep-fail" if [[ ! -f "${FILE}" ]]; then fail "${TEST_NAME}-file to be grepped does not exist." return fi # shellcheck disable=SC2086 if grep ${OPTS} -q -e "${BRE}" "${FILE}"; then mkdir -p "${TEST_LOG_DIR}" echo "ERROR: Found ${BRE} in ${FILE}" \ >"${TEST_LOG_DIR}/${TEST_NAME}.stderr" fail "${TEST_NAME}" return fi ok "${TEST_NAME}" } exists_ok() { local FILE="$1" local TEST_NAME TEST_NAME="$(basename "${FILE}")-file-exists-ok" if [[ -e $FILE ]]; then ok "${TEST_NAME}" return fi mkdir -p "${TEST_LOG_DIR}" echo "Does not exist: ${FILE}" >"${TEST_LOG_DIR}/${TEST_NAME}.stderr" fail "${TEST_NAME}" } exists_fail() { local FILE="$1" local TEST_NAME TEST_NAME="$(basename "${FILE}")-file-exists-fail" if [[ ! -e "${FILE}" ]]; then ok "${TEST_NAME}" return fi mkdir -p "${TEST_LOG_DIR}" echo "Exists: ${FILE}" >"${TEST_LOG_DIR}/${TEST_NAME}.stderr" fail "${TEST_NAME}" } graph_workflow() { # Generate a graphviz "plain" format graph of a workflow. local WORKFLOW_NAME="${1}" local OUTPUT_FILE="${2}" shift 2 mkdir -p "$(dirname "${OUTPUT_FILE}")" cylc graph --reference "${WORKFLOW_NAME}" "$@" \ >"${OUTPUT_FILE}" 2>"${OUTPUT_FILE}".err } workflow_id() { local TEST_NAME="${1:-${TEST_NAME_BASE}}" local TEST_SOURCE_BASE="${2:-${TEST_NAME}}" echo "${CYLC_TEST_REG_BASE}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME}" } init_workflow() { local TEST_NAME="$1" local FLOW_CONFIG="${2:--}" local RUN_NUMS="${3:-false}" WORKFLOW_NAME="$(workflow_id "${TEST_NAME}")" WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}" mkdir -p "${TEST_DIR}/${WORKFLOW_NAME}/" cat "${FLOW_CONFIG}" >"${TEST_DIR}/${WORKFLOW_NAME}/flow.cylc" cd "${TEST_DIR}/${WORKFLOW_NAME}" if "${RUN_NUMS}"; then cylc install "${TEST_DIR}/${WORKFLOW_NAME}" --workflow-name="${WORKFLOW_NAME}" else cylc install "${TEST_DIR}/${WORKFLOW_NAME}" --no-run-name --workflow-name="${WORKFLOW_NAME}" fi } install_workflow() { local TEST_NAME="${1:-${TEST_NAME_BASE}}" local TEST_SOURCE_BASE="${2:-${TEST_NAME}}" local RUN_NUMS="${3:-false}" WORKFLOW_NAME="$(workflow_id "${TEST_NAME}" "${TEST_SOURCE_BASE}")" WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}" mkdir -p "${TEST_DIR}/${WORKFLOW_NAME}/" # make source dir cp -r "${TEST_SOURCE_DIR}/${TEST_SOURCE_BASE}/"* "${TEST_DIR}/${WORKFLOW_NAME}/" cd "${TEST_DIR}/${WORKFLOW_NAME}" if "$RUN_NUMS"; then cylc install "${TEST_DIR}/${WORKFLOW_NAME}" --workflow-name="${WORKFLOW_NAME}" else cylc install "${TEST_DIR}/${WORKFLOW_NAME}" --no-run-name --workflow-name="${WORKFLOW_NAME}" fi } log_scan () { local TEST_NAME="$1" local FILE="$2" local REPS=$3 local DELAY=$4 local OPTS=${LOG_SCAN_GREP_OPTS:-} if ${CYLC_TEST_DEBUG:-false}; then local ERR=2 else local ERR=1 fi shift 4 local count=0 local success=false local position=0 local newposition= for pattern in "$@"; do count=$(( count + 1 )) success=false echo -n "scanning for '${pattern:0:30}'" >& $ERR for _ in $(seq 1 "${REPS}"); do echo -n '.' >& $ERR # shellcheck disable=SC2086 newposition=$(grep -n $OPTS "$pattern" "$FILE" | \ tail -n 1 | cut -d ':' -f 1) if (( newposition > position )); then position=$newposition echo ': succeeded' >& $ERR ok "${TEST_NAME}-${count}" success=true break fi sleep "${DELAY}" done shift if ! "${success}"; then echo ': failed' >& $ERR fail "${TEST_NAME}-${count}" if "${CYLC_TEST_DEBUG:-false}"; then cat "${FILE}" >&2 fi mkdir -p "${TEST_LOG_DIR}" ERR_FILE="${TEST_LOG_DIR}/${TEST_NAME}.stderr" echo "ERR_FILE=$ERR_FILE" >&2 echo "Could not find: ${pattern}" > "${ERR_FILE}" echo -e '\n=======================\n' >> "${ERR_FILE}" cat "${FILE}" >> "${ERR_FILE}" skip "$#" return 1 fi done } make_rnd_workflow() { # Create a randomly-named workflow source directory. # Define its run directory. # TODO: export a WORKFLOW_NAME variable so tests can get cylc-install # to install under the regular test hierarchy RND_WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" RND_WORKFLOW_SOURCE="$PWD/${RND_WORKFLOW_NAME}" mkdir -p "${RND_WORKFLOW_SOURCE}" touch "${RND_WORKFLOW_SOURCE}/flow.cylc" RND_WORKFLOW_RUNDIR="${RUN_DIR}/${RND_WORKFLOW_NAME}" export RND_WORKFLOW_NAME export RND_WORKFLOW_SOURCE export RND_WORKFLOW_RUNDIR } # shellcheck disable=2120 # these arguments are intentionally optional purge () { # workflow to remove - defaults to $WORKFLOW_NAME local WORKFLOW_NAME="${1:-$WORKFLOW_NAME}" # job platform to remove it on - defaults to $CYLC_TEST_PLATFORM local PLATFORM="${2:-$CYLC_TEST_PLATFORM}" if ((FAILURES != 0)); then # if tests have failed then don't clean up return 0 fi if [[ -z $WORKFLOW_NAME ]]; then echo 'no flow to purge' >&2 return 1 fi local WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}" # wait for local processes to let go of their file handles if ${HAS_LSOF}; then # NOTE: lsof can hang, so call with "timeout". # NOTE: lsof can raise warnings with some filesystems so ignore stderr # shellcheck disable=SC2034 for try in $(seq 1 5); do if ! grep -q "${WORKFLOW_RUN_DIR}" < <(timeout 5 lsof 2>/dev/null); then break fi sleep 1 done fi # shellcheck disable=SC2016 # shellcheck disable=SC2089 local CMD=' cd '~/cylc-run' && rm -rf "'"$WORKFLOW_NAME"'" && rmdir -p "$(dirname "'"$WORKFLOW_NAME"'")" ' # remove the workflow run directory on the platform if remote if [[ -n "$PLATFORM" && "${PLATFORM}" != _local* ]]; then local HOST HOST="$(get_host_from_platform "$PLATFORM")" # shellcheck disable=SC2029 # we want this to expand client side (ssh "$HOST" "$CMD" 2>/dev/null) || true fi # remove the workflow run directory tree locally - if empty # NOTE: do this after remote purge as this removes the test config too bash -c "$CMD" 2>/dev/null || true } purge_rnd_workflow() { if ((FAILURES != 0)); then # if tests have failed then don't clean up return 0 fi # Remove the workflow source created by make_rnd_workflow(). # And remove its run-directory too. rm -rf "${RND_WORKFLOW_SOURCE}" rm -rf "${RND_WORKFLOW_RUNDIR}" } poll() { local TIMEOUT="$(($(date +%s) + 60))" # wait 1 minute while ! "$@"; do sleep 1 if (($(date +%s) > TIMEOUT)); then echo "ERROR: poll timed out: $*" >&2 exit 1 fi done } poll_grep() { poll grep -s "$@" } poll_grep_workflow_log() { poll grep -s "$@" "${WORKFLOW_RUN_DIR}/log/scheduler/log" } poll_pid_done() { local TIMEOUT="$(($(date +%s) + 60))" # wait 1 minute while ps --no-headers "$@" 1>'/dev/null' 2>&1; do sleep 1 if (($(date +%s) > TIMEOUT)); then echo "ERROR: poll timed out: ! ps --no-headers $*" >&2 exit 1 fi done } poll_workflow_running() { poll test -e "${WORKFLOW_RUN_DIR}/.service/contact" } poll_workflow_stopped() { poll test '!' -e "${WORKFLOW_RUN_DIR}/.service/contact" } get_workflow_uuid() { sed -n 's|CYLC_WORKFLOW_UUID=\(.*\)|\1|p' "${WORKFLOW_RUN_DIR}/.service/contact" 2>/dev/null } poll_workflow_restart() { TIMEOUT="${1:-10}" STEP=1 UUID="${2:-$(get_workflow_uuid)}" TIME=0 while true; do if [[ ${TIME} -gt $TIMEOUT ]]; then return 1 fi if [[ $(get_workflow_uuid) != "${UUID}" ]]; then return 0 else TIME=$(( TIME + STEP )) sleep "${STEP}" fi done } skip() { local N_TO_SKIP="$1" shift 1 local COUNT=0 while ((COUNT++ < N_TO_SKIP)); do echo "ok $((++TEST_NUMBER)) # skip $*" done } skip_all() { echo "1..0 # SKIP $*" exit } skip_macos_gh_actions() { # https://github.com/cylc/cylc-flow/issues/6276 if [[ "${CI:-}" && "$OSTYPE" == "darwin"* ]]; then skip_all "Skipped due to performance issues on GH Actions MacOS runner" fi } ssh_install_cylc() { local RHOST="$1" local RHOST_CYLC_DIR= RHOST_CYLC_DIR=$(_ssh_mkdtemp_cylc_dir "${RHOST}") TEST_RHOST_CYLC_DIR="${RHOST}:${RHOST_CYLC_DIR}" rsync -a '--exclude=*.pyc' "${CYLC_REPO_DIR}"/* "${RHOST}:${RHOST_CYLC_DIR}/" # RHOST_CYLC_DIR is a variable of this function # shellcheck disable=SC2029 ssh -n -oBatchMode=yes -oConnectTimeout=5 "${RHOST}" \ "make -C '${RHOST_CYLC_DIR}' 'version'" 1>'/dev/null' 2>&1 } _ssh_mkdtemp_cylc_dir() { local RHOST="$1" ssh -oBatchMode=yes -oConnectTimeout=5 "${RHOST}" python3 - <<'__PYTHON__' import os from tempfile import mkdtemp print(mkdtemp(dir=os.path.expanduser("~"), prefix="cylc-")) __PYTHON__ } mock_smtpd_init() { # Logic borrowed from Rose local SMTPD_PORT= # Try several ports in case any are in use: for SMTPD_PORT in 8025 8125 8225 8325 8425 8525 8625 8725 8825 8925; do local SMTPD_LOG="${TEST_DIR}/smtpd.log" local SMTPD_HOST="localhost:${SMTPD_PORT}" # Set up fake SMTP server to catch outgoing mail & redirect to log: python3 -u -m 'aiosmtpd' \ --class aiosmtpd.handlers.Debugging stdout \ --debug --nosetuid \ --listen "${SMTPD_HOST}" \ 1>"${SMTPD_LOG}" 2>&1 & # Runs in background local SMTPD_PID="$!" while ! grep -q 'is listening' "${SMTPD_LOG}" 2>'/dev/null' do if ps "${SMTPD_PID}" 1>/dev/null 2>&1; then sleep 1 # Still waiting for fake server to start else # Server process failed rm -f "${SMTPD_LOG}" unset SMTPD_HOST SMTPD_LOG SMTPD_PID break fi done if [[ -n "${SMTPD_PID:-}" ]]; then # Variables used by tests # shellcheck disable=SC2034 TEST_SMTPD_HOST="${SMTPD_HOST}" # shellcheck disable=SC2034 TEST_SMTPD_LOG="${SMTPD_LOG}" TEST_SMTPD_PID="${SMTPD_PID}" break fi done } mock_smtpd_kill() { # Logic borrowed from Rose if [[ -n "${TEST_SMTPD_PID:-}" ]] && ps "${TEST_SMTPD_PID}" >'/dev/null' 2>&1 then kill "${TEST_SMTPD_PID}" wait "${TEST_SMTPD_PID}" 2>/dev/null || true unset TEST_SMTPD_HOST TEST_SMTPD_LOG TEST_SMTPD_PID fi } install_and_validate() { # First part of the reftest function, to allow separate use. # Expect 1 OK test. local TEST_NAME="${1:-${TEST_NAME_BASE}}" install_workflow "$@" run_ok "${TEST_NAME}-validate" cylc validate "${WORKFLOW_NAME}" } reftest_run() { # Guts of the reftest function, to allow separate use. # Expect 1 OK test. local TEST_NAME="${1:-${TEST_NAME_BASE}}-run" if [[ -n "${REFTEST_OPTS:-}" ]]; then # shellcheck disable=SC2086 workflow_run_ok "${TEST_NAME}" \ cylc play --reference-test --debug --no-detach \ ${REFTEST_OPTS} "${WORKFLOW_NAME}" else workflow_run_ok "${TEST_NAME}" \ cylc play --reference-test --debug --no-detach \ "${WORKFLOW_NAME}" fi } reftest() { # Install, validate, run, and purge, a reference test. # Expect 2 OK tests. install_and_validate "$@" reftest_run "$@" # shellcheck disable=SC2119 purge } _get_test_config_file() { # print the path to the most specific global-tests.cylc on the system # (or print nothing if no test configs are present) python3 -c " from pathlib import Path from cylc.flow.cfgspec.globalcfg import get_version_hierarchy for version in reversed(get_version_hierarchy('$(cylc version)')): path = Path('${HOME}/.cylc/flow', version, 'global-tests.cylc') if path.exists(): print(path) break " } create_test_global_config() { # (Documented in file header). local PRE= local POST= if (( $# == 1 )); then PRE=$1 elif (( $# == 2 )); then PRE=$1 POST=$2 elif (( $# > 2 )); then echo 'ERROR, create_test_global_config: too many args' >&2 exit 1 fi # Tidy in case of previous use of this function. rm -fr 'etc' mkdir 'etc' # Start with Jinja2 hashbang. Harmless if not needed. echo "#!Jinja2" >'etc/global.cylc' echo "$PRE" >>'etc/global.cylc' # add defaults cat >>'etc/global.cylc' <<__HERE__ # set a default timeout for all flow runs to avoid hanging tests [scheduler] [[events]] inactivity timeout = PT5M stall timeout = PT5M abort on inactivity timeout = true abort on workflow timeout = true [install] max depth = 5 __HERE__ # add global-tests.cylc USER_TESTS_CONF_FILE="$(_get_test_config_file)" if [[ -n "${USER_TESTS_CONF_FILE}" ]]; then cat "${USER_TESTS_CONF_FILE}" >>'etc/global.cylc' fi # add platform config if [[ -n "${CYLC_TEST_PLATFORM:-}" ]]; then _add_platform_to_test_global_conf "$CYLC_TEST_PLATFORM" fi echo "$POST" >>'etc/global.cylc' export CYLC_CONF_PATH="${PWD}/etc" } _parse_platform_spec () { # the requirement string: e.g. comms:tcp local requirement="$1" local locality='local' local job_runner='*' local file_system='*' local comms='*' for req in ${requirement}; do case "$req" in loc:*) locality="${req/loc:/}" ;; runner:*) job_runner="${req/runner:/}" ;; fs:*) file_system="${req/fs:/}" ;; comms:*) comms="${req/comms:/}" ;; *) echo "Invalid option $req" >&2 return 1 ;; esac done echo "_${locality}_${job_runner}_${file_system}_${comms}" } _get_test_platform () { # the requirement string: e.g. comms:tcp local requirement="$1" local platform local required_platform required_platform="$(_parse_platform_spec "${requirement}")" for platform in "${PLATFORMS[@]}"; do # we are purposefully allowing pattern matching here # shellcheck disable=SC2053 if [[ "$platform" == $required_platform ]]; then CYLC_TEST_PLATFORM="$platform" CYLC_TEST_JOB_RUNNER="$(cut -d '_' -f 3 <<< "$platform")" export CYLC_TEST_PLATFORM CYLC_TEST_JOB_RUNNER return 0 fi done return 1 } _get_test_host () { # extract a test host from the test platform # NOTE: this will cause a failure if the platform is not configured CYLC_TEST_HOST="$(get_host_from_platform "$CYLC_TEST_PLATFORM")" if [[ -z "${CYLC_TEST_HOST}" ]]; then echo "Could not find host for platform $CYLC_TEST_PLATFORM" >&2 exit 1 fi export CYLC_TEST_HOST } _get_test_install_target () { # extract a test host from the test platform # NOTE: this will cause a failure if the platform is not configured CYLC_TEST_INSTALL_TARGET="$( get_install_target_from_platform "$CYLC_TEST_PLATFORM" )" if [[ -z "${CYLC_TEST_INSTALL_TARGET}" ]]; then CYLC_TEST_INSTALL_TARGET="$CYLC_TEST_PLATFORM" fi export CYLC_TEST_INSTALL_TARGET } get_host_from_platform () { local platform="$1" cylc config -i "[platforms][${platform}]hosts" \ | cut -d ',' -f 1 } get_install_target_from_platform () { local platform="$1" cylc config -d -i "[platforms][${platform}]install target" \ | cut -d ',' -f 1 } localhost_fqdn () { python -c " from cylc.flow.hostuserutil import get_fqdn_by_host print(get_fqdn_by_host(None)) " } get_fqdn () { local HOST="${1}" ssh "$HOST" hostname -f } _get_test_platforms () { CYLC_CONF_PATH='' cylc config -i '[platforms]' \ | sed -n 's/\[\[\(_.*\)\]\]/\1/p' } _expand_test_platforms () { # convert string of space separated platforms to array read -r -a PLATFORMS <<< "${1}" local ret=() local pattern local platform local all_platforms # get list of all configured test platforms # NOTE: when we drop bash3 convert to # mapfile -t all_platforms < <(_get_test_platforms) all_platforms=() while IFS='' read -r line; do all_platforms+=("$line"); done < <(_get_test_platforms) # perform pattern matching on the PLATFORMS variable if [[ "${#PLATFORMS[@]}" -eq 0 ]]; then # default to all local platforms PLATFORMS=('_local_background*' '_local_at*') fi # perform pattern matching for pattern in "${PLATFORMS[@]}"; do for platform in "${all_platforms[@]}"; do # shellcheck disable=SC2053 # we are purposefully performing glob matching here if [[ ${platform} == ${pattern} ]]; then ret+=("${platform}") fi done done # set PLATFORMS to the unique results # NOTE: when we drop bash3 convert to # mapfile -t PLATFORMS < <( PLATFORMS=() while IFS='' read -r line; do PLATFORMS+=("$line"); done < <( for platform in "${ret[@]}"; do echo "${platform}"; done | sort -u ) export PLATFORMS } _check_test_requirements () { # check the test requirements against the array of test platforms if [[ -z "${REQUIRE_PLATFORM:-}" ]]; then # if no requirement string is provided default to local, background REQUIRE_PLATFORM='loc:local runner:background' fi if ! _get_test_platform "${REQUIRE_PLATFORM}"; then skip_all "requires $REQUIRE_PLATFORM" fi _add_platform_to_test_global_conf "$CYLC_TEST_PLATFORM" } _add_platform_to_test_global_conf () { # add a test platform from the global config to the test global config # NOTE: Uses global.cylc NOT global-tests.cylc # Do not configure test platforms in the global-tests.cylc file PLATFORM="$1" platform_cfg=$(CYLC_CONF_PATH='' cylc config -i "[platforms][$PLATFORM]") cat >> "${CYLC_CONF_PATH}/global.cylc" <<__HERE__ [platforms] [[$PLATFORM]] $platform_cfg __HERE__ if [[ -n "$platform_cfg" ]]; then # Ensure install target explicitly defined: cylc config -i "[platforms][${PLATFORM}]install target" || exit 1 fi } delete_db() { # Delete private/public workflow databases for a cold start local WORKFLOW_RUN_DIR="${1:-$WORKFLOW_RUN_DIR}" rm "${WORKFLOW_RUN_DIR}/.service/db" "${WORKFLOW_RUN_DIR}/log/db" } # tags PATH="${CYLC_REPO_DIR}/bin:${PATH}" RUN_DIR="${HOME}/cylc-run" export RUN_DIR TEST_NAME_BASE="$(basename "$0" '.t')" TEST_SOURCE_DIR="$(cd "$(dirname "$0")" && pwd)" TEST_DIR="$(mktemp -d)" cd "${TEST_DIR}" TEST_SOURCE_DIR_BASE="${TEST_SOURCE_DIR##*tests/}" # this prevents test runs started at the same time from conflicting TEST_LOG_DIR_BASE="${TMPDIR:-/tmp}/${USER}/${CYLC_TEST_REG_BASE}" TEST_LOG_DIR="${TEST_LOG_DIR_BASE}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BASE}" WORKFLOW_RUN_FAILS=false CYLC_TEST_SKIP="${CYLC_TEST_SKIP:-}" # Is this test in the skip list? THIS="${0#./}" THIS_DIR="$(dirname "${THIS}")" for SKIP in ${CYLC_TEST_SKIP}; do RSKIP="${SKIP#./}" if [[ "${THIS}" == "${RSKIP}" || "${THIS_DIR%/}" == "${RSKIP%/}" ]]; then # Deliberately print variable substitution syntax unexpanded # shellcheck disable=SC2016 skip_all 'this test is in $CYLC_TEST_SKIP.' fi done # Ignore the normal site/user global config, use global-tests.cylc. create_test_global_config "$@" # get array of test platforms to run tests with _expand_test_platforms "${PLATFORMS}" # run or skip tests based on the test requirements and provided test platforms # provides: CYLC_TEST_PLATFORM _check_test_requirements # extract a test host from the test platform # provides: CYLC_TEST_HOST _get_test_host # extract a test install target from the test platform # provides: CYLC_TEST_INSTALL_TARGET _get_test_install_target if ${CYLC_SHOW_TEST_EXECUTION:-false}; then # write the name of tests as they are run # (helpful for debugging hanging tests) echo "# $TEST_NAME_BASE" >&2 fi set +x set +e cylc-flow-8.7.0/tests/functional/lib/python/0000775000175000017500000000000015264420307021207 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/lib/python/diffr.py0000664000175000017500000002270615264420307022662 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Quick and simple JSON diff util. Examples: >>> bool(Diff({}, {})) True >>> diff = Diff({'a': 1, 'b': 2}, ... {'b': 3, 'd': 4}) >>> bool(diff) False >>> list(diff.added()) [(['d'], 4)] >>> list(diff.removed()) [(['a'], 1)] >>> list(diff.modified()) [(['b'], 2, 3)] >>> diff = Diff({'x': {'y': {'z': 1}}}, ... {'x': {'y': {'z': 2}}}) >>> list(diff.modified()) [(['x', 'y', 'z'], 1, 2)] """ import argparse import json import sys from typing import Callable, Optional, Tuple, Type class Diff: """Representation of a diff between two dictionaries.""" BRACES = { dict: ('{', '}'), list: ('[', ']') } def __init__(self, this, that, this_name='expected', that_name='got'): self.typ, self.changed = self.compute_diff(this, that) self.this_name = this_name self.that_name = that_name @classmethod def _diff_method( cls, this: object, that: object ) -> Tuple[Optional[Type], Optional[Callable]]: if isinstance(this, list) and isinstance(that, list): return list, cls.diff_list if isinstance(this, dict) and isinstance(that, dict): return dict, cls.diff_dict return None, None @classmethod def compute_diff(cls, this, that): """Return a list of differences between this and that. Entries take the form:: (symbol, (key, *value)) Where: symbol: * ``+``: for items in that but not in this. * ``-``: for items in this but not in that. * ``?``: for items common to both but with different values. key: * ``dict``: The key of a key, value pair. * ``list``: The index of an element. value: A tuple containing information about the change: * ``+ (key, value)`` * ``- (key, value)`` * ``? (key, this_value, that_value)`` """ typ, meth = cls._diff_method(this, that) if meth: return typ, meth(this, that) raise TypeError('%s Cannot compare %s and %s' % ( cls.__name__, type(this), type(that))) @classmethod def diff_list(cls, this, that): """Return differences between two lists. So unsurprisingly differencing nested lists is actually a little tricky so this is the simple and crude method, just mark the items as modified rather than going through the mess of working out how the lists could fit together. """ changed = [] for index, (this_item, that_item) in enumerate(zip(this, that)): if this_item != that_item: if cls._diff_method(this_item, that_item): changed.append((index, cls(this_item, that_item))) else: changed.append(('?', (index, this_item, that_item))) if len(this) > len(that): symbol = '-' additional = this else: symbol = '+' additional = that for index in range(min(len(this), len(that)), max(len(this), len(that))): changed.append((symbol, (index, additional[index],))) return changed @classmethod def diff_dict(cls, this, that): """Return differences between two dictionaries. As this is a JSON comparison tool ignore the order of keys. """ changed = [] for key, value in that.items(): if key not in this: changed.append(('+', (key, value))) elif isinstance(value, (dict, list)): if cls._diff_method(this[key], that[key]): this[key] = cls(this[key], that[key]) if this[key].changed: changed.append((key, this[key])) else: changed.append(('?', (key, this[key], value))) elif value != this[key]: changed.append(('?', (key, this[key], value))) for key, value in this.items(): if key not in that: changed.append(('-', (key, value))) return changed def __str__(self): return self.tostr() def tostr(self, indent=0): """Return unified(ish) diff.""" ret = '' if indent == 0: ret += '--- %s\n' % self.this_name ret += '+++ %s\n' % self.that_name ret += '============\n' ret += ' %s\n' % self.BRACES[self.typ][0] pre = ' ' * indent for itt, (symbol, item) in enumerate(self.changed): post = '' if itt != len(self.changed) - 1: post = ',' if isinstance(item, Diff): diff = item key = symbol ret += ' %s %s: %s\n' % (pre, key, self.BRACES[diff.typ][0]) ret += diff.tostr(indent + 1) ret += ' %s %s%s\n' % (pre, self.BRACES[diff.typ][1], post) else: if symbol == '?': key, before, after = item ret += f'{symbol}{pre} {key}: {before} => {after}{post}\n' elif symbol in ['+', '-']: if len(item) == 2: value = '%s: %s' % item else: value = item[0] ret += f'{symbol}{pre} {value}{post}\n' if indent == 0: ret += ' %s\n' % self.BRACES[self.typ][1] return ret def added(self): """Yield items present in this but not in that as (key, value).""" yield from self.filter('+') def removed(self): """Yield items present in that but not in this as (key, value).""" yield from self.filter('-') def modified(self): """Yield items change as (key, this_value, that_value).""" yield from self.filter('?') def filter(self, *symbols): """Filter items by change status (+,-,?).""" for symbol, item in self.changed: if symbol in symbols: yield ([item[0]], *item[1:]) elif isinstance(item, Diff): yield from (([symbol, *res[0]], *res[1:]) for res in item.filter(*symbols)) def __bool__(self): return not bool(self.changed) def __len__(self): return len(self.changed) def load_json(file1, file2=None): """Read in JSON, return python data structure. If file2 is None read from sys.stdin """ try: this = json.loads(file1) except json.decoder.JSONDecodeError as exc: sys.exit(f'Syntax error in file1: {exc}') try: if file2: that = json.loads(file2) else: that = json.load(sys.stdin) except json.decoder.JSONDecodeError as exc: sys.exit(f'Syntax error in file2: {exc}') return this, that def parse_args(): """Return CLI args.""" parser = argparse.ArgumentParser() parser.add_argument('file1') parser.add_argument('file2', nargs='?') parser.add_argument( '-1', action='store', dest='name1', default='expected', help=( 'name file 1, default=expected')) parser.add_argument( '-2', action='store', dest='name2', default='got', help=( 'name file 2, default=got')) parser.add_argument( '-c1', '--contains1', action='store_const', const=1, default=0, dest='contains', help=( 'test all (key, value) pairs in file1 are in file2')) parser.add_argument( '-c2', '--contains2', action='store_const', const=2, default=0, dest='contains', help=( 'test all (key, value) pairs in file2 are in file1')) parser.add_argument( '--color', '--colour', action='store', default='never', help=( 'Use color? Option in always, never'), dest='color') args = parser.parse_args() return args def main(args): """Implement dictdiff.""" this, that = load_json(args.file1, args.file2) if this == that: # skip all diff logic if possible sys.exit(0) if args.contains: # test if items from one file present in the other if args.contains == 2: this, that = that, this args.name1, args.name2 = args.name2, args.name1 diff = Diff(this, that, args.name1, args.name2) for _ in diff.filter('-', '?'): sys.stderr.write(str(diff)) sys.exit(1) sys.exit(0) diff = Diff(this, that, args.name1, args.name2) if diff: sys.exit(0) else: sys.stderr.write(str(diff)) sys.exit(len(diff)) if __name__ == '__main__': main(parse_args()) cylc-flow-8.7.0/tests/functional/lib/python/test_diffr.py0000664000175000017500000002612715264420307023722 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Test the diff.py interface.""" import json from subprocess import Popen, PIPE from textwrap import dedent import unittest from diffr import Diff, __file__ as SCRIPT class TestDiff(unittest.TestCase): @staticmethod def call_cli(this, that, *args): cmd = ( ['python', SCRIPT, json.dumps(this), json.dumps(that)] + list(args) ) proc = Popen(cmd, stdout=PIPE, stderr=PIPE) stdout, stderr = proc.communicate() return [proc.returncode, stdout.decode(), stderr.decode()] def test_dict_no_diff(self): for this in [ {'a': 1}, {'a': {'b': 1}}, {'a': {'b': 1, 'c': 2}}, {'a': [1, {'b': 2}]} ]: with self.subTest(this): self.assertTrue(Diff(this, this)) def test_dict_added(self): for this, that in [ ({}, {'b': 2}), ({'a': 1}, {'b': 2}), ({'a': 1}, {'a': 1, 'b': 2}) ]: with self.subTest((this, that)): diff = Diff(this, that) self.assertFalse(diff) self.assertEqual( list(diff.added()), [(['b'], 2)] ) def test_dict_removed(self): for this, that in [ ({}, {'b': 2}), ({'a': 1}, {'b': 2}), ({'a': 1}, {'a': 1, 'b': 2}) ]: with self.subTest((that, this)): diff = Diff(that, this) self.assertFalse(diff) self.assertEqual( list(diff.removed()), [(['b'], 2)] ) def test_dict_modified(self): for this, that in [ ({'a': 1}, {'a': 2}), ({'a': 1, 'b': 2}, {'a': 2, 'b': 2}), ]: with self.subTest((this, that)): diff = Diff(this, that) self.assertFalse(diff) self.assertEqual( list(diff.modified()), [(['a'], 1, 2)] ) def test_dict_str(self): diff = Diff( {'a': 1, 'b': 2, 'c': 3}, {'a': 1, 'b': 3, 'd': 4} ) self.assertFalse(diff) self.assertEqual( str(diff), dedent(''' +++ expected --- got ============ { ? b: 2 => 3, + d: 4, - c: 3 } ''').strip() + '\n' ) def test_dict_nested(self): diff = Diff( {'x': {'y': {'a': 1, 'b': 2, 'c': 3}}}, {'x': {'y': {'a': 1, 'b': 3, 'd': 4}}} ) self.assertFalse(diff) self.assertEqual( list(diff.added()), [(['x', 'y', 'd'], 4)] ) self.assertEqual( list(diff.removed()), [(['x', 'y', 'c'], 3)] ) self.assertEqual( list(diff.modified()), [(['x', 'y', 'b'], 2, 3)] ) self.assertEqual( str(diff), dedent(''' +++ expected --- got ============ { x: { y: { ? b: 2 => 3, + d: 4, - c: 3 } } } ''').strip() + '\n' ) def test_dict_types(self): diff = Diff( {'a': 4.2, 'b': False, 'c': 'a', 'd': 1, 'e': None}, {'a': 4.1, 'b': True, 'c': 'b', 'd': 2, 'e': 1} ) self.assertFalse(diff) self.assertEqual( list(diff.modified()), [ (['a'], 4.2, 4.1), (['b'], False, True), (['c'], 'a', 'b'), (['d'], 1, 2), (['e'], None, 1) ] ) self.assertEqual( str(diff), dedent(''' +++ expected --- got ============ { ? a: 4.2 => 4.1, ? b: False => True, ? c: a => b, ? d: 1 => 2, ? e: None => 1 } ''').strip() + '\n' ) def test_dict_names(self): diff = Diff({'a': 1}, {'a': 2}, this_name='foo', that_name='bar') self.assertEqual( str(diff), dedent(''' +++ foo --- bar ============ { ? a: 1 => 2 } ''').strip() + '\n' ) def test_list_no_diff(self): for this in [ [1], [1, 2, 3], [1, {'a': 1}, 3] ]: with self.subTest(this): self.assertTrue(Diff(this, this)) def test_list_equal_length(self): diff = Diff([1, 2, 3], [4, 5, 6]) self.assertFalse(diff) self.assertEqual( list(diff.modified()), [ ([0], 1, 4), ([1], 2, 5), ([2], 3, 6), ] ) diff = Diff([1, 2, 3], [3, 2, 1]) self.assertFalse(diff) self.assertEqual( list(diff.modified()), [ ([0], 1, 3), ([2], 3, 1), ] ) def test_list_different_length(self): that, this = [1, 2, 3, 4, 5, 6], [-1, 2, 3] diff = Diff(that, this) self.assertFalse(diff) self.assertEqual( list(diff.modified()), [ ([0], 1, -1), ] ) self.assertEqual( list(diff.added()), [] ) self.assertEqual( list(diff.removed()), [ ([3], 4), ([4], 5), ([5], 6) ] ) diff = Diff(this, that) self.assertFalse(diff) self.assertEqual( list(diff.modified()), [ ([0], -1, 1), ] ) self.assertEqual( list(diff.added()), [ ([3], 4), ([4], 5), ([5], 6) ] ) self.assertEqual( list(diff.removed()), [] ) def test_list_nested(self): diff = Diff( [1, [2, 3], 5, 6], [1, [2, 3, 4], 5] ) self.assertFalse(diff) self.assertEqual( list(diff.removed()), [ ([3], 6) ] ) self.assertEqual( list(diff.added()), [ ([1, 2], 4) ] ) self.assertEqual( list(diff.modified()), [] ) self.assertEqual( str(diff), dedent(''' +++ expected --- got ============ [ 1: [ + 2: 4 ], - 3: 6 ] ''').strip() + '\n' ) def test_mixed(self): diff = Diff( {'x': {'y': [0, 2, {'a': 1, 'b': 2, 'c': 3}]}}, {'x': {'y': [1, 2, {'a': 1, 'b': 3, 'd': 4}, 5]}} ) self.assertFalse(diff) self.assertEqual( list(diff.added()), [ (['x', 'y', 2, 'd'], 4), (['x', 'y', 3], 5) ] ) self.assertEqual( list(diff.removed()), [ (['x', 'y', 2, 'c'], 3), ] ) self.assertEqual( list(diff.modified()), [ (['x', 'y', 0], 0, 1), (['x', 'y', 2, 'b'], 2, 3) ] ) self.assertEqual( str(diff), dedent(''' +++ expected --- got ============ { x: { y: [ ? 0: 0 => 1, 2: { ? b: 2 => 3, + d: 4, - c: 3 }, + 3: 5 ] } } ''').strip() + '\n' ) def test_cli_no_diff(self): for this in [ {'a': 1}, {'a': {'b': 1}}, {'a': {'b': 1, 'c': 2}}, {'a': [1, {'b': 2}]}, [1, 2, 3], [1, {'a': 1}, 2] ]: ret, out, err = self.call_cli(this, this) self.assertEqual(ret, 0) self.assertEqual(out, '') self.assertEqual(err, '') def test_cli_nested(self): ret, out, err = self.call_cli( {'x': {'y': {'a': 1, 'b': 2, 'c': 3}}}, {'x': {'y': {'a': 1, 'b': 3, 'd': 4}}} ) self.assertEqual(ret, 1) self.assertEqual(out, '') self.assertEqual( err, dedent(''' +++ expected --- got ============ { x: { y: { ? b: 2 => 3, + d: 4, - c: 3 } } } ''').strip() + '\n' ) def test_cli_contains(self): this, that = ({'a': 1, 'b': 2}, {'a': 1}) ret, out, err = self.call_cli(this, that, '-c2') self.assertEqual(ret, 0) self.assertEqual(out, '') self.assertEqual(err, '') ret, out, err = self.call_cli(this, that, '-c1') # self.assertEqual(ret, 1) self.assertEqual(out, '') self.assertEqual( err, dedent(''' +++ expected --- got ============ { - b: 2 } ''').strip() + '\n' ) def test_cli_names(self): for args in [[], ['-c1']]: ret, out, err = self.call_cli( {'a': 1}, {'a': 2}, '-1', 'foo', '-2', 'bar', *args) self.assertEqual(ret, 1) self.assertEqual(out, '') self.assertEqual( err, dedent(''' +++ foo --- bar ============ { ? a: 1 => 2 } ''').strip() + '\n' ) cylc-flow-8.7.0/tests/functional/pre-initial/0000775000175000017500000000000015264420307021335 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/pre-initial/04-warm.t0000664000175000017500000000323115264420307022710 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test pre-initial cycling works under warm starts . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" warm-start #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --startcp=20130101T00 \ --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/pre-initial/07-simple-messaging.t0000664000175000017500000000174615264420307025222 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test removal of pre-initial cycle message dependencies. . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/pre-initial/warm-start/0000775000175000017500000000000015264420307023436 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/pre-initial/warm-start/reference.log0000664000175000017500000000047115264420307026101 0ustar alastairalastairFinal point: 20130102T0000Z 20130101T0000Z/foo -triggered off ['20121231T1800Z/foo'] 20130101T0600Z/foo -triggered off ['20130101T0000Z/foo'] 20130101T1200Z/foo -triggered off ['20130101T0600Z/foo'] 20130101T1800Z/foo -triggered off ['20130101T1200Z/foo'] 20130102T0000Z/foo -triggered off ['20130101T1800Z/foo'] cylc-flow-8.7.0/tests/functional/pre-initial/warm-start/flow.cylc0000664000175000017500000000032515264420307025261 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] initial cycle point = 20130101T00 final cycle point = 20130102T00 [[graph]] PT6H = "foo[-PT6H] => foo" [runtime] [[foo]] script = "true" cylc-flow-8.7.0/tests/functional/pre-initial/00-simple.t0000664000175000017500000000174315264420307023235 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test simple pre initial cycling dependencies removal . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/pre-initial/08-conditional-messaging.t0000664000175000017500000000175415264420307026234 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test removal of pre-initial cycle messages from conditionals. . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/pre-initial/07-simple-messaging/0000775000175000017500000000000015264420307025025 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/pre-initial/07-simple-messaging/reference.log0000664000175000017500000000025415264420307027467 0ustar alastairalastairInitial point: 20100808T0000Z Final point: 20100809T0000Z 20100808T0000Z/foo -triggered off ['20100807T0000Z/foo'] 20100809T0000Z/foo -triggered off ['20100808T0000Z/foo'] cylc-flow-8.7.0/tests/functional/pre-initial/07-simple-messaging/flow.cylc0000664000175000017500000000055015264420307026650 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] initial cycle point = 20100808T00 final cycle point = 20100809T00 [[graph]] T00 = "foo[-P1D]:restart1 => foo" [runtime] [[foo]] script = """ cylc__job__wait_cylc_message_started cylc message "restart files ready" """ [[[outputs]]] restart1 = "restart files ready" cylc-flow-8.7.0/tests/functional/pre-initial/warm-offset/0000775000175000017500000000000015264420307023567 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/pre-initial/warm-offset/reference.log0000664000175000017500000000043615264420307026233 0ustar alastairalastairInitial point: 20130101T0600Z Final point: 20130102T0000Z 20130101T0600Z/foo -triggered off ['20130101T0000Z/foo'] 20130101T1200Z/foo -triggered off ['20130101T0600Z/foo'] 20130101T1800Z/foo -triggered off ['20130101T1200Z/foo'] 20130102T0000Z/foo -triggered off ['20130101T1800Z/foo'] cylc-flow-8.7.0/tests/functional/pre-initial/warm-offset/flow.cylc0000664000175000017500000000032515264420307025412 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] initial cycle point = 20130101T00 final cycle point = 20130102T00 [[graph]] PT6H = "foo[-PT6H] => foo" [runtime] [[foo]] script = "true" cylc-flow-8.7.0/tests/functional/pre-initial/09-warm-iso.t0000664000175000017500000000322715264420307023512 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test pre-initial cycling works under warm starts . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" warm-start-iso #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --startcp=20130102T00 --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/pre-initial/warm-start-iso/0000775000175000017500000000000015264420307024226 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/pre-initial/warm-start-iso/reference.log0000664000175000017500000000106115264420307026665 0ustar alastairalastairInitial point: 20130101T0000Z Start point: 20130102T0000Z Final point: 20130103T0000Z 20130102T0000Z/qux -triggered off ['20130101T0000Z/foo', '20130101T1800Z/qux'] 20130102T0000Z/wibble -triggered off [] 20130102T0600Z/wibble -triggered off [] 20130102T0600Z/qux -triggered off ['20130101T0000Z/foo', '20130102T0000Z/qux'] 20130102T1200Z/qux -triggered off ['20130101T0000Z/foo', '20130102T0600Z/qux'] 20130102T1800Z/qux -triggered off ['20130101T0000Z/foo', '20130102T1200Z/qux'] 20130103T0000Z/qux -triggered off ['20130101T0000Z/foo', '20130102T1800Z/qux'] cylc-flow-8.7.0/tests/functional/pre-initial/warm-start-iso/flow.cylc0000664000175000017500000000053015264420307026047 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20130101T00 final cycle point = 20130103T00 [[graph]] R1 = "foo" R1/T18 = "bar => baz" R6//PT6H = "wibble" T00,T06,T12,T18 = "foo[^] & qux[-PT6H] => qux" [runtime] [[root]] script = "true" cylc-flow-8.7.0/tests/functional/pre-initial/12-warm-restart.t0000664000175000017500000000530015264420307024370 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test warm start persists across restarts . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 6 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" warm-start #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-run-paused workflow_run_ok "${TEST_NAME}" cylc play "${WORKFLOW_NAME}" --startcp=20130101T12 --pause #------------------------------------------------------------------------------- cylc stop --max-polls=10 --interval=2 "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-restart workflow_run_ok "${TEST_NAME}" cylc play "${WORKFLOW_NAME}" # Ensure workflow has started poll_workflow_running #------------------------------------------------------------------------------- # Check pre-reqs TEST_NAME=${TEST_NAME_BASE}-check-prereq run_ok "${TEST_NAME}" cylc show "${WORKFLOW_NAME}//20130101T1200Z/foo" --list-prereqs cmp_ok "${TEST_NAME}.stdout" <<'__OUT__' 20130101T0600Z/foo succeeded __OUT__ #------------------------------------------------------------------------------- # Stop workflow cylc stop --max-polls=10 --interval=2 "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- DB_FILE="${RUN_DIR}/${WORKFLOW_NAME}/log/db" NAME='database-entry' sqlite3 "${DB_FILE}" \ "SELECT value FROM workflow_params WHERE key=='startcp'" >"${NAME}" cmp_ok "${NAME}" <<<'20130101T12' #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/pre-initial/05-warm-new-ict.t0000664000175000017500000000336315264420307024263 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test pre-initial cycling works under warm starts with a new initial cycle # time that is later than the flow.cylc initial cycle point. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" warm-offset #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --startcp=20130101T0600Z \ --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/pre-initial/08-conditional-messaging/0000775000175000017500000000000015264420307026040 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/pre-initial/08-conditional-messaging/reference.log0000664000175000017500000000051115264420307030476 0ustar alastairalastairInitial point: 20100808T0000Z Final point: 20100809T0000Z 20100808T0000Z/bar -triggered off [] 20100808T0000Z/foo -triggered off ['20100807T0000Z/foo', '20100808T0000Z/bar'] 20100809T0000Z/bar -triggered off [] 20100809T0000Z/foo -triggered off ['20100808T0000Z/foo'] 20100809T0000Z/handled -triggered off ['20100809T0000Z/bar'] cylc-flow-8.7.0/tests/functional/pre-initial/08-conditional-messaging/flow.cylc0000664000175000017500000000117215264420307027664 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [[events]] expected task failures = 20100809T0000Z/bar [scheduling] initial cycle point = 20100808T00 final cycle point = 20100809T00 [[graph]] T00 = """ foo[-P1D]:restart1 | bar? => foo bar:fail? => handled """ [runtime] [[foo]] script = """ cylc__job__wait_cylc_message_started cylc message "restart files ready" """ [[[outputs]]] restart1 = "restart files ready" [[bar]] script = [[ "$(cylc cycle-point)" == 20100808T0000Z ]] cylc-flow-8.7.0/tests/functional/pre-initial/test_header0000777000175000017500000000000015264420307027652 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/pre-initial/00-simple/0000775000175000017500000000000015264420307023043 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/pre-initial/00-simple/reference.log0000664000175000017500000000024415264420307025504 0ustar alastairalastairInitial point: 20100101T0000Z Final point: 20100102T0000Z 20100101T0000Z/a -triggered off ['20091231T0000Z/a'] 20100102T0000Z/a -triggered off ['20100101T0000Z/a'] cylc-flow-8.7.0/tests/functional/pre-initial/00-simple/flow.cylc0000664000175000017500000000031515264420307024665 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] initial cycle point = 20100101T00 final cycle point = 20100102T00 [[graph]] T00 = "a[-PT24H] => a" [runtime] [[a]] script = true cylc-flow-8.7.0/tests/functional/jinja2/0000775000175000017500000000000015264420307020275 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/07-filters.t0000664000175000017500000000316615264420307022364 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # basic jinja2 expansion test . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" filters #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate -o 'flow.cylc.processed' "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-check-expansion cmp_ok 'flow.cylc.processed' "${TEST_DIR}/${WORKFLOW_NAME}/flow.cylc-expanded" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/jinja2/include-badsyntax/0000775000175000017500000000000015264420307023713 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/include-badsyntax/flow.cylc0000664000175000017500000000027715264420307025544 0ustar alastairalastair#!Jinja2 {% set MYVAR = "this is a set variable" %} [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = "a => FAM" [runtime] {% include 'runtime-bad.cylc' %} cylc-flow-8.7.0/tests/functional/jinja2/include-badsyntax/runtime-bad.cylc0000664000175000017500000000037515264420307027003 0ustar alastairalastair [[a]] script = echo {{ MYVAR }} [[FAM]] [[[environment]]] TITLE="member" {%- for num in ["0","1","2","3",4"] %} [[member_{{ num }}]] inherit = FAM script = echo I am $TITLE {{ num }} {%- endfor %} cylc-flow-8.7.0/tests/functional/jinja2/06-do-extension.t0000664000175000017500000000317215264420307023324 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # test that the jinja2 do extension is loaded. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" do-extension #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-list cylc list "${WORKFLOW_NAME}" > list.out cmp_ok list.out <<__ENDLIST mem_0 mem_1 mem_2 __ENDLIST #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/jinja2/00-simple.t0000664000175000017500000000316215264420307022172 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # basic jinja2 expansion test . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" simple #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate -o 'flow.cylc.processed' "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-check-expansion cmp_ok 'flow.cylc.processed' "${TEST_SOURCE_DIR}/simple/flow.cylc-expanded" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/jinja2/include-incomplete/0000775000175000017500000000000015264420307024055 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/include-incomplete/runtime-incomplete.cylc0000664000175000017500000000034315264420307030551 0ustar alastairalastair [[a]] script = echo {{ MYVAR }} [[FAM]] [[[environment]]] TITLE="member" {%- for num in range(5) %} [[member_{{ num }}]] inherit = FAM script = echo I am $TITLE {{ num }} cylc-flow-8.7.0/tests/functional/jinja2/include-incomplete/flow.cylc0000664000175000017500000000023215264420307025675 0ustar alastairalastair#!Jinja2 {% set MYVAR = "this is a set variable" %} [scheduling] [[graph]] R1 = "a => FAM" [runtime] {% include 'runtime-incomplete.cylc' %} cylc-flow-8.7.0/tests/functional/jinja2/08-local-lib-python.t0000664000175000017500000000327215264420307024070 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that a custom Jinja2 filter can load a local python module in lib/python. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate -o 'flow.cylc.processed' "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-check-expansion cmp_ok 'flow.cylc.processed' "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/flow.cylc.jproc" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/jinja2/09-custom-jinja2-filters/0000775000175000017500000000000015264420307024656 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/09-custom-jinja2-filters/reference.log0000664000175000017500000000043215264420307027316 0ustar alastairalastairInitial point: 01231212T1212Z Final point: None 01231212T1212Z/strftime -triggered off [] 01231212T1212Z/pad -triggered off ['01231212T1212Z/strftime'] 01231212T1212Z/duration_as -triggered off ['01231212T1212Z/pad'] 01231212T1212Z/end -triggered off ['01231212T1212Z/duration_as'] cylc-flow-8.7.0/tests/functional/jinja2/09-custom-jinja2-filters/flow.cylc0000664000175000017500000000167315264420307026510 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [scheduling] initial cycle point = 01231212T1212 [[graph]] R1 = strftime => pad => duration_as => end [runtime] [[strftime]] script = test $TEST_1 == '00'; test $TEST_2 == '30' [[[environment]]] TEST_1 = {{ '2000' | strftime('%H') }} TEST_2 = {{ '12-30-2000' | strftime('%d', '%m-%d-%Y') }} [[pad]] script = test {{ 42 | pad(3, 0) }} == '042' [[duration_as]] script = """ test $TEST_1 == '3600' test $TEST_2 == '1' test $TEST_3 == '1' test $TEST_4 == '1' """ [[[environment]]] TEST_1 = {{ 'PT1H' | duration_as('seconds') | int }} TEST_2 = {{ 'P7D' | duration_as('w') | int }} TEST_3 = {{ 'PT60M' | duration_as('hours') | int }} TEST_4 = {{ 'PT60S' | duration_as('minutes') | int }} [[end]] script = true cylc-flow-8.7.0/tests/functional/jinja2/do-extension/0000775000175000017500000000000015264420307022711 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/do-extension/flow.cylc0000664000175000017500000000055415264420307024540 0ustar alastairalastair#!Jinja2 [meta] title = "Jinja2 'do' extension test" {% set N_TASKS = N_TASKS | default(3) %} {% set TASKS = [] %} {% for I in range(0,N_TASKS|int) %} {% do TASKS.append( 'mem_' ~ I ) %} {% endfor %} [scheduling] [[graph]] R1 = "FAMILY" [runtime] [[FAMILY]] {% for T in TASKS %} [[ {{ T }} ]] inherit = FAMILY {% endfor %} cylc-flow-8.7.0/tests/functional/jinja2/03-bad.t0000664000175000017500000001077115264420307021436 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test Jinja2 errors # Ensure parsec can pull out the context lines even when they are behind # an include statement . "$(dirname "$0")/test_header" set_test_number 10 sub_tabs() { sed -i 's/\t/ /g' "${TEST_NAME}.stderr" } purge_workflow() { if ((FAILURES != 0)); then exit fi purge } #------------------------------------------------------------------------------ # Test syntax error in flow.cylc install_workflow "${TEST_NAME_BASE}" badsyntax TEST_NAME="${TEST_NAME_BASE}-badsyntax" run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" sub_tabs cmp_ok "${TEST_NAME}.stderr" <<__HERE__ Jinja2Error: Expected an expression, got 'end of statement block' File ${WORKFLOW_RUN_DIR}/flow.cylc #!Jinja2 # line before {% set x = %} <-- TemplateSyntaxError __HERE__ purge_workflow #------------------------------------------------------------------------------ # Test syntax error in an include file install_workflow "${TEST_NAME_BASE}" include-badsyntax TEST_NAME="${TEST_NAME_BASE}-include-badsyntax" run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" sub_tabs cmp_ok "${TEST_NAME}.stderr" <<__HERE__ Jinja2Error: unexpected char '"' at 142 Error in file "runtime-bad.cylc" File ${WORKFLOW_RUN_DIR}/runtime-bad.cylc [[FAM]] [[[environment]]] TITLE="member" {%- for num in ["0","1","2","3",4"] %} <-- TemplateSyntaxError File ${WORKFLOW_RUN_DIR}/flow.cylc [[graph]] R1 = "a => FAM" [runtime] {% include 'runtime-bad.cylc' %} <-- TemplateSyntaxError __HERE__ purge_workflow #------------------------------------------------------------------------------ # Test syntax error in an include file (included using the Cylc built-in # %include keyword) install_workflow "${TEST_NAME_BASE}" cylc-include-badsyntax TEST_NAME="${TEST_NAME_BASE}-cylc-include-badsyntax" run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" sub_tabs cmp_ok "${TEST_NAME}.stderr" <<__HERE__ Jinja2Error: Encountered unknown tag 'end'. Jinja was looking for the following tags: 'elif' or 'else' or 'endif'. The innermost block that needs to be closed is 'if'. File ${WORKFLOW_RUN_DIR}/flow.cylc # This is a bit of graph configuration. {% if true %} R1 = foo {% end if % <-- TemplateSyntaxError __HERE__ purge_workflow #------------------------------------------------------------------------------ # Test missing include file install_workflow "${TEST_NAME_BASE}" include-missing TEST_NAME="${TEST_NAME_BASE}-include-missing" run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" sub_tabs cmp_ok "${TEST_NAME}.stderr" <<__HERE__ Jinja2Error: runtime.cylc File ${WORKFLOW_RUN_DIR}/flow.cylc [[graph]] R1 = "a => FAM" [runtime] {% include 'runtime.cylc' %} <-- TemplateNotFound __HERE__ purge_workflow #------------------------------------------------------------------------------ # Test incomplete loop install_workflow "${TEST_NAME_BASE}" include-incomplete TEST_NAME="${TEST_NAME_BASE}-include-incomplete" run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" sub_tabs cmp_ok "${TEST_NAME}.stderr" << __HERE__ Jinja2Error: Unexpected end of template. Error in file "runtime-incomplete.cylc" Jinja was looking for the following tags: 'endfor' or 'else'. The innermost block that needs to be closed is 'for'. File ${WORKFLOW_RUN_DIR}/runtime-incomplete.cylc {%- for num in range(5) %} [[member_{{ num }}]] inherit = FAM script = echo I am \$TITLE {{ num }} <-- TemplateSyntaxError File ${WORKFLOW_RUN_DIR}/flow.cylc [[graph]] R1 = "a => FAM" [runtime] {% include 'runtime-incomplete.cylc' %} <-- TemplateSyntaxError __HERE__ purge_workflow cylc-flow-8.7.0/tests/functional/jinja2/08-local-lib-python/0000775000175000017500000000000015264420307023677 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/08-local-lib-python/lib/0000775000175000017500000000000015264420307024445 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/08-local-lib-python/lib/python/0000775000175000017500000000000015264420307025766 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/08-local-lib-python/lib/python/local_lookup.py0000664000175000017500000000161315264420307031024 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . lookup_table = { 'foo': 'fooble', 'bar': 'barble'} def lookup(arg): return lookup_table[arg] cylc-flow-8.7.0/tests/functional/jinja2/08-local-lib-python/flow.cylc0000664000175000017500000000021015264420307025513 0ustar alastairalastair#!Jinja2 [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = {{"foo" | qualify}} => {{"bar" | qualify}} cylc-flow-8.7.0/tests/functional/jinja2/08-local-lib-python/Jinja2Filters/0000775000175000017500000000000015264420307026345 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/08-local-lib-python/Jinja2Filters/qualify.py0000664000175000017500000000155415264420307030376 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import local_lookup def qualify(arg): return local_lookup.lookup(arg) cylc-flow-8.7.0/tests/functional/jinja2/08-local-lib-python/flow.cylc.jproc0000664000175000017500000000014515264420307026636 0ustar alastairalastair[scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = fooble => barble cylc-flow-8.7.0/tests/functional/jinja2/05-commandline.t0000664000175000017500000000366315264420307023202 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # jinja2 command line variables test . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" commandline-set #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-validate1 run_ok "${TEST_NAME}" cylc validate --set="TASKNAME='foo'" --set="STEP='2'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-validate2 run_ok "${TEST_NAME}" cylc validate \ --set-file="${TEST_DIR}/${WORKFLOW_NAME}/vars.txt" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --no-detach --reference-test \ --set-file="${TEST_DIR}/${WORKFLOW_NAME}/vars.txt" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/jinja2/09-custom-jinja2-filters.t0000664000175000017500000000254615264420307025052 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # basic jinja2 include and expand test . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 1 # Run Jinja2 custom filters workflow. TEST_NAME="${TEST_NAME_BASE}"-run-filters install_workflow "${TEST_NAME_BASE}-install-filter-workflow" "${TEST_NAME_BASE}" run_ok "${TEST_NAME}" cylc play "${WORKFLOW_NAME}" --reference-test --debug --no-detach #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/jinja2/commandline-set/0000775000175000017500000000000015264420307023354 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/commandline-set/vars.txt0000664000175000017500000000003015264420307025061 0ustar alastairalastairTASKNAME="foo" STEP="2" cylc-flow-8.7.0/tests/functional/jinja2/commandline-set/reference.log0000664000175000017500000000020415264420307026011 0ustar alastairalastairInitial point: 20100101T0000Z Final point: 20120101T0000Z 20100101T0000Z/foo -triggered off [] 20120101T0000Z/foo -triggered off [] cylc-flow-8.7.0/tests/functional/jinja2/commandline-set/flow.cylc0000664000175000017500000000075315264420307025204 0ustar alastairalastair#!Jinja2 [meta] title = "Jinja2 command line set test" description = """A couple of Jinja2 variables must be set on the command line or else this test will fail to validate: TASKNAME and STEP. Also, to pass the test run their values must be 'foo' and '2', respectively.""" [scheduler] UTC mode = True [scheduling] initial cycle point = 2010 final cycle point = 2012 [[graph]] P{{STEP}}Y = {{TASKNAME}} [runtime] [[ {{TASKNAME}} ]] script = true cylc-flow-8.7.0/tests/functional/jinja2/cylc-include-badsyntax/0000775000175000017500000000000015264420307024643 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/cylc-include-badsyntax/flow.cylc0000664000175000017500000000016715264420307026472 0ustar alastairalastair#!jinja2 [scheduler] allow implicit tasks = True [scheduling] [[graph]] %include flow-includeme.cylc [runtime] cylc-flow-8.7.0/tests/functional/jinja2/cylc-include-badsyntax/flow-includeme.cylc0000664000175000017500000000014315264420307030427 0ustar alastairalastair# This is a bit of graph configuration. {% if true %} R1 = foo {% end if % cylc-flow-8.7.0/tests/functional/jinja2/badsyntax/0000775000175000017500000000000015264420307022272 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/badsyntax/flow.cylc0000664000175000017500000000006215264420307024113 0ustar alastairalastair#!Jinja2 # line before {% set x = %} # line after cylc-flow-8.7.0/tests/functional/jinja2/filters/0000775000175000017500000000000015264420307021745 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/filters/flow.cylc-expanded0000664000175000017500000000023415264420307025355 0ustar alastairalastair# Hello from a filter! [scheduling] [[graph]] R1 = foo => bar [runtime] [[foo]] pre-script = true script = true [[bar]] cylc-flow-8.7.0/tests/functional/jinja2/filters/flow.cylc0000664000175000017500000000027015264420307023567 0ustar alastairalastair#!jinja2 # {{ ""|hello }} [scheduling] [[graph]] R1 = foo => bar [runtime] [[foo]] pre-script = {{ true|truly }} script = {{ false|truly }} [[bar]] cylc-flow-8.7.0/tests/functional/jinja2/filters/Jinja2Filters/0000775000175000017500000000000015264420307024413 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/filters/Jinja2Filters/hello.py0000664000175000017500000000147315264420307026075 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . def hello(foo): return 'Hello from a filter!' cylc-flow-8.7.0/tests/functional/jinja2/filters/Jinja2Filters/truly.py0000664000175000017500000000145215264420307026146 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . def truly(bar): return 'true' cylc-flow-8.7.0/tests/functional/jinja2/include/0000775000175000017500000000000015264420307021720 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/include/flow.cylc-expanded0000664000175000017500000000105415264420307025331 0ustar alastairalastair[scheduling] [[graph]] R1 = "a => FAM" [runtime] [[a]] script = echo this is a set variable [[FAM]] [[[environment]]] TITLE="member" [[member_0]] inherit = FAM script = echo I am $TITLE 0 [[member_1]] inherit = FAM script = echo I am $TITLE 1 [[member_2]] inherit = FAM script = echo I am $TITLE 2 [[member_3]] inherit = FAM script = echo I am $TITLE 3 [[member_4]] inherit = FAM script = echo I am $TITLE 4 cylc-flow-8.7.0/tests/functional/jinja2/include/flow.cylc0000664000175000017500000000022015264420307023535 0ustar alastairalastair#!Jinja2 {% set MYVAR = "this is a set variable" %} [scheduling] [[graph]] R1 = "a => FAM" [runtime] {% include 'runtime.cylc' %} cylc-flow-8.7.0/tests/functional/jinja2/include/runtime.cylc0000664000175000017500000000036115264420307024257 0ustar alastairalastair [[a]] script = echo {{ MYVAR }} [[FAM]] [[[environment]]] TITLE="member" {%- for num in range(5) %} [[member_{{ num }}]] inherit = FAM script = echo I am $TITLE {{ num }} {%- endfor %} cylc-flow-8.7.0/tests/functional/jinja2/simple/0000775000175000017500000000000015264420307021566 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/simple/flow.cylc-expanded0000664000175000017500000000105415264420307025177 0ustar alastairalastair[scheduling] [[graph]] R1 = "a => FAM" [runtime] [[a]] script = echo this is a set variable [[FAM]] [[[environment]]] TITLE="member" [[member_0]] inherit = FAM script = echo I am $TITLE 0 [[member_1]] inherit = FAM script = echo I am $TITLE 1 [[member_2]] inherit = FAM script = echo I am $TITLE 2 [[member_3]] inherit = FAM script = echo I am $TITLE 3 [[member_4]] inherit = FAM script = echo I am $TITLE 4 cylc-flow-8.7.0/tests/functional/jinja2/simple/flow.cylc0000664000175000017500000000054315264420307023413 0ustar alastairalastair#!Jinja2 {% set MYVAR = "this is a set variable" %} [scheduling] [[graph]] R1 = "a => FAM" [runtime] [[a]] script = echo {{ MYVAR }} [[FAM]] [[[environment]]] TITLE="member" {%- for num in range(5) %} [[member_{{ num }}]] inherit = FAM script = echo I am $TITLE {{ num }} {%- endfor %} cylc-flow-8.7.0/tests/functional/jinja2/test_header0000777000175000017500000000000015264420307026612 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/01-include.t0000664000175000017500000000317515264420307022331 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # basic jinja2 include and expand test . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" include #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate -o 'flow.cylc.processed' "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-check-expansion cmp_ok 'flow.cylc.processed' "${TEST_SOURCE_DIR}/include/flow.cylc-expanded" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/jinja2/11-logging.t0000664000175000017500000000261115264420307022327 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that the Cylc logger is available to Jinja2 at load time. . "$(dirname "$0")/test_header" set_test_number 2 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' #!Jinja2 {% from "cylc.flow" import LOG %} {% do LOG.debug("Hello World!") %} [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = foo __FLOW_CONFIG__ TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" --debug grep_ok 'Hello World!' "${TEST_NAME}.stderr" # debug lines go to stderr purge exit cylc-flow-8.7.0/tests/functional/jinja2/include-missing/0000775000175000017500000000000015264420307023367 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jinja2/include-missing/flow.cylc0000664000175000017500000000021715264420307025212 0ustar alastairalastair#!Jinja2 {% set MYVAR = "this is a set variable" %} [scheduling] [[graph]] R1 = "a => FAM" [runtime] {% include 'runtime.cylc' %} cylc-flow-8.7.0/tests/functional/cylc-get-resources/0000775000175000017500000000000015264420307022637 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-get-resources/00-error.t0000775000175000017500000000256415264420307024404 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Cylc get-resources doesn't overwrite a file with a dir, or vice-versa: . "$(dirname "$0")/test_header" set_test_number 4 TEST="${TEST_NAME_BASE}-overwrite-dir" mkdir cylc run_fail "${TEST}" cylc get-resources cylc grep_ok "Cannot extract file ${PWD}/cylc as there is an existing directory with the same name" "${TEST}.stderr" rm -r cylc touch syntax run_fail "${TEST}" cylc get-resources syntax grep_ok "Cannot extract directory ${PWD}/syntax as there is an existing file with the same name" "${TEST}.stderr" exit cylc-flow-8.7.0/tests/functional/cylc-get-resources/test_header0000777000175000017500000000000015264420307031154 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/jobscript/0000775000175000017500000000000015264420307021117 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jobscript/04-profiler.t0000664000175000017500000000611315264420307023350 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) NIWA & British Crown (Met Office) & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Cylc profile test. This test will run the Cylc profiler with mocked data # and test to see that the profiler data is received and processed correctly. . "$(dirname "$0")/test_header" if [[ "$OSTYPE" != "linux-gnu"* ]]; then skip_all "Tests not compatible with $OSTYPE" fi set_test_number 8 # Set up test data mkdir -p "${PWD}/cgroups_test_data" echo 'anon 12345678' > cgroups_test_data/memory.stat echo '123456789' > cgroups_test_data/memory.max printf "blah blah 123456\nusage_usec 56781234" > cgroups_test_data/cpu.stat export profiler_test_env_var='/cgroups_test_data' create_test_global_config " [platforms] [[localhost]] [[[profiler]]] activate = True cgroups path = ${PWD} " init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' #!Jinja2 [scheduling] [[graph]] R1 = the_good & the_bad? [runtime] [[the_good]] # this task should succeeded normally platform = localhost script = sleep 5 [[the_bad]] # this task should fail (it should still send profiling info) platform = localhost script = sleep 5; false __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --debug --no-detach "${WORKFLOW_NAME}" # ensure the cpu and memory messages were received and that these messages # were received before the succeeded message log_scan "${TEST_NAME_BASE}-task-succeeded" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" 1 0 \ '1/the_good.*(received)_cylc_profiler.*cpu_time' \ '1/the_good.*(received)succeeded' # ensure the cpu and memory messages were received and that these messages # were received before the failed message log_scan "${TEST_NAME_BASE}-task-failed" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" 1 0 \ '1/the_bad.*(received)_cylc_profiler.*cpu_time' \ '1/the_bad.*failed' # Verify the profiler data matches the mocked cgroup values: grep_workflow_log_ok "${TEST_NAME_BASE}-the_good-data" \ '1/the_good.*(received)_cylc_profiler.*"max_rss": 12345678.*"cpu_time": 56781.*"memory_allocated": 123456789' grep_workflow_log_ok "${TEST_NAME_BASE}-the_bad-data" \ '1/the_bad.*(received)_cylc_profiler.*"max_rss": 12345678.*"cpu_time": 56781.*"memory_allocated": 123456789' purge cylc-flow-8.7.0/tests/functional/jobscript/00-torture/0000775000175000017500000000000015264420307023040 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jobscript/00-torture/reference.log0000664000175000017500000000007015264420307025476 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] cylc-flow-8.7.0/tests/functional/jobscript/00-torture/flow.cylc0000664000175000017500000000430015264420307024660 0ustar alastairalastair[meta] title = "job script torture test" description = """Any task job script may fail regardless of user runtime settings if changes to cylc re-order the job script sections badly: e.g. "cylc task started" must be called after the CYLC_ environment variables are exported. Additionally, users may rely on the order of variable definition in each environment and script section: e.g. workflow bin directory must go in PATH before the task runtime environment is defined because workflow bin commands could be used in variable assignment expressions.""" [scheduling] [[graph]] R1 = "foo" [runtime] [[foo]] platform = localhost init-script = """ echo "HELLO FROM INIT-SCRIPT" # define a variable export VAR_IS=is""" pre-script = """ echo "HELLO FROM PRE-SCRIPT" # init-script must be done: echo VAR_IS is $VAR_IS # user environment must be done: echo E_ONE is $E_ONE echo E_TWO is $E_TWO echo E_THR is $E_THR echo E_FOU is $E_FOU echo E_FIV is $E_FIV # define a variable export VAR_PreCS=precs""" script = """ echo "HELLO FROM SCRIPT" # init-script must be done: echo VAR_IS is $VAR_IS # pre-script must be done: echo VAR_PreCS is $VAR_PreCS # environment must be done: echo E_ONE is $E_ONE echo E_TWO is $E_TWO echo E_THR is $E_THR echo E_FOU is $E_FOU echo E_FIV is $E_FIV # define a variable export VAR_CS=var_cs""" post-script = """ echo "HELLO FROM POST-SCRIPT" # init-script must be done: echo VAR_IS is $VAR_IS # pre-script must be done: echo VAR_PreCS is $VAR_PreCS # script must be done: echo VAR_CS is $VAR_CS # environment must be done: echo E_ONE is $E_ONE echo E_TWO is $E_TWO echo E_THR is $E_THR echo E_FOU is $E_FOU echo E_FIV is $E_FIV echo VAR_IS is $VAR_IS echo VAR_PreCS is $VAR_PreCS echo VAR_CS is $VAR_CS # define a variable export VAR_PostCS=postcs""" [[[environment]]] # path to cylc must be available: E_ONE = $(( RANDOM % 10 )) # init-script must be done: E_TWO = $VAR_IS # cylc-defined variables must be done: E_THR = $CYLC_WORKFLOW_SHARE_DIR E_FOU = $CYLC_TASK_NAME # the workflow bin must be in $PATH already: E_FIV = $( foo.sh ) cylc-flow-8.7.0/tests/functional/jobscript/00-torture/bin/0000775000175000017500000000000015264420307023610 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jobscript/00-torture/bin/foo.sh0000775000175000017500000000005115264420307024726 0ustar alastairalastair#!/usr/bin/env bash echo "Hello from $0" cylc-flow-8.7.0/tests/functional/jobscript/03-profiler-e2e.t0000664000175000017500000000646315264420307024030 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) NIWA & British Crown (Met Office) & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Cylc profile test # # Cylc profile test. This test will run the Cylc profiler with real data # and test to see that the profiler data is received and processed correctly. # NOTE: This test will run the Cylc profiler on the given test platform. # The test platform may need to be configured for this to work (e.g. # "cgroups path" may need to be set). export REQUIRE_PLATFORM='loc:* runner:?(pbs|slurm) comms:tcp' . "$(dirname "$0")/test_header" set_test_number 8 create_test_global_config " [platforms] [[${CYLC_TEST_PLATFORM}]] [[[profiler]]] activate = True polling interval = PT10S [[ugly_platform]] $(cylc config -i "[platforms][$CYLC_TEST_PLATFORM]") [[ugly_platform]] [[[profiler]]] activate = True polling interval = PT10S cgroups path = the/thing/that/should/not/be " init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' #!Jinja2 [scheduling] [[graph]] R1 = the_good & the_bad? & the_ugly [runtime] [[the_good]] # this task should succeeded normally platform = {{ environ['CYLC_TEST_PLATFORM'] }} script = sleep 5 [[the_bad]] # this task should fail (it should still send profiling info) platform = {{ environ['CYLC_TEST_PLATFORM'] }} script = sleep 5; false [[the_ugly]] # this task should succeed despite the broken profiler configuration platform = ugly_platform script = sleep 5 __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --debug --no-detach "${WORKFLOW_NAME}" # ensure the cpu and memory messages were received and that these messages # were received before the succeeded message log_scan "${TEST_NAME_BASE}-task-succeeded" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" 1 0 \ '1/the_good.*(received)_cylc_profiler.*cpu_time' \ '1/the_good.*(received)succeeded' # ensure the cpu and memory messages were received and that these messages # were received before the failed message log_scan "${TEST_NAME_BASE}-task-failed" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" 1 0 \ '1/the_bad.*(received)_cylc_profiler.*cpu_time' \ '1/the_bad.*failed' # ensure this task succeeded despite the broken profiler configuration grep_workflow_log_ok "${TEST_NAME_BASE}-broken" '1/the_ugly.*(received)succeeded' cylc cat-log "${WORKFLOW_NAME}//1/the_ugly" -f e > "${TEST_NAME_BASE}-ugly-stderr.txt" grep_ok 'Cgroup not found' "${TEST_NAME_BASE}-ugly-stderr.txt" purge cylc-flow-8.7.0/tests/functional/jobscript/02-share-cycle-cycle.t0000664000175000017500000000241715264420307025023 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Assert that workflow/share/cycle/ is created and exported. . "$(dirname "$0")/test_header" set_test_number 4 # Setup: install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}.validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}.play" cylc play --no-detach "${WORKFLOW_NAME}" grep_ok "Hello World" "${WORKFLOW_RUN_DIR}/share/cycle/19900211T0000Z/stuff" grep_ok "Hello World" "${WORKFLOW_RUN_DIR}/share/cycle/19900212T0000Z/stuff" purge cylc-flow-8.7.0/tests/functional/jobscript/01-share-lib-bin.t0000664000175000017500000000262215264420307024140 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # CYLC_WORKFLOW_SHARE_DIR/bin is added to the PATH # CYLC_WORKFLOW_SHARE_DIR/lib/python is added to PYTHONPATH # # Workflow creates scripts in CYLC_WORKFLOW_SHARE_DIR in `install_cold` task # Which are then used in subsequent tasks. We check for the output of those # tasks. . "$(dirname "$0")/test_header" set_test_number 2 # Setup: install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}.validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}.play" cylc play --no-detach "${WORKFLOW_NAME}" purge cylc-flow-8.7.0/tests/functional/jobscript/01-share-lib-bin/0000775000175000017500000000000015264420307023751 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jobscript/01-share-lib-bin/flow.cylc0000664000175000017500000000207015264420307025573 0ustar alastairalastair#!jinja2 [scheduler] [[events]] stall timeout = PT0M [scheduling] initial cycle point = 1266 [[graph]] R1 = install_cold => run_mypythonscript & run_myscript [runtime] [[install_cold]] script = """ mkdir -p "${CYLC_WORKFLOW_SHARE_DIR}/lib/python" mkdir -p "${CYLC_WORKFLOW_SHARE_DIR}/bin" cat > "${CYLC_WORKFLOW_SHARE_DIR}/lib/python/mypythonscript.py" <<__HERE__ def my_function(): print("I can speak Esperanto like a native.") __HERE__ cat > "${CYLC_WORKFLOW_SHARE_DIR}/bin/myscript.sh" <<__HERE__ #!/usr/bin/bash echo "A sure cure for seasickness is to sit under a tree." __HERE__ chmod +x "${CYLC_WORKFLOW_SHARE_DIR}/lib/python/mypythonscript.py" chmod +x "${CYLC_WORKFLOW_SHARE_DIR}/bin/myscript.sh" """ [[run_myscript]] script = myscript.sh [[run_mypythonscript]] script = python -c 'from mypythonscript import my_function; my_function()' cylc-flow-8.7.0/tests/functional/jobscript/02-share-cycle-cycle/0000775000175000017500000000000015264420307024632 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/jobscript/02-share-cycle-cycle/flow.cylc0000664000175000017500000000036115264420307026455 0ustar alastairalastair[scheduling] initial cycle point = 19900211 final cycle point = 19900212 [[graph]] P1D = task [runtime] [[task]] script = """ echo "Hello World" > "${CYLC_TASK_SHARE_CYCLE_DIR}/stuff" """ cylc-flow-8.7.0/tests/functional/jobscript/test_header0000777000175000017500000000000015264420307027434 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/jobscript/00-torture.t0000664000175000017500000000325515264420307023232 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # job script torture test . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" export PATH_TO_CYLC_BIN="/path/to/cylc/bin" create_test_global_config '' " [platforms] [[localhost]] cylc path = $PATH_TO_CYLC_BIN" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" purge cylc-flow-8.7.0/tests/functional/clock-trigger-inexact/0000775000175000017500000000000015264420307023305 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/clock-trigger-inexact/00-big-offset/0000775000175000017500000000000015264420307025547 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/clock-trigger-inexact/00-big-offset/flow.cylc0000664000175000017500000000215115264420307027371 0ustar alastairalastair#!Jinja2 # Test clock-trigger offset involving inexact intervals (months and years), # which requires adding the offset to the cycle point before conversion to # absolute seconds. Initial cycle point is next minute minus P1Y7M, with # opposite clock-offset to get real time triggering once per minute. {% set OFFSET = "P1Y7M" %} [scheduler] [[events]] # May take up to 60 secs to finish, allow some extra time. inactivity timeout = PT80S abort on inactivity timeout = True [scheduling] initial cycle point = next(T--00) - {{OFFSET}} # next minute - P1Y7M final cycle point = +P0Y [[xtriggers]] clock1 = wall_clock({{OFFSET}}) # xtrigger with arg clock2 = wall_clock(offset={{OFFSET}}) # xtrigger with kwarg [[special tasks]] clock-trigger = baz({{OFFSET}}) # old-style clock-triggered task [[graph]] PT1M = """ # These should all trigger at once, at the first minute boundary # after start-up @clock1 => foo @clock2 => bar baz """ [runtime] [[foo, bar, baz]] cylc-flow-8.7.0/tests/functional/clock-trigger-inexact/test_header0000777000175000017500000000000015264420307031622 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/clock-trigger-inexact/00-big-offset.t0000664000175000017500000000314315264420307025735 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test clock triggers (xtrigger and old-style) with a large inexact offset. . "$(dirname "$0")/test_header" skip_all 'TODO: fix test https://github.com/cylc/cylc-flow/issues/4633' set_test_number 5 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-val" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --no-detach "${WORKFLOW_NAME}" cylc cat-log "${WORKFLOW_NAME}" > log START_HOUR=$(grep 'Workflow:' log | cut -c 1-13) START_MINU=$(grep 'Workflow:' log | cut -c 15-16) TRIGG_MINU=$(( 10#${START_MINU} + 1)) [[ $START_MINU == 0* ]] && TRIGG_MINU=0${TRIGG_MINU} for NAME in foo bar baz; do grep_ok "${START_HOUR}:${TRIGG_MINU}.* INFO - \[.*/${NAME} .*\] => waiting$" log done purge cylc-flow-8.7.0/tests/functional/cylc-view/0000775000175000017500000000000015264420307021022 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-view/00-single-inc.t0000775000175000017500000001136415264420307023464 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test cat-view with a Jinja2 variable defined in a single cylc include-file # TODO - another test for nested file inclusion . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 5 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- # Just inline TEST_NAME=${TEST_NAME_BASE}-inline cylc view -i "${WORKFLOW_NAME}" > tmp.stdout cmp_ok tmp.stdout << EOF #!jinja2 [meta] title = "Jinja2 simple ensemble example" description = "Auto-generation of dependencies for related tasks." [scheduler] allow implicit tasks = True # Note that depending on the structure of the workflow you may be able to # SIMPLY use a task family name in the graph to represent the entire # ensemble (which would be defined by inheritance under [runtime]). {% set N_MEMBERS = 4 %} # line \\ continuation [scheduling] [[graph]] R1 = """ {# generate ensemble dependencies #} {% for I in range( 0, N_MEMBERS ) %} foo => mem_{{ I }} => post_{{ I }} => bar {% endfor %}""" EOF #------------------------------------------------------------------------------- # "cylc view -j/--jinja2" should imply "-i/inline" too: TEST_NAME=${TEST_NAME_BASE}-jinja2 cylc view -j "${WORKFLOW_NAME}" > tmp.stdout cmp_ok tmp.stdout << EOF [meta] title = "Jinja2 simple ensemble example" description = "Auto-generation of dependencies for related tasks." [scheduler] allow implicit tasks = True # Note that depending on the structure of the workflow you may be able to # SIMPLY use a task family name in the graph to represent the entire # ensemble (which would be defined by inheritance under [runtime]). # line \\ continuation [scheduling] [[graph]] R1 = """ foo => mem_0 => post_0 => bar foo => mem_1 => post_1 => bar foo => mem_2 => post_2 => bar foo => mem_3 => post_3 => bar """ EOF #------------------------------------------------------------------------------- # line continuation joining TEST_NAME=${TEST_NAME_BASE}-continuation cylc view -c "${WORKFLOW_NAME}" > tmp.stdout cmp_ok tmp.stdout << EOF #!jinja2 [meta] title = "Jinja2 simple ensemble example" description = "Auto-generation of dependencies for related tasks." [scheduler] allow implicit tasks = True # Note that depending on the structure of the workflow you may be able to # SIMPLY use a task family name in the graph to represent the entire # ensemble (which would be defined by inheritance under [runtime]). %include inc/default.jinja2 # line continuation [scheduling] [[graph]] R1 = """ {# generate ensemble dependencies #} {% for I in range( 0, N_MEMBERS ) %} foo => mem_{{ I }} => post_{{ I }} => bar {% endfor %}""" EOF #------------------------------------------------------------------------------- # all processing TEST_NAME=${TEST_NAME_BASE}-process cylc view -p "${WORKFLOW_NAME}" > tmp.stdout cmp_ok tmp.stdout << EOF [meta] title = "Jinja2 simple ensemble example" description = "Auto-generation of dependencies for related tasks." [scheduler] allow implicit tasks = True # Note that depending on the structure of the workflow you may be able to # SIMPLY use a task family name in the graph to represent the entire # ensemble (which would be defined by inheritance under [runtime]). # line continuation [scheduling] [[graph]] R1 = """ foo => mem_0 => post_0 => bar foo => mem_1 => post_1 => bar foo => mem_2 => post_2 => bar foo => mem_3 => post_3 => bar """ EOF #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/cylc-view/test_header0000777000175000017500000000000015264420307027337 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-view/00-single-inc/0000775000175000017500000000000015264420307023267 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-view/00-single-inc/flow.cylc0000664000175000017500000000120415264420307025107 0ustar alastairalastair#!jinja2 [meta] title = "Jinja2 simple ensemble example" description = "Auto-generation of dependencies for related tasks." [scheduler] allow implicit tasks = True # Note that depending on the structure of the workflow you may be able to # SIMPLY use a task family name in the graph to represent the entire # ensemble (which would be defined by inheritance under [runtime]). %include inc/default.jinja2 # line \ continuation [scheduling] [[graph]] R1 = """ {# generate ensemble dependencies #} {% for I in range( 0, N_MEMBERS ) %} foo => mem_{{ I }} => post_{{ I }} => bar {% endfor %}""" cylc-flow-8.7.0/tests/functional/cylc-view/00-single-inc/inc/0000775000175000017500000000000015264420307024040 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-view/00-single-inc/inc/default.jinja20000664000175000017500000000003015264420307026554 0ustar alastairalastair{% set N_MEMBERS = 4 %} cylc-flow-8.7.0/tests/functional/startup/0000775000175000017500000000000015264420307020622 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/startup/01-log-flow-config.t0000664000175000017500000000602615264420307024222 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "log/config/*-.cylc" files that are generated on workflow start up. . "$(dirname "$0")/test_header" set_test_number 10 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' #!Jinja2 [meta] title = a workflow that logs run, reload, and restart configs description = the weather is {{WEATHER | default("bad")}} [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S restart timeout = PT0S # shut down when restarted with empty pool [scheduling] [[graph]] R1 = reloader => whatever [runtime] [[reloader]] script = cylc reload "${CYLC_WORKFLOW_ID}" [[whatever]] script = true __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-val-1" cylc validate "${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}-val-2" \ cylc validate --set 'WEATHER="good"' "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --no-detach "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play --set 'WEATHER="good"' --no-detach "${WORKFLOW_NAME}" # Check for 4 generated *.cylc files LOGD="${RUN_DIR}/${WORKFLOW_NAME}/log/config" # shellcheck disable=SC2012 ls "${LOGD}" | sort >'ls.out' cmp_ok 'ls.out' <<'__OUT__' 01-start-01.cylc 02-reload-01.cylc 03-restart-02.cylc flow-processed.cylc __OUT__ LOGD="${RUN_DIR}/${WORKFLOW_NAME}/log/config" START_CONFIG="$(ls "${LOGD}/"*-start*.cylc)" REL_CONFIG="$(ls "${LOGD}/"*-reload*.cylc)" RES_CONFIG="$(ls "${LOGD}/"*-restart*.cylc)" mkdir start_config mkdir res_config cp "$START_CONFIG" start_config/flow.cylc cp "$RES_CONFIG" res_config/flow.cylc # The generated *-start*.cylc and *-reload*.cylc should be identical # The generated *.cylc files should validate cmp_ok "${START_CONFIG}" "${REL_CONFIG}" run_ok "${TEST_NAME_BASE}-validate-start-config" cylc validate ./start_config run_ok "${TEST_NAME_BASE}-validate-restart-config" cylc validate ./res_config rm -rf start_config res_config diff -u "${START_CONFIG}" "${RES_CONFIG}" >'diff.out' contains_ok 'diff.out' <<'__DIFF__' - description = the weather is bad + description = the weather is good __DIFF__ # Ensure that the start config is sparse. grep_fail "\[\[mail\]\]" "${START_CONFIG}" purge exit cylc-flow-8.7.0/tests/functional/startup/00-state-summary.t0000664000175000017500000000420715264420307024042 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that the state summary updates immediately on start-up. # See https://github.com/cylc/cylc-flow/pull/1756 . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- # Workflow runs and shuts down with a failed task. cylc play --no-detach "${WORKFLOW_NAME}" > /dev/null 2>&1 # Restart with a failed task and a succeeded task. cylc play "${WORKFLOW_NAME}" poll_grep_workflow_log -E '1/foo.* \(polled\)failed' cylc dump "${WORKFLOW_NAME}" > dump.out TEST_NAME=${TEST_NAME_BASE}-grep # State summary should not just say "Initializing..." grep_ok "state totals={'waiting': 0, 'expired': 0, 'preparing': 0, 'submit-failed': 0, 'submitted': 0, 'running': 0, 'failed': 1, 'succeeded': 0}" dump.out #------------------------------------------------------------------------------- cylc stop --max-polls=10 --interval=2 "${WORKFLOW_NAME}" purge cylc-flow-8.7.0/tests/functional/startup/00-state-summary/0000775000175000017500000000000015264420307023652 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/startup/00-state-summary/flow.cylc0000664000175000017500000000030215264420307025470 0ustar alastairalastair[scheduling] [[graph]] R1 = "foo & stopper" [runtime] [[foo]] script = false [[stopper]] script = """ cylc stop "${CYLC_WORKFLOW_ID}" """ cylc-flow-8.7.0/tests/functional/startup/test_header0000777000175000017500000000000015264420307027137 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/graph-equivalence/0000775000175000017500000000000015264420307022520 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/graph-equivalence/multiline_and2/0000775000175000017500000000000015264420307025426 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/graph-equivalence/multiline_and2/reference.log0000664000175000017500000000015615264420307030071 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off [] 1/c -triggered off ['1/a', '1/b'] cylc-flow-8.7.0/tests/functional/graph-equivalence/multiline_and2/flow.cylc0000664000175000017500000000017515264420307027254 0ustar alastairalastair[scheduling] [[graph]] R1 = """a => c b => c""" [runtime] [[a,b,c]] script = true cylc-flow-8.7.0/tests/functional/graph-equivalence/04-multiline_and2.t0000664000175000017500000000422015264420307026032 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test graph = """a => c # b => c""" # gives the same result as # graph = "a & b => c" . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" 'multiline_and2' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- delete_db TEST_NAME="${TEST_NAME_BASE}-check-c" cylc play "${WORKFLOW_NAME}" --hold-after=1 1>'out' 2>&1 poll_grep_workflow_log 'Setting hold cycle point' cylc show "${WORKFLOW_NAME}//1/c" | sed -n "/prerequisites/,/outputs/p" > 'c-prereqs' contains_ok "${TEST_SOURCE_DIR}/multiline_and_refs/c-ref-2" 'c-prereqs' cylc shutdown --max-polls=20 --interval=2 --now "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/graph-equivalence/00-oneline.t0000664000175000017500000000460415264420307024557 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test graph="a=>b=>c" gives the same result as # graph = """a => b\ # => c""" . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 5 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" test1 #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate \ --set="TEST_OUTPUT_PATH='${PWD}'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach \ --set="TEST_OUTPUT_PATH='${PWD}'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-check-a" cmp_ok "${TEST_SOURCE_DIR}/splitline_refs/a-ref" 'a-prereqs' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-check-b" poll_grep_workflow_log 'INFO - DONE' cmp_ok "${TEST_SOURCE_DIR}/splitline_refs/b-ref" 'b-prereqs' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-check-c" cmp_ok "${TEST_SOURCE_DIR}/splitline_refs/c-ref" 'c-prereqs' #------------------------------------------------------------------------------- cylc shutdown --max-polls=10 --interval=2 --now "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/graph-equivalence/test3/0000775000175000017500000000000015264420307023562 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/graph-equivalence/test3/reference.log0000664000175000017500000000015415264420307026223 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off ['1/a'] 1/c -triggered off ['1/b'] cylc-flow-8.7.0/tests/functional/graph-equivalence/test3/flow.cylc0000664000175000017500000000104315264420307025403 0ustar alastairalastair#!jinja2 [scheduling] [[graph]] R1 = """a => b \ => c""" [runtime] [[a]] script = """ cylc show "${CYLC_WORKFLOW_ID}//1/a" \ | sed -n "/prerequisites/,/outputs/p" > {{TEST_OUTPUT_PATH}}/a-prereqs """ [[b]] script = """ cylc show "${CYLC_WORKFLOW_ID}//1/b" \ | sed -n "/prerequisites/,/outputs/p" > {{TEST_OUTPUT_PATH}}/b-prereqs """ [[c]] script = """ cylc show "${CYLC_WORKFLOW_ID}//1/c" \ | sed -n "/prerequisites/,/outputs/p" > {{TEST_OUTPUT_PATH}}/c-prereqs """ cylc-flow-8.7.0/tests/functional/graph-equivalence/test1/0000775000175000017500000000000015264420307023560 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/graph-equivalence/test1/reference.log0000664000175000017500000000013315264420307026216 0ustar alastairalastairFinal point: 1 1/a -triggered off [] 1/b -triggered off ['1/a'] 1/c -triggered off ['1/b'] cylc-flow-8.7.0/tests/functional/graph-equivalence/test1/flow.cylc0000664000175000017500000000123615264420307025405 0ustar alastairalastair#!jinja2 [scheduling] [[graph]] R1 = "a => b => c" [runtime] [[a]] script = """ cylc show "${CYLC_WORKFLOW_ID}//1/a" \ | sed -n "/prerequisites/,/outputs/p" \ > {{TEST_OUTPUT_PATH}}/a-prereqs """ [[b]] script = """ cylc show "${CYLC_WORKFLOW_ID}//1/b" \ | sed -n "/prerequisites/,/outputs/p" \ > {{TEST_OUTPUT_PATH}}/b-prereqs """ [[c]] script = """ cylc show "${CYLC_WORKFLOW_ID}//1/c" \ | sed -n "/prerequisites/,/outputs/p" \ > {{TEST_OUTPUT_PATH}}/c-prereqs """ cylc-flow-8.7.0/tests/functional/graph-equivalence/02-splitline.t0000664000175000017500000000446515264420307025140 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test graph = """a => b\ # => c""" gives the same result as # graph = """a => b # b => c""" . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 5 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" test3 #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate \ --set="TEST_OUTPUT_PATH='${PWD}'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach \ --set="TEST_OUTPUT_PATH='${PWD}'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-check-a" cmp_ok "${TEST_SOURCE_DIR}/splitline_refs/a-ref" 'a-prereqs' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-check-b" cmp_ok "${TEST_SOURCE_DIR}/splitline_refs/b-ref" 'b-prereqs' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-check-c" cmp_ok "${TEST_SOURCE_DIR}/splitline_refs/c-ref" 'c-prereqs' #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/graph-equivalence/03-multiline_and1.t0000664000175000017500000000416115264420307026034 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test graph = "a & b => c" # gives the same result as # graph = """a => c # b => c""" . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" 'multiline_and1' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- delete_db TEST_NAME="${TEST_NAME_BASE}-check-c" cylc play "${WORKFLOW_NAME}" --hold-after=1 1>'out' 2>&1 poll_grep_workflow_log 'Setting hold cycle point' cylc show "${WORKFLOW_NAME}//1/c" | sed -n "/prerequisites/,/outputs/p" > 'c-prereqs' contains_ok "${TEST_SOURCE_DIR}/multiline_and_refs/c-ref" 'c-prereqs' cylc shutdown "${WORKFLOW_NAME}" --now #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/graph-equivalence/test_header0000777000175000017500000000000015264420307031035 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/graph-equivalence/multiline_and1/0000775000175000017500000000000015264420307025425 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/graph-equivalence/multiline_and1/reference.log0000664000175000017500000000015615264420307030070 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off [] 1/c -triggered off ['1/a', '1/b'] cylc-flow-8.7.0/tests/functional/graph-equivalence/multiline_and1/flow.cylc0000664000175000017500000000014315264420307027246 0ustar alastairalastair[scheduling] [[graph]] R1 = "a & b => c" [runtime] [[a,b,c]] script = true cylc-flow-8.7.0/tests/functional/graph-equivalence/test2/0000775000175000017500000000000015264420307023561 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/graph-equivalence/test2/reference.log0000664000175000017500000000015415264420307026222 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off ['1/a'] 1/c -triggered off ['1/b'] cylc-flow-8.7.0/tests/functional/graph-equivalence/test2/flow.cylc0000664000175000017500000000104215264420307025401 0ustar alastairalastair#!jinja2 [scheduling] [[graph]] R1 = """a => b b => c""" [runtime] [[a]] script = """ cylc show "${CYLC_WORKFLOW_ID}//1/a" \ | sed -n "/prerequisites/,/outputs/p" > {{TEST_OUTPUT_PATH}}/a-prereqs """ [[b]] script = """ cylc show "${CYLC_WORKFLOW_ID}//1/b" \ | sed -n "/prerequisites/,/outputs/p" > {{TEST_OUTPUT_PATH}}/b-prereqs """ [[c]] script = """ cylc show "${CYLC_WORKFLOW_ID}//1/c" \ | sed -n "/prerequisites/,/outputs/p" > {{TEST_OUTPUT_PATH}}/c-prereqs """ cylc-flow-8.7.0/tests/functional/graph-equivalence/multiline_and_refs/0000775000175000017500000000000015264420307026363 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/graph-equivalence/multiline_and_refs/c-ref-20000664000175000017500000000016715264420307027445 0ustar alastairalastairprerequisites: ('⨯': not satisfied) ⨯ 1/a succeeded ⨯ 1/b succeeded outputs: ('⨯': not completed) (None) cylc-flow-8.7.0/tests/functional/graph-equivalence/multiline_and_refs/c-ref0000664000175000017500000000016715264420307027306 0ustar alastairalastairprerequisites: ('⨯': not satisfied) ⨯ 1/a succeeded ⨯ 1/b succeeded outputs: ('⨯': not completed) (None) cylc-flow-8.7.0/tests/functional/graph-equivalence/01-twolines.t0000664000175000017500000000455015264420307024773 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test graph = """a => b # b => c""" gives the same result as # graph = "a => b => c" . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 5 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" test2 #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate \ --set="TEST_OUTPUT_PATH='${PWD}'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach \ --set="TEST_OUTPUT_PATH='${PWD}'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-check-a" cmp_ok "${TEST_SOURCE_DIR}/splitline_refs/a-ref" 'a-prereqs' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-check-b" poll_grep_workflow_log 'INFO - DONE' cmp_ok "${TEST_SOURCE_DIR}/splitline_refs/b-ref" 'b-prereqs' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-check-c" cmp_ok "${TEST_SOURCE_DIR}/splitline_refs/c-ref" 'c-prereqs' #------------------------------------------------------------------------------- cylc shutdown "${WORKFLOW_NAME}" --now purge cylc-flow-8.7.0/tests/functional/graph-equivalence/splitline_refs/0000775000175000017500000000000015264420307025542 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/graph-equivalence/splitline_refs/a-ref0000664000175000017500000000006615264420307026461 0ustar alastairalastairprerequisites: (None) outputs: ('⨯': not completed) cylc-flow-8.7.0/tests/functional/graph-equivalence/splitline_refs/b-ref0000664000175000017500000000013215264420307026454 0ustar alastairalastairprerequisites: ('⨯': not satisfied) ✓ 1/a succeeded outputs: ('⨯': not completed) cylc-flow-8.7.0/tests/functional/graph-equivalence/splitline_refs/c-ref0000664000175000017500000000013215264420307026455 0ustar alastairalastairprerequisites: ('⨯': not satisfied) ✓ 1/b succeeded outputs: ('⨯': not completed) cylc-flow-8.7.0/tests/functional/param_expand/0000775000175000017500000000000015264420307021557 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/param_expand/02-param_val.t0000664000175000017500000001635615264420307024140 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Check tasks and graph generated by parameter expansion. . "$(dirname "$0")/test_header" set_test_number 23 #------------------------------------------------------------------------------ cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = cat, dog, fish j = 1..5 [scheduling] [[graph]] R1 = """foo => bar bar => boo""" [runtime] [[foo, boo]] script = true [[FAM]] [[bar]] inherit = "FAM" [[bar]] inherit = "FAM" # implicit values here (takes i=cat,j=3) __WORKFLOW__ TNAME=${TEST_NAME_BASE}-1 # validate run_ok "${TNAME}" cylc validate . # family graph graph_workflow . "${TNAME}-graph-fam" --group="" cmp_ok "${TNAME}-graph-fam" "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/graph-fam-1.ref" # task graph graph_workflow . "${TNAME}-graph-exp" cmp_ok "${TNAME}-graph-exp" "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/graph-exp-1.ref" # inheritance graph graph_workflow . "${TNAME}-graph-nam" -n cmp_ok "${TNAME}-graph-nam" "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/graph-nam-1.ref" #------------------------------------------------------------------------------ cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = cat, dog, fish j = 1..5 [scheduling] [[graph]] R1 = """foo => bar bar => boo""" [runtime] [[foo, boo]] script = true [[FAM]] [[bar]] inherit = "FAM" [[bar]] inherit = "FAM" # same with explicit values __WORKFLOW__ TNAME=${TEST_NAME_BASE}-2 # validate run_ok "${TNAME}" cylc validate . # family graph graph_workflow . "${TNAME}-graph-fam" "--group=" cmp_ok "${TNAME}-graph-fam" "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/graph-fam-1.ref" # task graph graph_workflow . "${TNAME}-graph-exp" cmp_ok "${TNAME}-graph-exp" "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/graph-exp-1.ref" # inheritance graph graph_workflow . "${TNAME}-graph-nam" -n cmp_ok "${TNAME}-graph-nam" "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/graph-nam-1.ref" #------------------------------------------------------------------------------ # Same, with white space in the parameter syntax. cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = cat, dog, fish j = 1..5 [scheduling] [[graph]] R1 = """foo => bar< i ,j > bar< i = cat , j = 3 > => boo""" [runtime] [[foo, boo]] script = true [[FAM]] [[bar]] inherit = "FAM< i, j>" [[bar]] inherit = "FAM" __WORKFLOW__ TNAME=${TEST_NAME_BASE}-3 # validate run_ok "${TNAME}" cylc validate . # family graph graph_workflow . "${TNAME}-graph-fam" "--group=" cmp_ok "${TNAME}-graph-fam" "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/graph-fam-1.ref" # task graph graph_workflow . "${TNAME}-graph-exp" cmp_ok "${TNAME}-graph-exp" "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/graph-exp-1.ref" # inheritance graph graph_workflow . "${TNAME}-graph-nam" -n cmp_ok "${TNAME}-graph-nam" "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/graph-nam-1.ref" #------------------------------------------------------------------------------ cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = cat, dog, fish j = 1..5 [scheduling] [[graph]] R1 = """foo => bar bar => boo""" [runtime] [[foo, boo]] script = true [[FAM]] [[bar]] inherit = "FAM" [[bar]] inherit = "FAM" # different explicit values are legal __WORKFLOW__ TNAME=${TEST_NAME_BASE}-4 # validate run_ok "${TNAME}" cylc validate . # family graph graph_workflow . "${TNAME}-graph-fam" "--group=" cmp_ok "${TNAME}-graph-fam" "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/graph-fam-b.ref" # task graph graph_workflow . "${TNAME}-graph-exp" cmp_ok "${TNAME}-graph-exp" "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/graph-exp-b.ref" # inheritance graph graph_workflow . "${TNAME}-graph-nam" -n cmp_ok "${TNAME}-graph-nam" "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/graph-nam-b.ref" #------------------------------------------------------------------------------ cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = cat, dog, fish j = 1..5 [scheduling] [[graph]] R1 = """foo => bar bar => boo""" [runtime] [[foo]] script = true [[FAM]] [[bar]] inherit = "FAM" [[bar]] inherit = "FAM" [[boo]] inherit = "FAM" # OK (plain task can inherit from specific params) __WORKFLOW__ run_ok "${TEST_NAME_BASE}-5" cylc validate . #------------------------------------------------------------------------------ cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = cat, dog, fish j = 1..5 [scheduling] [[graph]] R1 = """foo => bar_baz""" [runtime] [[foo]] script = true [[BAR]] [[BAZ]] [[bar_baz]] # OK params separate inherit = BAR, BAZ __WORKFLOW__ run_ok "${TEST_NAME_BASE}-6" cylc validate . cylc graph --reference . 1>'06.graph' cmp_ok '06.graph' "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/06.graph.ref" #------------------------------------------------------------------------------ cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = cat, dog, fish j = 1..5 [scheduling] [[graph]] R1 = """foo => bar bar => boo""" [runtime] [[foo, boo]] script = true [[FAM]] [[bar]] inherit = "FAM" # ERROR: no frog [[bar]] inherit = "FAM" __WORKFLOW__ TNAME=${TEST_NAME_BASE}-err-1 run_fail "${TNAME}" cylc validate . cmp_ok "${TNAME}.stderr" - << __ERR__ ParamExpandError: illegal value 'i=frog' in 'inherit = FAM' __ERR__ #------------------------------------------------------------------------------ cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = cat, dog, fish j = 1..5 [scheduling] [[graph]] R1 = """foo => bar bar => boo""" [runtime] [[foo]] script = true [[FAM]] [[bar]] inherit = "FAM" [[bar]] inherit = "FAM" [[boo]] inherit = "FAM" # ERROR: i undefined here. __WORKFLOW__ TNAME="${TEST_NAME_BASE}-err-2" run_fail "${TNAME}" cylc validate . cmp_ok "${TNAME}.stderr" - << __ERR__ ParamExpandError: parameter 'i' undefined in 'inherit = FAM' __ERR__ cylc-flow-8.7.0/tests/functional/param_expand/03-env-tmpl.t0000775000175000017500000000203715264420307023733 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test parameter environment templates # Extended to include inheritance from parameterized parents (GitHub #4248) . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/param_expand/03-env-tmpl/0000775000175000017500000000000015264420307023541 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/param_expand/03-env-tmpl/t2-0099-that.ref0000664000175000017500000000002515264420307026116 0ustar alastairalastair99 stuff that closed cylc-flow-8.7.0/tests/functional/param_expand/03-env-tmpl/t1-0099-that.ref0000664000175000017500000000002315264420307026113 0ustar alastairalastair99 stuff that open cylc-flow-8.7.0/tests/functional/param_expand/03-env-tmpl/t1-0101-that.ref0000664000175000017500000000002415264420307026074 0ustar alastairalastair101 stuff that open cylc-flow-8.7.0/tests/functional/param_expand/03-env-tmpl/t1-0099-this.ref0000664000175000017500000000002315264420307026122 0ustar alastairalastair99 stuff this open cylc-flow-8.7.0/tests/functional/param_expand/03-env-tmpl/t1-0101-this.ref0000664000175000017500000000002415264420307026103 0ustar alastairalastair101 stuff this open cylc-flow-8.7.0/tests/functional/param_expand/03-env-tmpl/reference.log0000664000175000017500000000066615264420307026212 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1_num099_this -triggered off [] 1/t2_num099_this -triggered off ['1/t1_num099_this'] 1/t1_num099_that -triggered off [] 1/t2_num099_that -triggered off ['1/t1_num099_that'] 1/t1_num101_this -triggered off [] 1/t2_num101_this -triggered off ['1/t1_num101_this'] 1/t1_num101_that -triggered off [] 1/t2_num101_that -triggered off ['1/t1_num101_that'] 1/x_that -triggered off [] 1/x_this -triggered off [] cylc-flow-8.7.0/tests/functional/param_expand/03-env-tmpl/flow.cylc0000664000175000017500000000231415264420307025364 0ustar alastairalastair[task parameters] num = 99..101..2 stuff = this, that state = open, closed [scheduling] [[graph]] R1 = """ t1 => t2 x """ [runtime] [[T]] [[[environment]]] MYNUM = %(num)d MYSTUFF = stuff %(stuff)s MY_FILE = %(num)04d-%(stuff)s [[U]] [[[environment]]] STATUS = %(state)s [[t1]] inherit = T, U script = """ FILE="${CYLC_WORKFLOW_RUN_DIR}/t1-${MY_FILE}" echo "${MYNUM} ${MYSTUFF} ${STATUS}" >"${FILE}" diff ${FILE} ${FILE}.ref """ [[t2]] inherit = T, U script = """ FILE="${CYLC_WORKFLOW_RUN_DIR}/t2-${MY_FILE}" echo "${MYNUM} ${MYSTUFF} ${STATUS}" >"${FILE}" diff ${FILE} ${FILE}.ref """ # The following tests the example of GH #4248. We had wrongly assumed that # general comes before specific, for parameters in inherited environments. [[x]] inherit = U script = test $STATUS == closed [[x]] pre-script = "echo pre" cylc-flow-8.7.0/tests/functional/param_expand/03-env-tmpl/t2-0099-this.ref0000664000175000017500000000002515264420307026125 0ustar alastairalastair99 stuff this closed cylc-flow-8.7.0/tests/functional/param_expand/03-env-tmpl/t2-0101-that.ref0000664000175000017500000000002615264420307026077 0ustar alastairalastair101 stuff that closed cylc-flow-8.7.0/tests/functional/param_expand/03-env-tmpl/t2-0101-this.ref0000664000175000017500000000002615264420307026106 0ustar alastairalastair101 stuff this closed cylc-flow-8.7.0/tests/functional/param_expand/test_header0000777000175000017500000000000015264420307030074 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/param_expand/01-basic.t0000664000175000017500000002303315264420307023244 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check tasks and graph generated by parameter expansion. . "$(dirname "$0")/test_header" set_test_number 39 cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = cat, dog, fish j = 1..5 k = 1..10..4 [scheduling] [[graph]] R1 = """ foo => bar qux => waz """ [runtime] [[root]] script = true [[foo]] [[bar]] [[qux]] [[waz]] __WORKFLOW__ run_ok "${TEST_NAME_BASE}-01" cylc validate . cylc graph --reference . >'01.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/01.graph.ref" '01.graph' cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = 25, 30..35, 1..5, 110 [scheduling] [[graph]] R1 = """ foo => bar """ [runtime] [[root]] script = true [[foo]] [[bar]] __WORKFLOW__ run_ok "${TEST_NAME_BASE}-02" cylc validate . cylc graph --reference . >'02.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/02.graph.ref" '02.graph' cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = a-t, c-g [scheduling] [[graph]] R1 = """ foo => bar """ [runtime] [[root]] script = true [[foo]] [[bar]] __WORKFLOW__ run_ok "${TEST_NAME_BASE}-03" cylc validate . cylc graph --reference . >'03.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/03.graph.ref" '03.graph' cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = 100, hundred, one-hundred, 99+1 [scheduling] [[graph]] R1 = """ foo => bar """ [runtime] [[root]] script = true [[foo]] [[bar]] __WORKFLOW__ run_ok "${TEST_NAME_BASE}-04" cylc validate . cylc graph --reference . >'04.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/04.graph.ref" '04.graph' cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = space is dangerous [scheduling] [[graph]] R1 = """ foo => bar """ [runtime] [[root]] script = true [[foo]] [[bar]] __WORKFLOW__ run_fail "${TEST_NAME_BASE}-05" cylc validate . cmp_ok "${TEST_NAME_BASE}-05.stderr" <<'__ERR__' IllegalValueError: (type=parameter) [task parameters]i = space is dangerous - (space is dangerous: bad value) __ERR__ cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = mix, 1..10 [scheduling] [[graph]] R1 = """ foo => bar """ [runtime] [[root]] script = true [[foo]] [[bar]] __WORKFLOW__ run_fail "${TEST_NAME_BASE}-06" cylc validate . cmp_ok "${TEST_NAME_BASE}-06.stderr" <<'__ERR__' IllegalValueError: (type=parameter) [task parameters]i = mix, 1..10 - (mixing int range and str) __ERR__ cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = a, b #, c, d, e # comment [scheduling] [[graph]] R1 = """ foo => bar """ [runtime] [[root]] script = true [[foo]] [[bar]] __WORKFLOW__ run_ok "${TEST_NAME_BASE}-07" cylc validate . cylc graph --reference . >'07.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/07.graph.ref" '07.graph' cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = 1..2 3..4 [scheduling] [[graph]] R1 = """ foo => bar """ [runtime] [[root]] script = true [[foo]] [[bar]] __WORKFLOW__ run_fail "${TEST_NAME_BASE}-08" cylc validate . cmp_ok "${TEST_NAME_BASE}-08.stderr" <<'__ERR__' IllegalValueError: (type=parameter) [task parameters]i = 1..2 3..4 - (1..2 3..4: bad value) __ERR__ cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = [scheduling] [[graph]] R1 = """ foo => bar """ [runtime] [[root]] script = true [[foo]] [[bar]] __WORKFLOW__ run_fail "${TEST_NAME_BASE}-09" cylc validate . cmp_ok "${TEST_NAME_BASE}-09.stderr" <<'__ERR__' ParamExpandError: parameter i is not defined in foo __ERR__ cat >'flow.cylc' <<'__WORKFLOW__' [scheduling] [[graph]] R1 = """ foo => bar """ [runtime] [[root]] script = true [[foo]] [[bar]] __WORKFLOW__ run_fail "${TEST_NAME_BASE}-10" cylc validate . cmp_ok "${TEST_NAME_BASE}-10.stderr" <<'__ERR__' ParamExpandError: parameter i is not defined in : foo=>bar __ERR__ cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] j = +1..+5 [[templates]] j = @%(j)03d [scheduling] [[graph]] R1 = "foo => bar" [runtime] [[root]] script = true [[foo]] [[bar]] __WORKFLOW__ run_ok "${TEST_NAME_BASE}-11" cylc validate . cylc graph --reference . >'11.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/11.graph.ref" '11.graph' cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] j = 1..5 [[templates]] j = +%%j%(j)03d [scheduling] [[graph]] R1 = "foo => bar" [runtime] [[root]] script = true [[foo]] [[bar]] __WORKFLOW__ run_ok "${TEST_NAME_BASE}-12" cylc validate . cylc graph --reference . >'12.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/12.graph.ref" '12.graph' # Parameter with various meta characters cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] p = -minus, +plus, @at, %percent [[templates]] p = %(p)s [scheduling] [[graph]] R1 = "foo

=> bar

" [runtime] [[root]] script = true [[foo

]] [[bar

]] __WORKFLOW__ run_ok "${TEST_NAME_BASE}-13" cylc validate . cylc graph --reference . >'13.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/13.graph.ref" '13.graph' # Parameter as task name cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] i = 0..2 s = mercury, venus, earth, mars [[templates]] i = i%(i)d s = %(s)s [scheduling] [[graph]] R1 = """ foo => => bar foo => => bar """ [runtime] [[foo, bar, , ]] script = true __WORKFLOW__ run_ok "${TEST_NAME_BASE}-14" cylc validate . cylc graph --reference . >'14.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/14.graph.ref" '14.graph' # Parameter in middle of family name cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] s = mercury, venus, earth, mars [scheduling] [[graph]] R1 = XY [runtime] [[XY]] script = true [[xy]] inherit = XY __WORKFLOW__ run_ok "${TEST_NAME_BASE}-15" cylc validate . cylc graph --reference --group="" . >'15.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/15.graph.ref" '15.graph' # -ve offset on RHS cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] m = cat, dog [scheduling] [[graph]] R1 = "foo => foo" [runtime] [[root]] script = true [[foo]] __WORKFLOW__ run_ok "${TEST_NAME_BASE}-16" cylc validate . cylc graph --reference . >'16.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/16.graph.ref" '16.graph' # +ve offset cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] m = cat, dog [scheduling] [[graph]] R1 = "foo => foo" [runtime] [[root]] script = true [[foo]] __WORKFLOW__ run_ok "${TEST_NAME_BASE}-17" cylc validate . cylc graph --reference . >'17.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/17.graph.ref" '17.graph' # Negative integers cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] m = -12..12..6 [scheduling] [[graph]] R1 = "foo" [runtime] [[root]] script = true [[foo]] __WORKFLOW__ run_ok "${TEST_NAME_BASE}-18" cylc validate . cylc graph --reference . >'18.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/18.graph.ref" '18.graph' # Reference by value, with -+ meta characters cat >'flow.cylc' <<'__WORKFLOW__' [task parameters] lang = c++, fortran-2008 [[templates]] lang = %(lang)s [scheduling] [[graph]] R1 = " => " [runtime] [[]] script = true [[]] [[[environment]]] CC = gcc [[]] [[[environment]]] FC = gfortran __WORKFLOW__ run_ok "${TEST_NAME_BASE}-19" cylc validate --debug . cylc graph --reference . >'19.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/19.graph.ref" '19.graph' # Note: This also demonstrates current badness of "cylc config"... # Inconsistence between graph/runtime whitespace handling. # Inconsistence between graph/runtime parameter expansion. cylc config . >'19.cylc' cmp_ok '19.cylc' <<'__FLOW_CONFIG__' [task parameters] lang = c++, fortran-2008 [[templates]] lang = %(lang)s [scheduling] initial cycle point = 1 final cycle point = 1 cycling mode = integer [[graph]] R1 = => [runtime] [[root]] [[c++]] script = true completion = succeeded [[[environment]]] CC = gcc [[fortran-2008]] script = true completion = succeeded [[[environment]]] FC = gfortran __FLOW_CONFIG__ exit cylc-flow-8.7.0/tests/functional/param_expand/02-param_val/0000775000175000017500000000000015264420307023740 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/param_expand/02-param_val/graph-fam-1.ref0000664000175000017500000000204215264420307026434 0ustar alastairalastairedge "1/FAM_cat_j3" "1/boo" edge "1/foo" "1/FAM_cat_j1" edge "1/foo" "1/FAM_cat_j2" edge "1/foo" "1/FAM_cat_j3" edge "1/foo" "1/FAM_cat_j4" edge "1/foo" "1/FAM_cat_j5" edge "1/foo" "1/FAM_dog_j1" edge "1/foo" "1/FAM_dog_j2" edge "1/foo" "1/FAM_dog_j3" edge "1/foo" "1/FAM_dog_j4" edge "1/foo" "1/FAM_dog_j5" edge "1/foo" "1/FAM_fish_j1" edge "1/foo" "1/FAM_fish_j2" edge "1/foo" "1/FAM_fish_j3" edge "1/foo" "1/FAM_fish_j4" edge "1/foo" "1/FAM_fish_j5" graph node "1/FAM_cat_j1" "FAM_cat_j1\n1" node "1/FAM_cat_j2" "FAM_cat_j2\n1" node "1/FAM_cat_j3" "FAM_cat_j3\n1" node "1/FAM_cat_j4" "FAM_cat_j4\n1" node "1/FAM_cat_j5" "FAM_cat_j5\n1" node "1/FAM_dog_j1" "FAM_dog_j1\n1" node "1/FAM_dog_j2" "FAM_dog_j2\n1" node "1/FAM_dog_j3" "FAM_dog_j3\n1" node "1/FAM_dog_j4" "FAM_dog_j4\n1" node "1/FAM_dog_j5" "FAM_dog_j5\n1" node "1/FAM_fish_j1" "FAM_fish_j1\n1" node "1/FAM_fish_j2" "FAM_fish_j2\n1" node "1/FAM_fish_j3" "FAM_fish_j3\n1" node "1/FAM_fish_j4" "FAM_fish_j4\n1" node "1/FAM_fish_j5" "FAM_fish_j5\n1" node "1/boo" "boo\n1" node "1/foo" "foo\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/02-param_val/graph-fam-b.ref0000664000175000017500000000174215264420307026523 0ustar alastairalastairedge "1/FAM_dog_j1" "1/boo" edge "1/foo" "1/FAM_cat_j1" edge "1/foo" "1/FAM_cat_j2" edge "1/foo" "1/FAM_cat_j4" edge "1/foo" "1/FAM_cat_j5" edge "1/foo" "1/FAM_dog_j1" edge "1/foo" "1/FAM_dog_j2" edge "1/foo" "1/FAM_dog_j3" edge "1/foo" "1/FAM_dog_j4" edge "1/foo" "1/FAM_dog_j5" edge "1/foo" "1/FAM_fish_j1" edge "1/foo" "1/FAM_fish_j2" edge "1/foo" "1/FAM_fish_j3" edge "1/foo" "1/FAM_fish_j4" edge "1/foo" "1/FAM_fish_j5" graph node "1/FAM_cat_j1" "FAM_cat_j1\n1" node "1/FAM_cat_j2" "FAM_cat_j2\n1" node "1/FAM_cat_j4" "FAM_cat_j4\n1" node "1/FAM_cat_j5" "FAM_cat_j5\n1" node "1/FAM_dog_j1" "FAM_dog_j1\n1" node "1/FAM_dog_j2" "FAM_dog_j2\n1" node "1/FAM_dog_j3" "FAM_dog_j3\n1" node "1/FAM_dog_j4" "FAM_dog_j4\n1" node "1/FAM_dog_j5" "FAM_dog_j5\n1" node "1/FAM_fish_j1" "FAM_fish_j1\n1" node "1/FAM_fish_j2" "FAM_fish_j2\n1" node "1/FAM_fish_j3" "FAM_fish_j3\n1" node "1/FAM_fish_j4" "FAM_fish_j4\n1" node "1/FAM_fish_j5" "FAM_fish_j5\n1" node "1/boo" "boo\n1" node "1/foo" "foo\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/02-param_val/graph-exp-1.ref0000664000175000017500000000204215264420307026465 0ustar alastairalastairedge "1/bar_cat_j3" "1/boo" edge "1/foo" "1/bar_cat_j1" edge "1/foo" "1/bar_cat_j2" edge "1/foo" "1/bar_cat_j3" edge "1/foo" "1/bar_cat_j4" edge "1/foo" "1/bar_cat_j5" edge "1/foo" "1/bar_dog_j1" edge "1/foo" "1/bar_dog_j2" edge "1/foo" "1/bar_dog_j3" edge "1/foo" "1/bar_dog_j4" edge "1/foo" "1/bar_dog_j5" edge "1/foo" "1/bar_fish_j1" edge "1/foo" "1/bar_fish_j2" edge "1/foo" "1/bar_fish_j3" edge "1/foo" "1/bar_fish_j4" edge "1/foo" "1/bar_fish_j5" graph node "1/bar_cat_j1" "bar_cat_j1\n1" node "1/bar_cat_j2" "bar_cat_j2\n1" node "1/bar_cat_j3" "bar_cat_j3\n1" node "1/bar_cat_j4" "bar_cat_j4\n1" node "1/bar_cat_j5" "bar_cat_j5\n1" node "1/bar_dog_j1" "bar_dog_j1\n1" node "1/bar_dog_j2" "bar_dog_j2\n1" node "1/bar_dog_j3" "bar_dog_j3\n1" node "1/bar_dog_j4" "bar_dog_j4\n1" node "1/bar_dog_j5" "bar_dog_j5\n1" node "1/bar_fish_j1" "bar_fish_j1\n1" node "1/bar_fish_j2" "bar_fish_j2\n1" node "1/bar_fish_j3" "bar_fish_j3\n1" node "1/bar_fish_j4" "bar_fish_j4\n1" node "1/bar_fish_j5" "bar_fish_j5\n1" node "1/boo" "boo\n1" node "1/foo" "foo\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/02-param_val/graph-nam-b.ref0000664000175000017500000000356115264420307026534 0ustar alastairalastairedge "FAM_cat_j1" "bar_cat_j1" edge "FAM_cat_j2" "bar_cat_j2" edge "FAM_cat_j4" "bar_cat_j4" edge "FAM_cat_j5" "bar_cat_j5" edge "FAM_dog_j1" "bar_cat_j3" edge "FAM_dog_j1" "bar_dog_j1" edge "FAM_dog_j2" "bar_dog_j2" edge "FAM_dog_j3" "bar_dog_j3" edge "FAM_dog_j4" "bar_dog_j4" edge "FAM_dog_j5" "bar_dog_j5" edge "FAM_fish_j1" "bar_fish_j1" edge "FAM_fish_j2" "bar_fish_j2" edge "FAM_fish_j3" "bar_fish_j3" edge "FAM_fish_j4" "bar_fish_j4" edge "FAM_fish_j5" "bar_fish_j5" edge "root" "FAM_cat_j1" edge "root" "FAM_cat_j2" edge "root" "FAM_cat_j3" edge "root" "FAM_cat_j4" edge "root" "FAM_cat_j5" edge "root" "FAM_dog_j1" edge "root" "FAM_dog_j2" edge "root" "FAM_dog_j3" edge "root" "FAM_dog_j4" edge "root" "FAM_dog_j5" edge "root" "FAM_fish_j1" edge "root" "FAM_fish_j2" edge "root" "FAM_fish_j3" edge "root" "FAM_fish_j4" edge "root" "FAM_fish_j5" edge "root" "boo" edge "root" "foo" graph node "FAM_cat_j1" "FAM_cat_j1" node "FAM_cat_j2" "FAM_cat_j2" node "FAM_cat_j3" "FAM_cat_j3" node "FAM_cat_j4" "FAM_cat_j4" node "FAM_cat_j5" "FAM_cat_j5" node "FAM_dog_j1" "FAM_dog_j1" node "FAM_dog_j2" "FAM_dog_j2" node "FAM_dog_j3" "FAM_dog_j3" node "FAM_dog_j4" "FAM_dog_j4" node "FAM_dog_j5" "FAM_dog_j5" node "FAM_fish_j1" "FAM_fish_j1" node "FAM_fish_j2" "FAM_fish_j2" node "FAM_fish_j3" "FAM_fish_j3" node "FAM_fish_j4" "FAM_fish_j4" node "FAM_fish_j5" "FAM_fish_j5" node "bar_cat_j1" "bar_cat_j1" node "bar_cat_j2" "bar_cat_j2" node "bar_cat_j3" "bar_cat_j3" node "bar_cat_j4" "bar_cat_j4" node "bar_cat_j5" "bar_cat_j5" node "bar_dog_j1" "bar_dog_j1" node "bar_dog_j2" "bar_dog_j2" node "bar_dog_j3" "bar_dog_j3" node "bar_dog_j4" "bar_dog_j4" node "bar_dog_j5" "bar_dog_j5" node "bar_fish_j1" "bar_fish_j1" node "bar_fish_j2" "bar_fish_j2" node "bar_fish_j3" "bar_fish_j3" node "bar_fish_j4" "bar_fish_j4" node "bar_fish_j5" "bar_fish_j5" node "boo" "boo" node "foo" "foo" node "root" "root" stop cylc-flow-8.7.0/tests/functional/param_expand/02-param_val/graph-nam-1.ref0000664000175000017500000000356115264420307026453 0ustar alastairalastairedge "FAM_cat_j1" "bar_cat_j1" edge "FAM_cat_j2" "bar_cat_j2" edge "FAM_cat_j3" "bar_cat_j3" edge "FAM_cat_j4" "bar_cat_j4" edge "FAM_cat_j5" "bar_cat_j5" edge "FAM_dog_j1" "bar_dog_j1" edge "FAM_dog_j2" "bar_dog_j2" edge "FAM_dog_j3" "bar_dog_j3" edge "FAM_dog_j4" "bar_dog_j4" edge "FAM_dog_j5" "bar_dog_j5" edge "FAM_fish_j1" "bar_fish_j1" edge "FAM_fish_j2" "bar_fish_j2" edge "FAM_fish_j3" "bar_fish_j3" edge "FAM_fish_j4" "bar_fish_j4" edge "FAM_fish_j5" "bar_fish_j5" edge "root" "FAM_cat_j1" edge "root" "FAM_cat_j2" edge "root" "FAM_cat_j3" edge "root" "FAM_cat_j4" edge "root" "FAM_cat_j5" edge "root" "FAM_dog_j1" edge "root" "FAM_dog_j2" edge "root" "FAM_dog_j3" edge "root" "FAM_dog_j4" edge "root" "FAM_dog_j5" edge "root" "FAM_fish_j1" edge "root" "FAM_fish_j2" edge "root" "FAM_fish_j3" edge "root" "FAM_fish_j4" edge "root" "FAM_fish_j5" edge "root" "boo" edge "root" "foo" graph node "FAM_cat_j1" "FAM_cat_j1" node "FAM_cat_j2" "FAM_cat_j2" node "FAM_cat_j3" "FAM_cat_j3" node "FAM_cat_j4" "FAM_cat_j4" node "FAM_cat_j5" "FAM_cat_j5" node "FAM_dog_j1" "FAM_dog_j1" node "FAM_dog_j2" "FAM_dog_j2" node "FAM_dog_j3" "FAM_dog_j3" node "FAM_dog_j4" "FAM_dog_j4" node "FAM_dog_j5" "FAM_dog_j5" node "FAM_fish_j1" "FAM_fish_j1" node "FAM_fish_j2" "FAM_fish_j2" node "FAM_fish_j3" "FAM_fish_j3" node "FAM_fish_j4" "FAM_fish_j4" node "FAM_fish_j5" "FAM_fish_j5" node "bar_cat_j1" "bar_cat_j1" node "bar_cat_j2" "bar_cat_j2" node "bar_cat_j3" "bar_cat_j3" node "bar_cat_j4" "bar_cat_j4" node "bar_cat_j5" "bar_cat_j5" node "bar_dog_j1" "bar_dog_j1" node "bar_dog_j2" "bar_dog_j2" node "bar_dog_j3" "bar_dog_j3" node "bar_dog_j4" "bar_dog_j4" node "bar_dog_j5" "bar_dog_j5" node "bar_fish_j1" "bar_fish_j1" node "bar_fish_j2" "bar_fish_j2" node "bar_fish_j3" "bar_fish_j3" node "bar_fish_j4" "bar_fish_j4" node "bar_fish_j5" "bar_fish_j5" node "boo" "boo" node "foo" "foo" node "root" "root" stop cylc-flow-8.7.0/tests/functional/param_expand/02-param_val/graph-exp-b.ref0000664000175000017500000000204215264420307026546 0ustar alastairalastairedge "1/bar_cat_j3" "1/boo" edge "1/foo" "1/bar_cat_j1" edge "1/foo" "1/bar_cat_j2" edge "1/foo" "1/bar_cat_j3" edge "1/foo" "1/bar_cat_j4" edge "1/foo" "1/bar_cat_j5" edge "1/foo" "1/bar_dog_j1" edge "1/foo" "1/bar_dog_j2" edge "1/foo" "1/bar_dog_j3" edge "1/foo" "1/bar_dog_j4" edge "1/foo" "1/bar_dog_j5" edge "1/foo" "1/bar_fish_j1" edge "1/foo" "1/bar_fish_j2" edge "1/foo" "1/bar_fish_j3" edge "1/foo" "1/bar_fish_j4" edge "1/foo" "1/bar_fish_j5" graph node "1/bar_cat_j1" "bar_cat_j1\n1" node "1/bar_cat_j2" "bar_cat_j2\n1" node "1/bar_cat_j3" "bar_cat_j3\n1" node "1/bar_cat_j4" "bar_cat_j4\n1" node "1/bar_cat_j5" "bar_cat_j5\n1" node "1/bar_dog_j1" "bar_dog_j1\n1" node "1/bar_dog_j2" "bar_dog_j2\n1" node "1/bar_dog_j3" "bar_dog_j3\n1" node "1/bar_dog_j4" "bar_dog_j4\n1" node "1/bar_dog_j5" "bar_dog_j5\n1" node "1/bar_fish_j1" "bar_fish_j1\n1" node "1/bar_fish_j2" "bar_fish_j2\n1" node "1/bar_fish_j3" "bar_fish_j3\n1" node "1/bar_fish_j4" "bar_fish_j4\n1" node "1/bar_fish_j5" "bar_fish_j5\n1" node "1/boo" "boo\n1" node "1/foo" "foo\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/02-param_val/06.graph.ref0000664000175000017500000000224415264420307025765 0ustar alastairalastairedge "1/foo" "1/bar_cat_baz_j1" edge "1/foo" "1/bar_cat_baz_j2" edge "1/foo" "1/bar_cat_baz_j3" edge "1/foo" "1/bar_cat_baz_j4" edge "1/foo" "1/bar_cat_baz_j5" edge "1/foo" "1/bar_dog_baz_j1" edge "1/foo" "1/bar_dog_baz_j2" edge "1/foo" "1/bar_dog_baz_j3" edge "1/foo" "1/bar_dog_baz_j4" edge "1/foo" "1/bar_dog_baz_j5" edge "1/foo" "1/bar_fish_baz_j1" edge "1/foo" "1/bar_fish_baz_j2" edge "1/foo" "1/bar_fish_baz_j3" edge "1/foo" "1/bar_fish_baz_j4" edge "1/foo" "1/bar_fish_baz_j5" graph node "1/bar_cat_baz_j1" "bar_cat_baz_j1\n1" node "1/bar_cat_baz_j2" "bar_cat_baz_j2\n1" node "1/bar_cat_baz_j3" "bar_cat_baz_j3\n1" node "1/bar_cat_baz_j4" "bar_cat_baz_j4\n1" node "1/bar_cat_baz_j5" "bar_cat_baz_j5\n1" node "1/bar_dog_baz_j1" "bar_dog_baz_j1\n1" node "1/bar_dog_baz_j2" "bar_dog_baz_j2\n1" node "1/bar_dog_baz_j3" "bar_dog_baz_j3\n1" node "1/bar_dog_baz_j4" "bar_dog_baz_j4\n1" node "1/bar_dog_baz_j5" "bar_dog_baz_j5\n1" node "1/bar_fish_baz_j1" "bar_fish_baz_j1\n1" node "1/bar_fish_baz_j2" "bar_fish_baz_j2\n1" node "1/bar_fish_baz_j3" "bar_fish_baz_j3\n1" node "1/bar_fish_baz_j4" "bar_fish_baz_j4\n1" node "1/bar_fish_baz_j5" "bar_fish_baz_j5\n1" node "1/foo" "foo\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/01-basic/0000775000175000017500000000000015264420307023056 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/param_expand/01-basic/15.graph.ref0000664000175000017500000000021515264420307025077 0ustar alastairalastairgraph node "1/X_earthY" "X_earthY\n1" node "1/X_marsY" "X_marsY\n1" node "1/X_mercuryY" "X_mercuryY\n1" node "1/X_venusY" "X_venusY\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/01-basic/19.graph.ref0000664000175000017500000000014715264420307025107 0ustar alastairalastairedge "1/c++" "1/fortran-2008" graph node "1/c++" "c++\n1" node "1/fortran-2008" "fortran-2008\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/01-basic/01.graph.ref0000664000175000017500000000242115264420307025073 0ustar alastairalastairedge "1/foo_cat" "1/bar_j1" edge "1/foo_cat" "1/bar_j2" edge "1/foo_cat" "1/bar_j3" edge "1/foo_cat" "1/bar_j4" edge "1/foo_cat" "1/bar_j5" edge "1/foo_dog" "1/bar_j1" edge "1/foo_dog" "1/bar_j2" edge "1/foo_dog" "1/bar_j3" edge "1/foo_dog" "1/bar_j4" edge "1/foo_dog" "1/bar_j5" edge "1/foo_fish" "1/bar_j1" edge "1/foo_fish" "1/bar_j2" edge "1/foo_fish" "1/bar_j3" edge "1/foo_fish" "1/bar_j4" edge "1/foo_fish" "1/bar_j5" edge "1/qux_j1" "1/waz_k1" edge "1/qux_j1" "1/waz_k5" edge "1/qux_j1" "1/waz_k9" edge "1/qux_j2" "1/waz_k1" edge "1/qux_j2" "1/waz_k5" edge "1/qux_j2" "1/waz_k9" edge "1/qux_j3" "1/waz_k1" edge "1/qux_j3" "1/waz_k5" edge "1/qux_j3" "1/waz_k9" edge "1/qux_j4" "1/waz_k1" edge "1/qux_j4" "1/waz_k5" edge "1/qux_j4" "1/waz_k9" edge "1/qux_j5" "1/waz_k1" edge "1/qux_j5" "1/waz_k5" edge "1/qux_j5" "1/waz_k9" graph node "1/bar_j1" "bar_j1\n1" node "1/bar_j2" "bar_j2\n1" node "1/bar_j3" "bar_j3\n1" node "1/bar_j4" "bar_j4\n1" node "1/bar_j5" "bar_j5\n1" node "1/foo_cat" "foo_cat\n1" node "1/foo_dog" "foo_dog\n1" node "1/foo_fish" "foo_fish\n1" node "1/qux_j1" "qux_j1\n1" node "1/qux_j2" "qux_j2\n1" node "1/qux_j3" "qux_j3\n1" node "1/qux_j4" "qux_j4\n1" node "1/qux_j5" "qux_j5\n1" node "1/waz_k1" "waz_k1\n1" node "1/waz_k5" "waz_k5\n1" node "1/waz_k9" "waz_k9\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/01-basic/03.graph.ref0000664000175000017500000000027515264420307025102 0ustar alastairalastairedge "1/foo_a-t" "1/bar_a-t" edge "1/foo_c-g" "1/bar_c-g" graph node "1/bar_a-t" "bar_a-t\n1" node "1/bar_c-g" "bar_c-g\n1" node "1/foo_a-t" "foo_a-t\n1" node "1/foo_c-g" "foo_c-g\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/01-basic/18.graph.ref0000664000175000017500000000025315264420307025104 0ustar alastairalastairgraph node "1/foo_m+00" "foo_m+00\n1" node "1/foo_m+06" "foo_m+06\n1" node "1/foo_m+12" "foo_m+12\n1" node "1/foo_m-06" "foo_m-06\n1" node "1/foo_m-12" "foo_m-12\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/01-basic/07.graph.ref0000664000175000017500000000024515264420307025103 0ustar alastairalastairedge "1/foo_a" "1/bar_a" edge "1/foo_b" "1/bar_b" graph node "1/bar_a" "bar_a\n1" node "1/bar_b" "bar_b\n1" node "1/foo_a" "foo_a\n1" node "1/foo_b" "foo_b\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/01-basic/12.graph.ref0000664000175000017500000000100415264420307025071 0ustar alastairalastairedge "1/foo+%j001" "1/bar+%j001" edge "1/foo+%j002" "1/bar+%j002" edge "1/foo+%j003" "1/bar+%j003" edge "1/foo+%j004" "1/bar+%j004" edge "1/foo+%j005" "1/bar+%j005" graph node "1/bar+%j001" "bar+%j001\n1" node "1/bar+%j002" "bar+%j002\n1" node "1/bar+%j003" "bar+%j003\n1" node "1/bar+%j004" "bar+%j004\n1" node "1/bar+%j005" "bar+%j005\n1" node "1/foo+%j001" "foo+%j001\n1" node "1/foo+%j002" "foo+%j002\n1" node "1/foo+%j003" "foo+%j003\n1" node "1/foo+%j004" "foo+%j004\n1" node "1/foo+%j005" "foo+%j005\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/01-basic/17.graph.ref0000664000175000017500000000014415264420307025102 0ustar alastairalastairedge "1/foo_cat" "1/foo_dog" graph node "1/foo_cat" "foo_cat\n1" node "1/foo_dog" "foo_dog\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/01-basic/11.graph.ref0000664000175000017500000000071015264420307025073 0ustar alastairalastairedge "1/foo@001" "1/bar@001" edge "1/foo@002" "1/bar@002" edge "1/foo@003" "1/bar@003" edge "1/foo@004" "1/bar@004" edge "1/foo@005" "1/bar@005" graph node "1/bar@001" "bar@001\n1" node "1/bar@002" "bar@002\n1" node "1/bar@003" "bar@003\n1" node "1/bar@004" "bar@004\n1" node "1/bar@005" "bar@005\n1" node "1/foo@001" "foo@001\n1" node "1/foo@002" "foo@002\n1" node "1/foo@003" "foo@003\n1" node "1/foo@004" "foo@004\n1" node "1/foo@005" "foo@005\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/01-basic/14.graph.ref0000664000175000017500000000101715264420307025077 0ustar alastairalastairedge "1/earth" "1/bar" edge "1/foo" "1/earth" edge "1/foo" "1/i0" edge "1/foo" "1/i1" edge "1/foo" "1/i2" edge "1/foo" "1/mars" edge "1/foo" "1/mercury" edge "1/foo" "1/venus" edge "1/i0" "1/bar" edge "1/i1" "1/bar" edge "1/i2" "1/bar" edge "1/mars" "1/bar" edge "1/mercury" "1/bar" edge "1/venus" "1/bar" graph node "1/bar" "bar\n1" node "1/earth" "earth\n1" node "1/foo" "foo\n1" node "1/i0" "i0\n1" node "1/i1" "i1\n1" node "1/i2" "i2\n1" node "1/mars" "mars\n1" node "1/mercury" "mercury\n1" node "1/venus" "venus\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/01-basic/13.graph.ref0000664000175000017500000000062315264420307025100 0ustar alastairalastairedge "1/foo%percent" "1/bar%percent" edge "1/foo+plus" "1/bar+plus" edge "1/foo-minus" "1/bar-minus" edge "1/foo@at" "1/bar@at" graph node "1/bar%percent" "bar%percent\n1" node "1/bar+plus" "bar+plus\n1" node "1/bar-minus" "bar-minus\n1" node "1/bar@at" "bar@at\n1" node "1/foo%percent" "foo%percent\n1" node "1/foo+plus" "foo+plus\n1" node "1/foo-minus" "foo-minus\n1" node "1/foo@at" "foo@at\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/01-basic/16.graph.ref0000664000175000017500000000014415264420307025101 0ustar alastairalastairedge "1/foo_dog" "1/foo_cat" graph node "1/foo_cat" "foo_cat\n1" node "1/foo_dog" "foo_dog\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/01-basic/04.graph.ref0000664000175000017500000000067515264420307025107 0ustar alastairalastairedge "1/foo_100" "1/bar_100" edge "1/foo_99+1" "1/bar_99+1" edge "1/foo_hundred" "1/bar_hundred" edge "1/foo_one-hundred" "1/bar_one-hundred" graph node "1/bar_100" "bar_100\n1" node "1/bar_99+1" "bar_99+1\n1" node "1/bar_hundred" "bar_hundred\n1" node "1/bar_one-hundred" "bar_one-hundred\n1" node "1/foo_100" "foo_100\n1" node "1/foo_99+1" "foo_99+1\n1" node "1/foo_hundred" "foo_hundred\n1" node "1/foo_one-hundred" "foo_one-hundred\n1" stop cylc-flow-8.7.0/tests/functional/param_expand/01-basic/02.graph.ref0000664000175000017500000000233615264420307025101 0ustar alastairalastairedge "1/foo_i001" "1/bar_i001" edge "1/foo_i002" "1/bar_i002" edge "1/foo_i003" "1/bar_i003" edge "1/foo_i004" "1/bar_i004" edge "1/foo_i005" "1/bar_i005" edge "1/foo_i025" "1/bar_i025" edge "1/foo_i030" "1/bar_i030" edge "1/foo_i031" "1/bar_i031" edge "1/foo_i032" "1/bar_i032" edge "1/foo_i033" "1/bar_i033" edge "1/foo_i034" "1/bar_i034" edge "1/foo_i035" "1/bar_i035" edge "1/foo_i110" "1/bar_i110" graph node "1/bar_i001" "bar_i001\n1" node "1/bar_i002" "bar_i002\n1" node "1/bar_i003" "bar_i003\n1" node "1/bar_i004" "bar_i004\n1" node "1/bar_i005" "bar_i005\n1" node "1/bar_i025" "bar_i025\n1" node "1/bar_i030" "bar_i030\n1" node "1/bar_i031" "bar_i031\n1" node "1/bar_i032" "bar_i032\n1" node "1/bar_i033" "bar_i033\n1" node "1/bar_i034" "bar_i034\n1" node "1/bar_i035" "bar_i035\n1" node "1/bar_i110" "bar_i110\n1" node "1/foo_i001" "foo_i001\n1" node "1/foo_i002" "foo_i002\n1" node "1/foo_i003" "foo_i003\n1" node "1/foo_i004" "foo_i004\n1" node "1/foo_i005" "foo_i005\n1" node "1/foo_i025" "foo_i025\n1" node "1/foo_i030" "foo_i030\n1" node "1/foo_i031" "foo_i031\n1" node "1/foo_i032" "foo_i032\n1" node "1/foo_i033" "foo_i033\n1" node "1/foo_i034" "foo_i034\n1" node "1/foo_i035" "foo_i035\n1" node "1/foo_i110" "foo_i110\n1" stop cylc-flow-8.7.0/tests/functional/xtriggers/0000775000175000017500000000000015264420307021136 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/xtriggers/03-sequence.t0000664000175000017500000000370215264420307023355 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test xtrigger cycle-point specificity - # https://github.com/cylc/cylc-flow/issues/3283 . "$(dirname "$0")/test_header" set_test_number 2 # Test workflow uses built-in 'echo' xtrigger. init_workflow "${TEST_NAME_BASE}" << '__FLOW_CONFIG__' [scheduler] cycle point format = %Y [scheduling] initial cycle point = 2025 final cycle point = +P1Y [[xtriggers]] e1 = echo(name='bob', succeed=True) e2 = echo(name='alice', succeed=False) [[dependencies]] [[[R1]]] graph = "start" [[[R/^/P2Y]]] graph = "@e1 => foo" [[[R/^+P1Y/P2Y]]] graph = "@e2 => foo" [runtime] [[start]] [[foo]] __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-val" cylc validate "${WORKFLOW_NAME}" # Run workflow; it will stall waiting on the never-satisfied xtriggers. cylc play "${WORKFLOW_NAME}" poll_grep_workflow_log -E '2025/start.* => succeeded' cylc show "${WORKFLOW_NAME}//2026/foo" | grep -E '^ ⨯ xtrigger' > 2026.foo.log # 2026/foo should get only xtrigger e2. cmp_ok 2026.foo.log - <<__END__ ⨯ xtrigger "e2 = echo(name=alice, succeed=False)" __END__ cylc stop --now --max-polls=10 --interval=2 "${WORKFLOW_NAME}" purge cylc-flow-8.7.0/tests/functional/xtriggers/02-persistence/0000775000175000017500000000000015264420307023701 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/xtriggers/02-persistence/faker_fail.py0000664000175000017500000000160115264420307026334 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . def faker(name, debug=False): print("%s: failing" % name) return (False, {"name": name}) cylc-flow-8.7.0/tests/functional/xtriggers/02-persistence/faker_succ.py0000664000175000017500000000160315264420307026360 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . def faker(name, debug=False): print("%s: succeeding" % name) return (True, {"name": name}) cylc-flow-8.7.0/tests/functional/xtriggers/02-persistence/flow.cylc0000664000175000017500000000061415264420307025525 0ustar alastairalastair[scheduler] cycle point format = %Y [scheduling] initial cycle point = 2010 final cycle point = 2011 runahead limit = P0 [[xtriggers]] x1 = faker(name="bob") [[graph]] R1 = "@x1 => foo => shutdown" P1Y = "@x1 => foo" [runtime] [[foo]] script = "echo NAME is $x1_name" [[shutdown]] script = "cylc shutdown $CYLC_WORKFLOW_ID" cylc-flow-8.7.0/tests/functional/xtriggers/04-suite_state.t0000664000175000017500000000262015264420307024075 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test that deprecation warnings are printed appropriately for the suite_state # xtrigger. . "$(dirname "$0")/test_header" set_test_number 2 init_workflow "$TEST_NAME_BASE" << __FLOW_CONFIG__ [scheduling] initial cycle point = 2000 [[dependencies]] [[[R1]]] graph = @upstream => foo [[xtriggers]] upstream = suite_state(suite=thorin/oin/gloin, task=mithril, point=1) [runtime] [[foo]] __FLOW_CONFIG__ msg='WARNING - The suite_state xtrigger is deprecated' TEST_NAME="${TEST_NAME_BASE}-val" run_ok "$TEST_NAME" cylc validate "$WORKFLOW_NAME" grep_ok "$msg" "${TEST_NAME}.stderr" cylc-flow-8.7.0/tests/functional/xtriggers/02-persistence.t0000664000175000017500000000523015264420307024066 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test persistence of xtrigger results across restart. A cycling task depends # on a non cycle-point dependent custom xtrigger called "faker". In the first # cycle point the xtrigger succeeds and returns a result, then a task shuts # the workflow down. Then we replace the custom xtrigger function with one that # will fail if called again - which should not happen because the original # result should be remembered (as this xtrigger is not cycle point dependent). # Also test the correct result is broadcast to the dependent task before and # after workflow restart. . "$(dirname "$0")/test_header" set_test_number 6 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # Install the succeeding xtrigger function. cd "${WORKFLOW_RUN_DIR}" || exit 1 mkdir -p 'lib/python' cp "${WORKFLOW_RUN_DIR}/faker_succ.py" 'lib/python/faker.py' # Validate the test workflow. run_ok "${TEST_NAME_BASE}-val" cylc validate --debug "${WORKFLOW_NAME}" # Run the first cycle, till auto shutdown by task. TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --no-detach --debug "${WORKFLOW_NAME}" # Check the broadcast result of xtrigger. cylc cat-log "${WORKFLOW_NAME}//2010/foo" >'2010.foo.out' grep_ok 'NAME is bob' '2010.foo.out' # Replace the xtrigger function with one that will fail if called again. cp "${WORKFLOW_RUN_DIR}/faker_fail.py" 'lib/python/faker.py' # Validate again (with the new xtrigger function). run_ok "${TEST_NAME_BASE}-val2" cylc validate --debug "${WORKFLOW_NAME}" # Restart the workflow, to run the final cycle point. TEST_NAME="${TEST_NAME_BASE}-restart" workflow_run_ok "${TEST_NAME}" cylc play --no-detach "${WORKFLOW_NAME}" # Check the broadcast result has persisted from first run. cylc cat-log "${WORKFLOW_NAME}//2011/foo" >'2011.foo.out' grep_ok 'NAME is bob' '2011.foo.out' purge cylc-flow-8.7.0/tests/functional/xtriggers/04-sequential.t0000664000175000017500000000604115264420307023717 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test xtrigger sequential spawning - # . "$(dirname "$0")/test_header" set_test_number 7 # Test workflow uses built-in 'echo' xtrigger. init_workflow "${TEST_NAME_BASE}" << '__FLOW_CONFIG__' [scheduler] cycle point format = %Y allow implicit tasks = True [scheduling] initial cycle point = 3000 runahead limit = P5 sequential xtriggers = True [[xtriggers]] clock_1 = wall_clock(offset=P2Y, sequential=False) clock_2 = wall_clock() up_1 = workflow_state(\ workflow_task_id=%(workflow)s//%(point)s/b:succeeded, offset=-P1Y, sequential=False \ ):PT1S [[graph]] R1 = """ @clock_1 => a b """ +P1Y/P1Y = """ @clock_2 => a @clock_2 => b @up_1 => c """ __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-val" cylc validate "${WORKFLOW_NAME}" # Run workflow; it will stall waiting on the never-satisfied xtriggers. cylc play "${WORKFLOW_NAME}" poll_grep_workflow_log -E '3001/c/.* => succeeded' cylc stop --max-polls=10 --interval=2 "${WORKFLOW_NAME}" cylc play "${WORKFLOW_NAME}" cylc show "${WORKFLOW_NAME}//3001/a" | grep -E 'state: ' > 3001.a.log cylc show "${WORKFLOW_NAME}//3002/a" 2>&1 >/dev/null | grep -E 'No matching' > 3002.a.log # 3001/a should be spawned at both 3000/3001. cmp_ok 3001.a.log - <<__END__ state: waiting __END__ # 3002/a should not exist. cmp_ok 3002.a.log - <<__END__ No matching active tasks found: 3002/a __END__ cylc reload "${WORKFLOW_NAME}" cylc remove "${WORKFLOW_NAME}//3001/b" poll_grep_workflow_log 'Command "remove_tasks" actioned.' cylc show "${WORKFLOW_NAME}//3002/b" | grep -E 'state: ' > 3002.b.log cylc show "${WORKFLOW_NAME}//3003/b" 2>&1 >/dev/null | grep -E 'No matching' > 3003.b.log # 3002/b should be only at 3002. cmp_ok 3002.b.log - <<__END__ state: waiting __END__ cmp_ok 3003.b.log - <<__END__ No matching active tasks found: 3003/b __END__ cylc show "${WORKFLOW_NAME}//3002/c" | grep -E 'state: ' > 3002.c.log cylc show "${WORKFLOW_NAME}//3005/c" | grep -E 'state: ' > 3005.c.log # c should be from 3002-3005. cmp_ok 3002.c.log - <<__END__ state: waiting __END__ cmp_ok 3005.c.log - <<__END__ state: waiting __END__ cylc stop --now --max-polls=10 --interval=2 "${WORKFLOW_NAME}" purge cylc-flow-8.7.0/tests/functional/xtriggers/test_header0000777000175000017500000000000015264420307027453 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/xtriggers/04-respect-cylc-pythonpath.t0000664000175000017500000000302615264420307026336 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # An xtrigger added to $CYLC_PYTHONPATH should take precedence # over a `cylc.xtriggers` entry point of the same name . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # Install the succeeding xtrigger function. export CYLC_PYTHONPATH=${WORKFLOW_RUN_DIR}/dir:${CYLC_PYTHONPATH:-} # Validate the test workflow. run_ok "${TEST_NAME_BASE}-val" cylc validate --debug "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --no-detach --debug "${WORKFLOW_NAME}" # Check the result of xtrigger. grep_workflow_log_ok "${TEST_NAME_BASE}-grep" "echo overridden, args=('the_args',)" purge cylc-flow-8.7.0/tests/functional/xtriggers/04-respect-cylc-pythonpath/0000775000175000017500000000000015264420307026150 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/xtriggers/04-respect-cylc-pythonpath/dir/0000775000175000017500000000000015264420307026726 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/xtriggers/04-respect-cylc-pythonpath/dir/echo.py0000664000175000017500000000157415264420307030225 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . def echo(*args, **kwargs): print(f"echo overridden, args={args}") return (True, {}) cylc-flow-8.7.0/tests/functional/xtriggers/04-respect-cylc-pythonpath/flow.cylc0000664000175000017500000000021715264420307027773 0ustar alastairalastair[scheduling] [[xtriggers]] x1 = echo("the_args") [[graph]] R1 = @x1 => foo [runtime] [[foo]] script = true cylc-flow-8.7.0/tests/functional/runahead/0000775000175000017500000000000015264420307020707 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/runahead/04-no-final-cycle.t0000664000175000017500000000373415264420307024124 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test runahead limit is being enforced and doesn't break with no final cycle . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" no_final #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" run_fail "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-check-fail DB="$RUN_DIR/${WORKFLOW_NAME}/log/db" TASKS=$(sqlite3 "${DB}" "SELECT COUNT(*) FROM task_states WHERE status=='failed'") # manual comparison for the test if ((TASKS==2)); then ok "${TEST_NAME}" else fail "${TEST_NAME}" fi #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/runahead/default-future/0000775000175000017500000000000015264420307023643 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/runahead/default-future/flow.cylc0000664000175000017500000000147315264420307025473 0ustar alastairalastair#!jinja2 [scheduler] UTC mode = True allow implicit tasks = True [[events]] inactivity timeout = PT1M abort on inactivity timeout = True [scheduling] initial cycle point = 20100101T00 final cycle point = 20100105T00 [[xtriggers]] never = wall_clock(P100Y) [[graph]] R1 = spawner PT1H = """ @never => bar foo """ # If wibble gets into the pool, it will demand a +PT6H raise # of the 'runahead limit'. {{ FUTURE_TRIGGER_START_POINT }}/PT6H = """ foo[+PT6H] => wibble """ [runtime] [[root]] script = true [[spawner]] script = """ # spawn wibble cylc set $CYLC_WORKFLOW_ID 20100101T0800Z/foo """ [[foo]] script = false cylc-flow-8.7.0/tests/functional/runahead/runahead/0000775000175000017500000000000015264420307022476 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/runahead/runahead/flow.cylc0000664000175000017500000000135215264420307024322 0ustar alastairalastair[scheduler] allow implicit tasks = True [[events]] stall timeout = PT0S abort on stall timeout = True [scheduling] cycling mode = integer runahead limit = P3 initial cycle point = 1 final cycle point = 20 [[graph]] P1 = foo & run_ok => bar # foo fails on 1st cycle point only, succeeds on all others. # SoD: run_ok ensures bar spawns as waiting in 1st cycle pt, to # hold back the runahead. # As runahead limit is consecutive, even though cycle points 2 and # above succeed, workflow stalls after 4 cycle points. [runtime] [[root]] script = true [[foo]] script = if [[ "$CYLC_TASK_JOB" == '1/foo/01' ]]; then false; else true; fi cylc-flow-8.7.0/tests/functional/runahead/07-time-limit.t0000664000175000017500000000414215264420307023373 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test runahead limit is being enforced when specified as time limit . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 5 #------------------------------------------------------------------------------- install_workflow "$TEST_NAME_BASE" time-limit #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "$TEST_NAME" cylc validate "$WORKFLOW_NAME" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" run_fail "$TEST_NAME" cylc play --debug --no-detach "$WORKFLOW_NAME" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-max-cycle" DB="${WORKFLOW_RUN_DIR}/log/db" run_ok "$TEST_NAME" sqlite3 "$DB" \ "select max(cycle) from task_states where status!='waiting'" cmp_ok "${TEST_NAME}.stdout" <<< "20200101T0400Z" #------------------------------------------------------------------------------- grep_ok 'Workflow shutting down - "abort on stall timeout" is set' \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/runahead/default-simple/0000775000175000017500000000000015264420307023622 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/runahead/default-simple/flow.cylc0000664000175000017500000000101115264420307025436 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [[events]] stall timeout = PT0S abort on stall timeout = True [scheduling] initial cycle point = 20100101T00 final cycle point = 20100105T00 [[graph]] # Intervals are all 24 hours, but we really have a 6 hour repetition. # oops makes bar spawn as waiting to hold back the runahead T00, T06, T12, T18 = "foo & oops => bar" [runtime] [[foo]] script = false [[bar]] script = true cylc-flow-8.7.0/tests/functional/runahead/no_final/0000775000175000017500000000000015264420307022474 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/runahead/no_final/flow.cylc0000664000175000017500000000102615264420307024316 0ustar alastairalastair# Should shutdown stalled with 2 failed tasks due to P1 runahead limit [scheduler] cycle point time zone = Z [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT1M [scheduling] runahead limit = P1 initial cycle point = 20100101T00 [[xtriggers]] never = wall_clock(P100Y) [[graph]] PT6H = """ foo @never => bar """ [runtime] [[foo, bar]] script = false cylc-flow-8.7.0/tests/functional/runahead/00-runahead.t0000664000175000017500000000407715264420307023110 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test runahead limit is being enforced . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 5 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" runahead #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" run_fail "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-max-cycle" DB="${WORKFLOW_RUN_DIR}/log/db" run_ok "${TEST_NAME}" sqlite3 "${DB}" \ "select max(cycle) from task_states where status!='waiting'" cmp_ok "${TEST_NAME}.stdout" <<< "4" #------------------------------------------------------------------------------- grep_ok 'Workflow shutting down - "abort on stall timeout" is set' \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/runahead/01-check-default-simple.t0000664000175000017500000000456115264420307025306 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test default runahead limit behaviour is still the same . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 5 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" default-simple #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate -v "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" run_fail "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- # Testing runahead by counting tasks after a stall is tricky, so this test # might not be optimal. For instance, abort-on-stall aborts even if tasks # could immediately be released from the runahead pool. TEST_NAME="${TEST_NAME_BASE}-max-cycle" DB="${WORKFLOW_RUN_DIR}/log/db" run_ok "${TEST_NAME}" sqlite3 "${DB}" \ "select max(cycle) from task_states where status!='waiting'" cmp_ok "${TEST_NAME}.stdout" <<< "20100102T0000Z" # i.e. should have spawned 5 cycle points from initial 01T00 #------------------------------------------------------------------------------- grep_ok 'Workflow shutting down - "abort on stall timeout" is set' \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/runahead/test_header0000777000175000017500000000000015264420307027224 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/runahead/time-limit/0000775000175000017500000000000015264420307022761 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/runahead/time-limit/flow.cylc0000664000175000017500000000112715264420307024605 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] runahead limit = PT4H initial cycle point = 2020-01-01T00 final cycle point = 2020-01-02T00 [[graph]] PT1H = "foo & spawn-bar => bar" # foo fails first cycle point only. # bar waiting should cause a runahead stall at T04 # (PT4H limit allows T00, T01, T02, T03 [runtime] [[root]] script = true [[foo]] script = """[[ "$CYLC_TASK_JOB" != '20200101T0000Z/foo/01' ]]""" cylc-flow-8.7.0/tests/functional/runahead/06-release-update.t0000664000175000017500000000427015264420307024222 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that the datastore is updated when runahead tasks are released. # GitHub #1981 . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" 'release-update' #------------------------------------------------------------------------------- run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" 1>'out' 2>&1 & CYLC_RUN_PID="$!" poll_workflow_running YYYY="$(date +%Y)" NEXT1=$(( YYYY + 1 )) poll_grep_workflow_log -E "${NEXT1}/bar.* added to the n=0 window" # sleep a little to allow the datastore to update (`cylc dump` sees the # datastore) TODO can we avoid this flaky sleep somehow? sleep 10 # (gratuitous use of --flows for test coverage) cylc dump -l --flows -t "${WORKFLOW_NAME}" | awk '{print $1 $2 $3 $7}' >'log' # The scheduler task pool should contain: # NEXT1/foo - waiting on clock trigger # NEXT1/bar - waiting, partially satisfied # The n=1 data store should also contain: # YYYY/bar - succeeded cmp_ok 'log' - <<__END__ bar,$NEXT1,waiting,[1] foo,$NEXT1,waiting,[1] __END__ run_ok "${TEST_NAME_BASE}-stop" \ cylc stop --max-polls=10 --interval=6 "${WORKFLOW_NAME}" if ! wait "${CYLC_RUN_PID}" 1>'/dev/null' 2>&1; then cat 'out' >&2 fi #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/runahead/02-check-default-complex.t0000664000175000017500000000424115264420307025460 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test default runahead limit behaviour is still the same . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 5 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" default-complex #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate -v "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" run_fail "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-max-cycle DB="${WORKFLOW_RUN_DIR}/log/db" run_ok "${TEST_NAME}" sqlite3 "${DB}" \ "select max(cycle) from task_states where status!='waiting'" cmp_ok "${TEST_NAME}.stdout" <<< "20100101T1200Z" # i.e. should have spawned 5 cycle points from initial T00 #------------------------------------------------------------------------------- grep_ok 'Workflow shutting down - "abort on stall timeout" is set' \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/runahead/05-check-default-future-2.t0000664000175000017500000000463215264420307025471 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test default runahead limit behaviour is still the same . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 5 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" default-future #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate -v \ --set="FUTURE_TRIGGER_START_POINT='T02'" \ "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" run_fail "${TEST_NAME}" cylc play --debug --no-detach \ --set="FUTURE_TRIGGER_START_POINT='T02'" \ "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-max-cycle DB="$RUN_DIR/${WORKFLOW_NAME}/log/db" run_ok "${TEST_NAME}" sqlite3 "${DB}" \ "select max(cycle) from task_states where name=='foo' and status=='failed'" cmp_ok "${TEST_NAME}.stdout" <<< "20100101T1000Z" # i.e. should have spawned 5 cycle points from initial T00, and then raised # this by PT6H due to fact that wibble spawned. #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-check-aborted LOG="$RUN_DIR/${WORKFLOW_NAME}/log/scheduler/log" grep_ok 'Workflow shutting down - "abort on inactivity timeout" is set' "${LOG}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/runahead/release-update/0000775000175000017500000000000015264420307023607 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/runahead/release-update/flow.cylc0000664000175000017500000000060215264420307025430 0ustar alastairalastair[scheduler] allow implicit tasks = True cycle point format = %Y [scheduling] initial cycle point = now final cycle point = +P1Y runahead limit = P0 [[special tasks]] clock-trigger = foo(P0Y) [[graph]] P1Y = """ foo => bar bar[-P1Y] => bar """ # (or sequential bar) [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/runahead/default-complex/0000775000175000017500000000000015264420307024000 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/runahead/default-complex/flow.cylc0000664000175000017500000000120515264420307025621 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [[events]] abort on stall timeout = True stall timeout = PT0S stall timeout = PT30S abort on stall timeout = True [scheduling] initial cycle point = 20100101T00 final cycle point = 20100105T00 [[graph]] # T00, T07, T14, ... # oops makes bar spawn as waiting PT7H = "foo & oops => bar" # T00, T12, T18... T00, T12, T18 = "foo" # T04... T04 = "run_ok" # T05... T05 = "run_ok_2" [runtime] [[foo, fail]] script = false [[bar]] script = true cylc-flow-8.7.0/tests/functional/runahead/03-check-default-future.t0000664000175000017500000000445115264420307025327 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test default runahead limit behaviour is still the same . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 5 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" default-future #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate -v --set="FUTURE_TRIGGER_START_POINT='T04'" \ "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" run_fail "${TEST_NAME}" cylc play --debug --no-detach --set="FUTURE_TRIGGER_START_POINT='T04'" \ "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-max-cycle DB="$RUN_DIR/${WORKFLOW_NAME}/log/db" run_ok "${TEST_NAME}" sqlite3 "${DB}" \ "select max(cycle) from task_states where name=='foo' and status=='failed'" cmp_ok "${TEST_NAME}.stdout" <<< "20100101T0400Z" # i.e. should have spawned 5 cycle points from initial T00 (wibble not spawned) #------------------------------------------------------------------------------- grep_ok 'Workflow shutting down - "abort on inactivity timeout" is set' \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/workflow-state/0000775000175000017500000000000015264420307022110 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/workflow-state/output/0000775000175000017500000000000015264420307023450 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/workflow-state/output/reference.log0000664000175000017500000000021315264420307026105 0ustar alastairalastairInitial point: 20100101T0000Z Final point: None 20100101T0000Z/t1 -triggered off [] 20100101T0000Z/t2 -triggered off ['20100101T0000Z/t1'] cylc-flow-8.7.0/tests/functional/workflow-state/output/flow.cylc0000664000175000017500000000041615264420307025274 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] initial cycle point = 20100101T0000Z [[graph]] R1 = t1:out1 => t2 [runtime] [[t1]] script = cylc message "hello" [[[outputs]]] out1 = "hello" [[t2]] script = true cylc-flow-8.7.0/tests/functional/workflow-state/06a-noformat.t0000775000175000017500000000307315264420307024514 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test "cylc workflow-state" cycle point format conversion, when the target workflow # sets no explicit cycle point format, and the CLI does (the reverse of 06.t). . "$(dirname "$0")/test_header" set_test_number 3 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduler] UTC mode = True # (Use default cycle point format) [scheduling] initial cycle point = 20100101T0000Z [[graph]] R1 = foo [runtime] [[foo]] script = true __FLOW_CONFIG__ TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" TEST_NAME=${TEST_NAME_BASE}-cli-poll run_ok "${TEST_NAME}" cylc workflow-state "${WORKFLOW_NAME}//2010-01-01T00+00" contains_ok "${TEST_NAME}.stdout" <<__OUT__ 20100101T0000Z/foo:succeeded __OUT__ purge cylc-flow-8.7.0/tests/functional/workflow-state/11-multi.t0000664000175000017500000001053315264420307023650 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test all kinds of workflow-state DB checking. # shellcheck disable=SC2086 . "$(dirname "$0")/test_header" set_test_number 39 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # Create pre-8.3.0 and 8.3.0+ DBs for workflow-state checking. DBDIR="${WORKFLOW_RUN_DIR}/dbs" for x in c8a c8b; do mkdir -p "${DBDIR}/${x}/log" sqlite3 "${DBDIR}/${x}/log/db" < "${x}.sql" done run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" --set="ALT=\"${DBDIR}\"" grep_ok \ "WARNING - (8.3.0) Deprecated function signature used for workflow_state xtrigger was automatically upgraded" \ "${TEST_NAME_BASE}-validate.stderr" TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" \ cylc play "${WORKFLOW_NAME}" --set="ALT=\"${DBDIR}\"" \ --reference-test --debug --no-detach # Single poll. CMD="cylc workflow-state --run-dir=$DBDIR --max-polls=1" # Content of the c8b DB: # "select * from task_outputs" # 1|foo|[1]|{"submitted": "submitted", "started": "started", "succeeded": "succeeded", "x": "the quick brown"} # "select * from task_states" # foo|1|[1]|2024-06-05T16:34:02+12:00|2024-06-05T16:34:04+12:00|1|succeeded|0|0 #--------------- # Test the new-format command line (8.3.0+). T=${TEST_NAME_BASE}-cli-c8b run_ok "${T}-1" $CMD c8b run_ok "${T}-2" $CMD c8b//1 run_ok "${T}-3" $CMD c8b//1/foo run_fail "${T}-4" $CMD c8b//1/foo:waiting run_ok "${T}-4" $CMD c8b//1/foo:succeeded run_ok "${T}-5" $CMD "c8b//1/foo:the quick brown" --messages run_ok "${T}-6" $CMD "c8b//1/foo:x" --triggers run_ok "${T}-8" $CMD c8b//1 run_ok "${T}-9" $CMD c8b//1:succeeded run_fail "${T}-3" $CMD c8b//1/foo:failed run_fail "${T}-5" $CMD "c8b//1/foo:the quick brown" --triggers run_fail "${T}-5" $CMD "c8b//1/foo:x" --messages run_fail "${T}-1" $CMD c8b//1:failed run_fail "${T}-1" $CMD c8b//2 run_fail "${T}-1" $CMD c8b//2:failed #--------------- T=${TEST_NAME_BASE}-cli-c8a run_ok "${T}-1" $CMD "c8a//1/foo:the quick brown" --messages run_ok "${T}-2" $CMD "c8a//1/foo:the quick brown" --triggers # OK for 8.0 <= 8.3 run_fail "${T}-3" $CMD "c8a//1/foo:x" --triggers # not possible for 8.0 <= 8.3 #--------------- # Test the old-format command line (pre-8.3.0). T=${TEST_NAME_BASE}-cli-8b-compat run_ok "${T}-1" $CMD c8b run_ok "${T}-2" $CMD c8b --point=1 run_ok "${T}-3" $CMD c8b --point=1 --task=foo run_ok "${T}-4" $CMD c8b --point=1 --task=foo --status=succeeded run_ok "${T}-5" $CMD c8b --point=1 --task=foo --message="the quick brown" run_ok "${T}-6" $CMD c8b --point=1 --task=foo --output="the quick brown" run_fail "${T}-7" $CMD c8b --point=1 --task=foo --status=failed run_fail "${T}-8" $CMD c8b --point=1 --task=foo --message="x" run_fail "${T}-9" $CMD c8b --point=1 --task=foo --output="x" run_fail "${T}-10" $CMD c8b --point=2 run_fail "${T}-11" $CMD c8b --point=2 --task=foo --status="succeeded" #--------------- T=${TEST_NAME_BASE}-bad-cli TEST_NAME="${T}-1" run_fail "$TEST_NAME" $CMD c8b --status=succeeded --message="the quick brown" cmp_ok "${TEST_NAME}.stderr" <<__ERR__ InputError: set --status or --message, not both. __ERR__ TEST_NAME="${T}-2" run_fail "$TEST_NAME" $CMD c8b --task-point --point=1 cmp_ok "${TEST_NAME}.stderr" <<__ERR__ InputError: set --task-point or --point=CYCLE, not both. __ERR__ TEST_NAME="${T}-3" run_fail "$TEST_NAME" $CMD c8b --task-point cmp_ok "${TEST_NAME}.stderr" << "__ERR__" InputError: --task-point: $CYLC_TASK_CYCLE_POINT is not defined __ERR__ export CYLC_TASK_CYCLE_POINT=1 TEST_NAME="${T}-3" run_ok "$TEST_NAME" $CMD c8b --task-point purge cylc-flow-8.7.0/tests/functional/workflow-state/11-multi/0000775000175000017500000000000015264420307023461 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/workflow-state/11-multi/c8a.sql0000664000175000017500000001055015264420307024656 0ustar alastairalastairPRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE absolute_outputs(cycle TEXT, name TEXT, output TEXT); CREATE TABLE broadcast_events(time TEXT, change TEXT, point TEXT, namespace TEXT, key TEXT, value TEXT); CREATE TABLE broadcast_states(point TEXT, namespace TEXT, key TEXT, value TEXT, PRIMARY KEY(point, namespace, key)); CREATE TABLE inheritance(namespace TEXT, inheritance TEXT, PRIMARY KEY(namespace)); INSERT INTO inheritance VALUES('root','["root"]'); INSERT INTO inheritance VALUES('foo','["foo", "root"]'); CREATE TABLE task_action_timers(cycle TEXT, name TEXT, ctx_key TEXT, ctx TEXT, delays TEXT, num INTEGER, delay TEXT, timeout TEXT, PRIMARY KEY(cycle, name, ctx_key)); INSERT INTO task_action_timers VALUES('1','foo','"poll_timer"','["tuple", [[1, "running"]]]','[900.0]',1,'900.0','1717563116.69952'); INSERT INTO task_action_timers VALUES('1','foo','["try_timers", "submission-retry"]','null','[]',0,NULL,NULL); INSERT INTO task_action_timers VALUES('1','foo','["try_timers", "execution-retry"]','null','[]',0,NULL,NULL); CREATE TABLE task_events(name TEXT, cycle TEXT, time TEXT, submit_num INTEGER, event TEXT, message TEXT); INSERT INTO task_events VALUES('foo','1','2024-06-05T16:36:56+12:00',1,'submitted',''); INSERT INTO task_events VALUES('foo','1','2024-06-05T16:36:56+12:00',1,'started',''); INSERT INTO task_events VALUES('foo','1','2024-06-05T16:36:56+12:00',1,'x','the quick brown'); INSERT INTO task_events VALUES('foo','1','2024-06-05T16:36:57+12:00',1,'succeeded',''); CREATE TABLE task_jobs(cycle TEXT, name TEXT, submit_num INTEGER, flow_nums TEXT, is_manual_submit INTEGER, try_num INTEGER, time_submit TEXT, time_submit_exit TEXT, submit_status INTEGER, time_run TEXT, time_run_exit TEXT, run_signal TEXT, run_status INTEGER, platform_name TEXT, job_runner_name TEXT, job_id TEXT, PRIMARY KEY(cycle, name, submit_num)); INSERT INTO task_jobs VALUES('1','foo',1,'[1]',0,1,'2024-06-05T16:36:55+12:00','2024-06-05T16:36:56+12:00',0,'2024-06-05T16:36:56+12:00','2024-06-05T16:36:56+12:00',NULL,0,'localhost','background','21511'); CREATE TABLE task_late_flags(cycle TEXT, name TEXT, value INTEGER, PRIMARY KEY(cycle, name)); CREATE TABLE task_outputs(cycle TEXT, name TEXT, flow_nums TEXT, outputs TEXT, PRIMARY KEY(cycle, name, flow_nums)); INSERT INTO task_outputs VALUES('1','foo','[1]','["submitted", "started", "succeeded", "the quick brown"]'); CREATE TABLE task_pool(cycle TEXT, name TEXT, flow_nums TEXT, status TEXT, is_held INTEGER, PRIMARY KEY(cycle, name, flow_nums)); CREATE TABLE task_prerequisites(cycle TEXT, name TEXT, flow_nums TEXT, prereq_name TEXT, prereq_cycle TEXT, prereq_output TEXT, satisfied TEXT, PRIMARY KEY(cycle, name, flow_nums, prereq_name, prereq_cycle, prereq_output)); CREATE TABLE task_states(name TEXT, cycle TEXT, flow_nums TEXT, time_created TEXT, time_updated TEXT, submit_num INTEGER, status TEXT, flow_wait INTEGER, is_manual_submit INTEGER, PRIMARY KEY(name, cycle, flow_nums)); INSERT INTO task_states VALUES('foo','1','[1]','2024-06-05T16:36:55+12:00','2024-06-05T16:36:57+12:00',1,'succeeded',0,0); CREATE TABLE task_timeout_timers(cycle TEXT, name TEXT, timeout REAL, PRIMARY KEY(cycle, name)); CREATE TABLE tasks_to_hold(name TEXT, cycle TEXT); CREATE TABLE workflow_flows(flow_num INTEGER, start_time TEXT, description TEXT, PRIMARY KEY(flow_num)); INSERT INTO workflow_flows VALUES(1,'2024-06-05T16:36:55','original flow from 1'); CREATE TABLE workflow_params(key TEXT, value TEXT, PRIMARY KEY(key)); INSERT INTO workflow_params VALUES('uuid_str','cabb2bd8-bb36-4c7a-9c51-d2b1d456bc4e'); INSERT INTO workflow_params VALUES('cylc_version','8.3.0.dev'); INSERT INTO workflow_params VALUES('UTC_mode','0'); INSERT INTO workflow_params VALUES('n_restart','0'); INSERT INTO workflow_params VALUES('cycle_point_format',NULL); INSERT INTO workflow_params VALUES('is_paused','0'); INSERT INTO workflow_params VALUES('stop_clock_time',NULL); INSERT INTO workflow_params VALUES('stop_task',NULL); INSERT INTO workflow_params VALUES('icp',NULL); INSERT INTO workflow_params VALUES('fcp',NULL); INSERT INTO workflow_params VALUES('startcp',NULL); INSERT INTO workflow_params VALUES('stopcp',NULL); INSERT INTO workflow_params VALUES('run_mode',NULL); INSERT INTO workflow_params VALUES('cycle_point_tz','+1200'); CREATE TABLE workflow_template_vars(key TEXT, value TEXT, PRIMARY KEY(key)); CREATE TABLE xtriggers(signature TEXT, results TEXT, PRIMARY KEY(signature)); COMMIT; cylc-flow-8.7.0/tests/functional/workflow-state/11-multi/reference.log0000664000175000017500000000071315264420307026123 0ustar alastairalastair1/bar1 -triggered off [] in flow 1 1/bar2 -triggered off [] in flow 1 1/baz2 -triggered off [] in flow 1 1/f1 -triggered off [] in flow 1 1/f2 -triggered off [] in flow 1 1/f3 -triggered off [] in flow 1 1/f4 -triggered off [] in flow 1 1/f5 -triggered off [] in flow 1 1/f6 -triggered off [] in flow 1 1/f7 -triggered off [] in flow 1 1/g1 -triggered off ['1/bar1'] in flow 1 1/g2 -triggered off ['1/bar2'] in flow 1 1/g4 -triggered off ['1/baz2'] in flow 1 cylc-flow-8.7.0/tests/functional/workflow-state/11-multi/flow.cylc0000664000175000017500000000364615264420307025315 0ustar alastairalastair#!Jinja2 {# alt-cylc-run-dir default for easy validation #} {% set ALT = ALT | default("alt") %} [scheduling] cycling mode = integer initial cycle point = 1 final cycle point = 2 [[xtriggers]] # Cylc 7 xtrigger, Cylc 8 DB a1 = suite_state(c8b, foo, 1, offset=P0, cylc_run_dir={{ALT}}):PT1S # status=succeeded a2 = suite_state(c8b, foo, 1, offset=P0, message="the quick brown", cylc_run_dir={{ALT}}):PT1S # Cylc 8 back compat (pre-8.3.0) b1 = workflow_state(c8a, foo, 1, offset=P0, status=succeeded, cylc_run_dir={{ALT}}):PT1S b2 = workflow_state(c8a, foo, 1, offset=P0, message="the quick brown", cylc_run_dir={{ALT}}):PT1S # Cylc 8 new (from 8.3.0) c1 = workflow_state(c8b//1/foo, offset=P0, alt_cylc_run_dir={{ALT}}):PT1S c2 = workflow_state(c8b//1/foo:succeeded, offset=P0, alt_cylc_run_dir={{ALT}}):PT1S c3 = workflow_state(c8b//1/foo:x, offset=P0, alt_cylc_run_dir={{ALT}}, is_trigger=True):PT1S c4 = workflow_state(c8b//1/foo:"the quick brown", offset=P0, is_message=True, alt_cylc_run_dir={{ALT}}):PT1S [[graph]] R1 = """ # Deprecated workflow-state polling tasks. # (does not support %(suite_name)s templates or offsets # or output triggers - just messages) # status bar1 => g1 bar2 => g2 # output baz2 => g4 # message given in task definition @a1 => f1 @a2 => f2 @b1 => f3 @b2 => f4 @c1 => f5 @c2 => f6 @c3 => f7 """ [runtime] [[bar1, bar2]] [[[workflow state polling]]] alt-cylc-run-dir = {{ALT}} [[baz2]] [[[workflow state polling]]] message = "the quick brown" alt-cylc-run-dir = {{ALT}} [[f1, f2, f3, f4, f5, f6, f7]] [[g1, g2, g4]] cylc-flow-8.7.0/tests/functional/workflow-state/11-multi/c8b.sql0000664000175000017500000001062215264420307024657 0ustar alastairalastairPRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE absolute_outputs(cycle TEXT, name TEXT, output TEXT); CREATE TABLE broadcast_events(time TEXT, change TEXT, point TEXT, namespace TEXT, key TEXT, value TEXT); CREATE TABLE broadcast_states(point TEXT, namespace TEXT, key TEXT, value TEXT, PRIMARY KEY(point, namespace, key)); CREATE TABLE inheritance(namespace TEXT, inheritance TEXT, PRIMARY KEY(namespace)); INSERT INTO inheritance VALUES('root','["root"]'); INSERT INTO inheritance VALUES('foo','["foo", "root"]'); CREATE TABLE task_action_timers(cycle TEXT, name TEXT, ctx_key TEXT, ctx TEXT, delays TEXT, num INTEGER, delay TEXT, timeout TEXT, PRIMARY KEY(cycle, name, ctx_key)); INSERT INTO task_action_timers VALUES('1','foo','"poll_timer"','["tuple", [[1, "running"]]]','[900.0]',1,'900.0','1717562943.77014'); INSERT INTO task_action_timers VALUES('1','foo','["try_timers", "submission-retry"]','null','[]',0,NULL,NULL); INSERT INTO task_action_timers VALUES('1','foo','["try_timers", "execution-retry"]','null','[]',0,NULL,NULL); CREATE TABLE task_events(name TEXT, cycle TEXT, time TEXT, submit_num INTEGER, event TEXT, message TEXT); INSERT INTO task_events VALUES('foo','1','2024-06-05T16:34:03+12:00',1,'submitted',''); INSERT INTO task_events VALUES('foo','1','2024-06-05T16:34:03+12:00',1,'started',''); INSERT INTO task_events VALUES('foo','1','2024-06-05T16:34:03+12:00',1,'x','the quick brown'); INSERT INTO task_events VALUES('foo','1','2024-06-05T16:34:04+12:00',1,'succeeded',''); CREATE TABLE task_jobs(cycle TEXT, name TEXT, submit_num INTEGER, flow_nums TEXT, is_manual_submit INTEGER, try_num INTEGER, time_submit TEXT, time_submit_exit TEXT, submit_status INTEGER, time_run TEXT, time_run_exit TEXT, run_signal TEXT, run_status INTEGER, platform_name TEXT, job_runner_name TEXT, job_id TEXT, PRIMARY KEY(cycle, name, submit_num)); INSERT INTO task_jobs VALUES('1','foo',1,'[1]',0,1,'2024-06-05T16:34:02+12:00','2024-06-05T16:34:03+12:00',0,'2024-06-05T16:34:03+12:00','2024-06-05T16:34:03+12:00',NULL,0,'localhost','background','20985'); CREATE TABLE task_late_flags(cycle TEXT, name TEXT, value INTEGER, PRIMARY KEY(cycle, name)); CREATE TABLE task_outputs(cycle TEXT, name TEXT, flow_nums TEXT, outputs TEXT, PRIMARY KEY(cycle, name, flow_nums)); INSERT INTO task_outputs VALUES('1','foo','[1]','{"submitted": "submitted", "started": "started", "succeeded": "succeeded", "x": "the quick brown"}'); CREATE TABLE task_pool(cycle TEXT, name TEXT, flow_nums TEXT, status TEXT, is_held INTEGER, PRIMARY KEY(cycle, name, flow_nums)); CREATE TABLE task_prerequisites(cycle TEXT, name TEXT, flow_nums TEXT, prereq_name TEXT, prereq_cycle TEXT, prereq_output TEXT, satisfied TEXT, PRIMARY KEY(cycle, name, flow_nums, prereq_name, prereq_cycle, prereq_output)); CREATE TABLE task_states(name TEXT, cycle TEXT, flow_nums TEXT, time_created TEXT, time_updated TEXT, submit_num INTEGER, status TEXT, flow_wait INTEGER, is_manual_submit INTEGER, PRIMARY KEY(name, cycle, flow_nums)); INSERT INTO task_states VALUES('foo','1','[1]','2024-06-05T16:34:02+12:00','2024-06-05T16:34:04+12:00',1,'succeeded',0,0); CREATE TABLE task_timeout_timers(cycle TEXT, name TEXT, timeout REAL, PRIMARY KEY(cycle, name)); CREATE TABLE tasks_to_hold(name TEXT, cycle TEXT); CREATE TABLE workflow_flows(flow_num INTEGER, start_time TEXT, description TEXT, PRIMARY KEY(flow_num)); INSERT INTO workflow_flows VALUES(1,'2024-06-05T16:34:02','original flow from 1'); CREATE TABLE workflow_params(key TEXT, value TEXT, PRIMARY KEY(key)); INSERT INTO workflow_params VALUES('uuid_str','4185a45a-8faa-491f-ad35-2d221e780efa'); INSERT INTO workflow_params VALUES('cylc_version','8.3.0.dev'); INSERT INTO workflow_params VALUES('UTC_mode','0'); INSERT INTO workflow_params VALUES('n_restart','0'); INSERT INTO workflow_params VALUES('cycle_point_format',NULL); INSERT INTO workflow_params VALUES('is_paused','0'); INSERT INTO workflow_params VALUES('stop_clock_time',NULL); INSERT INTO workflow_params VALUES('stop_task',NULL); INSERT INTO workflow_params VALUES('icp',NULL); INSERT INTO workflow_params VALUES('fcp',NULL); INSERT INTO workflow_params VALUES('startcp',NULL); INSERT INTO workflow_params VALUES('stopcp',NULL); INSERT INTO workflow_params VALUES('run_mode',NULL); INSERT INTO workflow_params VALUES('cycle_point_tz','+1200'); CREATE TABLE workflow_template_vars(key TEXT, value TEXT, PRIMARY KEY(key)); CREATE TABLE xtriggers(signature TEXT, results TEXT, PRIMARY KEY(signature)); COMMIT; cylc-flow-8.7.0/tests/functional/workflow-state/11-multi/upstream/0000775000175000017500000000000015264420307025321 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/workflow-state/11-multi/upstream/suite.rc0000664000175000017500000000067615264420307027011 0ustar alastairalastair# Run this with Cylc 8 (pre-8.3.0), and 8 (8.3.0+) # to generate DBs for workflow state checks. # (The task_outputs table is different in each case). [scheduling] cycling mode = integer initial cycle point = 1 [[dependencies]] [[[R1]]] graph = """ foo """ [runtime] [[foo]] script = "cylc message - 'the quick brown'" [[[outputs]]] x = "the quick brown" cylc-flow-8.7.0/tests/functional/workflow-state/07-message2/0000775000175000017500000000000015264420307024042 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/workflow-state/07-message2/flow.cylc0000775000175000017500000000050415264420307025667 0ustar alastairalastair[scheduler] cycle point format = %Y [scheduling] initial cycle point = 2010 final cycle point = 2012 [[graph]] P1Y = "foo:x => bar" [runtime] [[foo]] script = cylc message "the quick brown fox" [[[outputs]]] x = "the quick brown fox" [[bar]] script = true cylc-flow-8.7.0/tests/functional/workflow-state/05-output.t0000775000175000017500000000234215264420307024063 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test cylc workflow-state for outputs (as opposed to statuses) . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" output TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" TEST_NAME=${TEST_NAME_BASE}-cli-check run_ok "${TEST_NAME}" cylc workflow-state "${WORKFLOW_NAME}//20100101T0000Z/t1:out1" --triggers --max-polls=1 purge cylc-flow-8.7.0/tests/functional/workflow-state/polling/0000775000175000017500000000000015264420307023554 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/workflow-state/polling/reference.log0000664000175000017500000000022215264420307026211 0ustar alastairalastairInitial point: 1 Final point: 1 1/l-good -triggered off [] 1/lbad -triggered off [] 1/l-mess -triggered off [] 1/done -triggered off ['1/l-mess'] cylc-flow-8.7.0/tests/functional/workflow-state/polling/flow.cylc0000664000175000017500000000123415264420307025377 0ustar alastairalastair#!jinja2 {# e.g. set OUTPUT = ":x" #} {% set OUTPUT = OUTPUT | default("") %} [meta] title = "polls for success and failure tasks in another workflow" [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = """ l-good<{{UPSTREAM}}::good-stuff> & lbad<{{UPSTREAM}}::bad:fail> l-mess<{{UPSTREAM}}::messenger{{OUTPUT}}> => done """ [runtime] [[l-good,lbad]] [[[workflow state polling]]] interval = PT2S max-polls = 20 [[l-mess]] [[[workflow state polling]]] interval = PT2S max-polls = 20 message = "the quick brown fox" cylc-flow-8.7.0/tests/functional/workflow-state/datetime/0000775000175000017500000000000015264420307023704 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/workflow-state/datetime/flow.cylc0000664000175000017500000000101615264420307025525 0ustar alastairalastair[scheduler] cycle point format = CCYY [scheduling] initial cycle point = 2050 final cycle point = 2052 [[graph]] P1Y = """ foo:x => bar """ [runtime] [[foo]] script = cylc message "hello" [[[outputs]]] x = "hello" [[bar]] script = """ if (( CYLC_TASK_CYCLE_POINT == 2052 )) && (( CYLC_TASK_SUBMIT_NUMBER == 1 )) then cylc trigger --flow=new $CYLC_WORKFLOW_ID//2052/foo fi """ cylc-flow-8.7.0/tests/functional/workflow-state/02-validate-blank-command-scripting.t0000775000175000017500000000254615264420307031020 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Validate blank script in automatic workflow polling task. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 1 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE} run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge exit 0 cylc-flow-8.7.0/tests/functional/workflow-state/00-polling.t0000664000175000017500000001007215264420307024156 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Validate and run the workflow-state/polling test workflow # The test workflow is in polling/; it depends on another workflow in upstream/ . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 7 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" 'polling' #------------------------------------------------------------------------------- # copy the upstream workflow to the test directory and install it cp -r "${TEST_SOURCE_DIR}/upstream" "${TEST_DIR}/" # use full range of characters in the workflow-to-be-polled name: UPSTREAM="${WORKFLOW_NAME}-up_stre.am" cylc install "${TEST_DIR}/upstream" --workflow-name="${UPSTREAM}" --no-run-name #------------------------------------------------------------------------------- # validate both workflows as tests TEST_NAME="${TEST_NAME_BASE}-validate-upstream" run_ok "${TEST_NAME}" cylc validate --debug "${UPSTREAM}" TEST_NAME=${TEST_NAME_BASE}-validate-polling-y run_fail "${TEST_NAME}" \ cylc validate --set="UPSTREAM='${UPSTREAM}'" --set="OUTPUT=':y'" "${WORKFLOW_NAME}" contains_ok "${TEST_NAME}.stderr" <<__ERR__ WorkflowConfigError: Polling task "l-mess" must configure a target status or output message in \ the graph (:y) or task definition (message = "the quick brown fox") but not both. __ERR__ TEST_NAME=${TEST_NAME_BASE}-validate-polling run_ok "${TEST_NAME}" \ cylc validate --debug --set="UPSTREAM='${UPSTREAM}'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- # run the upstream workflow and detach (not a test) cylc play "${UPSTREAM}" #------------------------------------------------------------------------------- # check auto-generated task script for lbad cylc config -d \ --set="UPSTREAM='${UPSTREAM}'" -i '[runtime][lbad]script' "${WORKFLOW_NAME}" \ >'lbad.script' cmp_ok 'lbad.script' << __END__ echo cylc workflow-state ${UPSTREAM}//\$CYLC_TASK_CYCLE_POINT/bad:failed --interval=2 --max-polls=20 cylc workflow-state ${UPSTREAM}//\$CYLC_TASK_CYCLE_POINT/bad:failed --interval=2 --max-polls=20 __END__ # check auto-generated task script for l-good cylc config -d \ --set="UPSTREAM='${UPSTREAM}'" -i '[runtime][l-good]script' "${WORKFLOW_NAME}" \ >'l-good.script' cmp_ok 'l-good.script' << __END__ echo cylc workflow-state ${UPSTREAM}//\$CYLC_TASK_CYCLE_POINT/good-stuff:succeeded --interval=2 --max-polls=20 cylc workflow-state ${UPSTREAM}//\$CYLC_TASK_CYCLE_POINT/good-stuff:succeeded --interval=2 --max-polls=20 __END__ #------------------------------------------------------------------------------- # run the workflow-state polling test workflow TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" \ cylc play --reference-test --debug --no-detach \ --set="UPSTREAM='${UPSTREAM}'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge #------------------------------------------------------------------------------- # clean up the upstream workflow # just in case (expect error message here, but exit 0): cylc stop --now "${UPSTREAM}" --max-polls=20 --interval=2 >'/dev/null' 2>&1 purge "${UPSTREAM}" exit cylc-flow-8.7.0/tests/functional/workflow-state/01-polling.t0000664000175000017500000000727715264420307024174 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Validate and run the workflow-state/polling test workflow # The test workflow is in polling/; it depends on another workflow in upstream/ . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 5 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" 'polling' #------------------------------------------------------------------------------- # copy the upstream workflow to the test directory and install it cp -r "${TEST_SOURCE_DIR}/upstream" "${TEST_DIR}/" # this version uses a simple Rose-style workflow name [\w-] UPSTREAM="${WORKFLOW_NAME}-upstream" cylc install "${TEST_DIR}/upstream" --workflow-name="${UPSTREAM}" --no-run-name #------------------------------------------------------------------------------- # validate both workflows as tests TEST_NAME="${TEST_NAME_BASE}-validate-upstream" run_ok "${TEST_NAME}" cylc validate --debug "${UPSTREAM}" TEST_NAME="${TEST_NAME_BASE}-validate-polling" run_ok "${TEST_NAME}" \ cylc validate --debug --set="UPSTREAM='${UPSTREAM}'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- # check auto-generated task script for lbad cylc config -d \ --set="UPSTREAM='${UPSTREAM}'" \ -i '[runtime][lbad]script' "${WORKFLOW_NAME}" >'lbad.script' cmp_ok 'lbad.script' << __END__ echo cylc workflow-state ${UPSTREAM}//\$CYLC_TASK_CYCLE_POINT/bad:failed --interval=2 --max-polls=20 cylc workflow-state ${UPSTREAM}//\$CYLC_TASK_CYCLE_POINT/bad:failed --interval=2 --max-polls=20 __END__ # check auto-generated task script for l-good cylc config -d \ --set="UPSTREAM='${UPSTREAM}'" \ -i '[runtime][l-good]script' "${WORKFLOW_NAME}" >'l-good.script' cmp_ok 'l-good.script' << __END__ echo cylc workflow-state ${UPSTREAM}//\$CYLC_TASK_CYCLE_POINT/good-stuff:succeeded --interval=2 --max-polls=20 cylc workflow-state ${UPSTREAM}//\$CYLC_TASK_CYCLE_POINT/good-stuff:succeeded --interval=2 --max-polls=20 __END__ #------------------------------------------------------------------------------- # run the upstream workflow and detach (not a test) cylc play "${UPSTREAM}" 1>'upstream.out' 2>&1 #------------------------------------------------------------------------------- # run the workflow-state polling test workflow TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" \ cylc play --reference-test --debug --no-detach --set="UPSTREAM='${UPSTREAM}'" \ "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge #------------------------------------------------------------------------------- # clean up the upstream workflow # just in case (expect error message here, but exit 0): cylc stop --now "${UPSTREAM}" --max-polls=20 --interval=2 >'/dev/null' 2>&1 purge "${UPSTREAM}" exit cylc-flow-8.7.0/tests/functional/workflow-state/08-integer.t0000775000175000017500000000652715264420307024174 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . . "$(dirname "$0")/test_header" set_test_number 19 install_workflow "${TEST_NAME_BASE}" integer # run one cycle TEST_NAME="${TEST_NAME_BASE}_run_1" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach --start-cycle-point=1 --stopcp=1 "${WORKFLOW_NAME}" # too many args TEST_NAME="${TEST_NAME_BASE}_cl_error" run_fail "${TEST_NAME}" cylc workflow-state --max-polls=1 "${WORKFLOW_NAME}-a" "${WORKFLOW_NAME}-b" TEST_NAME="${TEST_NAME_BASE}_check_1_status" run_ok "${TEST_NAME}" cylc workflow-state --max-polls=1 "${WORKFLOW_NAME}" contains_ok "${TEST_NAME}.stdout" <<__END__ 2/foo:waiting 1/foo:succeeded 1/bar:succeeded __END__ TEST_NAME="${TEST_NAME_BASE}_check_1_outputs" run_ok "${TEST_NAME}" cylc workflow-state --max-polls=1 --triggers "${WORKFLOW_NAME}" contains_ok "${TEST_NAME}.stdout" <<__END__ 1/foo:{'submitted': 'submitted', 'started': 'started', 'succeeded': 'succeeded', 'x': 'hello'} 2/foo:{} 1/bar:{'submitted': 'submitted', 'started': 'started', 'succeeded': 'succeeded'} __END__ TEST_NAME="${TEST_NAME_BASE}_poll_fail" run_fail "${TEST_NAME}" cylc workflow-state --max-polls=2 --interval=1 "${WORKFLOW_NAME}//2/foo:succeeded" contains_ok "${TEST_NAME}.stderr" <<__END__ ERROR - failed after 2 polls __END__ # finish the run TEST_NAME="${TEST_NAME_BASE}_run_2" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}_poll_succeed" run_ok "${TEST_NAME}" cylc workflow-state --max-polls=1 "${WORKFLOW_NAME}//2/foo:succeeded" contains_ok "${TEST_NAME}.stdout" <<__END__ 2/foo:succeeded __END__ TEST_NAME="${TEST_NAME_BASE}_int_offset" run_ok "${TEST_NAME}" cylc workflow-state --max-polls=1 "${WORKFLOW_NAME}//1/foo:succeeded" --offset=P1 contains_ok "${TEST_NAME}.stdout" <<__END__ 2/foo:succeeded __END__ TEST_NAME="${TEST_NAME_BASE}_wildcard_offset" run_fail "${TEST_NAME}" cylc workflow-state --max-polls=1 "${WORKFLOW_NAME}//*/foo:succeeded" --offset=P1 contains_ok "${TEST_NAME}.stderr" <<__END__ InputError: Cycle point "*" is not compatible with an offset. __END__ TEST_NAME="${TEST_NAME_BASE}_pre_start_cycle_point_complete_pre_scp" run_ok "${TEST_NAME}" cylc workflow-state --max-polls=1 "${WORKFLOW_NAME}//0/foo:succeeded" --triggers --accept-pre-start-tasks contains_ok "${TEST_NAME}.stdout" <<__END__ 0/foo:{"succeeded": "before start cycle point"} __END__ TEST_NAME="${TEST_NAME_BASE}_pre_start_cycle_point_no_option" run_fail "${TEST_NAME}" cylc workflow-state --max-polls=1 "${WORKFLOW_NAME}//0/foo:succeeded" --triggers contains_ok "${TEST_NAME}.stderr" <<__END__ ERROR - failed after 1 polls __END__ purge cylc-flow-8.7.0/tests/functional/workflow-state/07-message2.t0000775000175000017500000000257715264420307024245 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Originally (Cylc 7): test workflow-state query on a waiting task - GitHub #2440. # Now (Cylc 8): test result of a failed workflow-state query. . "$(dirname "$0")/test_header" set_test_number 4 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-val" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" TEST_NAME=${TEST_NAME_BASE}-query run_fail "${TEST_NAME}" cylc workflow-state "${WORKFLOW_NAME}//2013/foo:x" --triggers --max-polls=1 grep_ok "failed after 1 polls" "${TEST_NAME}.stderr" purge cylc-flow-8.7.0/tests/functional/workflow-state/09-datetime.t0000775000175000017500000001070415264420307024324 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . . "$(dirname "$0")/test_header" set_test_number 28 install_workflow "${TEST_NAME_BASE}" datetime # run one cycle TEST_NAME="${TEST_NAME_BASE}_run_1" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach --start-cycle-point=2051 --stopcp=2051 "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}_check_1_status" run_ok "${TEST_NAME}" cylc workflow-state --max-polls=1 "${WORKFLOW_NAME}" contains_ok "${TEST_NAME}.stdout" <<__END__ 2052/foo:waiting 2051/foo:succeeded 2051/bar:succeeded __END__ TEST_NAME="${TEST_NAME_BASE}_check_1_status_old_fmt" run_ok "${TEST_NAME}" cylc workflow-state --old-format --max-polls=1 "${WORKFLOW_NAME}" contains_ok "${TEST_NAME}.stdout" <<__END__ foo, 2052, waiting foo, 2051, succeeded bar, 2051, succeeded __END__ TEST_NAME="${TEST_NAME_BASE}_check_1_outputs" run_ok "${TEST_NAME}" cylc workflow-state --max-polls=1 --triggers "${WORKFLOW_NAME}" contains_ok "${TEST_NAME}.stdout" <<__END__ 2051/foo:{'submitted': 'submitted', 'started': 'started', 'succeeded': 'succeeded', 'x': 'hello'} 2052/foo:{} 2051/bar:{'submitted': 'submitted', 'started': 'started', 'succeeded': 'succeeded'} __END__ TEST_NAME="${TEST_NAME_BASE}_poll_fail" run_fail "${TEST_NAME}" cylc workflow-state --max-polls=2 --interval=1 "${WORKFLOW_NAME}//2052/foo:succeeded" contains_ok "${TEST_NAME}.stderr" <<__END__ ERROR - failed after 2 polls __END__ # finish the run TEST_NAME="${TEST_NAME_BASE}_run_2" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}_check_1_status_2" run_ok "${TEST_NAME}" cylc workflow-state --max-polls=1 "${WORKFLOW_NAME}" contains_ok "${TEST_NAME}.stdout" <<__END__ 2051/foo:succeeded 2052/foo:succeeded 2051/bar:succeeded 2052/bar:succeeded 2052/foo:succeeded(flows=2) 2052/bar:succeeded(flows=2) __END__ TEST_NAME="${TEST_NAME_BASE}_check_1_status_3" run_ok "${TEST_NAME}" cylc workflow-state --flow=2 --max-polls=1 "${WORKFLOW_NAME}" contains_ok "${TEST_NAME}.stdout" <<__END__ 2052/foo:succeeded(flows=2) 2052/bar:succeeded(flows=2) __END__ TEST_NAME="${TEST_NAME_BASE}_check_1_wildcard" run_ok "${TEST_NAME}" cylc workflow-state --flow=1 --max-polls=1 "${WORKFLOW_NAME}//*/foo" contains_ok "${TEST_NAME}.stdout" <<__END__ 2051/foo:succeeded __END__ TEST_NAME="${TEST_NAME_BASE}_poll_succeed" run_ok "${TEST_NAME}" cylc workflow-state --max-polls=1 "${WORKFLOW_NAME}//2052/foo:succeeded" contains_ok "${TEST_NAME}.stdout" <<__END__ 2052/foo:succeeded __END__ TEST_NAME="${TEST_NAME_BASE}_datetime_offset" run_ok "${TEST_NAME}" cylc workflow-state --max-polls=1 "${WORKFLOW_NAME}//2051/foo:succeeded" --offset=P1Y contains_ok "${TEST_NAME}.stdout" <<__END__ 2052/foo:succeeded __END__ TEST_NAME="${TEST_NAME_BASE}_datetime_format" run_ok "${TEST_NAME}" cylc workflow-state --max-polls=1 "${WORKFLOW_NAME}//20510101T0000Z/foo:succeeded" --offset=P1Y contains_ok "${TEST_NAME}.stdout" <<__END__ 2052/foo:succeeded __END__ TEST_NAME="${TEST_NAME_BASE}_bad_point" run_fail "${TEST_NAME}" cylc workflow-state --max-polls=1 "${WORKFLOW_NAME}//205/foo:succeeded" contains_ok "${TEST_NAME}.stderr" <<__END__ InputError: Cycle point "205" is not compatible with DB point format "CCYY" __END__ TEST_NAME="${TEST_NAME_BASE}_pre_start_cycle_point" run_ok "${TEST_NAME}" cylc workflow-state --max-polls=1 "${WORKFLOW_NAME}//2050/foo:succeeded" --messages --accept-pre-start-tasks contains_ok "${TEST_NAME}.stdout" <<__END__ 2050/foo:{"succeeded": "before start cycle point"} __END__ TEST_NAME="${TEST_NAME_BASE}_pre_start_cycle_point_no_option" run_fail "${TEST_NAME}" cylc workflow-state --max-polls=1 "${WORKFLOW_NAME}//2050/foo:succeeded" --messages contains_ok "${TEST_NAME}.stderr" <<__END__ ERROR - failed after 1 polls __END__ purge cylc-flow-8.7.0/tests/functional/workflow-state/06-format.t0000775000175000017500000000342315264420307024015 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test "cylc workflow-state" cycle point format conversion, when the target workflow # sets an explicit cycle point format, and the CLI does not. . "$(dirname "$0")/test_header" set_test_number 5 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduler] UTC mode = True cycle point format = CCYY-MM-DD [scheduling] initial cycle point = 20100101 [[graph]] R1 = foo [runtime] [[foo]] script = true __FLOW_CONFIG__ TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" TEST_NAME=${TEST_NAME_BASE}-cli-poll run_ok "${TEST_NAME}" cylc workflow-state "${WORKFLOW_NAME}//20100101T0000Z/foo:succeeded" --max-polls=1 contains_ok "${TEST_NAME}.stdout" <<__OUT__ 2010-01-01/foo:succeeded __OUT__ TEST_NAME=${TEST_NAME_BASE}-cli-dump run_ok "${TEST_NAME}" cylc workflow-state --old-format "${WORKFLOW_NAME}//20100101T0000Z" --max-polls=1 contains_ok "${TEST_NAME}.stdout" <<__OUT__ foo, 2010-01-01, succeeded __OUT__ purge cylc-flow-8.7.0/tests/functional/workflow-state/options/0000775000175000017500000000000015264420307023603 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/workflow-state/options/reference.log0000664000175000017500000000065715264420307026254 0ustar alastairalastairInitial point: 20100101T0000Z Final point: 20100103T0000Z 20100101T0000Z/foo -triggered off ['20091231T0000Z/foo'] 20100102T0000Z/foo -triggered off ['20100101T0000Z/foo'] 20100103T0000Z/foo -triggered off ['20100102T0000Z/foo'] 20100102T0000Z/env_polling -triggered off ['20100102T0000Z/foo'] 20100102T0000Z/offset_polling -triggered off ['20100102T0000Z/foo'] 20100102T0000Z/offset_polling2 -triggered off ['20100103T0000Z/foo'] cylc-flow-8.7.0/tests/functional/workflow-state/options/flow.cylc0000664000175000017500000000131015264420307025421 0ustar alastairalastair#!jinja2 [scheduler] UTC mode = True [scheduling] initial cycle point = 20100101T00Z final cycle point = 20100103T00Z [[graph]] T00 = "foo[-P1D] => foo" R1/20100102T00Z = """ foo => env_polling foo => offset_polling foo[+P1D] => offset_polling2 """ [runtime] [[foo]] script = true [[env_polling]] script = cylc workflow-state ${CYLC_WORKFLOW_ID}//$CYLC_TASK_CYCLE_POINT/foo:succeeded [[offset_polling]] script = cylc workflow-state ${CYLC_WORKFLOW_ID}//20100102T0000Z/foo --offset=P1D [[offset_polling2]] script = cylc workflow-state ${CYLC_WORKFLOW_ID}//20100102T0000Z/foo --offset=-P1D cylc-flow-8.7.0/tests/functional/workflow-state/test_header0000777000175000017500000000000015264420307030425 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/workflow-state/02-validate-blank-command-scripting/0000775000175000017500000000000015264420307030621 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/workflow-state/02-validate-blank-command-scripting/flow.cylc0000664000175000017500000000050515264420307032444 0ustar alastairalastair#!jinja2 [meta] title=Test validation of blank script in automatic workflow polling task [scheduling] [[graph]] R1=eat=>full [runtime] [[root]] script=true [[eat]] script= [[[workflow state polling]]] interval=PT2S max-polls=20 [[full]] cylc-flow-8.7.0/tests/functional/workflow-state/upstream/0000775000175000017500000000000015264420307023750 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/workflow-state/upstream/flow.cylc0000664000175000017500000000111015264420307025564 0ustar alastairalastair[meta] title = "One task takes 20 sec to succeed, another to fail, another to send a message." [scheduling] [[graph]] R1 = """ good-stuff & bad? bad:fail? => handler messenger:x => done """ [runtime] [[done]] [[good-stuff]] script = "sleep 20" [[bad]] script = "sleep 20; false" [[messenger]] script = """ sleep 20 cylc message 'the quick brown fox' """ [[[outputs]]] x = "the quick brown fox" [[handler]] script = true cylc-flow-8.7.0/tests/functional/workflow-state/integer/0000775000175000017500000000000015264420307023545 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/workflow-state/integer/flow.cylc0000664000175000017500000000043015264420307025365 0ustar alastairalastair[scheduling] cycling mode = integer initial cycle point = 0 final cycle point = 2 [[graph]] P1 = """ foo:x => bar """ [runtime] [[foo]] script = cylc message "hello" [[[outputs]]] x = "hello" [[bar]] cylc-flow-8.7.0/tests/functional/workflow-state/03-options.t0000775000175000017500000000260715264420307024220 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test running of cylc workflow-state with various CLI options . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 1 install_workflow "${TEST_NAME_BASE}" options #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE} workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge exit 0 cylc-flow-8.7.0/tests/functional/data-store/0000775000175000017500000000000015264420307021163 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/data-store/00-prune-optional-break.t0000775000175000017500000000530215264420307025626 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test data-store pruning . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- init_workflow "${TEST_NAME_BASE}" << __FLOW__ [scheduling] cycling mode = integer initial cycle point = 1 final cycle point = 1 [[graph]] P1 = """ a => b? => c? a => d => e """ [runtime] [[a,c,e]] script = true [[b]] script = false [[d]] script = """ cylc workflow-state \${CYLC_WORKFLOW_ID}//1/b:failed --interval=2 --max-polls=20 -v cylc pause \$CYLC_WORKFLOW_ID """ __FLOW__ # run workflow run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" cylc workflow-state "${WORKFLOW_NAME}//1/d:succeeded" --interval=2 --max-polls=60 -v # query workflow TEST_NAME="${TEST_NAME_BASE}-prune-optional-break" read -r -d '' optionalBreak <<_args_ { "request_string": " query { workflows { name taskProxies (sort: {keys: [\"name\"]}) { id state } } }", "variables": null } _args_ run_graphql_ok "${TEST_NAME}" "${WORKFLOW_NAME}" "${optionalBreak}" # stop workflow cylc stop --max-polls=10 --interval=2 --kill "${WORKFLOW_NAME}" RESPONSE="${TEST_NAME_BASE}-prune-optional-break.stdout" # compare to expectation cmp_json "${TEST_NAME}-out" "$RESPONSE" << __HERE__ { "workflows": [ { "name": "${WORKFLOW_NAME}", "taskProxies": [ { "id": "~${USER}/${WORKFLOW_NAME}//1/d", "state": "succeeded" }, { "id": "~${USER}/${WORKFLOW_NAME}//1/e", "state": "waiting" } ] } ] } __HERE__ purge cylc-flow-8.7.0/tests/functional/data-store/test_header0000777000175000017500000000000015264420307027500 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/special/0000775000175000017500000000000015264420307020540 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/special/clock-360/0000775000175000017500000000000015264420307022141 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/special/clock-360/flow.cylc0000664000175000017500000000057115264420307023767 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [[events]] abort on stall timeout = True stall timeout = PT0.5M [scheduling] initial cycle point = 20130101T00 final cycle point = 20150101T00 cycling mode = 360day [[special tasks]] clock-trigger = clock(0) [[graph]] T00 = "clock" [runtime] [[clock]] script = true cylc-flow-8.7.0/tests/functional/special/00-sequential.t0000664000175000017500000000170415264420307023316 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test sequential tasks . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/special/07-clock-triggered-360.t0000664000175000017500000000301615264420307024524 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test clock triggering is working . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" clock-360 #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" grep_ok "WorkflowConfigError: clock-trigger tasks require \[scheduling\]cycling mode=" \ "${TEST_NAME}.stderr" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/special/08-clock-trigger-retry.t0000664000175000017500000000350515264420307025054 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Old-style clock trigger should only go ahead if xtriggers succeeded # https://github.com/cylc/cylc-flow/issues/5217 . "$(dirname "$0")/test_header" set_test_number 4 init_workflow "${TEST_NAME_BASE}" << __FLOW__ [scheduling] initial cycle point = 2015 final cycle point = PT1S [[special tasks]] clock-trigger = foo(PT1H) [[graph]] T00 = foo [runtime] [[foo]] execution retry delays = PT5S # hopefully enough time to check task doesn't resubmit immediately script = ((CYLC_TASK_TRY_NUMBER > 1)) __FLOW__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "$WORKFLOW_NAME" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --no-detach "$WORKFLOW_NAME" log_scan "${TEST_NAME_BASE}-log-scan" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" 2 1 \ "\[20150101.*/foo.* retrying in PT5S" \ "xtrigger succeeded: _cylc_retry_20150101" # (if task resubmits immediately instead of waiting PT5S, xtrigger msg will not appear) purge cylc-flow-8.7.0/tests/functional/special/00-sequential/0000775000175000017500000000000015264420307023127 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/special/00-sequential/reference.log0000664000175000017500000000045215264420307025571 0ustar alastairalastairInitial point: 20100101T00 Final point: 20100101T18 20100101T00/monitor -triggered off [] 20100101T00/foo -triggered off ['20100101T00/monitor'] 20100101T06/foo -triggered off ['20100101T00/foo'] 20100101T12/foo -triggered off ['20100101T06/foo'] 20100101T18/foo -triggered off ['20100101T12/foo'] cylc-flow-8.7.0/tests/functional/special/00-sequential/flow.cylc0000664000175000017500000000160315264420307024752 0ustar alastairalastair[scheduler] cycle point format = %Y%m%dT%H [scheduling] initial cycle point = 20100101T00 final cycle point = 20100101T18 runahead limit = PT18H [[special tasks]] sequential = "foo" [[graph]] R1 = monitor:start => foo PT6H = foo [runtime] [[foo]] script = true [[monitor]] script = """ while true; do sleep 1 RUNNING=$(sqlite3 "$CYLC_WORKFLOW_RUN_DIR/log/db" "select count(*) from task_states where name is 'foo' and status is 'running'") || RUNNING=0 if ((RUNNING>1)); then false fi sleep 1 FINISHED=$(sqlite3 "$CYLC_WORKFLOW_RUN_DIR/log/db" "select count(*) from task_states where name is 'foo' and status is 'succeeded'") || FINISHED=0 if ((FINISHED==4)); then break fi done """ cylc-flow-8.7.0/tests/functional/special/test_header0000777000175000017500000000000015264420307027055 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/recurrence-min/0000775000175000017500000000000015264420307022036 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/recurrence-min/00-basic.t0000775000175000017500000000172615264420307023532 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test running with R/min(T00,...) syntax . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/recurrence-min/02-offset-truncated/0000775000175000017500000000000015264420307025532 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/recurrence-min/02-offset-truncated/reference.log0000664000175000017500000000012515264420307030171 0ustar alastairalastairInitial point: 20100101T0300Z Final point: None 20100101T0700Z/foo -triggered off [] cylc-flow-8.7.0/tests/functional/recurrence-min/02-offset-truncated/flow.cylc0000664000175000017500000000021315264420307027351 0ustar alastairalastair[scheduler] UTC mode = true [scheduling] initial cycle point = 20100101T03 [[graph]] R1/min(T00,T06+PT1H,T12,T18) = foo [runtime] [[foo]] cylc-flow-8.7.0/tests/functional/recurrence-min/02-offset-truncated.t0000775000175000017500000000173015264420307025723 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test running with R/min(+PTNH,...) syntax . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/recurrence-min/03-neg-offset-truncated/0000775000175000017500000000000015264420307026302 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/recurrence-min/03-neg-offset-truncated/reference.log0000664000175000017500000000012515264420307030741 0ustar alastairalastairInitial point: 20100101T0300Z Final point: None 20100101T0400Z/foo -triggered off [] cylc-flow-8.7.0/tests/functional/recurrence-min/03-neg-offset-truncated/flow.cylc0000664000175000017500000000021415264420307030122 0ustar alastairalastair[scheduler] UTC mode = true [scheduling] initial cycle point = 20100101T03 [[graph]] R1/min(T00-PT20H,T06,T12,T18) = foo [runtime] [[foo]] cylc-flow-8.7.0/tests/functional/recurrence-min/01-offset-initial.t0000775000175000017500000000173015264420307025362 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test running with R/min(+PTNH,...) syntax . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/recurrence-min/00-basic/0000775000175000017500000000000015264420307023334 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/recurrence-min/00-basic/reference.log0000664000175000017500000000012515264420307025773 0ustar alastairalastairInitial point: 20100101T0300Z Final point: None 20100101T0600Z/foo -triggered off [] cylc-flow-8.7.0/tests/functional/recurrence-min/00-basic/flow.cylc0000664000175000017500000000021215264420307025152 0ustar alastairalastair[scheduler] UTC mode = true [scheduling] initial cycle point = 20100101T03 [[graph]] R1/min(T00, T06, T12 , T18) = foo [runtime] [[foo]] cylc-flow-8.7.0/tests/functional/recurrence-min/test_header0000777000175000017500000000000015264420307030353 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/recurrence-min/03-neg-offset-truncated.t0000775000175000017500000000173015264420307026473 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test running with R/min(+PTNH,...) syntax . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/recurrence-min/01-offset-initial/0000775000175000017500000000000015264420307025171 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/recurrence-min/01-offset-initial/reference.log0000664000175000017500000000012515264420307027630 0ustar alastairalastairInitial point: 20100101T0300Z Final point: None 20100101T0400Z/foo -triggered off [] cylc-flow-8.7.0/tests/functional/recurrence-min/01-offset-initial/flow.cylc0000664000175000017500000000021415264420307027011 0ustar alastairalastair[scheduler] UTC mode = true [scheduling] initial cycle point = 20100101T03 [[graph]] R1/min(T00,+PT1H,T06,T12,T18) = foo [runtime] [[foo]] cylc-flow-8.7.0/tests/functional/include-files/0000775000175000017500000000000015264420307021643 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/include-files/00-basic.t0000664000175000017500000000365615264420307023340 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test include-file inlining . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" workflow #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" # test raw workflow validates run_ok "${TEST_NAME}.1" cylc validate "${WORKFLOW_NAME}" # test workflow validates as inlined during editing mkdir inlined cylc view --inline "${WORKFLOW_NAME}" > inlined/flow.cylc run_ok "${TEST_NAME}.2" cylc validate ./inlined #------------------------------------------------------------------------------- # compare inlined workflow def with reference copy TEST_NAME=${TEST_NAME_BASE}-compare cmp_ok inlined/flow.cylc "${TEST_SOURCE_DIR}/workflow/ref-inlined.cylc" rm -rf inlined #------------------------------------------------------------------------------- purge #------------------------------------------------------------------------------- cylc-flow-8.7.0/tests/functional/include-files/workflow/0000775000175000017500000000000015264420307023515 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/include-files/workflow/scheduling.cylc0000664000175000017500000000021715264420307026516 0ustar alastairalastair[scheduling] initial cycle point = 20130101T00 final cycle point = 20130101T00 [[graph]] R1 = foo => bar T00 = bar cylc-flow-8.7.0/tests/functional/include-files/workflow/body.cylc0000664000175000017500000000005715264420307025330 0ustar alastairalastair%include scheduling.cylc %include runtime.cylc cylc-flow-8.7.0/tests/functional/include-files/workflow/flow.cylc0000664000175000017500000000016715264420307025344 0ustar alastairalastair[meta] title = "A test with nested include-files" [scheduler] allow implicit tasks = True %include body.cylc cylc-flow-8.7.0/tests/functional/include-files/workflow/runtime.cylc0000664000175000017500000000007315264420307026054 0ustar alastairalastair[runtime] [[root]] script = "echo Hello World" cylc-flow-8.7.0/tests/functional/include-files/workflow/ref-inlined.cylc0000664000175000017500000000045615264420307026572 0ustar alastairalastair[meta] title = "A test with nested include-files" [scheduler] allow implicit tasks = True [scheduling] initial cycle point = 20130101T00 final cycle point = 20130101T00 [[graph]] R1 = foo => bar T00 = bar [runtime] [[root]] script = "echo Hello World" cylc-flow-8.7.0/tests/functional/include-files/test_header0000777000175000017500000000000015264420307030160 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-install/0000775000175000017500000000000015264420307021516 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-install/05-check-logs-git.t0000664000175000017500000000415315264420307024730 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test that we only log version control info on workflow source dir # when it is a subdir of a repo. . "$(dirname "$0")/test_header" if ! command -v 'git' > /dev/null; then skip_all 'git not installed' fi # set_test_number 6 set_test_number 3 WORKFLOW="$(date | md5sum | awk '{print $1}')" WORKFLOW_NAME="$(workflow_id "${TEST_NAME_BASE}")" WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}/runN" # Create a workflow in a subdirectory of the test tmpdir mkdir "${WORKFLOW}" cat > "${WORKFLOW}/flow.cylc" <<__HEREDOC__ [scheduler] implicit tasks allowed = True [scheduling] initial cycle point = 1649 [[graph]] R1 = foo __HEREDOC__ # Touch some non-functional files touch "${WORKFLOW}/test_file_in_workflow" touch test_file_outside_workflow # Initialize PWD as a git repo git init . git add . git commit -m "commit 0" # Make changes since commit: echo "Inside workflow" > "${WORKFLOW}/test_file_in_workflow" echo "Outside workflow" > test_file_outside_workflow # Carry out actual test with relpath: run_ok "${TEST_NAME_BASE}-install" \ cylc install "./${WORKFLOW}" --workflow-name "${WORKFLOW_NAME}" DIFF_FILE="${WORKFLOW_RUN_DIR}/log/version/01-uncommitted.diff" grep_ok "Inside workflow" "$DIFF_FILE" grep_fail "Outside workflow" "$DIFF_FILE" purge cylc-flow-8.7.0/tests/functional/cylc-install/00-simple.t0000775000175000017500000001334515264420307023422 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test workflow installation . "$(dirname "$0")/test_header" set_test_number 21 create_test_global_config "" " [install] source dirs = ${PWD}/cylc-src " mkdir "cylc-src" # ----------------------------------------------------------------------------- # Test default name: "cylc install" (flow in $PWD, no args) TEST_NAME="${TEST_NAME_BASE}-basic" make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 run_ok "${TEST_NAME}" cylc install contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE} __OUT__ popd || exit 1 purge_rnd_workflow # ----------------------------------------------------------------------------- # Test default name: "cylc install WORKFLOW_NAME" (flow in configured source dir) make_rnd_workflow # Before adding workflow to ~/cylc-src/, check install fails: TEST_NAME="${TEST_NAME_BASE}-WORKFLOW_NAME-fail-no-src-dir" run_fail "${TEST_NAME}" cylc install "${RND_WORKFLOW_NAME}" # Now add workflow to ~/cylc-src/ and test again RND_WORKFLOW_SOURCE="${PWD}/cylc-src/${RND_WORKFLOW_NAME}" mv "$RND_WORKFLOW_NAME" "${PWD}/cylc-src/" pushd "${RND_WORKFLOW_SOURCE}" || exit 1 TEST_NAME="${TEST_NAME_BASE}-WORKFLOW_NAME-install-ok" run_ok "${TEST_NAME}" cylc install "${RND_WORKFLOW_NAME}" contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE} __OUT__ popd || exit 1 purge_rnd_workflow # ----------------------------------------------------------------------------- # Test cylc install fails if suite.rc file in source dir # (also tests installing from absolute path) TEST_NAME="${TEST_NAME_BASE}-suite.rc" make_rnd_workflow rm -f "${RND_WORKFLOW_SOURCE}/flow.cylc" touch "${RND_WORKFLOW_SOURCE}/suite.rc" run_fail "${TEST_NAME}" cylc install "${RND_WORKFLOW_SOURCE}" --workflow-name="${RND_WORKFLOW_NAME}" grep_ok 'Support for suite.rc files was removed' "${TEST_NAME}.stderr" purge_rnd_workflow # ----------------------------------------------------------------------------- # Test default path: "cylc install" --no-run-name (flow in $PWD) TEST_NAME="${TEST_NAME_BASE}-pwd-no-run-name" make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 run_ok "${TEST_NAME}" cylc install --no-run-name contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME from ${RND_WORKFLOW_SOURCE} __OUT__ popd || exit 1 purge_rnd_workflow # ----------------------------------------------------------------------------- # Test "cylc install" flow-name given (flow in $PWD) TEST_NAME="${TEST_NAME_BASE}-flow-name" make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 run_ok "${TEST_NAME}" cylc install --workflow-name="${RND_WORKFLOW_NAME}-olaf" contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED ${RND_WORKFLOW_NAME}-olaf/run1 from ${RND_WORKFLOW_SOURCE} __OUT__ popd || exit 1 rm -rf "${RUN_DIR}/${RND_WORKFLOW_NAME}-olaf" purge_rnd_workflow # ----------------------------------------------------------------------------- # Test "cylc install" flow-name given, no run name (flow in $PWD) TEST_NAME="${TEST_NAME_BASE}-flow-name-no-run-name" make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 run_ok "${TEST_NAME}" cylc install --workflow-name="${RND_WORKFLOW_NAME}-olaf" --no-run-name contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED ${RND_WORKFLOW_NAME}-olaf from ${RND_WORKFLOW_SOURCE} __OUT__ popd || exit 1 rm -rf "${RUN_DIR}/${RND_WORKFLOW_NAME}-olaf" purge_rnd_workflow # ----------------------------------------------------------------------------- # Test running cylc install twice increments run dirs correctly TEST_NAME="${TEST_NAME_BASE}-install-twice-1" make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 run_ok "${TEST_NAME}" cylc install contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE} __OUT__ TEST_NAME="${TEST_NAME_BASE}-install-twice-2" run_ok "${TEST_NAME}" cylc install contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME/run2 from ${RND_WORKFLOW_SOURCE} __OUT__ popd || exit 1 purge_rnd_workflow # ----------------------------------------------------------------------------- # Test running cylc install with multi level name works correctly TEST_NAME="${TEST_NAME_BASE}-install-multi-level" pushd cylc-src || exit 1 make_rnd_workflow SUB_DIR="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" mkdir "${RND_WORKFLOW_NAME}/${SUB_DIR}" mv "${RND_WORKFLOW_NAME}/flow.cylc" "${RND_WORKFLOW_NAME}/${SUB_DIR}" run_ok "${TEST_NAME}" cylc install "${RND_WORKFLOW_NAME}/${SUB_DIR}" contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED ${RND_WORKFLOW_NAME}/${SUB_DIR}/run1 from ${RND_WORKFLOW_SOURCE}/$SUB_DIR __OUT__ TEST_NAME="${TEST_NAME_BASE}-multi-level-from-pwd" pushd "${RND_WORKFLOW_SOURCE}/$SUB_DIR" || exit 1 run_ok "${TEST_NAME}" cylc install contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED ${RND_WORKFLOW_NAME}/${SUB_DIR}/run2 from ${RND_WORKFLOW_SOURCE}/${SUB_DIR} __OUT__ popd || exit 1 purge_rnd_workflow cylc-flow-8.7.0/tests/functional/cylc-install/04-symlinks-not-resolved.t0000775000175000017500000000326015264420307026420 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Check that symlink is not resolved to its target on installation: # When ~/foo -> ~/bar and --symlink-dirs=run=~/bar DO NOT make run=~/foo . "$(dirname "$0")/test_header" set_test_number 3 cat > flow.cylc <<__HEREDOC__ [scheduler] allow implicit tasks = True [scheduling] initial cycle point = 1500 [[graph]] R1 = foo __HEREDOC__ run_ok "cylc validate ." # Create a temporary directory to put our symlinked foo and bar in: elsewhere=$(mktemp -d) mkdir -p "${elsewhere}/foo" ln -s "${elsewhere}/foo" "bar" # Install the workflow: run_ok "$TEST_NAME_BASE" cylc install --no-run-name --symlink-dirs=run="${elsewhere}/bar" # Check the installed workflow: ls -l "$RUN_DIR/$(basename "$PWD")" > list grep_ok "bar\/cylc-run" list # Tidy up: cylc clean "$(basename "$PWD")" 2> /dev/null rm -fr "${elsewhere}" exit cylc-flow-8.7.0/tests/functional/cylc-install/06-check-logs-svn.t0000664000175000017500000000431415264420307024753 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test that we only log version control info on workflow. . "$(dirname "$0")/test_header" svn --version || skip_all "svn not installed" set_test_number 3 WORKFLOW="$(date | md5sum | awk '{print $1}')" WORKFLOW_NAME="$(workflow_id "${TEST_NAME_BASE}")" WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}" WORKDIR1="${PWD}/workdir1" # Create a workflow in a subdirectory of the test tmpdir mkdir -p "${WORKDIR1}/${WORKFLOW}" cat > "${WORKDIR1}/${WORKFLOW}/flow.cylc" <<__HEREDOC__ [scheduler] implicit tasks allowed = True [scheduling] initial cycle point = 1649 [[graph]] R1 = foo __HEREDOC__ # Touch some non-functional files: touch "${WORKDIR1}/${WORKFLOW}/test_file_in_workflow" touch "${WORKDIR1}/test_file_outside_workflow" # Create an SVN repo: svnadmin create "myrepo" svn import "${WORKDIR1}" "file:///${PWD}/myrepo/trunk" -m "foo" svn co "file:///${PWD}/myrepo/trunk" "${PWD}/elephant" cd "elephant" || exit # Make changes since commit: echo "Inside workflow" > "${WORKFLOW}/test_file_in_workflow" echo "Outside workflow" > test_file_outside_workflow # Carry out actual test: run_ok "${TEST_NAME_BASE}-install" \ cylc install "./${WORKFLOW}" --no-run-name --workflow-name "${WORKFLOW_NAME}" DIFF_FILE="${WORKFLOW_RUN_DIR}/log/version/01-uncommitted.diff" grep_ok "Inside workflow" "$DIFF_FILE" grep_fail "Outside workflow" "$DIFF_FILE" purge cylc-flow-8.7.0/tests/functional/cylc-install/01-symlinks.t0000664000175000017500000001670015264420307023776 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test workflow installation symlinking localhost . "$(dirname "$0")/test_header" if [[ -z ${TMPDIR:-} || -z ${USER:-} || $TMPDIR/$USER == "$HOME" ]]; then skip_all '"TMPDIR" or "USER" not defined or "TMPDIR"/"USER" is "HOME"' fi set_test_number 32 create_test_global_config "" " [install] [[symlink dirs]] [[[localhost]]] run = \$TMPDIR/\$USER/test_cylc_symlink/cylctb_tmp_run_dir share = \$TMPDIR/\$USER/test_cylc_symlink/ log = \$TMPDIR/\$USER/test_cylc_symlink/ log/job = \$TMPDIR/\$USER/test_cylc_symlink/job_log_dir share/cycle = \$TMPDIR/\$USER/test_cylc_symlink/cylctb_tmp_share_dir work = \$TMPDIR/\$USER/test_cylc_symlink/ " # Test "cylc install" ensure symlinks are created TEST_NAME="${TEST_NAME_BASE}-symlinks-created" make_rnd_workflow run_ok "${TEST_NAME}" cylc install --workflow-name="${RND_WORKFLOW_NAME}" "${RND_WORKFLOW_SOURCE}" contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE} __OUT__ WORKFLOW_RUN_DIR="$HOME/cylc-run/${RND_WORKFLOW_NAME}/run1" TEST_SYM="${TEST_NAME_BASE}-run-glblcfg" run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}")" \ = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_run_dir/cylc-run/${RND_WORKFLOW_NAME}/run1" TEST_SYM="${TEST_NAME_BASE}-share-cycle-glblcfg" run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/share/cycle")" \ = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle" TEST_SYM="${TEST_NAME_BASE}-log-job-glblcfg" run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/log/job")" \ = "$TMPDIR/${USER}/test_cylc_symlink/job_log_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/log/job" for DIR in 'work' 'share' 'log'; do TEST_SYM="${TEST_NAME_BASE}-${DIR}-glbcfg" run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/${DIR}")" \ = "$TMPDIR/${USER}/test_cylc_symlink/cylc-run/${RND_WORKFLOW_NAME}/run1/${DIR}" done rm -rf "${TMPDIR}/${USER}/test_cylc_symlink/" purge_rnd_workflow # test cli --symlink-dirs overrides the glblcfg SYMDIR=${TMPDIR}/${USER}/test_cylc_cli_symlink/ TEST_NAME="${TEST_NAME_BASE}-cli-opt-install" make_rnd_workflow run_ok "${TEST_NAME}" cylc install "${RND_WORKFLOW_SOURCE}" \ --workflow-name="${RND_WORKFLOW_NAME}" \ --symlink-dirs="run= ${SYMDIR}cylctb_tmp_run_dir, log=${SYMDIR}, share=${SYMDIR}, \ work = ${SYMDIR}" contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE} __OUT__ WORKFLOW_RUN_DIR="$HOME/cylc-run/${RND_WORKFLOW_NAME}/run1" TEST_SYM="${TEST_NAME_BASE}-run-cli" run_ok "$TEST_SYM" test "$(readlink "${WORKFLOW_RUN_DIR}")" \ = "$TMPDIR/${USER}/test_cylc_cli_symlink/cylctb_tmp_run_dir/cylc-run/${RND_WORKFLOW_NAME}/run1" for DIR in 'work' 'share' 'log'; do TEST_SYM="${TEST_NAME_BASE}-${DIR}-cli" run_ok "$TEST_SYM" test "$(readlink "${WORKFLOW_RUN_DIR}/${DIR}")" \ = "${TMPDIR}/${USER}/test_cylc_cli_symlink/cylc-run/${RND_WORKFLOW_NAME}/run1/${DIR}" done INSTALL_LOG="$(find "${WORKFLOW_RUN_DIR}/log/install" -type f -name '*.log')" for DIR in 'work' 'share' 'log'; do grep_ok "${TMPDIR}/${USER}/test_cylc_cli_symlink/cylc-run/${RND_WORKFLOW_NAME}/run1/${DIR}" "${INSTALL_LOG}" done # test cylc play symlinks after cli opts (mapping to different directories) pushd "${WORKFLOW_RUN_DIR}" || exit 1 cat > 'flow.cylc' << __FLOW__ [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = true __FLOW__ popd || exit 1 run_ok "${TEST_NAME_BASE}-play" cylc play "${RND_WORKFLOW_NAME}/runN" --debug --no-detach # test ensure symlinks, not in cli install are not created from glbl cfg. TEST_SYM="${TEST_NAME_BASE}-share-cycle-cli" run_fail "$TEST_SYM" test "$(readlink "${WORKFLOW_RUN_DIR}/share/cycle")" \ = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle" rm -rf "${TMPDIR}/${USER}/test_cylc_cli_symlink/" purge_rnd_workflow # test no symlinks created with --symlink-dirs="" TEST_NAME="${TEST_NAME_BASE}-no-sym-dirs-cli" make_rnd_workflow run_ok "${TEST_NAME}" cylc install "${RND_WORKFLOW_SOURCE}" \ --workflow-name="${RND_WORKFLOW_NAME}" --symlink-dirs="" contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE} __OUT__ WORKFLOW_RUN_DIR="$HOME/cylc-run/${RND_WORKFLOW_NAME}/run1" TEST_SYM="${TEST_NAME}-run" run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}")" \ = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_run_dir/cylc-run/${RND_WORKFLOW_NAME}/run1" TEST_SYM="${TEST_NAME}-share-cycle" run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/share/cycle")" \ = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle" TEST_SYM="${TEST_NAME}-log-job" run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/log/job")" \ = "$TMPDIR/${USER}/test_cylc_symlink/job_log_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/log/job" for DIR in 'work' 'share' 'log'; do TEST_SYM="${TEST_NAME}-${DIR}" run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/${DIR}")" \ = "$TMPDIR/${USER}/test_cylc_symlink/cylc-run/${RND_WORKFLOW_NAME}/run1/${DIR}" done pushd "${WORKFLOW_RUN_DIR}" || exit 1 cat > 'flow.cylc' << __FLOW__ [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = true __FLOW__ popd || exit 1 run_ok "${TEST_NAME_BASE}-play" cylc play "${RND_WORKFLOW_NAME}/runN" --debug --no-detach # test ensure localhost symlink dirs skipped for installed workflows. TEST_SYM="${TEST_NAME_BASE}-installed-workflow-skips-symdirs" run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}")" \ = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_run_dir/cylc-run/${RND_WORKFLOW_NAME}/run1" rm -rf "${TMPDIR}/${USER}/test_cylc_cli_symlink/" purge_rnd_workflow # test share and share/cycle same symlinks don't error SYMDIR=${TMPDIR}/${USER}/test_cylc_cli_symlink/ TEST_NAME="${TEST_NAME_BASE}-share-share-cycle-same-dirs" make_rnd_workflow # check install runs without failure run_ok "${TEST_NAME}" cylc install "${RND_WORKFLOW_SOURCE}" \ --workflow-name="${RND_WORKFLOW_NAME}" \ --symlink-dirs="share/cycle=${SYMDIR}, share=${SYMDIR}" contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE} __OUT__ WORKFLOW_RUN_DIR="$HOME/cylc-run/${RND_WORKFLOW_NAME}/run1" TEST_SYM="${TEST_NAME_BASE}-share-cli" run_ok "$TEST_SYM" test "$(readlink "${WORKFLOW_RUN_DIR}/share")" \ = "${TMPDIR}/${USER}/test_cylc_cli_symlink/cylc-run/${RND_WORKFLOW_NAME}/run1/share" rm -rf "${TMPDIR}/${USER}/test_cylc_cli_symlink/" purge_rnd_workflow cylc-flow-8.7.0/tests/functional/cylc-install/02-failures.t0000664000175000017500000002214615264420307023741 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test workflow installation failures . "$(dirname "$0")/test_header" set_test_number 43 create_test_global_config '' ' [install] max depth = 6 ' # Test source directory between runs that are not consistent result in error TEST_NAME="${TEST_NAME_BASE}-forbid-inconsistent-source-dir-between-runs" SOURCE_DIR_1="test-install-${CYLC_TEST_TIME_INIT}/${TEST_NAME_BASE}" WORKFLOW_NAME="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR##*tests/}/${TEST_NAME}" mkdir -p "${PWD}/${SOURCE_DIR_1}" pushd "${SOURCE_DIR_1}" || exit 1 touch flow.cylc run_ok "${TEST_NAME}" cylc install --workflow-name "$WORKFLOW_NAME" popd || exit 1 SOURCE_DIR_2="test-install-${CYLC_TEST_TIME_INIT}2/${TEST_NAME_BASE}" WORKFLOW_NAME="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR##*tests/}/${TEST_NAME}" mkdir -p "${PWD}/${SOURCE_DIR_2}" pushd "${SOURCE_DIR_2}" || exit 1 touch flow.cylc run_fail "${TEST_NAME}" cylc install --workflow-name "$WORKFLOW_NAME" grep_ok "previous installations were from" "${TEST_NAME}.stderr" rm -rf "${PWD:?}/${SOURCE_DIR_1}" "${PWD:?}/${SOURCE_DIR_2}" purge popd || exit # ----------------------------------------------------------------------------- # Test fail no flow.cylc file make_rnd_workflow TEST_NAME="${TEST_NAME_BASE}-no-flow-file" rm -f "${RND_WORKFLOW_SOURCE}/flow.cylc" run_fail "${TEST_NAME}" cylc install --workflow-name="${RND_WORKFLOW_NAME}" "${RND_WORKFLOW_SOURCE}" contains_ok "${TEST_NAME}.stderr" <<__ERR__ WorkflowFilesError: No flow.cylc file in ${RND_WORKFLOW_SOURCE} __ERR__ # ----------------------------------------------------------------------------- # Test fail no workflow source dir TEST_NAME="${TEST_NAME_BASE}-nodir" rm -rf "${RND_WORKFLOW_SOURCE}" run_fail "${TEST_NAME}" cylc install --workflow-name="${RND_WORKFLOW_NAME}" --no-run-name "${RND_WORKFLOW_SOURCE}" contains_ok "${TEST_NAME}.stderr" <<__ERR__ WorkflowFilesError: No flow.cylc file in ${RND_WORKFLOW_SOURCE} __ERR__ purge_rnd_workflow # ----------------------------------------------------------------------------- # Test cylc install fails when given flow-name that is an absolute path make_rnd_workflow TEST_NAME="${TEST_NAME_BASE}-no-abs-path-flow-name" run_fail "${TEST_NAME}" cylc install --workflow-name="${RND_WORKFLOW_SOURCE}" "${RND_WORKFLOW_SOURCE}" contains_ok "${TEST_NAME}.stderr" <<__ERR__ WorkflowFilesError: workflow name cannot be an absolute path: ${RND_WORKFLOW_SOURCE} __ERR__ # Test cylc install fails when given forbidden run-name TEST_NAME="${TEST_NAME_BASE}-run-name-forbidden" run_fail "${TEST_NAME}" cylc install --run-name=_cylc-install "${RND_WORKFLOW_SOURCE}" cmp_ok "${TEST_NAME}.stderr" <<__ERR__ WorkflowFilesError: Workflow/run name cannot contain a directory named '_cylc-install' (that filename is reserved) __ERR__ # Test cylc install invalid flow-name TEST_NAME="${TEST_NAME_BASE}-invalid-flow-name" run_fail "${TEST_NAME}" cylc install --workflow-name=".invalid" "${RND_WORKFLOW_SOURCE}" contains_ok "${TEST_NAME}.stderr" <<__ERR__ WorkflowFilesError: invalid workflow name '.invalid' - cannot start with: \`.\`, \`-\`, numbers __ERR__ # Test --run-name and --no-run-name options are mutually exclusive TEST_NAME="${TEST_NAME_BASE}--no-run-name-and--run-name-forbidden" pushd "${RND_WORKFLOW_SOURCE}" || exit 1 run_fail "${TEST_NAME}" cylc install --run-name="${RND_WORKFLOW_NAME}" --no-run-name cmp_ok "${TEST_NAME}.stderr" <<__ERR__ InputError: options --no-run-name and --run-name are mutually exclusive. __ERR__ popd || exit 1 purge_rnd_workflow # ----------------------------------------------------------------------------- # Test source dir can not contain '_cylc-install, log, share, work' dirs for DIR in 'work' 'share' 'log' '_cylc-install'; do TEST_NAME="${TEST_NAME_BASE}-${DIR}-forbidden-in-source" make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 mkdir ${DIR} run_fail "${TEST_NAME}" cylc install contains_ok "${TEST_NAME}.stderr" <<__ERR__ WorkflowFilesError: ${RND_WORKFLOW_NAME} installation failed - ${DIR} exists in source directory. __ERR__ purge_rnd_workflow popd || exit 1 done # ----------------------------------------------------------------------------- # Test running cylc install twice, first using --run-name, followed by standard run results in error TEST_NAME="${TEST_NAME_BASE}-install-twice-mix-options-1-1st-install" make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 run_ok "${TEST_NAME}" cylc install --run-name=olaf contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED ${RND_WORKFLOW_NAME}/olaf from ${RND_WORKFLOW_SOURCE} __OUT__ TEST_NAME="${TEST_NAME_BASE}-install-twice-mix-options-1-2nd-install" run_fail "${TEST_NAME}" cylc install contains_ok "${TEST_NAME}.stderr" <<__ERR__ WorkflowFilesError: Path: "${RND_WORKFLOW_RUNDIR}" contains an installed workflow. Use --run-name to create a new run. __ERR__ popd || exit 1 purge_rnd_workflow # ----------------------------------------------------------------------------- # Test running cylc install twice, first using standard run, followed by --run-name results in error TEST_NAME="${TEST_NAME_BASE}-install-twice-mix-options-2-1st-install" make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 run_ok "${TEST_NAME}" cylc install contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED ${RND_WORKFLOW_NAME}/run1 from ${RND_WORKFLOW_SOURCE} __OUT__ TEST_NAME="${TEST_NAME_BASE}-install-twice-mix-options-2-2nd-install" run_fail "${TEST_NAME}" cylc install --run-name=olaf contains_ok "${TEST_NAME}.stderr" <<__ERR__ WorkflowFilesError: --run-name option not allowed as '${RND_WORKFLOW_RUNDIR}' contains installed numbered runs. __ERR__ popd || exit 1 purge_rnd_workflow # ----------------------------------------------------------------------------- # Test running cylc install twice, using the same --run-name results in error TEST_NAME="${TEST_NAME_BASE}-install-twice-same-run-name-1st-install" make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 run_ok "${TEST_NAME}" cylc install --run-name=olaf contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED ${RND_WORKFLOW_NAME}/olaf from ${RND_WORKFLOW_SOURCE} __OUT__ TEST_NAME="${TEST_NAME_BASE}-install-twice-same-run-name-2nd-install" run_fail "${TEST_NAME}" cylc install --run-name=olaf contains_ok "${TEST_NAME}.stderr" <<__ERR__ WorkflowFilesError: '${RND_WORKFLOW_RUNDIR}/olaf' already exists __ERR__ popd || exit 1 purge_rnd_workflow # ----------------------------------------------------------------------------- # Test cylc install fails if installation would result in nested run dirs TEST_NAME="${TEST_NAME_BASE}-nested-rundir" make_rnd_workflow mkdir -p "${RND_WORKFLOW_RUNDIR}/.service" run_fail "${TEST_NAME}-install" cylc install "${RND_WORKFLOW_SOURCE}" \ --workflow-name="${RND_WORKFLOW_NAME}/nested" cmp_ok "${TEST_NAME}-install.stderr" <<__ERR__ WorkflowFilesError: Nested run directories not allowed - cannot install workflow in '${RND_WORKFLOW_RUNDIR}/nested/run1' as '${RND_WORKFLOW_RUNDIR}' is already a valid run directory. __ERR__ # Test moving source dir results in error TEST_NAME="${TEST_NAME_BASE}-install-moving-src-dir" make_rnd_workflow run_ok "${TEST_NAME}" cylc install "./${RND_WORKFLOW_NAME}" contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME/run1 from ${PWD}/${RND_WORKFLOW_NAME} __OUT__ rm -rf "${RND_WORKFLOW_SOURCE}" ALT_SOURCE="${TMPDIR}/${USER}/cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" mkdir -p "${ALT_SOURCE}/${RND_WORKFLOW_NAME}" touch "${ALT_SOURCE}/${RND_WORKFLOW_NAME}/flow.cylc" TEST_NAME="${TEST_NAME_BASE}-install-twice-moving-src-dir-raises-error" run_fail "${TEST_NAME}" cylc install "${ALT_SOURCE}/${RND_WORKFLOW_NAME}" grep_ok "WorkflowFilesError: Symlink broken" "${TEST_NAME}.stderr" rm -rf "${ALT_SOURCE}" purge_rnd_workflow # ----------------------------------------------------------------------------- # --run-name cannot be a path make_rnd_workflow TEST_NAME="${TEST_NAME_BASE}-forbid-cylc-run-dir-install" BASE_NAME="test-install-${CYLC_TEST_TIME_INIT}" mkdir -p "${RUN_DIR}/${BASE_NAME}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME}" && cd "$_" || exit touch flow.cylc run_fail "${TEST_NAME}" cylc install --run-name=foo/bar/baz --workflow-name "$RND_WORKFLOW_NAME" contains_ok "${TEST_NAME}.stderr" <<__ERR__ WorkflowFilesError: Run name cannot be a path. (You used foo/bar/baz) __ERR__ cd "${RUN_DIR}" || exit rm -rf "${BASE_NAME}" purge_rnd_workflow exit cylc-flow-8.7.0/tests/functional/cylc-install/07-no-recursive-installations.t0000775000175000017500000000440115264420307027434 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test workflow installation . "$(dirname "$0")/test_header" set_test_number 9 cat > flow.cylc <<__HEREDOC__ [scheduler] allow implicit tasks = true [scheduling] [[graph]] R1 = foo __HEREDOC__ run_ok "$TEST_NAME_BASE" cylc validate "$PWD" TEST_FOLDERS=() MSG="Nested install directories not allowed" TEST_FOLDER=cylctb-$(uuidgen) TEST_FOLDERS+=("$TEST_FOLDER") cylc install "$PWD" --workflow-name "${TEST_FOLDER}/" TEST_NAME="${TEST_NAME_BASE}-child" run_fail "$TEST_NAME" cylc install "$PWD" --workflow-name "${TEST_FOLDER}/child/grandchild" grep_ok "$MSG" "${TEST_NAME}.stderr" TEST_NAME="${TEST_NAME_BASE}-child-no-run-name" run_fail "$TEST_NAME" cylc install "$PWD" --workflow-name "${TEST_FOLDER}/child/grandchild" --no-run-name grep_ok "$MSG" "${TEST_NAME}.stderr" TEST_FOLDER=cylctb-$(uuidgen) TEST_FOLDERS+=("$TEST_FOLDER") cylc install "$PWD" --workflow-name "${TEST_FOLDER}/child/grandchild" TEST_NAME="${TEST_NAME_BASE}-parent" run_fail "$TEST_NAME" cylc install "$PWD" --workflow-name "${TEST_FOLDER}/" grep_ok "$MSG" "${TEST_NAME}.stderr" TEST_NAME="${TEST_NAME_BASE}-parent-no-run-name" run_fail "$TEST_NAME" cylc install "$PWD" --workflow-name "${TEST_FOLDER}/" --no-run-name grep_ok "$MSG" "${TEST_NAME}.stderr" # Cleanup all the test folders added to the array. # shellcheck disable=SC2048 for TEST_FOLDER in ${TEST_FOLDERS[*]}; do rm -fr "${RUN_DIR}/${TEST_FOLDER:-}" done exit cylc-flow-8.7.0/tests/functional/cylc-install/test_header0000777000175000017500000000000015264420307030033 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-install/03-file-transfer.t0000664000175000017500000000672515264420307024676 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test rsync of workflow installation . "$(dirname "$0")/test_header" if ! command -v 'tree' >'/dev/null'; then skip_all '"tree" command not available' fi set_test_number 9 # Need to override any symlink dirs set in global.cylc: create_test_global_config "" " [install] [[symlink dirs]] [[[localhost]]] run = log = work = share = share/cycle = " # Test cylc install copies files to run dir successfully. TEST_NAME="${TEST_NAME_BASE}-basic" make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 mkdir .git .svn dir1 dir2 touch .git/file1 .svn/file1 dir1/file1 dir2/file1 file1 file2 run_ok "${TEST_NAME}" cylc install --no-run-name # If rose-cylc plugin is installed add install files to tree. export ROSE_FILES='' tree -a -v -I '*.log|03-file-transfer*' --charset=ascii --noreport "${RND_WORKFLOW_RUNDIR}/" > 'basic-tree.out' cmp_ok 'basic-tree.out' <<__OUT__ ${RND_WORKFLOW_RUNDIR}/ |-- _cylc-install | \`-- source -> ${RND_WORKFLOW_SOURCE} |-- dir1 | \`-- file1 |-- dir2 | \`-- file1 |-- file1 |-- file2 |-- flow.cylc \`-- log \`-- install __OUT__ contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME from ${RND_WORKFLOW_SOURCE} __OUT__ popd || exit 1 purge_rnd_workflow # Test cylc install copies files to run dir successfully, excluding files from # .cylcignore file. # Should work if we run "cylc install" from source dir or not (see GH #5066) for RUN_IN_SRC_DIR in true false; do TEST_NAME="${TEST_NAME_BASE}-cylcignore-${RUN_IN_SRC_DIR}" make_rnd_workflow pushd "${RND_WORKFLOW_SOURCE}" || exit 1 mkdir .git .svn dir1 dir2 extradir1 extradir2 touch .git/file1 .svn/file1 dir1/file1 dir2/file1 extradir1/file1 extradir2/file1 file1 file2 .cylcignore cat > .cylcignore <<__END__ dir* extradir* file2 __END__ if ${RUN_IN_SRC_DIR}; then run_ok "${TEST_NAME}" cylc install --no-run-name CWD="${PWD}" else DTMP=$(mktemp -d) pushd "${DTMP}" || exit 1 run_ok "${TEST_NAME}" cylc install --no-run-name "${RND_WORKFLOW_SOURCE}" CWD="${PWD}" popd || exit 1 fi OUT="cylc-ignore-tree-${RUN_IN_SRC_DIR}.out" tree -a -v -I '*.log|03-file-transfer*' --charset=ascii --noreport "${RND_WORKFLOW_RUNDIR}/" > "$OUT" cmp_ok "$OUT" <<__OUT__ ${RND_WORKFLOW_RUNDIR}/ |-- _cylc-install | \`-- source -> ${RND_WORKFLOW_SOURCE} |-- file1 |-- flow.cylc \`-- log \`-- install __OUT__ contains_ok "${CWD}/${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME from ${RND_WORKFLOW_SOURCE} __OUT__ popd || exit 1 purge_rnd_workflow done cylc-flow-8.7.0/tests/functional/cylc-get-workflow-contact/0000775000175000017500000000000015264420307024130 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-get-workflow-contact/00-basic.t0000664000175000017500000000352415264420307025617 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc get-workflow-contact" basic usage. . "$(dirname "$0")/test_header" set_test_number 6 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduler] cycle point format = %Y [scheduling] initial cycle point = 2016 [[graph]] R1 = t1 [runtime] [[t1]] script = true __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" run_fail "${TEST_NAME_BASE}-get-workflow-contact-1" \ cylc get-workflow-contact "${WORKFLOW_NAME}" cmp_ok "${TEST_NAME_BASE}-get-workflow-contact-1.stderr" <<__ERR__ CylcError: ${WORKFLOW_NAME}: cannot get contact info, workflow not running? __ERR__ run_ok "${TEST_NAME_BASE}-run-pause" cylc play --pause "${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}-get-workflow-contact-2" \ cylc get-workflow-contact "${WORKFLOW_NAME}" contains_ok "${TEST_NAME_BASE}-get-workflow-contact-2.stdout" \ "${WORKFLOW_RUN_DIR}/.service/contact" cylc stop --max-polls=60 --interval=1 "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/cylc-get-workflow-contact/test_header0000777000175000017500000000000015264420307032445 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/offset/0000775000175000017500000000000015264420307020406 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/offset/03-final-next-chain/0000775000175000017500000000000015264420307023753 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/offset/03-final-next-chain/reference.log0000664000175000017500000000036515264420307026420 0ustar alastairalastairInitial point: 20100101T0000+01 Final point: 20100101T1200+01 20100101T0000+01/foo -triggered off ['20091231T1800+01/foo'] 20100101T0600+01/foo -triggered off ['20100101T0000+01/foo'] 20100101T1200+01/foo -triggered off ['20100101T0600+01/foo'] cylc-flow-8.7.0/tests/functional/offset/03-final-next-chain/flow.cylc0000664000175000017500000000035615264420307025602 0ustar alastairalastair[scheduler] cycle point time zone = +01 [scheduling] initial cycle point = 20100101T00 final cycle point = T06+PT6H [[graph]] T00, T06, T12, T18 = "foo[-PT6H] => foo" [runtime] [[foo]] script = true cylc-flow-8.7.0/tests/functional/offset/05-long-final-chain.t0000664000175000017500000000170315264420307024124 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test a 3 item offset . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/offset/04-cycle-offset-chain/0000775000175000017500000000000015264420307024272 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/offset/04-cycle-offset-chain/reference.log0000664000175000017500000000036515264420307026737 0ustar alastairalastairInitial point: 20100101T0000+01 Final point: 20100101T1200+01 20100101T0000+01/foo -triggered off ['20091231T1800+01/foo'] 20100101T0600+01/foo -triggered off ['20100101T0000+01/foo'] 20100101T1200+01/foo -triggered off ['20100101T0600+01/foo'] cylc-flow-8.7.0/tests/functional/offset/04-cycle-offset-chain/flow.cylc0000664000175000017500000000036415264420307026120 0ustar alastairalastair[scheduler] cycle point time zone = +01 [scheduling] initial cycle point = 20100101T00 final cycle point = T06+PT06H [[graph]] T00, T06, T12, T18 = "foo[-P1D+PT18H] => foo" [runtime] [[foo]] script = true cylc-flow-8.7.0/tests/functional/offset/01-final-next.t0000664000175000017500000000174615264420307023066 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test specifying final cycle as the next instance of T06 . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/offset/03-final-next-chain.t0000664000175000017500000000176515264420307024151 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test specifying final cycle as an offset to the next occurrence of T06 . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/offset/01-final-next/0000775000175000017500000000000015264420307022671 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/offset/01-final-next/reference.log0000664000175000017500000000027015264420307025331 0ustar alastairalastairInitial point: 20100101T0000+01 Final point: 20100101T0600+01 20100101T0000+01/foo -triggered off ['20091231T1800+01/foo'] 20100101T0600+01/foo -triggered off ['20100101T0000+01/foo'] cylc-flow-8.7.0/tests/functional/offset/01-final-next/flow.cylc0000664000175000017500000000063215264420307024515 0ustar alastairalastair[scheduler] cycle point time zone = +01 [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT3M [scheduling] initial cycle point = 20100101T00 final cycle point = T06 [[graph]] T00 = "foo[-PT6H] => foo" T06 = "foo[-PT6H] => foo" [runtime] [[foo]] script = true cylc-flow-8.7.0/tests/functional/offset/05-long-final-chain/0000775000175000017500000000000015264420307023736 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/offset/05-long-final-chain/reference.log0000664000175000017500000000075115264420307026402 0ustar alastairalastairInitial point: 20100101T0000+01 Final point: 20100102T1200+01 20100101T0000+01/foo -triggered off ['20091231T1800+01/foo'] 20100101T0600+01/foo -triggered off ['20100101T0000+01/foo'] 20100101T1200+01/foo -triggered off ['20100101T0600+01/foo'] 20100101T1800+01/foo -triggered off ['20100101T1200+01/foo'] 20100102T0000+01/foo -triggered off ['20100101T1800+01/foo'] 20100102T0600+01/foo -triggered off ['20100102T0000+01/foo'] 20100102T1200+01/foo -triggered off ['20100102T0600+01/foo'] cylc-flow-8.7.0/tests/functional/offset/05-long-final-chain/flow.cylc0000664000175000017500000000036215264420307025562 0ustar alastairalastair[scheduler] cycle point time zone = +01 [scheduling] initial cycle point = 20100101T00 final cycle point = T06+P1D+PT6H [[graph]] T00, T06, T12, T18 = "foo[-PT6H] => foo" [runtime] [[foo]] script = true cylc-flow-8.7.0/tests/functional/offset/04-cycle-offset-chain.t0000664000175000017500000000173615264420307024466 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test specifying an offset as a chain of periods . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/offset/00-final-simple/0000775000175000017500000000000015264420307023203 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/offset/00-final-simple/reference.log0000664000175000017500000000065415264420307025651 0ustar alastairalastairInitial point: 20100101T0000+01 Final point: 20100106T0000+01 20100101T0000+01/foo -triggered off ['20091231T0000+01/foo'] 20100102T0000+01/foo -triggered off ['20100101T0000+01/foo'] 20100103T0000+01/foo -triggered off ['20100102T0000+01/foo'] 20100104T0000+01/foo -triggered off ['20100103T0000+01/foo'] 20100105T0000+01/foo -triggered off ['20100104T0000+01/foo'] 20100106T0000+01/foo -triggered off ['20100105T0000+01/foo'] cylc-flow-8.7.0/tests/functional/offset/00-final-simple/flow.cylc0000664000175000017500000000033115264420307025023 0ustar alastairalastair[scheduler] cycle point time zone = +01 [scheduling] initial cycle point = 20100101T00 final cycle point = +P5D [[graph]] T00 = "foo[-P1D] => foo" [runtime] [[foo]] script = true cylc-flow-8.7.0/tests/functional/offset/02-final-chain.t0000664000175000017500000000173515264420307023171 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test specifying final cycle as an offset chain . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/offset/00-final-simple.t0000664000175000017500000000172715264420307023377 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test specifying final cycle as an offset . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/offset/test_header0000777000175000017500000000000015264420307026723 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/offset/02-final-chain/0000775000175000017500000000000015264420307022776 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/offset/02-final-chain/reference.log0000664000175000017500000000065415264420307025444 0ustar alastairalastairInitial point: 20100101T0000+01 Final point: 20100102T0600+01 20100101T0000+01/foo -triggered off ['20091231T1800+01/foo'] 20100101T0600+01/foo -triggered off ['20100101T0000+01/foo'] 20100101T1200+01/foo -triggered off ['20100101T0600+01/foo'] 20100101T1800+01/foo -triggered off ['20100101T1200+01/foo'] 20100102T0000+01/foo -triggered off ['20100101T1800+01/foo'] 20100102T0600+01/foo -triggered off ['20100102T0000+01/foo'] cylc-flow-8.7.0/tests/functional/offset/02-final-chain/flow.cylc0000664000175000017500000000035715264420307024626 0ustar alastairalastair[scheduler] cycle point time zone = +01 [scheduling] initial cycle point = 20100101T00 final cycle point = +P1D+PT6H [[graph]] T00, T06, T12, T18 = "foo[-PT6H] => foo" [runtime] [[foo]] script = true cylc-flow-8.7.0/tests/functional/cylc-config/0000775000175000017500000000000015264420307021315 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-config/09-platforms.t0000775000175000017500000000455715264420307023755 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test cylc config --platform-names and --platforms . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 7 #------------------------------------------------------------------------------- cat > "global.cylc" <<__HEREDOC__ [platforms] [[foo]] job runner = slurm hosts = of_melkor, of_valar [[bar]] job runner = slurm hosts = of_orcs, of_gondor [platform groups] [[FOO]] platforms = foo, bar [task events] # Just make sure this doesn't get included __HEREDOC__ export CYLC_CONF_PATH="${PWD}" TEST_NAME="${TEST_NAME_BASE}-names" run_ok "${TEST_NAME}" cylc config --platform-names cmp_ok "${TEST_NAME}.stdout" <<__HEREDOC__ localhost foo bar FOO __HEREDOC__ cmp_ok "${TEST_NAME}.stderr" <<__HEREDOC__ Configured names are regular expressions; any match is a valid platform. They are searched from the bottom up, until the first match is found. Platforms --------- Platform Groups -------------- __HEREDOC__ TEST_NAME="${TEST_NAME_BASE}-s" head -n 10 > just_platforms < global.cylc run_ok "${TEST_NAME}" cylc config --platforms cmp_ok "${TEST_NAME}.stdout" "just_platforms" #------------------------------------------------------------------------------- # test with no platforms or platform groups defined rm "global.cylc" touch "global.cylc" TEST_NAME="${TEST_NAME_BASE}-names" run_ok "${TEST_NAME}" cylc config --platform-names cmp_ok "${TEST_NAME}.stdout" <<__HEREDOC__ localhost __HEREDOC__ exit cylc-flow-8.7.0/tests/functional/cylc-config/00-simple.t0000775000175000017500000000661415264420307023222 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test cylc config . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 19 #------------------------------------------------------------------------------- init_workflow "${TEST_NAME_BASE}" "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/flow.cylc" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-all" run_ok "${TEST_NAME}" cylc config -d "${WORKFLOW_NAME}" mkdir tmp_src cp "${TEST_NAME}.stdout" tmp_src/flow.cylc run_ok "${TEST_NAME}-validate" cylc validate --check-circular ./tmp_src cmp_ok "${TEST_NAME}.stderr" <'/dev/null' rm -rf tmp_src #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-section1" run_ok "${TEST_NAME}" cylc config -d --item=[scheduling] "${WORKFLOW_NAME}" cmp_ok "${TEST_NAME}.stdout" "$TEST_SOURCE_DIR/${TEST_NAME_BASE}/section1.stdout" cmp_ok "${TEST_NAME}.stderr" - VAR __OUT__ cmp_ok "${TEST_NAME}.stderr" - VAR __OUT__ cmp_ok "${TEST_NAME}.stderr" - "flow_path.out" cmp_ok "flow_path.out" <<< "${WORKFLOW_RUN_DIR}/flow.cylc" purge exit cylc-flow-8.7.0/tests/functional/cylc-config/06-compat.t0000664000175000017500000000331415264420307023211 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test compat following https://github.com/cylc/cylc-flow/pull/3191 . "$(dirname "$0")/test_header" set_test_number 3 cat >'flow.cylc' <<'__FLOW_CONFIG__' [scheduler] allow implicit tasks = True [scheduling] initial cycle point = next(T00) [[dependencies]] [[[R1]]] graph = r1 [[[R1]]] graph = 'r2' [[[ R1 ]]] graph = """r3""" [[[R1]]] graph = """ r4 => r5 r6 => r7 """ [[[T06, T12]]] graph = t1 => t2 [[[ P1D!(01T, 11T) ]]] graph = t3 __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate . run_ok "${TEST_NAME_BASE}-dependencies" \ cylc config --item='[scheduling][graph]' . cmp_ok "${TEST_NAME_BASE}-dependencies.stdout" <<'__OUT__' R1 = """ r1 r2 r3 r4 => r5 r6 => r7 """ T06, T12 = t1 => t2 P1D!(01T, 11T) = t3 __OUT__ exit cylc-flow-8.7.0/tests/functional/cylc-config/05-param-vars/0000775000175000017500000000000015264420307023610 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-config/05-param-vars/flow.cylc0000664000175000017500000000050615264420307025434 0ustar alastairalastair[task parameters] t = 1..5 u = right,left [scheduling] [[graph]] R1 = foo [runtime] [[root]] script = true [[[environment]]] PARAM1 = 'something' [[foo]] [[[environment]]] PARAM1 = $CYLC_TASK_PARAM_t PARAM2 = $CYLC_TASK_PARAM_u cylc-flow-8.7.0/tests/functional/cylc-config/07-failif-item-in-platforms-and-groups.t0000775000175000017500000000320715264420307030607 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that Platforms and Platform Groups return an error listing items # defined in both. # Not having additional tests for cases with no overlap or platforms only # because other tests will fail. # Not testing for platform groups only because the really should never happen. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 4 create_test_global_config '' ''' [platforms] [[Thomas]] [[Percy]] [[Edward]] [[Skarloey]] [platform groups] [[Thomas]] [[Percy]] [[Gordon]] [[Rhenas]] ''' run_fail "${TEST_NAME_BASE}" cylc config grep_ok "GlobalConfigError" "${TEST_NAME_BASE}.stderr" grep_ok "\* Thomas" "${TEST_NAME_BASE}.stderr" grep_ok "\* Percy" "${TEST_NAME_BASE}.stderr" exit cylc-flow-8.7.0/tests/functional/cylc-config/02-cycling/0000775000175000017500000000000015264420307023164 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-config/02-cycling/flow.cylc0000664000175000017500000000031415264420307025005 0ustar alastairalastair[scheduling] initial cycle point = 2020 final cycle point = 2030 [[graph]] P1Y = foo [runtime] [[foo]] script = """ echo this echo that """ cylc-flow-8.7.0/tests/functional/cylc-config/02-cycling.t0000775000175000017500000000242115264420307023353 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc config" on a cycling workflow, result should validate. . "$(dirname "$0")/test_header" set_test_number 2 init_workflow "${TEST_NAME_BASE}" "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/flow.cylc" run_ok "${TEST_NAME_BASE}" cylc config "${WORKFLOW_NAME}" mkdir temp cp "${TEST_NAME_BASE}.stdout" temp/flow.cylc run_ok "${TEST_NAME_BASE}-validate" \ cylc validate --check-circular ./temp rm -rf temp purge exit cylc-flow-8.7.0/tests/functional/cylc-config/08-item-not-found.t0000775000175000017500000000443415264420307024604 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test cylc config . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 9 #------------------------------------------------------------------------------- cat >>'global.cylc' <<__HERE__ [platforms] [[foo]] __HERE__ OLD="$CYLC_CONF_PATH" export CYLC_CONF_PATH="${PWD}" # Control Run run_ok "${TEST_NAME_BASE}-ok" cylc config -i "[platforms][foo]" # If item not settable in config (platforms is misspelled): run_fail "${TEST_NAME_BASE}-not-in-config-spec" cylc config -i "[platfroms][foo]" cmp_ok "${TEST_NAME_BASE}-not-in-config-spec.stderr" << __HERE__ InvalidConfigError: "platfroms" is not a valid configuration for global.cylc. __HERE__ # If item settable in config but not set. run_fail "${TEST_NAME_BASE}-not-defined" cylc config -i "[scheduler]" cmp_ok "${TEST_NAME_BASE}-not-defined.stderr" << __HERE__ ItemNotFoundError: You have not set "scheduler" in this config. __HERE__ run_fail "${TEST_NAME_BASE}-not-defined-2" cylc config -i "[platforms][bar]" cmp_ok "${TEST_NAME_BASE}-not-defined-2.stderr" << __HERE__ ItemNotFoundError: You have not set "[platforms]bar" in this config. __HERE__ run_fail "${TEST_NAME_BASE}-not-defined-3" cylc config -i "[platforms][foo]hosts" cmp_ok "${TEST_NAME_BASE}-not-defined-3.stderr" << __HERE__ ItemNotFoundError: You have not set "[platforms][foo]hosts" in this config. __HERE__ rm global.cylc export CYLC_CONF_PATH="$OLD" cylc-flow-8.7.0/tests/functional/cylc-config/03-icp.t0000775000175000017500000000260315264420307022501 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test for "cylc config --icp=CYCLE_POINT". . "$(dirname "$0")/test_header" set_test_number 3 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduler] UTC mode = True [scheduling] [[graph]] R1 = foo => bar [runtime] [[foo, bar]] script = true __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}" cylc config --icp=20200101T0000Z "${WORKFLOW_NAME}" contains_ok "${TEST_NAME_BASE}.stdout" <<__OUT__ initial cycle point = 20200101T0000Z __OUT__ cmp_ok "${TEST_NAME_BASE}.stderr" <'/dev/null' purge exit cylc-flow-8.7.0/tests/functional/cylc-config/01-no-final/0000775000175000017500000000000015264420307023236 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-config/01-no-final/flow.cylc0000664000175000017500000000027115264420307025061 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] initial cycle point = 20100808T06 final cycle point = [[graph]] T06 = foo [runtime] [[foo]] script = true cylc-flow-8.7.0/tests/functional/cylc-config/test_header0000777000175000017500000000000015264420307027632 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-config/01-no-final.t0000775000175000017500000000307315264420307023431 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test cylc config with a workflow with an explicitly empty final cycle point . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- init_workflow "${TEST_NAME_BASE}" "$TEST_SOURCE_DIR/${TEST_NAME_BASE}/flow.cylc" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-all run_ok "${TEST_NAME}" cylc config "${WORKFLOW_NAME}" --item='[scheduling]final cycle point' cmp_ok "${TEST_NAME}.stdout" - << __OUT__ __OUT__ #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/cylc-config/10-platform-expansion.t0000775000175000017500000000301015264420307025543 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test cylc config expansion of platform section. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- cat > "global.cylc" <<__HEREDOC__ [platforms] [[ \ foo, bar..., \ baz\d\d, qux\S\S \ ]] hosts = of_melkor, of_valar job runner = slurm __HEREDOC__ export CYLC_CONF_PATH="${PWD}" TEST_NAME="${TEST_NAME_BASE}-names" run_ok "${TEST_NAME}" cylc config --platform-names cmp_ok "${TEST_NAME}.stdout" <<__HEREDOC__ localhost foo bar... baz\d\d qux\S\S __HEREDOC__ exit cylc-flow-8.7.0/tests/functional/cylc-config/00-simple/0000775000175000017500000000000015264420307023023 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-config/00-simple/section1.stdout0000664000175000017500000000071615264420307026020 0ustar alastairalastairinitial cycle point = 1 final cycle point = 1 initial cycle point constraints = final cycle point constraints = hold after cycle point = stop after cycle point = cycling mode = integer runahead limit = P4 sequential xtriggers = False [[queues]] [[[default]]] limit = 100 members = [[special tasks]] clock-trigger = external-trigger = clock-expire = sequential = [[xtriggers]] [[graph]] R1 = OPS:finish-all => VAR cylc-flow-8.7.0/tests/functional/cylc-config/00-simple/flow.cylc0000664000175000017500000000164215264420307024651 0ustar alastairalastair[meta] title = "multiple inheritance example" description = """To see how multiple inheritance works: % cylc list -tb[m] WORKFLOW # list namespaces % cylc graph -n WORKFLOW # graph namespaces % cylc graph WORKFLOW # dependencies, collapse on first-parent namespaces % cylc config --item [runtime]ops_s1 WORKFLOW % cylc config --item [runtime]var_p2 foo""" [scheduling] [[graph]] R1 = "OPS:finish-all => VAR" [runtime] [[root]] [[OPS]] script = echo "RUN: run-ops.sh" [[VAR]] script = echo "RUN: run-var.sh" [[SERIAL]] [[[directives]]] job_type = serial [[PARALLEL]] [[[directives]]] job_type = parallel [[ops_s1, ops_s2]] inherit = OPS, SERIAL [[ops_p1, ops_p2]] inherit = OPS, PARALLEL [[var_s1, var_s2]] inherit = VAR, SERIAL [[var_p1, var_p2]] inherit = VAR, PARALLEL cylc-flow-8.7.0/tests/functional/cylc-config/00-simple/section2.stdout0000664000175000017500000006360115264420307026023 0ustar alastairalastair[[root]] completion = platform = inherit = script = init-script = env-script = err-script = exit-script = pre-script = post-script = work sub-directory = execution polling intervals = execution retry delays = execution time limit = submission polling intervals = submission retry delays = run mode = live [[[meta]]] title = description = URL = [[[skip]]] outputs = disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = time limit buffer = PT30S fail cycle points = fail try 1 only = True disable task event handlers = True [[[environment filter]]] include = exclude = [[[job]]] batch system = batch submit command template = [[[remote]]] host = owner = retrieve job logs = retrieve job logs max size = retrieve job logs retry delays = [[[events]]] handlers = handler events = handler retry delays = mail events = execution timeout = submission timeout = expired handlers = late offset = late handlers = submitted handlers = started handlers = succeeded handlers = failed handlers = submission failed handlers = warning handlers = critical handlers = retry handlers = submission retry handlers = execution timeout handlers = submission timeout handlers = custom handlers = [[[mail]]] from = to = [[[workflow state polling]]] interval = max-polls = message = alt-cylc-run-dir = verbose mode = [[[environment]]] [[[directives]]] [[[outputs]]] [[[parameter environment templates]]] [[OPS]] script = echo "RUN: run-ops.sh" completion = platform = inherit = init-script = env-script = err-script = exit-script = pre-script = post-script = work sub-directory = execution polling intervals = execution retry delays = execution time limit = submission polling intervals = submission retry delays = run mode = live [[[meta]]] title = description = URL = [[[skip]]] outputs = disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = time limit buffer = PT30S fail cycle points = fail try 1 only = True disable task event handlers = True [[[environment filter]]] include = exclude = [[[job]]] batch system = batch submit command template = [[[remote]]] host = owner = retrieve job logs = retrieve job logs max size = retrieve job logs retry delays = [[[events]]] handlers = handler events = handler retry delays = mail events = execution timeout = submission timeout = expired handlers = late offset = late handlers = submitted handlers = started handlers = succeeded handlers = failed handlers = submission failed handlers = warning handlers = critical handlers = retry handlers = submission retry handlers = execution timeout handlers = submission timeout handlers = custom handlers = [[[mail]]] from = to = [[[workflow state polling]]] interval = max-polls = message = alt-cylc-run-dir = verbose mode = [[[environment]]] [[[directives]]] [[[outputs]]] [[[parameter environment templates]]] [[VAR]] script = echo "RUN: run-var.sh" completion = platform = inherit = init-script = env-script = err-script = exit-script = pre-script = post-script = work sub-directory = execution polling intervals = execution retry delays = execution time limit = submission polling intervals = submission retry delays = run mode = live [[[meta]]] title = description = URL = [[[skip]]] outputs = disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = time limit buffer = PT30S fail cycle points = fail try 1 only = True disable task event handlers = True [[[environment filter]]] include = exclude = [[[job]]] batch system = batch submit command template = [[[remote]]] host = owner = retrieve job logs = retrieve job logs max size = retrieve job logs retry delays = [[[events]]] handlers = handler events = handler retry delays = mail events = execution timeout = submission timeout = expired handlers = late offset = late handlers = submitted handlers = started handlers = succeeded handlers = failed handlers = submission failed handlers = warning handlers = critical handlers = retry handlers = submission retry handlers = execution timeout handlers = submission timeout handlers = custom handlers = [[[mail]]] from = to = [[[workflow state polling]]] interval = max-polls = message = alt-cylc-run-dir = verbose mode = [[[environment]]] [[[directives]]] [[[outputs]]] [[[parameter environment templates]]] [[SERIAL]] completion = platform = inherit = script = init-script = env-script = err-script = exit-script = pre-script = post-script = work sub-directory = execution polling intervals = execution retry delays = execution time limit = submission polling intervals = submission retry delays = run mode = live [[[directives]]] job_type = serial [[[meta]]] title = description = URL = [[[skip]]] outputs = disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = time limit buffer = PT30S fail cycle points = fail try 1 only = True disable task event handlers = True [[[environment filter]]] include = exclude = [[[job]]] batch system = batch submit command template = [[[remote]]] host = owner = retrieve job logs = retrieve job logs max size = retrieve job logs retry delays = [[[events]]] handlers = handler events = handler retry delays = mail events = execution timeout = submission timeout = expired handlers = late offset = late handlers = submitted handlers = started handlers = succeeded handlers = failed handlers = submission failed handlers = warning handlers = critical handlers = retry handlers = submission retry handlers = execution timeout handlers = submission timeout handlers = custom handlers = [[[mail]]] from = to = [[[workflow state polling]]] interval = max-polls = message = alt-cylc-run-dir = verbose mode = [[[environment]]] [[[outputs]]] [[[parameter environment templates]]] [[PARALLEL]] completion = platform = inherit = script = init-script = env-script = err-script = exit-script = pre-script = post-script = work sub-directory = execution polling intervals = execution retry delays = execution time limit = submission polling intervals = submission retry delays = run mode = live [[[directives]]] job_type = parallel [[[meta]]] title = description = URL = [[[skip]]] outputs = disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = time limit buffer = PT30S fail cycle points = fail try 1 only = True disable task event handlers = True [[[environment filter]]] include = exclude = [[[job]]] batch system = batch submit command template = [[[remote]]] host = owner = retrieve job logs = retrieve job logs max size = retrieve job logs retry delays = [[[events]]] handlers = handler events = handler retry delays = mail events = execution timeout = submission timeout = expired handlers = late offset = late handlers = submitted handlers = started handlers = succeeded handlers = failed handlers = submission failed handlers = warning handlers = critical handlers = retry handlers = submission retry handlers = execution timeout handlers = submission timeout handlers = custom handlers = [[[mail]]] from = to = [[[workflow state polling]]] interval = max-polls = message = alt-cylc-run-dir = verbose mode = [[[environment]]] [[[outputs]]] [[[parameter environment templates]]] [[ops_s1]] script = echo "RUN: run-ops.sh" inherit = OPS, SERIAL completion = succeeded or failed platform = init-script = env-script = err-script = exit-script = pre-script = post-script = work sub-directory = execution polling intervals = execution retry delays = execution time limit = submission polling intervals = submission retry delays = run mode = live [[[directives]]] job_type = serial [[[meta]]] title = description = URL = [[[skip]]] outputs = disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = time limit buffer = PT30S fail cycle points = fail try 1 only = True disable task event handlers = True [[[environment filter]]] include = exclude = [[[job]]] batch system = batch submit command template = [[[remote]]] host = owner = retrieve job logs = retrieve job logs max size = retrieve job logs retry delays = [[[events]]] handlers = handler events = handler retry delays = mail events = execution timeout = submission timeout = expired handlers = late offset = late handlers = submitted handlers = started handlers = succeeded handlers = failed handlers = submission failed handlers = warning handlers = critical handlers = retry handlers = submission retry handlers = execution timeout handlers = submission timeout handlers = custom handlers = [[[mail]]] from = to = [[[workflow state polling]]] interval = max-polls = message = alt-cylc-run-dir = verbose mode = [[[environment]]] [[[outputs]]] [[[parameter environment templates]]] [[ops_s2]] script = echo "RUN: run-ops.sh" inherit = OPS, SERIAL completion = succeeded or failed platform = init-script = env-script = err-script = exit-script = pre-script = post-script = work sub-directory = execution polling intervals = execution retry delays = execution time limit = submission polling intervals = submission retry delays = run mode = live [[[directives]]] job_type = serial [[[meta]]] title = description = URL = [[[skip]]] outputs = disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = time limit buffer = PT30S fail cycle points = fail try 1 only = True disable task event handlers = True [[[environment filter]]] include = exclude = [[[job]]] batch system = batch submit command template = [[[remote]]] host = owner = retrieve job logs = retrieve job logs max size = retrieve job logs retry delays = [[[events]]] handlers = handler events = handler retry delays = mail events = execution timeout = submission timeout = expired handlers = late offset = late handlers = submitted handlers = started handlers = succeeded handlers = failed handlers = submission failed handlers = warning handlers = critical handlers = retry handlers = submission retry handlers = execution timeout handlers = submission timeout handlers = custom handlers = [[[mail]]] from = to = [[[workflow state polling]]] interval = max-polls = message = alt-cylc-run-dir = verbose mode = [[[environment]]] [[[outputs]]] [[[parameter environment templates]]] [[ops_p1]] script = echo "RUN: run-ops.sh" inherit = OPS, PARALLEL completion = succeeded or failed platform = init-script = env-script = err-script = exit-script = pre-script = post-script = work sub-directory = execution polling intervals = execution retry delays = execution time limit = submission polling intervals = submission retry delays = run mode = live [[[directives]]] job_type = parallel [[[meta]]] title = description = URL = [[[skip]]] outputs = disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = time limit buffer = PT30S fail cycle points = fail try 1 only = True disable task event handlers = True [[[environment filter]]] include = exclude = [[[job]]] batch system = batch submit command template = [[[remote]]] host = owner = retrieve job logs = retrieve job logs max size = retrieve job logs retry delays = [[[events]]] handlers = handler events = handler retry delays = mail events = execution timeout = submission timeout = expired handlers = late offset = late handlers = submitted handlers = started handlers = succeeded handlers = failed handlers = submission failed handlers = warning handlers = critical handlers = retry handlers = submission retry handlers = execution timeout handlers = submission timeout handlers = custom handlers = [[[mail]]] from = to = [[[workflow state polling]]] interval = max-polls = message = alt-cylc-run-dir = verbose mode = [[[environment]]] [[[outputs]]] [[[parameter environment templates]]] [[ops_p2]] script = echo "RUN: run-ops.sh" inherit = OPS, PARALLEL completion = succeeded or failed platform = init-script = env-script = err-script = exit-script = pre-script = post-script = work sub-directory = execution polling intervals = execution retry delays = execution time limit = submission polling intervals = submission retry delays = run mode = live [[[directives]]] job_type = parallel [[[meta]]] title = description = URL = [[[skip]]] outputs = disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = time limit buffer = PT30S fail cycle points = fail try 1 only = True disable task event handlers = True [[[environment filter]]] include = exclude = [[[job]]] batch system = batch submit command template = [[[remote]]] host = owner = retrieve job logs = retrieve job logs max size = retrieve job logs retry delays = [[[events]]] handlers = handler events = handler retry delays = mail events = execution timeout = submission timeout = expired handlers = late offset = late handlers = submitted handlers = started handlers = succeeded handlers = failed handlers = submission failed handlers = warning handlers = critical handlers = retry handlers = submission retry handlers = execution timeout handlers = submission timeout handlers = custom handlers = [[[mail]]] from = to = [[[workflow state polling]]] interval = max-polls = message = alt-cylc-run-dir = verbose mode = [[[environment]]] [[[outputs]]] [[[parameter environment templates]]] [[var_s1]] script = echo "RUN: run-var.sh" inherit = VAR, SERIAL completion = succeeded platform = init-script = env-script = err-script = exit-script = pre-script = post-script = work sub-directory = execution polling intervals = execution retry delays = execution time limit = submission polling intervals = submission retry delays = run mode = live [[[directives]]] job_type = serial [[[meta]]] title = description = URL = [[[skip]]] outputs = disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = time limit buffer = PT30S fail cycle points = fail try 1 only = True disable task event handlers = True [[[environment filter]]] include = exclude = [[[job]]] batch system = batch submit command template = [[[remote]]] host = owner = retrieve job logs = retrieve job logs max size = retrieve job logs retry delays = [[[events]]] handlers = handler events = handler retry delays = mail events = execution timeout = submission timeout = expired handlers = late offset = late handlers = submitted handlers = started handlers = succeeded handlers = failed handlers = submission failed handlers = warning handlers = critical handlers = retry handlers = submission retry handlers = execution timeout handlers = submission timeout handlers = custom handlers = [[[mail]]] from = to = [[[workflow state polling]]] interval = max-polls = message = alt-cylc-run-dir = verbose mode = [[[environment]]] [[[outputs]]] [[[parameter environment templates]]] [[var_s2]] script = echo "RUN: run-var.sh" inherit = VAR, SERIAL completion = succeeded platform = init-script = env-script = err-script = exit-script = pre-script = post-script = work sub-directory = execution polling intervals = execution retry delays = execution time limit = submission polling intervals = submission retry delays = run mode = live [[[directives]]] job_type = serial [[[meta]]] title = description = URL = [[[skip]]] outputs = disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = time limit buffer = PT30S fail cycle points = fail try 1 only = True disable task event handlers = True [[[environment filter]]] include = exclude = [[[job]]] batch system = batch submit command template = [[[remote]]] host = owner = retrieve job logs = retrieve job logs max size = retrieve job logs retry delays = [[[events]]] handlers = handler events = handler retry delays = mail events = execution timeout = submission timeout = expired handlers = late offset = late handlers = submitted handlers = started handlers = succeeded handlers = failed handlers = submission failed handlers = warning handlers = critical handlers = retry handlers = submission retry handlers = execution timeout handlers = submission timeout handlers = custom handlers = [[[mail]]] from = to = [[[workflow state polling]]] interval = max-polls = message = alt-cylc-run-dir = verbose mode = [[[environment]]] [[[outputs]]] [[[parameter environment templates]]] [[var_p1]] script = echo "RUN: run-var.sh" inherit = VAR, PARALLEL completion = succeeded platform = init-script = env-script = err-script = exit-script = pre-script = post-script = work sub-directory = execution polling intervals = execution retry delays = execution time limit = submission polling intervals = submission retry delays = run mode = live [[[directives]]] job_type = parallel [[[meta]]] title = description = URL = [[[skip]]] outputs = disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = time limit buffer = PT30S fail cycle points = fail try 1 only = True disable task event handlers = True [[[environment filter]]] include = exclude = [[[job]]] batch system = batch submit command template = [[[remote]]] host = owner = retrieve job logs = retrieve job logs max size = retrieve job logs retry delays = [[[events]]] handlers = handler events = handler retry delays = mail events = execution timeout = submission timeout = expired handlers = late offset = late handlers = submitted handlers = started handlers = succeeded handlers = failed handlers = submission failed handlers = warning handlers = critical handlers = retry handlers = submission retry handlers = execution timeout handlers = submission timeout handlers = custom handlers = [[[mail]]] from = to = [[[workflow state polling]]] interval = max-polls = message = alt-cylc-run-dir = verbose mode = [[[environment]]] [[[outputs]]] [[[parameter environment templates]]] [[var_p2]] script = echo "RUN: run-var.sh" inherit = VAR, PARALLEL completion = succeeded platform = init-script = env-script = err-script = exit-script = pre-script = post-script = work sub-directory = execution polling intervals = execution retry delays = execution time limit = submission polling intervals = submission retry delays = run mode = live [[[directives]]] job_type = parallel [[[meta]]] title = description = URL = [[[skip]]] outputs = disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = time limit buffer = PT30S fail cycle points = fail try 1 only = True disable task event handlers = True [[[environment filter]]] include = exclude = [[[job]]] batch system = batch submit command template = [[[remote]]] host = owner = retrieve job logs = retrieve job logs max size = retrieve job logs retry delays = [[[events]]] handlers = handler events = handler retry delays = mail events = execution timeout = submission timeout = expired handlers = late offset = late handlers = submitted handlers = started handlers = succeeded handlers = failed handlers = submission failed handlers = warning handlers = critical handlers = retry handlers = submission retry handlers = execution timeout handlers = submission timeout handlers = custom handlers = [[[mail]]] from = to = [[[workflow state polling]]] interval = max-polls = message = alt-cylc-run-dir = verbose mode = [[[environment]]] [[[outputs]]] [[[parameter environment templates]]] cylc-flow-8.7.0/tests/functional/cylc-config/05-param-vars.t0000775000175000017500000000242515264420307024003 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test parameter vars are not defined with user env GitHub #2225 . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------ TEST_NAME="${TEST_NAME_BASE}-config" run_ok "${TEST_NAME}" \ cylc config -i "[runtime][foo_t1_right]environment" "${WORKFLOW_NAME}" cmp_ok "${TEST_NAME}.stdout" - <<__END__ PARAM1 = \$CYLC_TASK_PARAM_t PARAM2 = \$CYLC_TASK_PARAM_u __END__ purge cylc-flow-8.7.0/tests/functional/execution-time-limit/0000775000175000017500000000000015264420307023173 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/execution-time-limit/03-pbs.t0000775000175000017500000000271315264420307024372 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test execution time limit setting, PBS job JOB_RUNNER="${0##*\/??-}" export REQUIRE_PLATFORM="runner:${JOB_RUNNER%%.t}" . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" LOGD="$RUN_DIR/${WORKFLOW_NAME}/log/job/1/foo" grep_ok '#PBS -l walltime=70' "${LOGD}/01/job" purge exit cylc-flow-8.7.0/tests/functional/execution-time-limit/02-slurm/0000775000175000017500000000000015264420307024554 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/execution-time-limit/02-slurm/reference.log0000664000175000017500000000012015264420307027206 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/foo -triggered off [] cylc-flow-8.7.0/tests/functional/execution-time-limit/02-slurm/flow.cylc0000664000175000017500000000065615264420307026406 0ustar alastairalastair#!jinja2 [scheduler] [[events]] inactivity timeout = PT2S [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = """ if [[ "${CYLC_TASK_SUBMIT_NUMBER}" == '1' ]]; then sleep 300 fi """ platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[[job]]] execution time limit = PT5S execution retry delays = PT0S cylc-flow-8.7.0/tests/functional/execution-time-limit/03-pbs0000777000175000017500000000000015264420307025421 202-slurmustar alastairalastaircylc-flow-8.7.0/tests/functional/execution-time-limit/test_header0000777000175000017500000000000015264420307031510 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/execution-time-limit/02-slurm.t0000775000175000017500000000271715264420307024753 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test execution time limit setting, slurm job JOB_RUNNER="${0##*\/??-}" export REQUIRE_PLATFORM="runner:${JOB_RUNNER%%.t}" . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" LOGD="$RUN_DIR/${WORKFLOW_NAME}/log/job/1/foo" grep_ok '#SBATCH --time=0:05' "${LOGD}/01/job" purge exit cylc-flow-8.7.0/tests/functional/execution-time-limit/04-polling-intervals.t0000664000175000017500000000506315264420307027256 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test execution time limit works correctly with polling intervals . "$(dirname "$0")/test_header" set_test_number 6 init_workflow "${TEST_NAME_BASE}" << __FLOW__ [scheduling] [[graph]] R1 = nolimit & limit95M & limit1H & limit10M & limit70S [runtime] [[root]] script = "echo Hello" execution polling intervals = 3*PT30S, PT10M, PT1H [[nolimit]] [[limit95M]] execution time limit = PT95M [[limit1H]] execution time limit = PT1H [[limit10M]] execution time limit = PT10M [[limit70S]] execution time limit = PT70S __FLOW__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" cylc play --debug "${WORKFLOW_NAME}" poll_grep_workflow_log "INFO - DONE" # NOTE: execution timeout polling is delayed by PT1M to let things settle # PT10M = (3*PT3S + PT9M30S) - PT1M grep_workflow_log_ok grep-limit10M "\[1/limit10M/01:running\] health: execution timeout=None, polling intervals=3\*PT30S,PT9M30S,PT2M,PT7M,..." # PT60M = (3*PT3S + PT10M + PT49M30S) - PT1M grep_workflow_log_ok grep-limit1H "\[1/limit1H/01:running\] health: execution timeout=None, polling intervals=3\*PT30S,PT10M,PT49M30S,PT2M,PT7M,..." # PT70S = (2*PT30S + PT1M10S) - PT1M grep_workflow_log_ok grep-limit70S "\[1/limit70S/01:running\] health: execution timeout=None, polling intervals=2\*PT30S,PT1M10S,PT2M,PT7M,..." # PT95M = (3*PT3S + PT10M + PT1H + PT24M30S) - PT1M grep_workflow_log_ok grep-limit95M "\[1/limit95M/01:running\] health: execution timeout=None, polling intervals=3\*PT30S,PT10M,PT1H,PT24M30S,PT2M,PT7M,..." grep_workflow_log_ok grep-no-limit "\[1/nolimit/01:running\] health: execution timeout=None, polling intervals=3\*PT30S,PT10M,PT1H,..." purge cylc-flow-8.7.0/tests/functional/queues/0000775000175000017500000000000015264420307020427 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/queues/02-queueorder/0000775000175000017500000000000015264420307023026 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/queues/02-queueorder/reference.log0000664000175000017500000000114415264420307025467 0ustar alastairalastairInitial point: 1 Final point: 1 1/delay_n1 -triggered off [] 1/hold -triggered off [] 1/delay_n2 -triggered off ['1/delay_n1'] 1/delay_n3 -triggered off ['1/delay_n2'] 1/delay_n4 -triggered off ['1/delay_n3'] 1/delay_n5 -triggered off ['1/delay_n4'] 1/proc_n1 -triggered off ['1/delay_n1'] 1/delay_n6 -triggered off ['1/delay_n5'] 1/proc_n2 -triggered off ['1/delay_n2'] 1/delay_n7 -triggered off ['1/delay_n6'] 1/proc_n3 -triggered off ['1/delay_n3'] 1/proc_n4 -triggered off ['1/delay_n4'] 1/proc_n5 -triggered off ['1/delay_n5'] 1/proc_n6 -triggered off ['1/delay_n6'] 1/proc_n7 -triggered off ['1/delay_n7'] cylc-flow-8.7.0/tests/functional/queues/02-queueorder/flow.cylc0000664000175000017500000000052615264420307024654 0ustar alastairalastair[task parameters] n = 1..7 [scheduling] [[queues]] [[[q1]]] limit = 1 members = proc, hold [[graph]] R1 = """ delay => delay delay => proc hold """ [runtime] [[delay]] [[proc]] [[hold]] script = sleep 7 cylc-flow-8.7.0/tests/functional/queues/test_header0000777000175000017500000000000015264420307026744 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/queues/02-queueorder.t0000664000175000017500000000255415264420307023221 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test job script sets dependencies in environment. . "$(dirname "${0}")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}-run" \ cylc play "${WORKFLOW_NAME}" --reference-test --debug --no-detach --timestamp run_ok "${TEST_NAME_BASE}-test" bash -o pipefail -c " cylc cat-log '${WORKFLOW_NAME}' | grep 'proc_n.*submitted at' | sort --key=4,4 --check" purge exit cylc-flow-8.7.0/tests/functional/cylc-set/0000775000175000017500000000000015264420307020643 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-set/04-switch.t0000664000175000017500000000411315264420307022551 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # "cylc set" proposal examples: 5 - Set and complete a future switch task with the "--wait" flag # https://cylc.github.io/cylc-admin/proposal-cylc-set.html#5-set-switch-tasks-at-an-optional-branch-point-to-direct-the-future-flow . "$(dirname "$0")/test_header" set_test_number 5 install_and_validate reftest_run # The branch-point task foo should be recorded as succeeded. sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \ "SELECT status FROM task_states WHERE name is 'foo'" > db-foo.2 cmp_ok "db-foo.2" - << __OUT__ succeeded __OUT__ # the outputs of foo should be recorded as: # a, succeeded # and the implied outputs (of succeeded) as well: # submitted, started sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \ "SELECT outputs FROM task_outputs WHERE name is 'foo'" > db-foo.1 # Json string list of outputs from the db may not be ordered correctly. python3 - << __END__ > db-foo.2 import json with open("db-foo.1", 'r') as f: print( ','.join( sorted( json.load(f) ) ) ) __END__ cmp_ok "db-foo.2" - << __OUT__ a,started,submitted,succeeded __OUT__ # Check the flow-wait worked grep_workflow_log_ok check-wait "1/foo.* spawning outputs after flow-wait" -E purge cylc-flow-8.7.0/tests/functional/cylc-set/08-switch2.t0000664000175000017500000000220015264420307022632 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # "cylc set" proposal examples: 5 - Set and complete a future switch task. # https://cylc.github.io/cylc-admin/proposal-cylc-set.html#5-set-switch-tasks-at-an-optional-branch-point-to-direct-the-future-flow . "$(dirname "$0")/test_header" set_test_number 2 reftest purge cylc-flow-8.7.0/tests/functional/cylc-set/01-off-flow-pre/0000775000175000017500000000000015264420307023364 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-set/01-off-flow-pre/reference.log0000664000175000017500000000073315264420307026030 0ustar alastairalastair1/c_cold -triggered off [] in flow 1 1/a_cold -triggered off [] in flow 1 1/b_cold -triggered off [] in flow 1 1/a -triggered off ['1/a_cold'] in flow 1 1/b -triggered off ['1/a', '1/b_cold'] in flow 1 1/c -triggered off ['1/b', '1/c_cold'] in flow 1 1/reflow -triggered off ['1/c'] in flow 1 1/a -triggered off ['1/a_cold'] in flow 2 1/b -triggered off ['1/a', '1/b_cold'] in flow 2 1/c -triggered off ['1/b', '1/c_cold'] in flow 2 1/reflow -triggered off ['1/c'] in flow 2 cylc-flow-8.7.0/tests/functional/cylc-set/01-off-flow-pre/flow.cylc0000664000175000017500000000203715264420307025211 0ustar alastairalastair# start a new flow after setting off-flow prerequites to avoid stall. [scheduler] [[events]] stall timeout = PT0S abort on stall timeout = True inactivity timeout = PT30S abort on inactivity timeout = True [scheduling] [[graph]] R1 = """ # the tasks we want the flow to run a => b => c => reflow # the off-flow prerequisites a_cold => a b_cold => b c_cold => c """ [runtime] [[a, b, c]] [[a_cold, b_cold, c_cold]] [[reflow]] script = """ if (( CYLC_TASK_SUBMIT_NUMBER == 1 )); then # set off-flow prerequisites (and trigger 1/a) cylc set --flow=new \ --pre=1/a_cold:succeeded \ --pre=1/b_cold:succeeded \ --pre=1/c_cold:succeeded \ ${CYLC_WORKFLOW_ID}//1/a \ ${CYLC_WORKFLOW_ID}//1/b \ ${CYLC_WORKFLOW_ID}//1/c fi """ cylc-flow-8.7.0/tests/functional/cylc-set/02-off-flow-out.t0000664000175000017500000000404715264420307023600 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # "cylc set" proposal examples: 2 - Set off-flow outputs to prevent a new flow from stalling. # https://cylc.github.io/cylc-admin/proposal-cylc-set.html#2-set-off-flow-prerequisites-to-prep-for-a-new-flow . "$(dirname "$0")/test_header" set_test_number 11 install_and_validate reftest_run # Check that we set: # - all the required outputs of a_cold # - the requested and implied outputs of b_cold and c_cold grep_workflow_log_ok "${TEST_NAME_BASE}-grep-a1" '1/a_cold.* setting implied output: submitted' grep_workflow_log_ok "${TEST_NAME_BASE}-grep-a2" '1/a_cold.* setting implied output: started' grep_workflow_log_ok "${TEST_NAME_BASE}-grep-a3" '1/a_cold.* completed' grep_workflow_log_ok "${TEST_NAME_BASE}-grep-a1" '1/b_cold.* setting implied output: submitted' grep_workflow_log_ok "${TEST_NAME_BASE}-grep-a2" '1/b_cold.* setting implied output: started' grep_workflow_log_ok "${TEST_NAME_BASE}-grep-b3" '1/b_cold.* completed' grep_workflow_log_ok "${TEST_NAME_BASE}-grep-a1" '1/c_cold.* setting implied output: submitted' grep_workflow_log_ok "${TEST_NAME_BASE}-grep-a2" '1/c_cold.* setting implied output: started' grep_workflow_log_ok "${TEST_NAME_BASE}-grep-c3" '1/c_cold.* completed' purge cylc-flow-8.7.0/tests/functional/cylc-set/01-off-flow-pre.t0000664000175000017500000000327515264420307023560 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # # "cylc set" proposal examples: 2 - Set off-flow prerequisites to prevent a new flow from stalling. # https://cylc.github.io/cylc-admin/proposal-cylc-set.html#2-set-off-flow-prerequisites-to-prep-for-a-new-flow . "$(dirname "$0")/test_header" set_test_number 8 install_and_validate reftest_run grep_workflow_log_ok "${TEST_NAME_BASE}-ab" '1/a does not depend on "1/b_cold:succeeded"' grep_workflow_log_ok "${TEST_NAME_BASE}-ac" '1/a does not depend on "1/c_cold:succeeded"' grep_workflow_log_ok "${TEST_NAME_BASE}-ba" '1/b does not depend on "1/a_cold:succeeded"' grep_workflow_log_ok "${TEST_NAME_BASE}-bc" '1/b does not depend on "1/c_cold:succeeded"' grep_workflow_log_ok "${TEST_NAME_BASE}-ca" '1/c does not depend on "1/a_cold:succeeded"' grep_workflow_log_ok "${TEST_NAME_BASE}-cb" '1/c does not depend on "1/b_cold:succeeded"' purge cylc-flow-8.7.0/tests/functional/cylc-set/05-expire/0000775000175000017500000000000015264420307022361 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-set/05-expire/reference.log0000664000175000017500000000012315264420307025016 0ustar alastairalastair1/expirer -triggered off [] in flow 1 1/foo -triggered off ['1/expirer'] in flow 1 cylc-flow-8.7.0/tests/functional/cylc-set/05-expire/flow.cylc0000664000175000017500000000107315264420307024205 0ustar alastairalastair# Expire an inactive task, so it won't run. [scheduler] [[events]] inactivity timeout = PT1M abort on inactivity timeout = True stall timeout = PT0S abort on stall timeout = True [scheduling] [[graph]] R1 = """ # bar and baz should not run if bar expires expirer => foo => bar? => baz bar:expired? """ [runtime] [[expirer]] script = """ cylc set --output=expired ${CYLC_WORKFLOW_ID}//1/bar """ [[foo]] [[bar, baz]] script = false cylc-flow-8.7.0/tests/functional/cylc-set/06-parentless/0000775000175000017500000000000015264420307023246 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-set/06-parentless/reference.log0000664000175000017500000000030415264420307025704 0ustar alastairalastairStart task: ['1800/a'] 18000101T0000Z/a -triggered off [] in flow 1 18000101T0000Z/x -triggered off [] in flow 1 18000101T0000Z/b -triggered off ['18000101T0000Z/a', '18000101T0000Z/x'] in flow 1 cylc-flow-8.7.0/tests/functional/cylc-set/06-parentless/flow.cylc0000664000175000017500000000073615264420307025077 0ustar alastairalastair# Start this with --start-task=1800/a. # Task a's script should spawn x. # The log should show a clock-trigger check before x runs. [scheduler] [[events]] inactivity timeout = PT30S abort on inactivity timeout = True [scheduling] initial cycle point = 1800 [[graph]] R1 = """ a => b @wall_clock => x => b """ [runtime] [[a]] script = cylc set --pre=all "${CYLC_WORKFLOW_ID}//1800/x" [[b, x]] cylc-flow-8.7.0/tests/functional/cylc-set/04-switch/0000775000175000017500000000000015264420307022365 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-set/04-switch/reference.log0000664000175000017500000000011615264420307025024 0ustar alastairalastair1/switcher -triggered off [] in flow 1 1/a -triggered off ['1/foo'] in flow 1 cylc-flow-8.7.0/tests/functional/cylc-set/04-switch/flow.cylc0000664000175000017500000000146015264420307024211 0ustar alastairalastair# Set outputs of inactive task to direct the flow at an optional branch point. [scheduler] [[events]] inactivity timeout = PT1M abort on inactivity timeout = True stall timeout = PT0S abort on stall timeout = True [scheduling] [[graph]] R1 = """ switcher => foo foo:a? => a foo:b? => b """ [runtime] [[switcher]] script = """ cylc set --output=a,succeeded --wait ${CYLC_WORKFLOW_ID}//1/foo # wait for command actioned, to avoid race condition cylc__job__poll_grep_workflow_log "actioned" """ [[foo]] script = "cylc message b" # always go b-way if I run [[[outputs]]] a = a b = b [[a]] [[b]] script = false cylc-flow-8.7.0/tests/functional/cylc-set/05-expire.t0000664000175000017500000000270115264420307022546 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # "cylc set" proposal examples: 6 - check that forced task expiry works # https://cylc.github.io/cylc-admin/proposal-cylc-set.html#6-expire-a-task . "$(dirname "$0")/test_header" set_test_number 4 install_and_validate reftest_run sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \ "SELECT status FROM task_states WHERE name is 'bar'" > db-bar.1 cmp_ok "db-bar.1" - << __OUT__ expired __OUT__ sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \ "SELECT outputs FROM task_outputs WHERE name is 'bar'" > db-bar.2 cmp_ok "db-bar.2" - << __OUT__ {"expired": "(manually completed)"} __OUT__ purge cylc-flow-8.7.0/tests/functional/cylc-set/06-parentless.t0000664000175000017500000000236615264420307023442 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # "cylc set" proposal examples: 7 - Check spawning a parentless task without ignoring xtriggers. # https://cylc.github.io/cylc-admin/proposal-cylc-set.html#7-spawning-parentless-tasks . "$(dirname "$0")/test_header" set_test_number 3 install_and_validate REFTEST_OPTS="--start-task=1800/a" reftest_run grep_workflow_log_ok "${TEST_NAME_BASE}-clock" "xtrigger succeeded: wall_clock" purge cylc-flow-8.7.0/tests/functional/cylc-set/test_header0000777000175000017500000000000015264420307027160 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-set/02-off-flow-out/0000775000175000017500000000000015264420307023406 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-set/02-off-flow-out/reference.log0000664000175000017500000000073315264420307026052 0ustar alastairalastair1/c_cold -triggered off [] in flow 1 1/a_cold -triggered off [] in flow 1 1/b_cold -triggered off [] in flow 1 1/a -triggered off ['1/a_cold'] in flow 1 1/b -triggered off ['1/a', '1/b_cold'] in flow 1 1/c -triggered off ['1/b', '1/c_cold'] in flow 1 1/reflow -triggered off ['1/c'] in flow 1 1/a -triggered off ['1/a_cold'] in flow 2 1/b -triggered off ['1/a', '1/b_cold'] in flow 2 1/c -triggered off ['1/b', '1/c_cold'] in flow 2 1/reflow -triggered off ['1/c'] in flow 2 cylc-flow-8.7.0/tests/functional/cylc-set/02-off-flow-out/flow.cylc0000664000175000017500000000161415264420307025233 0ustar alastairalastair# start a new flow after setting off-flow outputs to avoid stall. [scheduler] [[events]] stall timeout = PT0S abort on stall timeout = True inactivity timeout = PT30S abort on inactivity timeout = True [scheduling] [[graph]] R1 = """ # the tasks we want the flow to run a => b => c => reflow # the off-flow prerequisites a_cold => a b_cold => b c_cold => c """ [runtime] [[a, b, c]] [[a_cold, b_cold, c_cold]] [[reflow]] script = """ if (( CYLC_TASK_SUBMIT_NUMBER == 1 )); then # set off-flow outputs of x_cold cylc set --flow=new \ ${CYLC_WORKFLOW_ID}//1/a_cold \ ${CYLC_WORKFLOW_ID}//1/b_cold \ ${CYLC_WORKFLOW_ID}//1/c_cold fi """ cylc-flow-8.7.0/tests/functional/cylc-set/03-set-failed/0000775000175000017500000000000015264420307023100 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-set/03-set-failed/flow.cylc0000664000175000017500000000060615264420307024725 0ustar alastairalastair# A single task that dies silently, requiring set to failed [scheduler] [[events]] inactivity timeout = PT1M abort on inactivity timeout = True [scheduling] [[graph]] R1 = "foo" [runtime] [[foo]] init-script = cylc__job__disable_fail_signals script = """ cylc__job__wait_cylc_message_started exit 1 """ cylc-flow-8.7.0/tests/functional/cylc-set/03-set-failed.t0000664000175000017500000000406715264420307023274 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # "cylc set" proposal examples: 4 -check that we can set a dead orphaned job to failed. # https://cylc.github.io/cylc-admin/proposal-cylc-set.html#4-set-jobs-to-failed-when-a-job-platform-is-known-to-be-down . "$(dirname "$0")/test_header" set_test_number 3 install_and_validate run_ok play-it cylc play --debug "${WORKFLOW_NAME}" poll_grep_workflow_log -E "1/foo.* \(internal\)submitted" cylc set -o failed "${WORKFLOW_NAME}//1/foo" # Check the log for: # - set completion message # - implied outputs reported as already completed poll_grep_workflow_log -E "1/foo.* => failed" poll_grep_workflow_log -E "1/foo.* did not complete the required outputs" cylc stop --now --now --interval=2 --max-polls=5 "${WORKFLOW_NAME}" # Check the DB records all the outputs. sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \ "SELECT outputs FROM task_outputs WHERE name is 'foo'" > db-foo.1 # Json string list of outputs from the db may not be ordered correctly. python3 - << __END__ > db-foo.2 import json with open("db-foo.1", 'r') as f: print( ','.join( sorted( json.load(f) ) ) ) __END__ cmp_ok "db-foo.2" - << __OUT__ failed,started,submitted __OUT__ purge cylc-flow-8.7.0/tests/functional/cylc-set/00-set-succeeded/0000775000175000017500000000000015264420307023575 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-set/00-set-succeeded/reference.log0000664000175000017500000000034515264420307026240 0ustar alastairalastair1/setter -triggered off ['1/bar', '1/foo'] in flow 1 1/foo -triggered off [] in flow 1 1/bar -triggered off [] in flow 1 1/post_m1 -triggered off ['1/bar', '1/foo'] in flow 1 1/post_m2 -triggered off ['1/bar', '1/foo'] in flow 1 cylc-flow-8.7.0/tests/functional/cylc-set/00-set-succeeded/flow.cylc0000664000175000017500000000211315264420307025415 0ustar alastairalastair# 1. foo and bar fail incomplete. # 2. setter sets foo and bar to succeeded. # 3. foo and bar are completed, post runs, scheduler shuts down. [scheduler] [[events]] inactivity timeout = PT30S abort on inactivity timeout = True expected task failures = 1/foo, 1/bar [task parameters] m = 1..2 [scheduling] [[graph]] R1 = """ foo & bar => post foo:started & bar:started => setter """ [runtime] [[post]] [[foo, bar]] script = false [[setter]] script = """ # wait for foo and bar to fail. for TASK in foo bar do cylc workflow-state \ ${CYLC_WORKFLOW_ID}//${CYLC_TASK_CYCLE_POINT}/${TASK}:failed \ --max-polls=20 --interval=1 done # set foo succeeded (via --output) cylc set -o succeeded $CYLC_WORKFLOW_ID//$CYLC_TASK_CYCLE_POINT/foo # set bar succeeded (via default) cylc set $CYLC_WORKFLOW_ID//$CYLC_TASK_CYCLE_POINT/bar """ cylc-flow-8.7.0/tests/functional/cylc-set/09-set-skip.t0000664000175000017500000000224215264420307023015 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # # Skip Mode proposal example: # https://github.com/cylc/cylc-admin/blob/master/docs/proposal-skip-mode.md # The cylc set --out option should accept the skip value # which should set the outputs defined in # [runtime][][skip]outputs. . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/cylc-set/08-switch2/0000775000175000017500000000000015264420307022453 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-set/08-switch2/reference.log0000664000175000017500000000130415264420307025112 0ustar alastairalastair# 1/a runs naturally and generates the output "x" 1/a -triggered off [] in flow 1 1/x -triggered off ['1/a'] in flow 1 1/z -triggered off ['1/x'] in flow 1 # 1/a is artificially completed with the output "y" 2/y -triggered off ['2/a'] in flow 1 2/z -triggered off ['2/y'] in flow 1 # 1/a has the output "y" is artificially set but is not completed # (so 1/a will re-run and generate the output "x" naturally) 3/a -triggered off [] in flow 1 3/x -triggered off ['3/a'] in flow 1 3/y -triggered off ['3/a'] in flow 1 3/z -triggered off ['3/y'] in flow 1 # 1/a runs naturally and generates the output "x" 4/a -triggered off [] in flow 1 4/x -triggered off ['4/a'] in flow 1 4/z -triggered off ['4/x'] in flow 1 cylc-flow-8.7.0/tests/functional/cylc-set/08-switch2/flow.cylc0000664000175000017500000000241415264420307024277 0ustar alastairalastair # Complete a parentless switch task that already exists in the pool but is # beyond the runahead limit. Cylc should auto-spawn its next instance to # avoid premature shutdown when it is removed as complete. # (We only spawn the first runahead-limited instance of parentless tasks). [scheduler] allow implicit tasks = True [scheduling] initial cycle point = 1 final cycle point = 4 cycling mode = integer runahead limit = P0 [[graph]] P1 = """ a:x? => x a:y? => y x | y => z """ [runtime] [[a]] script = """ cylc__job__wait_cylc_message_started cylc message -- x # always go x-path """ [[[outputs]]] x = x y = y [[z]] script = """ if (( CYLC_TASK_CYCLE_POINT == 1 )); then # mark 2/a as succeeded with output y # (task will be skipped) cylc set "${CYLC_WORKFLOW_ID}//2/a" --out=y,succeeded elif (( CYLC_TASK_CYCLE_POINT == 2 )); then # mark 2/a as having generated output y # (task will re-run and generate output x in the prociess) cylc set "${CYLC_WORKFLOW_ID}//3/a" --out=y fi """ cylc-flow-8.7.0/tests/functional/cylc-set/09-set-skip/0000775000175000017500000000000015264420307022630 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-set/09-set-skip/reference.log0000664000175000017500000000035215264420307025271 0ustar alastairalastair1/bar -triggered off [] in flow 1 1/foo -triggered off [] in flow 1 1/do_skip -triggered off ['1/bar', '1/foo'] in flow 1 1/foo_done -triggered off ['1/foo', '1/foo', '1/foo'] in flow 1 1/bar_failed -triggered off ['1/bar'] in flow 1 cylc-flow-8.7.0/tests/functional/cylc-set/09-set-skip/flow.cylc0000664000175000017500000000253115264420307024454 0ustar alastairalastair[meta] test_description = """ Test that cylc set --out skip satisfies all outputs which are required by the graph. """ proposal url = https://github.com/cylc/cylc-admin/blob/master/docs/proposal-skip-mode.md [scheduler] allow implicit tasks = true [[events]] expected task failures = 1/bar [scheduling] [[graph]] R1 = """ # Optional out not created by set --out skip foo:no? => not_this_task # set --out skip creates required, started, submitted # and succeeded (unless failed is set): foo:yes => foo_done foo:submitted => foo_done foo:succeeded => foo_done foo:started => do_skip # set --out skip creates failed if that is required # by skip mode settings: bar:started => do_skip bar:failed? => bar_failed """ [runtime] [[foo]] script = sleep 100 [[[skip]]] outputs = yes [[[outputs]]] no = Don't require this task yes = Require this task [[bar]] script = sleep 100 [[[skip]]] outputs = failed [[do_skip]] script = """ cylc set --out skip ${CYLC_WORKFLOW_ID}//1/foo \ ${CYLC_WORKFLOW_ID}//1/bar """ cylc-flow-8.7.0/tests/functional/cylc-set/00-set-succeeded.t0000664000175000017500000000312315264420307023761 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # "cylc set" proposal examples: 1 - Carry on as if a failed task had succeeded # https://cylc.github.io/cylc-admin/proposal-cylc-set.html#1-carry-on-as-if-a-failed-task-had-succeeded . "$(dirname "$0")/test_header" set_test_number 6 install_and_validate reftest_run for TASK in foo bar do sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \ "SELECT status FROM task_states WHERE name is '$TASK'" > "${TASK}.1" cmp_ok ${TASK}.1 - << "${TASK}.2" cmp_json \ "check-${TASK}-outputs" \ "${TASK}.2" \ "${TASK}.2"<<<'["submitted", "started", "succeeded", "hello"]' done purge cylc-flow-8.7.0/tests/functional/periodicals/0000775000175000017500000000000015264420307021416 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/periodicals/03-monthly.t0000664000175000017500000000174115264420307023520 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test Monthly cycling . "$(dirname "$0")/test_header" set_test_number 2 reftest "${TEST_NAME_BASE}" 'Monthly' exit cylc-flow-8.7.0/tests/functional/periodicals/README0000664000175000017500000000125315264420307022277 0ustar alastairalastairStepped Daily, Monthly, and Yearly cycling tests: Cycler(,) Each test workflow defines a single task on a 3-unit cycle: Daily(20100110,3): 20100110, 20100113, 20100116, ... Monthly(201001,3): 201001, 201004, 201007, ... Yearly(2010,3) : 2010, 2013, 2016, ... The reference log is for a test run starting one unit in from the anchor cycle time. Cylc should get exactly the same result for test runs that start one, two, or three units in because the initial cycle point gets adjusted up to the next on-sequence cycle time, so additional tests are done to confirm this - by modifying the test start time in the installed reference log on the fly. cylc-flow-8.7.0/tests/functional/periodicals/Monthly/0000775000175000017500000000000015264420307023050 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/periodicals/Monthly/reference.log0000664000175000017500000000020015264420307025501 0ustar alastairalastairInitial point: 20100201T0000Z Final point: 20100801T0000Z 20100401T0000Z/a -triggered off [] 20100701T0000Z/a -triggered off [] cylc-flow-8.7.0/tests/functional/periodicals/Monthly/flow.cylc0000664000175000017500000000033015264420307024667 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] initial cycle point = 2010-02 final cycle point = 2010-08 runahead limit = P0 [[graph]] 2010-01/P3M = "a" [runtime] [[a]] script = true cylc-flow-8.7.0/tests/functional/periodicals/05-monthly.t0000664000175000017500000000302415264420307023516 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test Monthly cycling . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 1 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" Monthly #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" perl -pi -e 's/(Initial point: ).*$/${1}2010-04/' \ "${TEST_DIR}/${WORKFLOW_NAME}/reference.log" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/periodicals/06-yearly.t0000664000175000017500000000173715264420307023343 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test Yearly cycling . "$(dirname "$0")/test_header" set_test_number 2 reftest "${TEST_NAME_BASE}" 'Yearly' exit cylc-flow-8.7.0/tests/functional/periodicals/Monthly-reorder/0000775000175000017500000000000015264420307024510 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/periodicals/Monthly-reorder/reference.log0000664000175000017500000000101115264420307027142 0ustar alastairalastairInitial point: 20130130T0000Z Final point: 20130202T0000Z 20130130T0000Z/dummy -triggered off ['20130129T0000Z/daily'] 20130130T0000Z/daily -triggered off [] 20130131T0000Z/dummy -triggered off ['20130130T0000Z/daily'] 20130131T0000Z/daily -triggered off [] 20130201T0000Z/dummy -triggered off ['20130131T0000Z/daily'] 20130201T0000Z/daily -triggered off [] 20130201T0000Z/monthly -triggered off ['20130201T0000Z/dummy'] 20130202T0000Z/dummy -triggered off ['20130201T0000Z/daily'] 20130202T0000Z/daily -triggered off [] cylc-flow-8.7.0/tests/functional/periodicals/Monthly-reorder/flow.cylc0000664000175000017500000000070615264420307026336 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20130130T00 final cycle point = 20130202T00 runahead limit = P0 [[graph]] # (this triggers a monthly task off the last daily task each month) T00 = """ daily daily[-PT24H] => dummy """ # marker1 R/01T/P1M = "dummy => monthly" # marker2 [runtime] [[root]] script = true cylc-flow-8.7.0/tests/functional/periodicals/Daily/0000775000175000017500000000000015264420307022460 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/periodicals/Daily/reference.log0000664000175000017500000000024315264420307025120 0ustar alastairalastairInitial point: 20140107T0000Z Final point: 20140118T0000Z 20140110T0600Z/a -triggered off [] 20140113T0600Z/a -triggered off [] 20140116T0600Z/a -triggered off [] cylc-flow-8.7.0/tests/functional/periodicals/Daily/flow.cylc0000664000175000017500000000044615264420307024307 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] initial cycle point = 20140107 final cycle point = 20140118 runahead limit = P0 [[graph]] # daily cycling *with a non-zero hour* is a more stringent test 20140110T06/P3D = "a" [runtime] [[a]] script = true cylc-flow-8.7.0/tests/functional/periodicals/01-daily.t0000664000175000017500000000301615264420307023123 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test Daily cycling . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 1 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" Daily #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" perl -pi -e 's/(Initial point: ).*$/${1}20140106T06/' "${TEST_DIR}/${WORKFLOW_NAME}/reference.log" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/periodicals/08-yearly.t0000664000175000017500000000301715264420307023336 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test Yearly cycling . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 1 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" Yearly #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" perl -pi -e 's/(Initial point: ).*$/${1}2013/' \ "${TEST_DIR}/${WORKFLOW_NAME}/reference.log" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/periodicals/09-monthly-reorder.t0000664000175000017500000000404215264420307025163 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test Monthly cycling . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" Monthly-reorder #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- delete_db TEST_NAME="${TEST_NAME_BASE}-run" # Swapping the Monthly and Hours-of-the-day cycling sections # should make no difference. perl -pi -e 'undef $/; s/( *\[\[\[00.*marker1)\n( *\[\[\[Monthly.*marker2)/${2}\n${1}/smg' "${TEST_DIR}/${WORKFLOW_NAME}/flow.cylc" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/periodicals/00-daily.t0000664000175000017500000000173515264420307023130 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test Daily cycling . "$(dirname "$0")/test_header" set_test_number 2 reftest "${TEST_NAME_BASE}" 'Daily' exit cylc-flow-8.7.0/tests/functional/periodicals/02-daily.t0000664000175000017500000000302415264420307023123 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test Daily cycling . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 1 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" Daily #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" perl -pi -e 's/(Initial point: ).*$/${1}20140105T06/' \ "${TEST_DIR}/${WORKFLOW_NAME}/reference.log" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/periodicals/test_header0000777000175000017500000000000015264420307027733 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/periodicals/04-monthly.t0000664000175000017500000000302415264420307023515 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test Monthly cycling . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 1 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" Monthly #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" perl -pi -e 's/(Initial point: ).*$/${1}2010-03/' \ "${TEST_DIR}/${WORKFLOW_NAME}/reference.log" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/periodicals/07-yearly.t0000664000175000017500000000301715264420307023335 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test Yearly cycling . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 1 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" Yearly #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" perl -pi -e 's/(Initial point: ).*$/${1}2012/' \ "${TEST_DIR}/${WORKFLOW_NAME}/reference.log" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/periodicals/Yearly/0000775000175000017500000000000015264420307022663 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/periodicals/Yearly/reference.log0000664000175000017500000000020015264420307025314 0ustar alastairalastairInitial point: 20110101T0000Z Final point: 20180101T0000Z 20130101T0000Z/a -triggered off [] 20160101T0000Z/a -triggered off [] cylc-flow-8.7.0/tests/functional/periodicals/Yearly/flow.cylc0000664000175000017500000000031715264420307024507 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] initial cycle point = 2011 final cycle point = 2018 runahead limit = P0 [[graph]] 2010/P3Y = "a" [runtime] [[a]] script = true cylc-flow-8.7.0/tests/functional/message-triggers/0000775000175000017500000000000015264420307022370 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/message-triggers/00-basic.t0000664000175000017500000000175315264420307024061 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test new-style simplified message triggers (see GitHub #1761) . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/message-triggers/02-action/0000775000175000017500000000000015264420307024064 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/message-triggers/02-action/flow.cylc0000664000175000017500000000071215264420307025707 0ustar alastairalastair[scheduler] [[events]] abort on inactivity timeout = True inactivity timeout = PT30S [scheduling] [[graph]] R1 = """ foo:a => bar foo:b & bar => baz""" [runtime] [[foo]] script = """ cylc message "the quick brown fox" cylc message "jumped over the lazy dog" """ [[[outputs]]] a = "the quick brown fox" b = "jumped over the lazy dog" [[bar]] [[baz]] cylc-flow-8.7.0/tests/functional/message-triggers/02-action.t0000664000175000017500000000331715264420307024255 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that message triggers (custom outputs) are actioned immediately even if # nothing else is happening at the time - GitHub #2548. . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" # The workflow tests that two tasks suicide immediately on message triggers. TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --no-detach --abort-if-any-task-fails "${WORKFLOW_NAME}" # Check that final task pool indicates bar and baz ran # TODO: some final null task pool tests would be better on task_states table! TEST_NAME=${TEST_NAME_BASE}-cmp-task-pool sqlite3 "${WORKFLOW_RUN_DIR}/log/db" 'select cycle, name, status from task_pool;' > task-pool.log cmp_ok task-pool.log - <'/dev/null' purge cylc-flow-8.7.0/tests/functional/message-triggers/00-basic/0000775000175000017500000000000015264420307023666 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/message-triggers/00-basic/reference.log0000664000175000017500000000077715264420307026342 0ustar alastairalastairInitial point: 20140801T0000Z Final point: 20141201T0000Z 20140801T0000Z/baz -triggered off ['20140601T0000Z/foo'] 20140801T0000Z/foo -triggered off [] 20141001T0000Z/foo -triggered off [] 20140801T0000Z/bar -triggered off ['20140801T0000Z/foo'] 20141201T0000Z/foo -triggered off [] 20141001T0000Z/baz -triggered off ['20140801T0000Z/foo'] 20141001T0000Z/bar -triggered off ['20141001T0000Z/foo'] 20141201T0000Z/baz -triggered off ['20141001T0000Z/foo'] 20141201T0000Z/bar -triggered off ['20141201T0000Z/foo'] cylc-flow-8.7.0/tests/functional/message-triggers/00-basic/flow.cylc0000664000175000017500000000117715264420307025517 0ustar alastairalastair[meta] title = "test workflow for cylc-6 message triggers" [scheduler] UTC mode = True [scheduling] initial cycle point = 20140801T00 final cycle point = 20141201T00 [[graph]] P2M = """ foo:out1 => bar foo[-P2M]:out2 => baz """ [runtime] [[foo]] script = """ cylc__job__wait_cylc_message_started cylc message -- "${CYLC_WORKFLOW_ID} "${CYLC_TASK_JOB} "file 1 done" cylc message -- "${CYLC_WORKFLOW_ID} "${CYLC_TASK_JOB} "file 2 done" """ [[[outputs]]] out1 = "file 1 done" out2 = "file 2 done" [[bar, baz]] script = true cylc-flow-8.7.0/tests/functional/message-triggers/test_header0000777000175000017500000000000015264420307030705 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/empy/0000775000175000017500000000000015264420307020072 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/empy/test_header0000777000175000017500000000000015264420307026407 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/empy/00-simple/0000775000175000017500000000000015264420307021600 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/empy/00-simple/flow.cylc-expanded0000664000175000017500000000105415264420307025211 0ustar alastairalastair[scheduling] [[graph]] R1 = "a => FAM" [runtime] [[a]] script = echo this is a set variable [[FAM]] [[[environment]]] TITLE="member" [[member_0]] inherit = FAM script = echo I am $TITLE 0 [[member_1]] inherit = FAM script = echo I am $TITLE 1 [[member_2]] inherit = FAM script = echo I am $TITLE 2 [[member_3]] inherit = FAM script = echo I am $TITLE 3 [[member_4]] inherit = FAM script = echo I am $TITLE 4 cylc-flow-8.7.0/tests/functional/cylc-poll/0000775000175000017500000000000015264420307021016 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/05-poll-multi-messages.t0000775000175000017500000000171115264420307025333 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test poll multiple messages . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/cylc-poll/13-comm-method/0000775000175000017500000000000015264420307023450 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/13-comm-method/reference.log0000664000175000017500000000011615264420307026107 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] 1/t2 -triggered off [] cylc-flow-8.7.0/tests/functional/cylc-poll/13-comm-method/flow.cylc0000664000175000017500000000100415264420307025266 0ustar alastairalastair#!Jinja2 [scheduling] [[graph]] R1 = t1 & t2 [runtime] [[root]] script = """ wait # sleep for twice the polling interval to make sure # the started message gets picked up before the succeeded # message is issued sleep 12 """ platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[t1]] [[t2]] [[[job]]] submission polling intervals = 10*PT6S execution polling intervals = 10*PT6S cylc-flow-8.7.0/tests/functional/cylc-poll/01-task-failed.t0000775000175000017500000000175315264420307023616 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that polling a failed task sets the task state correctly . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/cylc-poll/08-slurm/0000775000175000017500000000000015264420307022405 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/08-slurm/reference.log0000664000175000017500000000012115264420307025040 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off ['1/a'] cylc-flow-8.7.0/tests/functional/cylc-poll/08-slurm/flow.cylc0000664000175000017500000000034115264420307024226 0ustar alastairalastair#!Jinja2 [scheduling] [[graph]] R1 = a:start => b [runtime] [[a]] script = sleep 20 platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[b]] script = cylc poll "$CYLC_WORKFLOW_ID//*/a" cylc-flow-8.7.0/tests/functional/cylc-poll/00-basic.t0000775000175000017500000000172615264420307022512 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test cat-check against workflow database . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/cylc-poll/01-task-failed/0000775000175000017500000000000015264420307023420 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/01-task-failed/reference.log0000664000175000017500000000016215264420307026060 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off ['1/a'] 1/handled -triggered off ['1/a'] cylc-flow-8.7.0/tests/functional/cylc-poll/01-task-failed/flow.cylc0000664000175000017500000000150315264420307025242 0ustar alastairalastair[meta] title = "Test workflow for task state change on poll result." description = """ Task A fails silently - it will be stuck in 'running' unless polled. Task B then polls A to find it has failed, allowing A to suicide via a :fail trigger, and the workflow to shut down successfully. """ [scheduler] [[events]] expected task failures = 1/a [scheduling] [[graph]] R1 = """ a:start => b a:fail => handled """ [runtime] [[a]] init-script = cylc__job__disable_fail_signals ERR EXIT script = """ cylc__job__wait_cylc_message_started exit 1 """ [[b]] script = cylc poll "$CYLC_WORKFLOW_ID//*/a" [[handled]] # (allows a:fail to be removed as handled) script = true cylc-flow-8.7.0/tests/functional/cylc-poll/02-task-submit-failed/0000775000175000017500000000000015264420307024722 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/02-task-submit-failed/reference.log0000664000175000017500000000026315264420307027364 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/kill_foo_submit -triggered off ['1/foo'] 1/poll_foo -triggered off ['1/kill_foo_submit'] 1/stop -triggered off ['1/foo'] cylc-flow-8.7.0/tests/functional/cylc-poll/02-task-submit-failed/flow.cylc0000664000175000017500000000135615264420307026552 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] expected task failures = 1/foo [scheduling] [[graph]] R1 = """ foo:submit? => kill_foo_submit => poll_foo foo:submit-fail? => stop """ [runtime] [[foo]] platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[poll_foo]] script = """ cylc__job__wait_cylc_message_started cylc poll "$CYLC_WORKFLOW_ID//*/foo" """ [[stop]] script = cylc stop $CYLC_WORKFLOW_ID [[kill_foo_submit]] script = """ cylc__job__wait_cylc_message_started ID=$(sed -n "s/^CYLC_JOB_ID=//p" \ $CYLC_WORKFLOW_RUN_DIR/log/job/1/foo/01/job.status) atrm $ID """ cylc-flow-8.7.0/tests/functional/cylc-poll/10-basic-compat.t0000775000175000017500000000172615264420307023774 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test cat-check against workflow database . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/cylc-poll/17-pbs-cant-connect/0000775000175000017500000000000015264420307024401 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/17-pbs-cant-connect/lib/0000775000175000017500000000000015264420307025147 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/17-pbs-cant-connect/lib/python/0000775000175000017500000000000015264420307026470 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/17-pbs-cant-connect/lib/python/my_pbs.py0000664000175000017500000000223615264420307030336 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import os from cylc.flow.job_runner_handlers.pbs import PBSHandler class MyPBSHandler(PBSHandler): """For testing poll command connection refused.""" @staticmethod def get_poll_many_cmd(_): """Always print PBSHandler.POLL_CANT_CONNECT_ERR to STDERR.""" return os.path.join(os.path.dirname(__file__), 'badqstat') JOB_RUNNER_HANDLER = MyPBSHandler() cylc-flow-8.7.0/tests/functional/cylc-poll/17-pbs-cant-connect/lib/python/badqstat0000775000175000017500000000007115264420307030217 0ustar alastairalastair#!/usr/bin/env bash echo 'Connection refused' >&2 exit 1 cylc-flow-8.7.0/tests/functional/cylc-poll/17-pbs-cant-connect/reference.log0000664000175000017500000000006715264420307027045 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/cylc-poll/17-pbs-cant-connect/flow.cylc0000664000175000017500000000040515264420307026223 0ustar alastairalastair#!Jinja2 [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = sleep 60 platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[[job]]] execution time limit = PT2M execution polling intervals = PT20S cylc-flow-8.7.0/tests/functional/cylc-poll/09-lsf/0000775000175000017500000000000015264420307022030 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/09-lsf/reference.log0000664000175000017500000000012115264420307024463 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off ['1/a'] cylc-flow-8.7.0/tests/functional/cylc-poll/09-lsf/flow.cylc0000664000175000017500000000034115264420307023651 0ustar alastairalastair#!Jinja2 [scheduling] [[graph]] R1 = a:start => b [runtime] [[a]] script = sleep 20 platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[b]] script = cylc poll "$CYLC_WORKFLOW_ID//*/a" cylc-flow-8.7.0/tests/functional/cylc-poll/08-slurm.t0000777000175000017500000000000015264420307025473 206-loadleveler.tustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/11-event-time.t0000775000175000017500000000302715264420307023504 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test delayed poll gives the correct event time . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" RUND="$RUN_DIR/${WORKFLOW_NAME}" sed -n 's/CYLC_JOB_EXIT_TIME=//p' "${RUND}/log/job/1/w1/NN/job.status" >'st-time.txt' sqlite3 "${RUND}/log/db" " SELECT time_run_exit FROM task_jobs WHERE cycle=='1' AND name=='w1' AND submit_num=='1'" >'db-time.txt' run_ok "${TEST_NAME_BASE}-time-run-exit" diff -u 'st-time.txt' 'db-time.txt' purge exit cylc-flow-8.7.0/tests/functional/cylc-poll/00-basic/0000775000175000017500000000000015264420307022314 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/00-basic/reference.log0000664000175000017500000000012115264420307024747 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off ['1/a'] cylc-flow-8.7.0/tests/functional/cylc-poll/00-basic/flow.cylc0000664000175000017500000000041215264420307024134 0ustar alastairalastair[scheduling] [[graph]] R1 = a:start => b [runtime] [[a]] script = """ while ! grep -qF 'CYLC_JOB_EXIT' "${CYLC_WORKFLOW_RUN_DIR}/log/job/1/b/NN/job.status" do sleep 1 done """ [[b]] script = cylc poll "$CYLC_WORKFLOW_ID//1/a" cylc-flow-8.7.0/tests/functional/cylc-poll/10-basic-compat/0000775000175000017500000000000015264420307023576 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/10-basic-compat/reference.log0000664000175000017500000000012115264420307026231 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off ['1/a'] cylc-flow-8.7.0/tests/functional/cylc-poll/10-basic-compat/flow.cylc0000664000175000017500000000035615264420307025425 0ustar alastairalastair[scheduling] [[graph]] R1 = a:start => b [runtime] [[a]] script = sleep 20 [[b]] script = """ cylc__job__wait_cylc_message_started cylc poll "$CYLC_WORKFLOW_ID//1/a" """ cylc-flow-8.7.0/tests/functional/cylc-poll/04-poll-multi-hosts.t0000775000175000017500000000341515264420307024666 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test poll multiple jobs on localhost and a remote host export REQUIRE_PLATFORM='loc:remote comms:tcp' . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" RUN_DIR="$RUN_DIR/${WORKFLOW_NAME}" LOG="${RUN_DIR}/log/scheduler/log" sed -n 's/^.*\(cylc jobs-poll\)/\1/p' "${LOG}" | sort -u >'edited-workflow-log' sort >'edited-workflow-log-ref' <<__LOG__ cylc jobs-poll --debug -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 1/remote-fail-1/01 1/remote-success-1/01 1/remote-success-2/01 cylc jobs-poll --debug -- '\$HOME/cylc-run/${WORKFLOW_NAME}/log/job' 1/local-fail-1/01 1/local-fail-2/01 1/local-success-1/01 __LOG__ cmp_ok 'edited-workflow-log' 'edited-workflow-log-ref' purge exit cylc-flow-8.7.0/tests/functional/cylc-poll/05-poll-multi-messages/0000775000175000017500000000000015264420307025143 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/05-poll-multi-messages/reference.log0000664000175000017500000000032415264420307027603 0ustar alastairalastairInitial point: 1 Final point: 1 1/speaker1 -triggered off [] 1/speaker2 -triggered off [] 1/poller -triggered off ['1/speaker1', '1/speaker2'] 1/finisher -triggered off ['1/speaker1', '1/speaker1', '1/speaker2'] cylc-flow-8.7.0/tests/functional/cylc-poll/05-poll-multi-messages/flow.cylc0000664000175000017500000000316615264420307026774 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [scheduling] [[graph]] R1 = """ speaker1:start & speaker2:start => poller speaker1:hello1 & speaker1:hello2 & speaker2:greet => finisher """ [runtime] [[speaker1]] script = """ # Wait for "cylc message started" command cylc__job__wait_cylc_message_started # Simulate "cylc message", messages written to status file but failed to # get sent back to the workflow { echo "CYLC_MESSAGE=$(date +%FT%H:%M:%SZ)|INFO|hello1" echo "CYLC_MESSAGE=$(date +%FT%H:%M:%SZ)|INFO|hello2" } >>"${CYLC_TASK_LOG_ROOT}.status" cylc__job__poll_grep_workflow_log -E '1/speaker1/01:running.* \(polled\)hello1' cylc__job__poll_grep_workflow_log -E '1/speaker1/01:running.* \(polled\)hello2' """ [[[outputs]]] hello1 = "hello1" hello2 = "hello2" [[speaker2]] script=""" # Wait for "cylc message started" command cylc__job__wait_cylc_message_started # Simulate "cylc message", messages written to status file but failed to # get sent back to the workflow echo "CYLC_MESSAGE=$(date +%FT%H:%M:%SZ)|INFO|greet" \ >>"${CYLC_TASK_LOG_ROOT}.status" cylc__job__poll_grep_workflow_log -E '1/speaker2/01:running.* \(polled\)greet' """ [[[outputs]]] greet = "greet" [[finisher]] script=true [[poller]] script=cylc poll "${CYLC_WORKFLOW_ID}//*/speaker[12]" cylc-flow-8.7.0/tests/functional/cylc-poll/06-loadleveler.t0000775000175000017500000000230615264420307023730 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc poll" for loadleveler, slurm, or pbs jobs. # TODO Check this test on a dockerized system or VM. JOB_RUNNER="${0##*\/??-}" export REQUIRE_PLATFORM="runner:${JOB_RUNNER%%.t} comms:tcp" . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 reftest purge exit cylc-flow-8.7.0/tests/functional/cylc-poll/02-task-submit-failed.t0000775000175000017500000000224715264420307025117 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that polling a submit-failed task sets the task state correctly export REQUIRE_PLATFORM='runner:at comms:tcp' . "$(dirname "$0")/test_header" set_test_number 2 create_test_global_config "" " [platforms] [[$CYLC_TEST_PLATFORM]] job runner command template = at noon tomorrow " reftest purge exit cylc-flow-8.7.0/tests/functional/cylc-poll/07-pbs.t0000777000175000017500000000000015264420307025114 206-loadleveler.tustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/17-pbs-cant-connect.t0000775000175000017500000000415415264420307024575 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test poll PBS connection refused export REQUIRE_PLATFORM="runner:pbs" . "$(dirname "$0")/test_header" set_test_number 4 create_test_global_config "" " [platforms] [[${CYLC_TEST_PLATFORM}]] job runner = my_pbs hosts = ${CYLC_TEST_BATCH_TASK_HOST} " install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" if [[ "${CYLC_TEST_HOST}" != 'localhost' ]]; then # shellcheck disable=SC2029 ssh -n "${CYLC_TEST_HOST}" "mkdir -p 'cylc-run/${WORKFLOW_NAME}/'" rsync -a 'lib' "${CYLC_TEST_HOST}:cylc-run/${WORKFLOW_NAME}/" fi run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" # ssh security warnings may appear between outputs => check separately too. sed -n 's/^.*\(\[jobs-poll err\]\) \(Connection refused\).*$/\1\n\2/p; s/^.*\(\[jobs-poll err\]\).*$/\1/p; s/^.*\(Connection refused\).*$/\1/p; s/^.*\(INFO - \[1/t1\] status=running: (polled)started\).*$/\1/p' \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" >'sed-log.out' contains_ok 'sed-log.out' <<'__LOG__' [jobs-poll err] Connection refused __LOG__ contains_ok 'sed-log.out' <<'__LOG__' INFO - [1/t1] status=running: (polled)started __LOG__ purge exit cylc-flow-8.7.0/tests/functional/cylc-poll/test_header0000777000175000017500000000000015264420307027333 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/13-comm-method.t0000775000175000017500000000377015264420307023647 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test poll intervals is used from both global.cylc and flow.cylc . "$(dirname "${0}")/test_header" #------------------------------------------------------------------------------- set_test_number 6 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" create_test_global_config " [platforms] [[$CYLC_TEST_PLATFORM]] communication method = poll execution polling intervals = 10*PT6S submission polling intervals = 10*PT6S " workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- LOG_FILE="${WORKFLOW_RUN_DIR}/log/scheduler/log" PRE_MSG='health:' POST_MSG='.*, polling intervals=10\*PT6S...' for INDEX in 1 2; do for STAGE in 'submission' 'execution'; do grep_ok "1/t${INDEX}.* ${PRE_MSG} ${STAGE}${POST_MSG}" "${LOG_FILE}" -E done done #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/cylc-poll/07-pbs/0000775000175000017500000000000015264420307022026 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/07-pbs/reference.log0000664000175000017500000000012115264420307024461 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off ['1/a'] cylc-flow-8.7.0/tests/functional/cylc-poll/07-pbs/flow.cylc0000664000175000017500000000034115264420307023647 0ustar alastairalastair#!Jinja2 [scheduling] [[graph]] R1 = a:start => b [runtime] [[a]] script = sleep 20 platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[b]] script = cylc poll "$CYLC_WORKFLOW_ID//*/a" cylc-flow-8.7.0/tests/functional/cylc-poll/15-job-st-file-no-batch.t0000775000175000017500000000274015264420307025240 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test polling with "job.status" missing job runner information. . "$(dirname "${0}")/test_header" set_test_number 4 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" LOG="${WORKFLOW_RUN_DIR}/log/scheduler/log" run_ok "${TEST_NAME_BASE}-log-1" \ grep -F '[jobs-poll err] 1/t1/01/job.status: incomplete job runner info' "${LOG}" run_ok "${TEST_NAME_BASE}-log-2" \ grep -E '1/t1/01:running.*\(polled\)failed' "${LOG}" purge exit cylc-flow-8.7.0/tests/functional/cylc-poll/09-lsf.t0000777000175000017500000000000015264420307025116 206-loadleveler.tustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/14-intervals.t0000775000175000017500000000411215264420307023435 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test the correct intervals are used . "$(dirname "${0}")/test_header" #------------------------------------------------------------------------------- set_test_number 6 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" create_test_global_config ' [platforms] [[localhost]] submission polling intervals = PT2S,6*PT10S execution polling intervals = 2*PT1S,10*PT6S' workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- LOG_FILE="${WORKFLOW_RUN_DIR}/log/scheduler/log" PRE_MSG='health:' for INDEX in 1 2; do for STAGE in 'submission' 'execution'; do POLL_INT='PT2S,6\*PT10S,' if [[ "${STAGE}" == 'execution' ]]; then POLL_INT='2\*PT1S,10\*PT6S,' fi POST_MSG=".*, polling intervals=${POLL_INT}..." grep_ok "1/t${INDEX}.*${PRE_MSG} ${STAGE}${POST_MSG}" "${LOG_FILE}" -E done done #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/cylc-poll/04-poll-multi-hosts/0000775000175000017500000000000015264420307024473 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/04-poll-multi-hosts/reference.log0000664000175000017500000000060315264420307027133 0ustar alastairalastairInitial point: 1 Final point: 1 1/remote-success-1 -triggered off [] 1/remote-fail-1 -triggered off [] 1/local-fail-2 -triggered off [] 1/remote-success-2 -triggered off [] 1/local-success-1 -triggered off [] 1/local-fail-1 -triggered off [] 1/poller -triggered off ['1/local-fail-1', '1/local-fail-2', '1/local-success-1', '1/remote-fail-1', '1/remote-success-1', '1/remote-success-2'] cylc-flow-8.7.0/tests/functional/cylc-poll/04-poll-multi-hosts/flow.cylc0000664000175000017500000000411215264420307026314 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [[events]] expected task failures = 1/local-fail-1, 1/local-fail-2, 1/remote-fail-1 [scheduling] [[graph]] R1 = """ POLLABLE:start-all => poller POLLABLE:succeed-any # (make member success optional) """ [runtime] # turn off error trapping to disable "cylc message" calls # (this prevents the "succeeded" / "failed" messages getting through) [[POLLABLE]] init-script = cylc__job__disable_fail_signals ERR EXIT # these tasks will fail (but not send the failed message or write it to the # job.status file) [[FAIL]] inherit = POLLABLE script = """ cylc__job__wait_cylc_message_started echo 'I am failing...' >&2 exit 1 """ [[local-fail-1, local-fail-2]] inherit = FAIL [[remote-fail-1]] inherit = FAIL platform = {{ environ['CYLC_TEST_PLATFORM'] }} # these tasks will succeed (but not send the succeeded message or write it # to the job.status file) [[SUCCESS]] inherit = POLLABLE script = """ cylc__job__wait_cylc_message_started echo 'I am OK.' { echo 'CYLC_JOB_EXIT=SUCCEEDED' echo "CYLC_JOB_EXIT_TIME=$(date +%FT%H:%M:%SZ)" } >>"${CYLC_TASK_LOG_ROOT}.status" exit 1 """ [[local-success-1]] inherit = SUCCESS [[remote-success-1, remote-success-2]] inherit = SUCCESS platform = {{ environ['CYLC_TEST_PLATFORM'] }} # this task will poll the active jobs and hopefully detect their # success / failure [[poller]] script = """ # give a bit of grace time for the job to leave the job runner queue sleep 5 # poll the job (it should change to a final state) cylc poll "${CYLC_WORKFLOW_ID}//*/POLLABLE" # shut down the scheduler (there should not be any active tasks # left to delay shutdown) cylc stop "${CYLC_WORKFLOW_ID}" """ cylc-flow-8.7.0/tests/functional/cylc-poll/15-job-st-file-no-batch/0000775000175000017500000000000015264420307025045 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/15-job-st-file-no-batch/reference.log0000664000175000017500000000006715264420307027511 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/cylc-poll/15-job-st-file-no-batch/flow.cylc0000664000175000017500000000111515264420307026666 0ustar alastairalastair[scheduler] UTC mode = True [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] init-script = cylc__job__disable_fail_signals ERR EXIT script = """ cylc__job__wait_cylc_message_started ST_FNAME="${CYLC_TASK_LOG_ROOT}.status" sed -i '/\(CYLC_JOB_RUNNER_NAME\|CYLC_JOB_ID\)/d' "${ST_FNAME}" #echo 'CYLC_JOB_EXIT=SUCCEEDED' >>"${ST_FNAME}" #echo "CYLC_JOB_EXIT_TIME=$(date -u +%FT%H:%M:%SZ)" >>"${ST_FNAME}" exit 1 """ [[[events]]] execution timeout = PT15S cylc-flow-8.7.0/tests/functional/cylc-poll/11-event-time/0000775000175000017500000000000015264420307023312 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/11-event-time/reference.log0000664000175000017500000000012415264420307025750 0ustar alastairalastairInitial point: 1 Final point: 1 1/w1 -triggered off [] 1/w2 -triggered off ['1/w1'] cylc-flow-8.7.0/tests/functional/cylc-poll/11-event-time/flow.cylc0000664000175000017500000000116415264420307025137 0ustar alastairalastair[scheduling] [[graph]] R1="w1:started => w2" [runtime] [[w1]] init-script=cylc__job__disable_fail_signals ERR EXIT script=""" cylc__job__wait_cylc_message_started # Append to job.status cat >>"${CYLC_TASK_LOG_ROOT}.status" <<__STATUS__ CYLC_JOB_EXIT=SUCCEEDED CYLC_JOB_EXIT_TIME=$(date -u +'%FT%H:%M:%SZ') __STATUS__ # Exit without trap exit 1 """ [[w2]] script = """ cylc__job__wait_cylc_message_started cylc poll "${CYLC_WORKFLOW_ID}//1/w1" """ cylc-flow-8.7.0/tests/functional/cylc-poll/06-loadleveler/0000775000175000017500000000000015264420307023537 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/06-loadleveler/reference.log0000664000175000017500000000012115264420307026172 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off ['1/a'] cylc-flow-8.7.0/tests/functional/cylc-poll/06-loadleveler/flow.cylc0000664000175000017500000000066415264420307025370 0ustar alastairalastair#!Jinja2 [scheduling] [[graph]] R1 = a:start => b [runtime] [[a]] script = sleep 20 platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[[directives]]] class=serial job_type=serial notification=never resources=ConsumableCpus(1) ConsumableMemory(64mb) wall_clock_limit=180,120 [[b]] script = cylc poll "$CYLC_WORKFLOW_ID//*/a" cylc-flow-8.7.0/tests/functional/cylc-poll/14-intervals/0000775000175000017500000000000015264420307023247 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-poll/14-intervals/reference.log0000664000175000017500000000011615264420307025706 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] 1/t2 -triggered off [] cylc-flow-8.7.0/tests/functional/cylc-poll/14-intervals/flow.cylc0000664000175000017500000000051415264420307025072 0ustar alastairalastair[scheduling] [[graph]] R1 = t1 & t2 [runtime] [[t1]] env-script = """ # Simulate disappearing after job submission export CYLC_TASK_COMMS_METHOD=poll sleep 10 """ script = true [[t2]] script = """ # Simulate disappearing after job running export CYLC_TASK_COMMS_METHOD=poll sleep 10 """ cylc-flow-8.7.0/tests/functional/validate/0000775000175000017500000000000015264420307020711 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/validate/23-fail-old-syntax-7/0000775000175000017500000000000015264420307024312 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/validate/23-fail-old-syntax-7/flow.cylc0000664000175000017500000000042215264420307026133 0ustar alastairalastair[scheduler] [[events]] timeout = 4320 [scheduling] initial cycle time = 2014030300 final cycle time = 2014030306 [[graph]] 0,6,12,18 = A [runtime] [[A]] script = "sleep 10" execution retry delays = 2*PT30M, PT60M cylc-flow-8.7.0/tests/functional/validate/43-jinja2-template-error-main.t0000775000175000017500000000272615264420307026373 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validation for a filter Jinja2 error with no line number. . "$(dirname "$0")/test_header" set_test_number 2 cat >'flow.cylc' <<'__FLOW_CONFIG__' #!jinja2 {% set foo = {} %} [scheduling] [[graph]] R1 = {{ foo|dictsort(by='by') }} [runtime] [[foo]] script = sleep 1 __FLOW_CONFIG__ run_fail "${TEST_NAME_BASE}" cylc validate . cmp_ok_re "${TEST_NAME_BASE}.stderr" <<'__ERROR__' Jinja2Error: You can only sort by either "key" or "value" File.* {% set foo = {} %} \[scheduling\] \[\[graph\]\] R1 = {{ foo\|dictsort\(by='by'\) }} <-- FilterArgumentError __ERROR__ exit cylc-flow-8.7.0/tests/functional/validate/71-task-proxy-sequence-bounds-err.t0000775000175000017500000000351515264420307027337 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test for handling task proxy sequence bounds error. #2735 . "$(dirname "$0")/test_header" set_test_number 4 cat > flow.cylc <<__END__ [scheduler] UTC mode = True [scheduling] initial cycle point = 2000 [[graph]] R1//1999 = t1 [runtime] [[t1]] script = true __END__ TEST_NAME="${TEST_NAME_BASE}-single" run_ok "$TEST_NAME" cylc validate . cmp_ok "${TEST_NAME}.stderr" <<'__ERR__' WARNING - R1/P0Y/19990101T0000Z: sequence out of bounds for initial cycle point 20000101T0000Z __ERR__ cat > flow.cylc <<__END__ [scheduler] UTC mode = True [scheduling] initial cycle point = 2000 [[graph]] R1//1996, R1//1997, R1//1998, R1//1999 = t1 [runtime] [[t1]] script = true __END__ TEST_NAME="${TEST_NAME_BASE}-multiple" run_ok "$TEST_NAME" cylc validate . contains_ok "${TEST_NAME}.stderr" <<__ERR__ WARNING - multiple sequences out of bounds for initial cycle point 20000101T0000Z: ${LOG_INDENT}R1/P0Y/19960101T0000Z, R1/P0Y/19970101T0000Z, R1/P0Y/19980101T0000Z, ${LOG_INDENT}R1/P0Y/19990101T0000Z __ERR__ exit cylc-flow-8.7.0/tests/functional/validate/67-relative-icp.t0000775000175000017500000000254515264420307023725 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Checking that syntax of relative initial cycle point is validated. # Note: remember to update this test after 01/01/2117 . "$(dirname "$0")/test_header" set_test_number 3 cat >'flow.cylc' <<'__FLOW_CONFIG__' [scheduler] UTC mode = true allow implicit tasks = True [scheduling] initial cycle point = previous(-17T1200Z; -18T1200Z) - P1D [[graph]] P1D = t1 __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-val" cylc validate . TEST_NAME="${TEST_NAME_BASE}-graph" run_ok "$TEST_NAME" cylc graph --reference . grep_ok "20171231T1200Z/t1" "${TEST_NAME}.stdout" exit cylc-flow-8.7.0/tests/functional/validate/23-fail-old-syntax-7.t0000775000175000017500000000304015264420307024477 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validation with a prev-style cycle syntax and post-style retry syntax . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}" run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" grep_ok 'IllegalItemError: \[scheduling\]initial cycle time.*' "${TEST_NAME}.stderr" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/validate/08-whitespace.t0000775000175000017500000000303315264420307023461 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validation with a lot of whitespace added - trailing, around # section headings, around list item delimiters, in include-files, and # added by jinja2. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 1 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE} run_ok "${TEST_NAME}" cylc validate -v "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/validate/61-include-missing-quote.t0000775000175000017500000000220515264420307025551 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Error message for missing quote in include statement . "$(dirname "$0")/test_header" set_test_number 2 cat >'flow.cylc' <<'__FLOW_CONFIG__' %include 'foo.cylc __FLOW_CONFIG__ run_fail "${TEST_NAME_BASE}" cylc validate . cmp_ok "${TEST_NAME_BASE}.stderr" <<__ERR__ FileParseError: mismatched quotes (in $PWD/flow.cylc): %include 'foo.cylc __ERR__ exit cylc-flow-8.7.0/tests/functional/validate/75-CYLC_TEMPLATE_VARS.t0000664000175000017500000000241115264420307024205 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test CYLC_TEMPLATE_VARS exported. . "$(dirname "$0")/test_header" set_test_number 2 cat > 'flow.cylc' <<__HEREDOC__ #!jinja2 [scheduling] initial cycle point = 2020 [[graph]] R1 = foo [runtime] [[foo]] __HEREDOC__ run_ok "${TEST_NAME_BASE}-validate" cylc validate . --debug grep_ok "CYLC_TEMPLATE_VARS={'CYLC_VERSION': '.*', 'CYLC_TEMPLATE_VARS': {...}" \ "${TEST_NAME_BASE}-validate.stderr" cylc-flow-8.7.0/tests/functional/validate/37-special-implicit-task.t0000664000175000017500000000270515264420307025521 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validation fails for special implicit tasks . "$(dirname "$0")/test_header" set_test_number 2 cat >'flow.cylc' <<'__FLOW_CONFIG__' [scheduling] initial cycle point = 20200101 [[special tasks]] clock-trigger = foo(PT0M) [[graph]] T00 = bar [runtime] [[bar]] script = true __FLOW_CONFIG__ run_fail "${TEST_NAME_BASE}" cylc validate "${PWD}" cmp_ok "${TEST_NAME_BASE}.stderr" << '__ERR__' WorkflowConfigError: implicit tasks detected (no entry under [runtime]): * foo To allow implicit tasks, use 'flow.cylc[scheduler]allow implicit tasks' __ERR__ exit cylc-flow-8.7.0/tests/functional/validate/08-whitespace/0000775000175000017500000000000015264420307023272 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/validate/08-whitespace/flow.cylc0000664000175000017500000000056515264420307025123 0ustar alastairalastair#!jinja2 {% set HELLO=False %} [ scheduling ] initial cycle point = 20140101T00 [[ special tasks ]] sequential = foo , bar , baz [[ graph ]] T00, T12 =""" a => b foo => bar & baz """ [ runtime ] [[baz ]] [[ foo,bar , a, b ]] %include "inc.cylc" cylc-flow-8.7.0/tests/functional/validate/08-whitespace/inc.cylc0000664000175000017500000000004715264420307024720 0ustar alastairalastair [ runtime ] # nothing ... cylc-flow-8.7.0/tests/functional/validate/64-circular.t0000775000175000017500000000741315264420307023141 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validation of a workflow with self-edges fails. . "$(dirname "$0")/test_header" set_test_number 13 cat >'flow.cylc' <<'__FLOW_CONFIG__' [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = a => a __FLOW_CONFIG__ run_fail "${TEST_NAME_BASE}-simple-1" cylc validate . contains_ok "${TEST_NAME_BASE}-simple-1.stderr" <<'__ERR__' WorkflowConfigError: self-edge detected: a:succeeded => a __ERR__ cat >'flow.cylc' <<'__FLOW_CONFIG__' [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = a => b => c => d => a => z __FLOW_CONFIG__ run_fail "${TEST_NAME_BASE}-simple-2" cylc validate . contains_ok "${TEST_NAME_BASE}-simple-2.stderr" <<'__ERR__' WorkflowConfigError: circular edges detected: 1/d => 1/a 1/a => 1/b 1/b => 1/c 1/c => 1/d __ERR__ cat >'flow.cylc' <<'__FLOW_CONFIG__' [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = FAM:succeed-all => f & g => z [runtime] [[FAM]] [[f,g,h]] inherit = FAM __FLOW_CONFIG__ run_fail "${TEST_NAME_BASE}-simple-fam" cylc validate . contains_ok "${TEST_NAME_BASE}-simple-fam.stderr" <<'__ERR__' WorkflowConfigError: self-edge detected: f:succeeded => f __ERR__ cat >'flow.cylc' <<'__FLOW_CONFIG__' [scheduler] allow implicit tasks = True cycle point format = %Y [scheduling] initial cycle point = 2001 final cycle point = 2010 [[graph]] P1Y = ''' a[-P1Y] => a a[+P1Y] => a ''' __FLOW_CONFIG__ run_fail "${TEST_NAME_BASE}-intercycle-1" cylc validate . contains_ok "${TEST_NAME_BASE}-intercycle-1.stderr" <<'__ERR__' WorkflowConfigError: circular edges detected: 2002/a => 2001/a 2001/a => 2002/a 2003/a => 2002/a 2002/a => 2003/a __ERR__ cat >'flow.cylc' <<'__FLOW_CONFIG__' [scheduler] allow implicit tasks = True [scheduling] cycling mode = integer initial cycle point = 1 [[graph]] 2/P3 = foo => bar => baz 8/P1 = baz => foo __FLOW_CONFIG__ run_fail "${TEST_NAME_BASE}-intercycle-2" cylc validate . contains_ok "${TEST_NAME_BASE}-intercycle-2.stderr" <<'__ERR__' WorkflowConfigError: circular edges detected: 8/foo => 8/bar 8/bar => 8/baz 8/baz => 8/foo __ERR__ cat >'flow.cylc' <<'__FLOW_CONFIG__' [scheduler] allow implicit tasks = True [task parameters] foo = 1..5 [scheduling] [[graph]] R1 = """ fool => fool fool => fool """ __FLOW_CONFIG__ run_fail "${TEST_NAME_BASE}-param-1" cylc validate . contains_ok "${TEST_NAME_BASE}-param-1.stderr" <<'__ERR__' WorkflowConfigError: circular edges detected: 1/fool_foo2 => 1/fool_foo1 1/fool_foo1 => 1/fool_foo2 __ERR__ cat >'flow.cylc' <<'__FLOW_CONFIG__' [scheduler] allow implicit tasks = True [scheduling] cycling mode = integer initial cycle point = 1 [[graph]] 1/P3 = foo => bar 2/P3 = bar => foo __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-param-2" cylc validate . exit cylc-flow-8.7.0/tests/functional/validate/03-incomplete-quotes/0000775000175000017500000000000015264420307024606 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/validate/03-incomplete-quotes/flow.cylc0000664000175000017500000000013215264420307026425 0ustar alastairalastair[scheduling] [[graph]] R1 = "foo" [runtime] [[foo]] script = "foo cylc-flow-8.7.0/tests/functional/validate/74-templatevar-types.t0000664000175000017500000000306415264420307025017 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validating xtrigger names in workflow. . "$(dirname "$0")/test_header" set_test_number 5 TEST_NAME="${TEST_NAME_BASE}-val" # test a valid xtrigger cat >'flow.cylc' <<'__FLOW_CONFIG__' #!Jinja2 [scheduler] allow implicit tasks = True [scheduling] initial cycle point = {{ ICP - 1 }} cycling mode = integer [[graph]] R1 = foo __FLOW_CONFIG__ run_fail "${TEST_NAME}-valid" cylc validate . run_fail "${TEST_NAME}-valid" cylc validate . -s 'ICP="2000"' run_ok "${TEST_NAME}-valid" cylc validate . -s 'ICP=2000' cat >'template' <<'__TEMPLATE__' ICP=2000 __TEMPLATE__ run_fail "${TEST_NAME}-valid" cylc validate . run_ok "${TEST_NAME}-valid" cylc validate . --set-file=template exit cylc-flow-8.7.0/tests/functional/validate/07-null-parentage.t0000775000175000017500000000313515264420307024245 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validation of workflow for tasks with inherit = None . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- init_workflow "${TEST_NAME_BASE}" << __FLOW__ [scheduling] [[graph]] R1 = "foo" [runtime] [[foo]] inherit = None __FLOW__ #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-val" run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" grep_ok 'WorkflowConfigError: null parentage for foo' "${TEST_NAME}.stderr" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/validate/06-implicit-missing.t0000775000175000017500000000304315264420307024605 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validation of workflow for tasks without runtime entries # (accidental implicit tasks) . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 1 #------------------------------------------------------------------------------- init_workflow "${TEST_NAME_BASE}" << __FLOW__ [scheduling] [[graph]] R1 = "foo => bar" [runtime] [[foo]] __FLOW__ #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-val" run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/validate/45-jinja2-type-error.t0000775000175000017500000000342715264420307024620 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validation for some StandardError while doing Jinja2 processing. . "$(dirname "$0")/test_header" set_test_number 4 TEST_NAME="${TEST_NAME_BASE}-type-error" cat >'flow.cylc' <<'__FLOW_CONFIG__' #!jinja2 [scheduling] [[graph]] R1 = foo {{ 1 / 'foo' }} __FLOW_CONFIG__ run_fail "${TEST_NAME}" cylc validate . cmp_ok_re "${TEST_NAME}.stderr" <<'__ERROR__' Jinja2Error: unsupported operand type\(s\) .* 'int' and 'str' File.* \[scheduling\] \[\[graph\]\] R1 = foo {{ 1 / 'foo' }} <-- TypeError __ERROR__ TEST_NAME="${TEST_NAME}-value-error" cat >'flow.cylc' <<'__FLOW_CONFIG__' #!Jinja2 {% set foo = [1, 2] %} {% set a, b, c = foo %} __FLOW_CONFIG__ run_fail "${TEST_NAME}" cylc validate . cmp_ok_re "${TEST_NAME}.stderr" <<'__ERROR__' Jinja2Error: not enough values to unpack \(expected 3, got 2\) File.* #!Jinja2 {% set foo = \[1, 2\] %} {% set a, b, c = foo %} <-- ValueError __ERROR__ exit cylc-flow-8.7.0/tests/functional/validate/03-incomplete-quotes.t0000775000175000017500000000264515264420307025005 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validating: script = "foo fails . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 1 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-val" run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/validate/40-jinja2-template-syntax-error-main/0000775000175000017500000000000015264420307027515 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/validate/40-jinja2-template-syntax-error-main/flow.cylc0000664000175000017500000000027215264420307031341 0ustar alastairalastair#!jinja2 [scheduler] allow implicit tasks = True [scheduling] [[graph]] {% if true %} R1 = foo {% end if % [runtime] [[foo]] script = sleep 1 cylc-flow-8.7.0/tests/functional/validate/40-jinja2-template-syntax-error-main.t0000775000175000017500000000326415264420307027712 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validation for a bad Jinja2 TemplateSyntaxError workflow. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-val" run_fail "${TEST_NAME}" cylc validate . cmp_ok_re "${TEST_NAME}.stderr" <<'__ERROR__' Jinja2Error: Encountered unknown tag 'end'. Jinja was looking for the following tags: 'elif' or 'else' or 'endif'. The innermost block that needs to be closed is 'if'. File.* \[\[graph\]\] {% if true %} R1 = foo {% end if % <-- TemplateSyntaxError __ERROR__ #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/validate/66-fail-consec-spaces.t0000775000175000017500000000260115264420307024770 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test consecutive spaces in a __MANY__ name fails validation (GitHub #2417). . "$(dirname "$0")/test_header" set_test_number 2 TEST_NAME="${TEST_NAME_BASE}-val" cat > flow.cylc <<__END__ [scheduling] [[graph]] R1 = task1 [runtime] [[HPC]] [[[directives]]] -l select=1:ncpus=1:mem=5GB [[task1]] inherit = HPC [[[directives]]] -l select=1:ncpus=24:mem=20GB # ERROR! __END__ run_fail "${TEST_NAME}" cylc validate . cmp_ok "${TEST_NAME}.stderr" <<__END__ IllegalItemError: [runtime][task1][directives]-l select - (consecutive spaces) __END__ cylc-flow-8.7.0/tests/functional/validate/54-self-suicide.t0000775000175000017500000000404315264420307023704 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validation, graph for workflow with self-suiciding task . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- init_workflow "${TEST_NAME_BASE}" << __FLOW__ [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = """ foo? => bar => baz foo:fail? => qux => baz foo:fail? & baz => !foo """ __FLOW__ #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-val" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-graph-check run_ok "${TEST_NAME}" cylc graph --reference "${WORKFLOW_NAME}" cmp_ok "${TEST_NAME}.stdout" <<'__OUT__' edge "1/bar" "1/baz" edge "1/foo" "1/bar" edge "1/foo" "1/qux" edge "1/qux" "1/baz" graph node "1/bar" "bar\n1" node "1/baz" "baz\n1" node "1/foo" "foo\n1" node "1/qux" "qux\n1" stop __OUT__ #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/validate/test_header0000777000175000017500000000000015264420307027226 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/validate/05-implicit-typo.t0000775000175000017500000000306015264420307024125 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validation of workflow with case mismatches in task names # (accidental implicit tasks) . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 1 #------------------------------------------------------------------------------- init_workflow "${TEST_NAME_BASE}" << __FLOW__ [scheduling] [[graph]] R1 = "Foo => bar" [runtime] [[foo]] [[bar]] __FLOW__ #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-val" run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/validate/49-jinja2-undefined-error/0000775000175000017500000000000015264420307025406 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/validate/49-jinja2-undefined-error/flow.cylc0000664000175000017500000000020415264420307027225 0ustar alastairalastair#!jinja2 [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = foo [[[{{ UNDEFINED_WHATEVER }}]]] cylc-flow-8.7.0/tests/functional/validate/68-trailing_whitespace.t0000664000175000017500000000340215264420307025355 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validation of a workflow with self-edges fails. . "$(dirname "$0")/test_header" set_test_number 3 # Test example with trailing whitespace cat >'flow.cylc' <<'__FLOW_CONFIG__' [scheduler] allow implicit tasks = True [scheduling] initial cycle point = 20000101T06 final cycle point = 20010101T18 [[graph]] T00 = """ # NOTE: don't let editor strip trailing space on next line foo | bar \ => baz & qux pub """ T12 = """ qux baz """ __FLOW_CONFIG__ run_fail "${TEST_NAME_BASE}-simple-fail" cylc validate . cmp_ok "${TEST_NAME_BASE}-simple-fail.stderr" <<'__ERR__' FileParseError: Syntax error line 9: Whitespace after the line continuation character (\). __ERR__ # Test example with correct syntax sed -i 's/\\ /\\/' 'flow.cylc' run_ok "${TEST_NAME_BASE}-simple-pass" cylc validate . exit cylc-flow-8.7.0/tests/functional/validate/09-include-missing.t0000775000175000017500000000255515264420307024430 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validation missing include-file. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 echo '%include foo.cylc' >flow.cylc echo '%include bar.cylc' >foo.cylc run_fail "${TEST_NAME_BASE}" cylc validate . cmp_ok "${TEST_NAME_BASE}.stderr" <<__ERR__ IncludeFileNotFoundError: bar.cylc via foo.cylc from $PWD/flow.cylc __ERR__ #------------------------------------------------------------------------------- rm flow.cylc rm foo.cylc cylc-flow-8.7.0/tests/functional/validate/49-jinja2-undefined-error.t0000775000175000017500000000311015264420307025571 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validation for a Jinja2 type error, with no line number info. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-val" run_fail "${TEST_NAME}" cylc validate . cmp_ok_re "${TEST_NAME}.stderr" <<'__ERROR__' Jinja2Error: 'UNDEFINED_WHATEVER' is undefined File.* \[scheduling\] \[\[graph\]\] R1 = foo \[\[\[{{ UNDEFINED_WHATEVER }}\]\]\] <-- UndefinedError __ERROR__ #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/remote/0000775000175000017500000000000015264420307020413 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/remote/basic/0000775000175000017500000000000015264420307021474 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/remote/basic/reference.log0000664000175000017500000000007015264420307024132 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] cylc-flow-8.7.0/tests/functional/remote/basic/flow.cylc0000664000175000017500000000032715264420307023321 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] stall timeout = PT0S [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = hostname -f platform = {{ environ['CYLC_TEST_PLATFORM'] }} cylc-flow-8.7.0/tests/functional/remote/08-symlink-dir-target-exist.t0000664000175000017500000000443715264420307025715 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # ----------------------------------------------------------------------------- # Test remote init fails if symlink dir target already exists export REQUIRE_PLATFORM='loc:remote fs:indep comms:tcp' . "$(dirname "$0")/test_header" set_test_number 6 mapfile -d ' ' -t SSH_CMD < <(cylc config -d -i "[platforms][${CYLC_TEST_PLATFORM}]ssh command") TEST_NAME="$(basename "$0")" create_test_global_config "" " [install] [[symlink dirs]] [[[${CYLC_TEST_INSTALL_TARGET}]]] run = \$HOME/cylctb-symlinks/$TEST_NAME/ " install_workflow "${TEST_NAME_BASE}" basic # shellcheck disable=SC2016 run_ok "${TEST_NAME_BASE}-mkdir" \ "${SSH_CMD[@]}" "$CYLC_TEST_HOST" 'mkdir -p $HOME/cylc-symlink-test' run_ok "${TEST_NAME_BASE}-val" cylc validate "$WORKFLOW_NAME" # Run once to setup symlink dirs on remote install target workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --no-detach "$WORKFLOW_NAME" # Remove remote run dir symlink (but not its target) "${SSH_CMD[@]}" "$CYLC_TEST_HOST" "rm -rf ~/cylc-run/${WORKFLOW_NAME}" # New run should abort delete_db TEST_NAME="${TEST_NAME_BASE}-run-again" workflow_run_fail "$TEST_NAME" cylc play --no-detach "$WORKFLOW_NAME" grep_ok "ERROR - platform: .* initialisation did not complete" "${TEST_NAME}.stderr" grep_ok "WorkflowFilesError: Symlink dir target already exists" "${TEST_NAME}.stderr" # Clean up remote symlink dir target # shellcheck disable=SC2016 "${SSH_CMD[@]}" "$CYLC_TEST_HOST" 'rm -rf "${TMPDIR}/${USER}/sym-run" "${HOME}/cylctb-symlinks/$TEST_NAME/"' purge cylc-flow-8.7.0/tests/functional/remote/00-basic.t0000664000175000017500000000516315264420307022103 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test job execution on a remote platform # # Note: We cannot run this test with job runners that may be configured to # submit the job to a compute node (e.g. pbs, slurm, etc) as we are unable # to check the hostname export REQUIRE_PLATFORM='loc:remote runner:?(background|at)' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 4 create_test_global_config "" " [platforms] [[$CYLC_TEST_PLATFORM]] retrieve job logs = True " #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" basic #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-platform sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ "SELECT platform_name FROM task_jobs WHERE name=='foo'" >'foo-host.txt' cmp_ok 'foo-host.txt' <<<"${CYLC_TEST_PLATFORM}" #------------------------------------------------------------------------------- # Check that the remote job has actually been run on the correct remote by # checking it's job.out file for @CYLC_TEST_HOST REMOTE_HOST_FQDN="$(ssh "${CYLC_TEST_HOST}" hostname -f)" TEST_NAME=${TEST_NAME_BASE}-ensure-remote-run grep_ok \ "^$REMOTE_HOST_FQDN" \ "${WORKFLOW_RUN_DIR}/log/job/1/foo/NN/job.out" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/remote/01-file-install.t0000664000175000017500000000702215264420307023402 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # test file installation to remote platforms export REQUIRE_PLATFORM='loc:remote comms:?(tcp|ssh)' . "$(dirname "$0")/test_header" set_test_number 6 create_files () { # dump some files into the run dir for DIR in "bin" "ana" "app" "etc" "lib" "dir1" "dir2" "etc/share" "share" do mkdir -p "${WORKFLOW_RUN_DIR}/${DIR}" touch "${WORKFLOW_RUN_DIR}/${DIR}/moo" done for FILE in "file1" "file2" do touch "${WORKFLOW_RUN_DIR}/${FILE}" done } # Test configured files/directories along with default files/directories # (ana, app, bin, etc, lib) are correctly installed on the remote platform. TEST_NAME="${TEST_NAME_BASE}-default-paths" init_workflow "${TEST_NAME}" <<__FLOW_CONFIG__ [scheduling] [[graph]] R1 = foo [runtime] [[foo]] platform = $CYLC_TEST_PLATFORM __FLOW_CONFIG__ RUN_DIR_REL="${WORKFLOW_RUN_DIR#"${HOME}"/}" create_files # run the flow run_ok "${TEST_NAME}-validate" cylc validate "${WORKFLOW_NAME}" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" workflow_run_ok "${TEST_NAME}-run1" cylc play "${WORKFLOW_NAME}" \ --no-detach \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" # ensure these files get installed on the remote platform SSH="$(cylc config -d -i "[platforms][$CYLC_TEST_PLATFORM]ssh command")" ${SSH} "${CYLC_TEST_HOST}" \ find "${RUN_DIR_REL}/"{ana,app,bin,etc,lib} -type f | sort > 'find.out' cmp_ok 'find.out' <<__OUT__ ${RUN_DIR_REL}/ana/moo ${RUN_DIR_REL}/app/moo ${RUN_DIR_REL}/bin/moo ${RUN_DIR_REL}/etc/moo ${RUN_DIR_REL}/etc/share/moo ${RUN_DIR_REL}/lib/moo __OUT__ purge # ----------------------------------------------------------------------------- # Test the [scheduler]install configuration TEST_NAME="${TEST_NAME_BASE}-configured-paths" init_workflow "${TEST_NAME}" <<__FLOW_CONFIG__ [scheduler] install = dir1/, dir2/, file1, file2 [scheduling] [[graph]] R1 = foo [runtime] [[foo]] platform = $CYLC_TEST_PLATFORM __FLOW_CONFIG__ RUN_DIR_REL="${WORKFLOW_RUN_DIR#"${HOME}"/}" create_files run_ok "${TEST_NAME}-validate" cylc validate "${WORKFLOW_NAME}" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" workflow_run_ok "${TEST_NAME}-run2" cylc play "${WORKFLOW_NAME}" \ --no-detach \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" ${SSH} "${CYLC_TEST_HOST}" \ find "${RUN_DIR_REL}/"{ana,app,bin,dir1,dir2,file1,file2,etc,lib} -type f | sort > 'find.out' cmp_ok 'find.out' <<__OUT__ ${RUN_DIR_REL}/ana/moo ${RUN_DIR_REL}/app/moo ${RUN_DIR_REL}/bin/moo ${RUN_DIR_REL}/dir1/moo ${RUN_DIR_REL}/dir2/moo ${RUN_DIR_REL}/etc/moo ${RUN_DIR_REL}/etc/share/moo ${RUN_DIR_REL}/file1 ${RUN_DIR_REL}/file2 ${RUN_DIR_REL}/lib/moo __OUT__ purge exit cylc-flow-8.7.0/tests/functional/remote/05-remote-init.t0000664000175000017500000000415315264420307023261 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test remote initialisation - when remote init fails for an install target, # check other platforms with same install target can be initialised. export REQUIRE_PLATFORM='loc:remote fs:indep comms:tcp' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 5 create_test_global_config "" " [platforms] [[belle]] hosts = ${CYLC_TEST_HOST} install target = ${CYLC_TEST_INSTALL_TARGET} ssh command = garbage " #------------------------------------------------------------------------------- install_workflow run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" NAME='select-task-jobs.out' DB_FILE="${WORKFLOW_RUN_DIR}/log/db" sqlite3 "${DB_FILE}" \ 'SELECT name, submit_status, run_status, platform_name FROM task_jobs ORDER BY name' \ >"${NAME}" cmp_ok "${NAME}" <<__SELECT__ a|1||belle b|1||belle e|0|0|${CYLC_TEST_PLATFORM} f|0|0|${CYLC_TEST_PLATFORM} g|0|0|localhost __SELECT__ grep_ok "ERROR - Incomplete tasks:" "${TEST_NAME_BASE}-run.stderr" grep_ok "1/a did not complete the required outputs" "${TEST_NAME_BASE}-run.stderr" purge exit cylc-flow-8.7.0/tests/functional/remote/07-slow-file-install/0000775000175000017500000000000015264420307024204 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/remote/07-slow-file-install/flow.cylc0000664000175000017500000000122015264420307026022 0ustar alastairalastair[scheduler] install = dir1/, dir2/ [[events]] abort on stall timeout = true stall timeout = PT0S abort on inactivity timeout = true [scheduling] [[graph]] R1 = olaf => sven [runtime] [[olaf]] # task dependent on file install already being complete script = """ cat ${CYLC_WORKFLOW_RUN_DIR}/dir1/moo """ platform = $CYLC_TEST_PLATFORM [[sven]] # task dependent on file install already being complete script = """ rm -r ${CYLC_WORKFLOW_RUN_DIR}/dir1 ${CYLC_WORKFLOW_RUN_DIR}/dir2 """ platform = $CYLC_TEST_PLATFORM cylc-flow-8.7.0/tests/functional/remote/07-slow-file-install/bin/0000775000175000017500000000000015264420307024754 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/remote/07-slow-file-install/bin/my-rsync.sh0000775000175000017500000000005515264420307027074 0ustar alastairalastair#!/usr/bin/env bash sleep 30 exec rsync "$@" cylc-flow-8.7.0/tests/functional/remote/09-restart-running-file-install.t0000664000175000017500000000570515264420307026560 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restart remote init, file install and task messaging works for running tasks # The test works as follows: # - Starts workflow off with a remote starter task which will not complete # until restarted. It greps for updated file which will be remote installed on # restart. # - On start of this starter task, update the file that is included in # remote file install for starter task. # - Stop the workflow - this will leave starter task orphaned # - Restart, starter task should remote install, authentication keys will be # updated which is verified by checking task messaging works (checks for # "(received)succeeded" in the logs). export REQUIRE_PLATFORM='loc:remote fs:indep comms:?(tcp|ssh)' . "$(dirname "$0")/test_header" set_test_number 6 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduler] install = changing-file [[events]] abort on inactivity timeout = True inactivity timeout = PT1M [scheduling] [[graph]] R1 = """ starter:start => file-changer => stopper """ [runtime] [[starter]] platform = ${CYLC_TEST_PLATFORM} script = """ while ! grep 'Restart Play' "${CYLC_WORKFLOW_RUN_DIR}/changing-file"; do sleep 1 done """ [[stopper]] script = cylc stop --now "${CYLC_WORKFLOW_ID}" [[file-changer]] script = echo Restart Play > ${CYLC_WORKFLOW_RUN_DIR}/changing-file __FLOW_CONFIG__ echo "First Play" > "${WORKFLOW_RUN_DIR}/changing-file" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-start" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" LOG="${WORKFLOW_RUN_DIR}/log/scheduler/log" grep_ok "remote file install complete" "${LOG}" grep_ok "\[1/starter/01:running\] (received)succeeded" "${LOG}" ls "${WORKFLOW_RUN_DIR}/log/remote-install" > 'ls.out' cmp_ok ls.out <<__RLOGS__ 01-start-${CYLC_TEST_INSTALL_TARGET}.log 02-restart-${CYLC_TEST_INSTALL_TARGET}.log __RLOGS__ purge exit cylc-flow-8.7.0/tests/functional/remote/07-slow-file-install.t0000664000175000017500000000334215264420307024373 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test file install completes before dependent tasks are executed export REQUIRE_PLATFORM='loc:remote comms:?(tcp|ssh)' . "$(dirname "$0")/test_header" set_test_number 2 create_test_global_config "" " [platforms] [[${CYLC_TEST_PLATFORM}]] rsync command = my-rsync.sh " TEST_NAME="${TEST_NAME_BASE}-installation-timing" install_workflow "${TEST_NAME}" "${TEST_NAME_BASE}" for DIR in "dir1" "dir2"; do mkdir -p "${WORKFLOW_RUN_DIR}/${DIR}" echo "hello" > "${WORKFLOW_RUN_DIR}/${DIR}/moo" done run_ok "${TEST_NAME}-validate" cylc validate "${WORKFLOW_NAME}" export PATH="${WORKFLOW_RUN_DIR}/bin:$PATH" # shellcheck disable=SC2029 ssh -n "${CYLC_TEST_HOST}" "mkdir -p 'cylc-run/${WORKFLOW_NAME}/'" rsync -a 'bin' "${CYLC_TEST_HOST}:cylc-run/${WORKFLOW_NAME}/" workflow_run_ok "${TEST_NAME}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/remote/06-poll.t0000664000175000017500000000372115264420307021774 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test remote host settings. export REQUIRE_PLATFORM='loc:remote comms:poll' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 4 create_test_global_config "" " [platforms] [[$CYLC_TEST_PLATFORM]] retrieve job logs = True " #------------------------------------------------------------------------------- init_workflow "${TEST_NAME_BASE}" <<__HERE__ [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = cylc message -- foo platform = $CYLC_TEST_PLATFORM [[[outputs]]] foo = foo __HERE__ #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" \ cylc play "${WORKFLOW_NAME}" \ --debug \ --no-detach log_scan \ "${TEST_NAME_BASE}-poll" \ "$(cylc cat-log -m p "$WORKFLOW_NAME")" \ 10 \ 1 \ '\[1/foo.* (polled)foo' \ '\[1/foo.* (polled)succeeded' purge exit cylc-flow-8.7.0/tests/functional/remote/02-install-target.t0000664000175000017500000000334415264420307023755 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test remote installation only happens when appropriate export REQUIRE_PLATFORM='loc:remote fs:shared comms:?(tcp|ssh)' . "$(dirname "$0")/test_header" set_test_number 3 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' #!jinja2 [scheduling] [[graph]] graph = remote [runtime] [[remote]] # this should not require remote-init because the platform # has a shared filesystem (same install target) script = true platform = {{CYLC_TEST_PLATFORM}} __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --debug \ --no-detach \ "${WORKFLOW_NAME}" -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" grep_ok "REMOTE INIT NOT REQUIRED for localhost" "${WORKFLOW_RUN_DIR}/log/scheduler/log" purge exit cylc-flow-8.7.0/tests/functional/remote/04-symlink-dirs.t0000664000175000017500000001306215264420307023450 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Checks configured symlinks are created for run, work, share, share/cycle, log # # directories on localhost and the remote platform. export REQUIRE_PLATFORM='loc:remote comms:tcp fs:indep' . "$(dirname "$0")/test_header" if [[ -z ${TMPDIR:-} || -z ${USER:-} || $TMPDIR/$USER == "$HOME" ]]; then skip_all '"TMPDIR" or "USER" not defined or "TMPDIR"/"USER" is "HOME"' fi set_test_number 15 mapfile -d ' ' -t SSH_CMD < <(cylc config -d -i "[platforms][${CYLC_TEST_PLATFORM}]ssh command") TEST_NAME="$(basename "$0")" create_test_global_config "" " [install] [[symlink dirs]] [[[localhost]]] run = \$TMPDIR/\$USER/cylctb_tmp_run_dir share = \$TMPDIR/\$USER log = \$TMPDIR/\$USER log/job = \$TMPDIR/\$USER/cylctb_tmp_log_job_dir share/cycle = \$TMPDIR/\$USER/cylctb_tmp_share_dir work = \$TMPDIR/\$USER [[[$CYLC_TEST_INSTALL_TARGET]]] run = \$HOME/cylctb-symlinks/$TEST_NAME/ctb_tmp_run_dir share = \$HOME/cylctb-symlinks/$TEST_NAME/ log = \$HOME/cylctb-symlinks/$TEST_NAME/ log/job = \$HOME/cylctb_tmp_log_job_dir share/cycle = \$HOME/cylctb-symlinks/$TEST_NAME/ctb_tmp_share_dir work = \$HOME/cylctb-symlinks/$TEST_NAME/ " # shellcheck disable=SC2016 run_ok \ "${TEST_NAME_BASE}-mkdir" \ "${SSH_CMD[@]}" "$CYLC_TEST_HOST" \ 'mkdir -p $HOME/cylc-symlink-test' install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" workflow_run_ok "${TEST_NAME_BASE}-run-ok" cylc play "${WORKFLOW_NAME}" \ -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" --debug poll_grep_workflow_log 'remote file install complete' TEST_SYM="${TEST_NAME_BASE}-run-symlink-exists-ok" if [[ $(readlink "$HOME/cylc-run/${WORKFLOW_NAME}") == \ "$TMPDIR/$USER/cylctb_tmp_run_dir/cylc-run/${WORKFLOW_NAME}" ]]; then ok "$TEST_SYM.localhost" else fail "$TEST_SYM.localhost" fi TEST_SYM="${TEST_NAME_BASE}-share/cycle-symlink-exists-ok" if [[ $(readlink "$HOME/cylc-run/${WORKFLOW_NAME}/share/cycle") == \ "$TMPDIR/$USER/cylctb_tmp_share_dir/cylc-run/${WORKFLOW_NAME}/share/cycle" ]]; then ok "$TEST_SYM.localhost" else fail "$TEST_SYM.localhost" fi TEST_SYM="${TEST_NAME_BASE}-log/job-symlink-exists-ok" if [[ $(readlink "$HOME/cylc-run/${WORKFLOW_NAME}/log/job") == \ "$TMPDIR/$USER/cylctb_tmp_log_job_dir/cylc-run/${WORKFLOW_NAME}/log/job" ]]; then ok "$TEST_SYM.localhost" else fail "$TEST_SYM.localhost" fi for DIR in 'work' 'share' 'log'; do TEST_SYM="${TEST_NAME_BASE}-${DIR}-symlink-exists-ok" if [[ $(readlink "$HOME/cylc-run/${WORKFLOW_NAME}/${DIR}") == \ "$TMPDIR/$USER/cylc-run/${WORKFLOW_NAME}/${DIR}" ]]; then ok "$TEST_SYM.localhost" else fail "$TEST_SYM.localhost" fi done # shellcheck disable=SC2016 LINK="$("${SSH_CMD[@]}" "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"'"')" if [[ "$LINK" == *"/cylctb-symlinks/$TEST_NAME/ctb_tmp_run_dir/cylc-run/${WORKFLOW_NAME}" ]]; then ok "${TEST_NAME_BASE}-run-symlink-exists-ok.remotehost" else echo "Link points to $LINK" >&2 fail "${TEST_NAME_BASE}-run-symlink-exists-ok.remotehost" fi # shellcheck disable=SC2016 LINK="$("${SSH_CMD[@]}" "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/share/cycle'"')" if [[ "$LINK" == *"/cylctb-symlinks/$TEST_NAME/ctb_tmp_share_dir/cylc-run/${WORKFLOW_NAME}/share/cycle" ]]; then ok "${TEST_NAME_BASE}-share/cycle-symlink-exists-ok.remotehost" else echo "Link points to $LINK" >&2 fail "${TEST_NAME_BASE}-share/cycle-symlink-exists-ok.remotehost" fi # shellcheck disable=SC2016 LINK="$("${SSH_CMD[@]}" "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/log/job'"')" if [[ "$LINK" == *"/cylctb_tmp_log_job_dir/cylc-run/${WORKFLOW_NAME}/log/job" ]]; then ok "${TEST_NAME_BASE}-log/job-symlink-exists-ok.remotehost" else echo "Link points to $LINK" >&2 fail "${TEST_NAME_BASE}-log/job-symlink-exists-ok.remotehost" fi for DIR in 'work' 'share' 'log'; do # shellcheck disable=SC2016 LINK="$("${SSH_CMD[@]}" "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/$DIR'"')" if [[ "$LINK" == *"/cylctb-symlinks/$TEST_NAME/cylc-run/${WORKFLOW_NAME}/${DIR}" ]]; then ok "${TEST_NAME_BASE}-${DIR}-symlink-exists-ok.remotehost" else echo "Link points to $LINK" >&2 fail "${TEST_NAME_BASE}-${DIR}-symlink-exists-ok.remotehost" fi done # clean up remote # shellcheck disable=SC2016 "${SSH_CMD[@]}" "${CYLC_TEST_HOST}" \ 'rm -rf "${TMPDIR}/${USER}/cylctb-symlinks/" "${TMPDIR}/${USER}/sym-run" "${HOME}/cylctb-symlinks/$TEST_NAME/"' purge exit cylc-flow-8.7.0/tests/functional/remote/04-symlink-dirs/0000775000175000017500000000000015264420307023261 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/remote/04-symlink-dirs/flow.cylc0000664000175000017500000000030515264420307025102 0ustar alastairalastair#!jinja2 [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = holder => held [runtime] [[holder]] script = true platform = {{CYLC_TEST_PLATFORM}} cylc-flow-8.7.0/tests/functional/remote/05-remote-init/0000775000175000017500000000000015264420307023071 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/remote/05-remote-init/flow.cylc0000664000175000017500000000205615264420307024717 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] abort on stall timeout = true stall timeout = PT0S abort on inactivity timeout = true [scheduling] [[graph]] R1 = """ a & b & g # a & b setup to submit-fail, g (localhost) does not require remote init b:submitted? => c b:start => d # c & d should not be triggered b:submit-fail? => e & f # e and f on a same install target but with a different platform name - these should execute """ [runtime] [[task]] script="sleep 1; echo hello" [[a]] inherit = task platform = belle [[b]] inherit = task platform = belle [[c]] inherit = task platform = belle [[d]] inherit=task platform = belle [[e]] inherit=task # ariel platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[f]] inherit=task # ariel platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[g]] inherit=task platform = localhost cylc-flow-8.7.0/tests/functional/remote/03-polled-task-started/0000775000175000017500000000000015264420307024516 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/remote/03-polled-task-started/reference.log0000664000175000017500000000024315264420307027156 0ustar alastairalastair1/picard -triggered off [] 1/worf -triggered off ['1/picard'] 1/riker -triggered off ['1/picard'] 1/janeway -triggered off [] 1/tuvok -triggered off ['1/janeway'] cylc-flow-8.7.0/tests/functional/remote/03-polled-task-started/flow.cylc0000664000175000017500000000157015264420307026344 0ustar alastairalastair#!Jinja2 [scheduler] allow implicit tasks = True [[events]] expected task failures = 1/janeway [scheduling] cycling mode = integer [[graph]] R1 = """ picard:start => worf picard => riker janeway:start => tuvok janeway:fail => !janeway # Makes test workflow shut down without a fuss """ [runtime] [[root]] platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[picard, janeway]] # Longer polling so that they finish before the first poll is done submission polling intervals = PT10S execution polling intervals = PT10S [[picard]] script = true [[[events]]] started handlers = echo "THERE ARE FOUR LIGHTS" [[janeway]] script = false [[[events]]] started handlers = echo "THERE'S COFFEE IN THAT NEBULA" cylc-flow-8.7.0/tests/functional/remote/03-polled-task-started.t0000664000175000017500000000323615264420307024707 0ustar alastairalastair#!/bin/bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # ----------------------------------------------------------------------------- # Test that a quickly finishing task's `:start` trigger does not get missed # when using polling to get remote task status. export REQUIRE_PLATFORM='loc:remote comms:poll' . "$(dirname "$0")/test_header" set_test_number 4 install_workflow run_ok "${TEST_NAME_BASE}-validate" cylc validate "$WORKFLOW_NAME" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --reference-test --no-detach "$WORKFLOW_NAME" # Check 'started' event handlers ran PICARD_ACTIVITY_LOG="${WORKFLOW_RUN_DIR}/log/job/1/picard/01/job-activity.log" grep_ok "[(('event-handler-00', 'started'), 1) out] THERE ARE FOUR LIGHTS" "$PICARD_ACTIVITY_LOG" -F JANEWAY_ACTIVITY_LOG="${WORKFLOW_RUN_DIR}/log/job/1/janeway/01/job-activity.log" grep_ok "[(('event-handler-00', 'started'), 1) out] THERE'S COFFEE IN THAT NEBULA" "$JANEWAY_ACTIVITY_LOG" -F purge exit cylc-flow-8.7.0/tests/functional/remote/test_header0000777000175000017500000000000015264420307026730 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/database/0000775000175000017500000000000015264420307020664 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/database/08-broadcast-upgrade/0000775000175000017500000000000015264420307024500 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/database/08-broadcast-upgrade/flow.cylc0000664000175000017500000000036715264420307026331 0ustar alastairalastair[scheduling] [[graph]] R1 = foo [runtime] [[foo]] # NOTE: "execution retry delays" set by broadcast script = """ if (( $CYLC_TASK_TRY_NUMBER < 2 )); then false fi """ cylc-flow-8.7.0/tests/functional/database/08-broadcast-upgrade/db.sqlite30000664000175000017500000000627115264420307026401 0ustar alastairalastairPRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE absolute_outputs(cycle TEXT, name TEXT, output TEXT); CREATE TABLE broadcast_events(time TEXT, change TEXT, point TEXT, namespace TEXT, key TEXT, value TEXT); INSERT INTO broadcast_events VALUES('2022-11-11T11:18:20.229805Z','+','1','foo','execution retry delays','[60.0, 120.0, 86400.0]'); CREATE TABLE broadcast_states(point TEXT, namespace TEXT, key TEXT, value TEXT, PRIMARY KEY(point, namespace, key)); INSERT INTO broadcast_states VALUES('1','foo','execution retry delays','[1.0, 120.0, 86400.0]'); CREATE TABLE inheritance(namespace TEXT, inheritance TEXT, PRIMARY KEY(namespace)); INSERT INTO inheritance VALUES('root','["root"]'); INSERT INTO inheritance VALUES('foo','["foo", "root"]'); CREATE TABLE task_action_timers(cycle TEXT, name TEXT, ctx_key TEXT, ctx TEXT, delays TEXT, num INTEGER, delay TEXT, timeout TEXT, PRIMARY KEY(cycle, name, ctx_key)); CREATE TABLE task_events(name TEXT, cycle TEXT, time TEXT, submit_num INTEGER, event TEXT, message TEXT); CREATE TABLE task_jobs(cycle TEXT, name TEXT, submit_num INTEGER, is_manual_submit INTEGER, try_num INTEGER, time_submit TEXT, time_submit_exit TEXT, submit_status INTEGER, time_run TEXT, time_run_exit TEXT, run_signal TEXT, run_status INTEGER, platform_name TEXT, job_runner_name TEXT, job_id TEXT, PRIMARY KEY(cycle, name, submit_num)); CREATE TABLE task_late_flags(cycle TEXT, name TEXT, value INTEGER, PRIMARY KEY(cycle, name)); CREATE TABLE task_outputs(cycle TEXT, name TEXT, flow_nums TEXT, outputs TEXT, PRIMARY KEY(cycle, name, flow_nums)); INSERT INTO task_outputs VALUES('1','foo','[1]','[]'); CREATE TABLE task_pool(cycle TEXT, name TEXT, flow_nums TEXT, status TEXT, is_held INTEGER, PRIMARY KEY(cycle, name, flow_nums)); INSERT INTO task_pool VALUES('1','foo','[1]','waiting',0); CREATE TABLE task_prerequisites(cycle TEXT, name TEXT, flow_nums TEXT, prereq_name TEXT, prereq_cycle TEXT, prereq_output TEXT, satisfied TEXT, PRIMARY KEY(cycle, name, flow_nums, prereq_name, prereq_cycle, prereq_output)); CREATE TABLE task_states(name TEXT, cycle TEXT, flow_nums TEXT, time_created TEXT, time_updated TEXT, submit_num INTEGER, status TEXT, flow_wait INTEGER, is_manual_submit INTEGER, PRIMARY KEY(name, cycle, flow_nums)); INSERT INTO task_states VALUES('foo','1','[1]','2022-11-11T11:17:54Z','2022-11-11T11:17:54Z',0,'waiting',0,0); CREATE TABLE task_timeout_timers(cycle TEXT, name TEXT, timeout REAL, PRIMARY KEY(cycle, name)); CREATE TABLE tasks_to_hold(name TEXT, cycle TEXT); CREATE TABLE workflow_flows(flow_num INTEGER, start_time TEXT, description TEXT, PRIMARY KEY(flow_num)); INSERT INTO workflow_flows VALUES(1,'2022-11-11 11:17:54','original flow from 1'); CREATE TABLE workflow_params(key TEXT, value TEXT, PRIMARY KEY(key)); INSERT INTO workflow_params VALUES('uuid_str','486ec217-34f0-449e-aa23-7fe56abf0290'); INSERT INTO workflow_params VALUES('cylc_version','8.0.4.dev'); INSERT INTO workflow_params VALUES('UTC_mode','0'); INSERT INTO workflow_params VALUES('n_restart','0'); INSERT INTO workflow_params VALUES('cycle_point_tz','Z'); CREATE TABLE workflow_template_vars(key TEXT, value TEXT, PRIMARY KEY(key)); CREATE TABLE xtriggers(signature TEXT, results TEXT, PRIMARY KEY(signature)); COMMIT; cylc-flow-8.7.0/tests/functional/database/04-lock-recover/0000775000175000017500000000000015264420307023500 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/database/04-lock-recover/reference.log0000664000175000017500000000075415264420307026147 0ustar alastairalastairInitial point: 1 Final point: 1 1/locker -triggered off [] 1/t6 -triggered off ['1/locker'] 1/t5 -triggered off ['1/locker'] 1/t0 -triggered off ['1/locker'] 1/t1 -triggered off ['1/locker'] 1/t8 -triggered off ['1/locker'] 1/t9 -triggered off ['1/locker'] 1/t4 -triggered off ['1/locker'] 1/t7 -triggered off ['1/locker'] 1/t2 -triggered off ['1/locker'] 1/t3 -triggered off ['1/locker'] 1/done -triggered off ['1/t0', '1/t1', '1/t2', '1/t3', '1/t4', '1/t5', '1/t6', '1/t7', '1/t8', '1/t9'] cylc-flow-8.7.0/tests/functional/database/04-lock-recover/flow.cylc0000664000175000017500000000077715264420307025336 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] [[graph]] R1 = """ locker:lock => TRUES TRUES:succeed-all => done """ [runtime] [[locker]] script = cylc__job__wait_cylc_message_started; timeout 60 cylc-db-lock [[[outputs]]] lock = "I have locked the public database file" [[TRUES]] script = true [[t0, t1, t2, t3, t4, t5, t6, t7, t8, t9]] inherit = TRUES [[done]] script = """ cylc__job__poll_grep_workflow_log -F 'db: recovered' """ cylc-flow-8.7.0/tests/functional/database/04-lock-recover/bin/0000775000175000017500000000000015264420307024250 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/database/04-lock-recover/bin/cylc-db-lock0000775000175000017500000000114415264420307026441 0ustar alastairalastair#!/usr/bin/env python3 """Lock a workflow's database file.""" from fcntl import lockf, LOCK_SH import os from subprocess import call def main(): handle = open( os.path.join(os.getenv("CYLC_WORKFLOW_RUN_DIR"), "log", "db")) lockf(handle, LOCK_SH) call([ "cylc", "task-message", "I have locked the public database file"]) workflow_log_dir = os.getenv("CYLC_WORKFLOW_LOG_DIR") while True: for line in open(os.path.join(workflow_log_dir, "log")): if "write attempt (1) did not complete" in line: return if __name__ == "__main__": main() cylc-flow-8.7.0/tests/functional/database/03-remote/0000775000175000017500000000000015264420307022377 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/database/03-remote/reference.log0000664000175000017500000000020215264420307025032 0ustar alastairalastairInitial point: 20200101T0000Z Final point: 20200101T0000Z 20200101T0000Z/t2 -triggered off [] 20200101T0000Z/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/database/03-remote/flow.cylc0000664000175000017500000000047715264420307024232 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [scheduling] initial cycle point = 2020 final cycle point = 2020 [[graph]] P1Y = """ t1 t2 """ [runtime] [[t1]] script = true [[t2]] script = true platform = {{ environ['CYLC_TEST_PLATFORM'] }} cylc-flow-8.7.0/tests/functional/database/05-lock-recover-100.t0000775000175000017500000000314515264420307024172 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Workflow database content, "task_jobs" table after a task retries. . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" DB_FILE="$RUN_DIR/${WORKFLOW_NAME}/log/db" NAME='select-task-states.out' sqlite3 "${DB_FILE}" \ 'SELECT cycle, name, status FROM task_states ORDER BY name' \ >"${NAME}" cmp_ok "${NAME}" <<'__SELECT__' 1|locker|succeeded 1|t0|succeeded 1|t1|succeeded 1|t2|succeeded 1|t3|succeeded 1|t4|succeeded 1|t5|succeeded 1|t6|succeeded 1|t7|succeeded 1|t8|succeeded 1|t9|succeeded __SELECT__ purge exit cylc-flow-8.7.0/tests/functional/database/05-lock-recover-100/0000775000175000017500000000000015264420307023777 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/database/05-lock-recover-100/reference.log0000664000175000017500000000060515264420307026441 0ustar alastairalastairInitial point: 1 Final point: 1 1/locker -triggered off [] 1/t6 -triggered off ['1/locker'] 1/t5 -triggered off ['1/locker'] 1/t0 -triggered off ['1/locker'] 1/t1 -triggered off ['1/locker'] 1/t8 -triggered off ['1/locker'] 1/t9 -triggered off ['1/locker'] 1/t4 -triggered off ['1/locker'] 1/t7 -triggered off ['1/locker'] 1/t2 -triggered off ['1/locker'] 1/t3 -triggered off ['1/locker'] cylc-flow-8.7.0/tests/functional/database/05-lock-recover-100/flow.cylc0000664000175000017500000000054515264420307025626 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] [[graph]] R1 = """ locker:lock => TRUES """ [runtime] [[locker]] script = timeout 400 cylc-db-lock [[[outputs]]] lock = "I have locked the public database file" [[TRUES]] script = true [[t0, t1, t2, t3, t4, t5, t6, t7, t8, t9]] inherit = TRUES cylc-flow-8.7.0/tests/functional/database/05-lock-recover-100/bin/0000775000175000017500000000000015264420307024547 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/database/05-lock-recover-100/bin/cylc-db-lock0000775000175000017500000000116615264420307026744 0ustar alastairalastair#!/usr/bin/env python3 """Lock a workflow's database file.""" from fcntl import lockf, LOCK_SH import os import time from subprocess import call def main(): handle = open( os.path.join(os.getenv("CYLC_WORKFLOW_RUN_DIR"), "log", "db")) lockf(handle, LOCK_SH) call([ "cylc", "task-message", "I have locked the public database file"]) workflow_log_dir = os.getenv("CYLC_WORKFLOW_LOG_DIR") while True: for line in open(os.path.join(workflow_log_dir, "log")): if "db: recovered from" in line: return time.sleep(1) if __name__ == "__main__": main() cylc-flow-8.7.0/tests/functional/database/07-incompatible.t0000664000175000017500000000402215264420307023741 0ustar alastairalastair#!/bin/bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # ----------------------------------------------------------------------------- # Test that operating on a Cylc 7 workflow fails due to database incompatibility, # and that suitable error message is given. . "$(dirname "$0")/test_header" set_test_number 6 install_workflow # install the cylc7 restart database SRV_DIR="${WORKFLOW_RUN_DIR}/.service" mkdir "$SRV_DIR" sqlite3 "${SRV_DIR}/db" < 'db.sqlite3' sqlite3 "${SRV_DIR}/db" '.tables' > orig_tables.txt run_ok "${TEST_NAME_BASE}-validate" cylc validate "$WORKFLOW_NAME" TEST_NAME="${TEST_NAME_BASE}-play-fail" run_fail "$TEST_NAME" cylc play "$WORKFLOW_NAME" grep_ok \ 'Workflow database is incompatible with Cylc .*, or is corrupted' \ "${TEST_NAME}.stderr" # Check no new tables have been created cmp_ok orig_tables.txt <<< "$(sqlite3 "${SRV_DIR}/db" '.tables')" TEST_NAME="${TEST_NAME_BASE}-clean-fail" run_fail "$TEST_NAME" cylc clean "$WORKFLOW_NAME" grep_ok \ 'This database is either corrupted or not compatible with this' \ "${TEST_NAME}.stderr" purge # Note: The test for "trying to restart a workflow without a database gives. # suitable error message" was removed with the change from cylc run/restart to # cylc play, as if the database is not present it will simply do a cold start. exit cylc-flow-8.7.0/tests/functional/database/07-incompatible/0000775000175000017500000000000015264420307023556 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/database/07-incompatible/flow.cylc0000664000175000017500000000036615264420307025406 0ustar alastairalastair#!jinja2 [scheduler] cycle point time zone = Z allow implicit tasks = True [scheduling] cycling mode = integer initial cycle point = 1 final cycle point = 2 [[dependencies]] [[[P1]]] graph = foo => bar cylc-flow-8.7.0/tests/functional/database/07-incompatible/db.sqlite30000664000175000017500000000544515264420307025461 0ustar alastairalastairPRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE suite_params(key TEXT, value TEXT, PRIMARY KEY(key)); INSERT INTO suite_params VALUES('uuid_str','0869fe59-918f-4a32-889c-743fd9b516b9'); INSERT INTO suite_params VALUES('run_mode','live'); INSERT INTO suite_params VALUES('cylc_version','7.8.6-20-g5f91b'); INSERT INTO suite_params VALUES('UTC_mode','0'); INSERT INTO suite_params VALUES('is_held','1'); CREATE TABLE task_jobs(cycle TEXT, name TEXT, submit_num INTEGER, is_manual_submit INTEGER, try_num INTEGER, time_submit TEXT, time_submit_exit TEXT, submit_status INTEGER, time_run TEXT, time_run_exit TEXT, run_signal TEXT, run_status INTEGER, user_at_host TEXT, job_runner_name TEXT, job_id TEXT, PRIMARY KEY(cycle, name, submit_num)); CREATE TABLE task_late_flags(cycle TEXT, name TEXT, value INTEGER, PRIMARY KEY(cycle, name)); CREATE TABLE inheritance(namespace TEXT, inheritance TEXT, PRIMARY KEY(namespace)); INSERT INTO inheritance VALUES('root','["root"]'); INSERT INTO inheritance VALUES('foo','["foo", "root"]'); INSERT INTO inheritance VALUES('bar','["bar", "root"]'); CREATE TABLE task_outputs(cycle TEXT, name TEXT, flow_nums TEXT, outputs TEXT, PRIMARY KEY(cycle, name, flow_nums)); CREATE TABLE broadcast_states(point TEXT, namespace TEXT, key TEXT, value TEXT, PRIMARY KEY(point, namespace, key)); CREATE TABLE task_timeout_timers(cycle TEXT, name TEXT, timeout REAL, PRIMARY KEY(cycle, name)); CREATE TABLE task_states(name TEXT, cycle TEXT, time_created TEXT, time_updated TEXT, submit_num INTEGER, status TEXT, PRIMARY KEY(name, cycle)); INSERT INTO task_states VALUES('foo','1','2020-10-15T17:47:34+01:00','2020-10-15T17:47:34+01:00',0,'held'); INSERT INTO task_states VALUES('bar','1','2020-10-15T17:47:34+01:00','2020-10-15T17:47:34+01:00',0,'held'); CREATE TABLE broadcast_events(time TEXT, change TEXT, point TEXT, namespace TEXT, key TEXT, value TEXT); CREATE TABLE task_events(name TEXT, cycle TEXT, time TEXT, submit_num INTEGER, event TEXT, message TEXT); CREATE TABLE suite_template_vars(key TEXT, value TEXT, PRIMARY KEY(key)); CREATE TABLE task_pool(cycle TEXT, name TEXT, spawned INTEGER, status TEXT, hold_swap TEXT, PRIMARY KEY(cycle, name)); INSERT INTO task_pool VALUES('1','foo',0,'held','waiting'); INSERT INTO task_pool VALUES('1','bar',0,'held','waiting'); CREATE TABLE xtriggers(signature TEXT, results TEXT, PRIMARY KEY(signature)); CREATE TABLE task_action_timers(cycle TEXT, name TEXT, ctx_key TEXT, ctx TEXT, delays TEXT, num INTEGER, delay TEXT, timeout TEXT, PRIMARY KEY(cycle, name, ctx_key)); CREATE TABLE absolute_outputs(cycle TEXT, name TEXT, output TEXT); CREATE TABLE task_prerequisites(cycle TEXT, name TEXT, flow_nums TEXT, prereq_name TEXT, prereq_cycle TEXT, prereq_output TEXT, satisfied TEXT, PRIMARY KEY(cycle, name, flow_nums, prereq_name, prereq_cycle, prereq_output)); COMMIT; cylc-flow-8.7.0/tests/functional/database/03-remote.t0000775000175000017500000000325215264420307022571 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Workflow database content, "task_jobs" table with a remote job. export REQUIRE_PLATFORM='loc:remote' . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" DB_FILE="${RUN_DIR}/${WORKFLOW_NAME}/log/db" NAME='select-task-jobs.out' sqlite3 "${DB_FILE}" \ 'SELECT cycle, name, submit_num, try_num, submit_status, run_status, platform_name, job_runner_name FROM task_jobs ORDER BY name' \ >"${NAME}" cmp_ok "${NAME}" <<__SELECT__ 20200101T0000Z|t1|1|1|0|0|localhost|background 20200101T0000Z|t2|1|1|0|0|${CYLC_TEST_PLATFORM}|${CYLC_TEST_JOB_RUNNER} __SELECT__ purge exit cylc-flow-8.7.0/tests/functional/database/06-task-message/0000775000175000017500000000000015264420307023473 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/database/06-task-message/reference.log0000664000175000017500000000006715264420307026137 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/database/06-task-message/flow.cylc0000664000175000017500000000057015264420307025320 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = """ sleep 1 cylc message -p 'WARNING' 'You have been warned' sleep 1 cylc message -p 'CRITICAL' 'You are being critical' sleep 1 cylc message 'You are normal' sleep 1 """ cylc-flow-8.7.0/tests/functional/database/test_header0000777000175000017500000000000015264420307027201 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/database/06-task-message.t0000775000175000017500000000321515264420307023664 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Workflow database content, "task_events" table contains unhandled task messages . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" DB_FILE="$RUN_DIR/${WORKFLOW_NAME}/log/db" NAME='select-task-events.out' sqlite3 "${DB_FILE}" " SELECT cycle, name, event, message FROM task_events WHERE event GLOB 'message *' ORDER BY event " >"${NAME}" cmp_ok "${NAME}" <<'__SELECT__' 1|t1|message critical|You are being critical 1|t1|message info|You are normal 1|t1|message warning|You have been warned __SELECT__ purge exit cylc-flow-8.7.0/tests/functional/database/04-lock-recover.t0000775000175000017500000000415315264420307023673 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Workflow database content, "task_jobs" table after a task retries. . "$(dirname "$0")/test_header" set_test_number 4 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" # Ensure that DB statement and its args are printed to STDERR grep -A 3 -F 'INFO - cannot execute database statement:' \ "${TEST_NAME_BASE}-run.stderr" > "${TEST_NAME_BASE}-run.stderr.grep" # The following "sed" turns the value for "time_submit_exit" to "?" sed -i "s/, '[^T']*T[^Z']*Z',/, '?',/" "${TEST_NAME_BASE}-run.stderr.grep" # Cannot use cmp_ok as the error message is prefixed by a timestamp. grep_ok "INFO - cannot execute database statement:" \ "${TEST_NAME_BASE}-run.stderr.grep" DB_FILE="$RUN_DIR/${WORKFLOW_NAME}/log/db" NAME='select-task-states.out' sqlite3 "${DB_FILE}" \ 'SELECT cycle, name, status FROM task_states ORDER BY name' \ >"${NAME}" cmp_ok "${NAME}" <<'__SELECT__' 1|done|succeeded 1|locker|succeeded 1|t0|succeeded 1|t1|succeeded 1|t2|succeeded 1|t3|succeeded 1|t4|succeeded 1|t5|succeeded 1|t6|succeeded 1|t7|succeeded 1|t8|succeeded 1|t9|succeeded __SELECT__ purge exit cylc-flow-8.7.0/tests/functional/database/08-broadcast-upgrade.t0000664000175000017500000000251315264420307024666 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restarting when there are broadcasts saved in the DB that need upgrading . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" mkdir "${WORKFLOW_RUN_DIR}/.service" DB_PATH="${WORKFLOW_RUN_DIR}/.service/db" sqlite3 "$DB_PATH" < db.sqlite3 run_ok "${TEST_NAME_BASE}-val" cylc validate "$WORKFLOW_NAME" workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play -vv --no-detach --upgrade "$WORKFLOW_NAME" purge exit cylc-flow-8.7.0/tests/functional/cylc-get-cylc-version/0000775000175000017500000000000015264420307023242 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-get-cylc-version/00-basic.t0000664000175000017500000000171215264420307024726 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Basic get-cylc-version test. . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/cylc-get-cylc-version/00-basic/0000775000175000017500000000000015264420307024540 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-get-cylc-version/00-basic/reference.log0000664000175000017500000000007015264420307027176 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] cylc-flow-8.7.0/tests/functional/cylc-get-cylc-version/00-basic/flow.cylc0000664000175000017500000000053515264420307026366 0ustar alastairalastair[meta] title = Test for the get-cylc-version command. description = """A task compares its own cylc version to that running the test workflow (should be the same).""" [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = """ diff -u <(cylc --version) <(cylc get-cylc-version "${CYLC_WORKFLOW_ID}") """ cylc-flow-8.7.0/tests/functional/cylc-get-cylc-version/test_header0000777000175000017500000000000015264420307031557 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/0000775000175000017500000000000015264420307021062 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/08-space.t0000775000175000017500000000203515264420307022572 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test broadcast -s '[foo] bar=baz' syntax. cylc/cylc-flow#1680 . "$(dirname "$0")/test_header" set_test_number 2 export REFTEST_OPTS="--abort-if-any-task-fails" reftest exit cylc-flow-8.7.0/tests/functional/broadcast/00-simple.t0000775000175000017500000000504615264420307022765 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test broadcasts . "$(dirname "$0")/test_header" set_test_number 5 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --no-detach --reference-test "${WORKFLOW_NAME}" sort "${WORKFLOW_RUN_DIR}/share/broadcast.log" >'broadcast.log.sorted' cmp_ok 'broadcast.ref' 'broadcast.log.sorted' DB_FILE="${WORKFLOW_RUN_DIR}/log/db" NAME='select-broadcast-events.out' sqlite3 "${DB_FILE}" \ 'SELECT change, point, namespace, key, value FROM broadcast_events ORDER BY time, change, point, namespace, key' >"${NAME}" cmp_ok "${NAME}" <<'__SELECT__' +|*|root|[environment]BCAST|ROOT +|20100808T00|foo|[environment]BCAST|FOO +|*|bar|[environment]BCAST|BAR +|20100809T00|baz|[environment]BCAST|BAZ +|20100809T00|qux|[environment]BCAST|QUX -|20100809T00|qux|[environment]BCAST|QUX +|*|wibble|[environment]BCAST|WIBBLE -|*|wibble|[environment]BCAST|WIBBLE +|*|ENS|[environment]BCAST|ENS +|*|ENS1|[environment]BCAST|ENS1 +|20100809T00|m2|[environment]BCAST|M2 +|*|m7|[environment]BCAST|M7 +|*|m8|[environment]BCAST|M8 +|*|m9|[environment]BCAST|M9 __SELECT__ NAME='select-broadcast-states.out' sqlite3 "${DB_FILE}" \ 'SELECT point, namespace, key, value FROM broadcast_states ORDER BY point, namespace, key' >"${NAME}" cmp_ok "${NAME}" <<'__SELECT__' *|ENS|[environment]BCAST|ENS *|ENS1|[environment]BCAST|ENS1 *|bar|[environment]BCAST|BAR *|m7|[environment]BCAST|M7 *|m8|[environment]BCAST|M8 *|m9|[environment]BCAST|M9 *|root|[environment]BCAST|ROOT 20100808T00|foo|[environment]BCAST|FOO 20100809T00|baz|[environment]BCAST|BAZ 20100809T00|m2|[environment]BCAST|M2 __SELECT__ purge exit cylc-flow-8.7.0/tests/functional/broadcast/09-remote.t0000775000175000017500000000200715264420307022772 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test broadcast from remote task job. export REQUIRE_PLATFORM='loc:remote comms:tcp' . "$(dirname "$0")/test_header" set_test_number 2 reftest purge exit cylc-flow-8.7.0/tests/functional/broadcast/05-bad-point.t0000775000175000017500000000231715264420307023354 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that broadcast to an invalid cycle point fails. . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}" cylc play --debug --no-detach --abort-if-any-task-fails "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/broadcast/13-file-cancel/0000775000175000017500000000000015264420307023445 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/13-file-cancel/reference.log0000664000175000017500000000012415264420307026103 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] 1/t2 -triggered off ['1/t1'] cylc-flow-8.7.0/tests/functional/broadcast/13-file-cancel/flow.cylc0000664000175000017500000000057315264420307025275 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] [[graph]] R1 = "t1 => t2" [runtime] [[t1]] script = """ cylc broadcast -n 't2' -F "${CYLC_WORKFLOW_RUN_DIR}/broadcast-1.cylc" "${CYLC_WORKFLOW_ID}" cylc broadcast -n 't2' -G "${CYLC_WORKFLOW_RUN_DIR}/broadcast-2.cylc" "${CYLC_WORKFLOW_ID}" """ [[t2]] script = false cylc-flow-8.7.0/tests/functional/broadcast/13-file-cancel/broadcast-1.cylc0000664000175000017500000000017515264420307026424 0ustar alastairalastairscript=printenv CYLC_FOOBAR && ! printenv CYLC_QUX [environment] CYLC_FOOBAR=""" foo bar baz """ CYLC_QUX=quux corge cylc-flow-8.7.0/tests/functional/broadcast/13-file-cancel/broadcast-2.cylc0000664000175000017500000000004615264420307026422 0ustar alastairalastair[environment] CYLC_QUX=quux corge cylc-flow-8.7.0/tests/functional/broadcast/09-remote/0000775000175000017500000000000015264420307022603 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/09-remote/reference.log0000664000175000017500000000022515264420307025243 0ustar alastairalastairInitial point: 19990101T0000Z Final point: 19990101T0000Z 19990101T0000Z/t1 -triggered off [] 19990101T0000Z/t2 -triggered off ['19990101T0000Z/t1'] cylc-flow-8.7.0/tests/functional/broadcast/09-remote/flow.cylc0000664000175000017500000000061715264420307024432 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [scheduling] initial cycle point = 1999 final cycle point = 1999 [[graph]] P1Y = t1 => t2 [runtime] [[t1]] script = """ cylc broadcast -v -v --debug "${CYLC_WORKFLOW_ID}" \ -n t2 -s 'script=true' """ platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[t2]] script = false cylc-flow-8.7.0/tests/functional/broadcast/06-bad-namespace.t0000775000175000017500000000231715264420307024160 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that broadcast to an undefined namespace fails. . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}" cylc play --debug --no-detach --abort-if-any-task-fails "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/broadcast/02-inherit/0000775000175000017500000000000015264420307022743 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/02-inherit/reference.log0000664000175000017500000000034315264420307025404 0ustar alastairalastairInitial point: 20140101T00 Final point: 20140101T00 20140101T00/broadcast_all -triggered off [] 20140101T00/broadcast_tag -triggered off ['20140101T00/broadcast_all'] 20140101T00/t1 -triggered off ['20140101T00/broadcast_tag'] cylc-flow-8.7.0/tests/functional/broadcast/02-inherit/flow.cylc0000664000175000017500000000167515264420307024577 0ustar alastairalastair#!jinja2 [meta] title=broadcast description=Test Broadcast Inheritance [scheduler] cycle point format = %Y%m%dT%H [scheduling] initial cycle point = 20140101T00 final cycle point = 20140101T00 [[graph]] R1 = broadcast_all => broadcast_tag T00 = broadcast_tag => t1 [runtime] [[broadcast_all]] script=""" cylc broadcast -s "[environment]ALL_0=true" -n F1 $CYLC_WORKFLOW_ID cylc broadcast -s "[environment]ALL_1=true" -n t1 $CYLC_WORKFLOW_ID """ [[broadcast_tag]] script=""" cylc broadcast -s "[environment]TAG_0=true" -n F1 -p $CYLC_TASK_CYCLE_POINT \ $CYLC_WORKFLOW_ID cylc broadcast -s "[environment]TAG_1=true" -n t1 -p $CYLC_TASK_CYCLE_POINT \ $CYLC_WORKFLOW_ID """ [[F1]] script=""" $ALL_0 $ALL_1 $TAG_0 $TAG_1 """ [[[environment]]] ALL_0=false ALL_1=false TAG_0=false TAG_1=false [[t1]] inherit=F1 cylc-flow-8.7.0/tests/functional/broadcast/12-file-stdin.t0000775000175000017500000000172115264420307023531 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test broadcast settings from STDIN . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/broadcast/03-expire/0000775000175000017500000000000015264420307022576 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/03-expire/reference.log0000664000175000017500000000111215264420307025232 0ustar alastairalastairInitial point: 20200101T0000Z Final point: 20250101T0000Z 20200101T0000Z/broadcast -triggered off [] 20250101T0000Z/broadcast -triggered off [] 20200101T0000Z/t1 -triggered off ['20200101T0000Z/broadcast'] 20250101T0000Z/t1 -triggered off ['20250101T0000Z/broadcast'] 20200101T0000Z/broadcast-expire -triggered off ['20150101T0000Z/t2', '20200101T0000Z/t1'] 20200101T0000Z/t2 -triggered off ['20200101T0000Z/broadcast-expire'] 20250101T0000Z/broadcast-expire -triggered off ['20200101T0000Z/t2', '20250101T0000Z/t1'] 20250101T0000Z/t2 -triggered off ['20250101T0000Z/broadcast-expire'] cylc-flow-8.7.0/tests/functional/broadcast/03-expire/flow.cylc0000664000175000017500000000355515264420307024431 0ustar alastairalastair[meta] title=broadcast expire description=Test broadcast expire option [scheduler] UTC mode = True [scheduling] initial cycle point = 2020 final cycle point = 2025 [[graph]] P5Y=""" broadcast => t1 => broadcast-expire => t2 t2[-P5Y] => broadcast-expire """ [runtime] [[broadcast]] script = """ cylc broadcast \ -s '[environment]FABRIC=Wool' \ -s "[environment]ORGANISM=sheep" \ -p "${CYLC_TASK_CYCLE_POINT}" \ -n 'F1' \ "${CYLC_WORKFLOW_ID}" \ | tee 'broadcast.out' """ post-script = """ diff -u - 'broadcast.out' <<__OUT__ Broadcast set: + [${CYLC_TASK_CYCLE_POINT}/F1] [environment]FABRIC=Wool + [${CYLC_TASK_CYCLE_POINT}/F1] [environment]ORGANISM=sheep __OUT__ """ [[broadcast-expire]] script = """ NEXT_CYCLE_POINT=$(cylc cycletime --offset=P5Y) cylc broadcast --expire="${NEXT_CYCLE_POINT}" "${CYLC_WORKFLOW_ID}" \ | tee 'broadcast.out' """ post-script = """ diff -u - 'broadcast.out' <<__OUT__ Broadcast cancelled: - [${CYLC_TASK_CYCLE_POINT}/F1] [environment]FABRIC=Wool - [${CYLC_TASK_CYCLE_POINT}/F1] [environment]ORGANISM=sheep __OUT__ """ [[F1]] script = """ echo "${FABRIC} is from ${ORGANISM}." | tee 'echo.out' """ [[[environment]]] FABRIC=Silk ORGANISM=silk worm [[t1]] inherit=F1 post-script=""" diff -u - 'echo.out' <<<'Wool is from sheep.' """ [[t2]] inherit=F1 post-script=""" diff -u - 'echo.out' <<<'Silk is from silk worm.' """ cylc-flow-8.7.0/tests/functional/broadcast/11-file-2/0000775000175000017500000000000015264420307022357 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/11-file-2/reference.log0000664000175000017500000000012415264420307025015 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] 1/t2 -triggered off ['1/t1'] cylc-flow-8.7.0/tests/functional/broadcast/11-file-2/flow.cylc0000664000175000017500000000050015264420307024175 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] [[graph]] R1 = "t1 => t2" [runtime] [[t1]] script = """ cylc broadcast -n 't2' \ -F "${CYLC_WORKFLOW_RUN_DIR}/broadcast-1.cylc" \ -F "${CYLC_WORKFLOW_RUN_DIR}/broadcast-2.cylc" \ "${CYLC_WORKFLOW_ID}" """ [[t2]] script = false cylc-flow-8.7.0/tests/functional/broadcast/11-file-2/broadcast-1.cylc0000664000175000017500000000003415264420307025330 0ustar alastairalastairscript=printenv CYLC_FOOBAR cylc-flow-8.7.0/tests/functional/broadcast/11-file-2/broadcast-2.cylc0000664000175000017500000000006215264420307025332 0ustar alastairalastair[environment] CYLC_FOOBAR=""" foo bar baz """ cylc-flow-8.7.0/tests/functional/broadcast/04-empty.t0000775000175000017500000000172015264420307022631 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test broadcast of an empty string . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/broadcast/03-expire.t0000775000175000017500000000172215264420307022770 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test broadcast --expire=CYCLE_POINT . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/broadcast/04-empty/0000775000175000017500000000000015264420307022441 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/04-empty/reference.log0000664000175000017500000000014215264420307025077 0ustar alastairalastairInitial point: 1 Final point: 1 1/broadcast -triggered off [] 1/t1 -triggered off ['1/broadcast'] cylc-flow-8.7.0/tests/functional/broadcast/04-empty/flow.cylc0000664000175000017500000000123615264420307024266 0ustar alastairalastair[meta] title = broadcast empty description = Test broadcast of an empty string [scheduling] [[graph]] R1 = "broadcast => t1" [runtime] [[broadcast]] script = """ cylc broadcast -s '[environment]EMPTY=' -p '1' -n 't1' "${CYLC_WORKFLOW_ID}" \ | tee 'broadcast.out' diff -u - 'broadcast.out' <<__OUT__ Broadcast set: + [${CYLC_TASK_CYCLE_POINT}/t1] [environment]EMPTY= __OUT__ """ [[t1]] script = """ printenv EMPTY | tee 'echo.out' diff -u - 'echo.out' <<<'' """ [[[environment]]] EMPTY=full cylc-flow-8.7.0/tests/functional/broadcast/07-timeout/0000775000175000017500000000000015264420307022774 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/07-timeout/reference.log0000664000175000017500000000026215264420307025435 0ustar alastairalastairInitial point: 20100808T0000Z Final point: 20100809T0000Z 20100808T0000Z/send_broadcast -triggered off [] 20100808T0000Z/timeout -triggered off ['20100808T0000Z/send_broadcast'] cylc-flow-8.7.0/tests/functional/broadcast/07-timeout/flow.cylc0000664000175000017500000000122515264420307024617 0ustar alastairalastair[meta] title = "test workflow for broadcast timeout functionality" [scheduler] UTC mode = True [scheduling] initial cycle point = 20100808T0000Z final cycle point = 20100808T0000Z [[graph]] R1 = send_broadcast => timeout [runtime] [[send_broadcast]] script = """ cylc broadcast -n timeout --point=20100808T0000Z --set='[events]execution timeout=PT1S' $CYLC_WORKFLOW_ID """ [[timeout]] script = """ cylc__job__poll_grep_workflow_log -E \ "${CYLC_TASK_ID}.* execution timeout after PT1S" """ [[[events]]] execution timeout = PT1M cylc-flow-8.7.0/tests/functional/broadcast/12-file-stdin/0000775000175000017500000000000015264420307023340 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/12-file-stdin/reference.log0000664000175000017500000000012415264420307025776 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] 1/t2 -triggered off ['1/t1'] cylc-flow-8.7.0/tests/functional/broadcast/12-file-stdin/flow.cylc0000664000175000017500000000044615264420307025167 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] [[graph]] R1 = "t1 => t2" [runtime] [[t1]] script = """ cylc broadcast -n 't2' -F - "${CYLC_WORKFLOW_ID}" \ <"${CYLC_WORKFLOW_RUN_DIR}/broadcast.cylc" """ [[t2]] script = false cylc-flow-8.7.0/tests/functional/broadcast/12-file-stdin/broadcast.cylc0000664000175000017500000000011615264420307026154 0ustar alastairalastairscript=printenv CYLC_FOOBAR [environment] CYLC_FOOBAR=""" foo bar baz """ cylc-flow-8.7.0/tests/functional/broadcast/10-file-1/0000775000175000017500000000000015264420307022355 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/10-file-1/reference.log0000664000175000017500000000016115264420307025014 0ustar alastairalastairInitial point: 1 Final point: 1 1/t1 -triggered off [] 1/t2 -triggered off ['1/t1'] 1/t2 -triggered off ['1/t1'] cylc-flow-8.7.0/tests/functional/broadcast/10-file-1/flow.cylc0000664000175000017500000000037515264420307024205 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] [[graph]] R1 = "t1 => t2" [runtime] [[t1]] script = """ cylc broadcast -n 't2' -F "${CYLC_WORKFLOW_RUN_DIR}/broadcast.cylc" "${CYLC_WORKFLOW_ID}" """ [[t2]] script = false cylc-flow-8.7.0/tests/functional/broadcast/10-file-1/broadcast.cylc0000664000175000017500000000045515264420307025177 0ustar alastairalastairscript=""" printenv CYLC_FOOBAR # This hash char should not cause the rest of the script to be stripped out # - https://github.com/cylc/cylc-flow/pull/5933 if (($CYLC_TASK_TRY_NUMBER < 2 )); then false fi """ execution retry delays = PT1S, PT2S [environment] CYLC_FOOBAR=""" foo bar baz """ cylc-flow-8.7.0/tests/functional/broadcast/06-bad-namespace/0000775000175000017500000000000015264420307023765 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/06-bad-namespace/flow.cylc0000664000175000017500000000101215264420307025602 0ustar alastairalastair[meta] title=broadcast bad namespace description=Test broadcast to an undefined namespace fails. [scheduler] [[events]] abort on stall timeout = True stall timeout=PT1M [scheduling] initial cycle point = 20150808 final cycle point = 20150808 [[graph]] P1M = broadcast [runtime] [[broadcast]] script=""" # Broadcast to an undefined namespace; fail if the broadcast succeeds (it # should fail). ! cylc broadcast -s 'title=foo' -n 'zilch' "${CYLC_WORKFLOW_ID}" """ cylc-flow-8.7.0/tests/functional/broadcast/14-display-format.t0000664000175000017500000000453015264420307024426 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test the --format option. . "$(dirname "$0")/test_header" set_test_number 6 init_workflow "${TEST_NAME_BASE}" << '__EOF__' [scheduler] [[events]] stall timeout = PT0S abort on stall timeout = True [scheduling] [[graph]] R1 = foo [runtime] [[foo]] pre-script = """ cylc broadcast "$CYLC_WORKFLOW_ID" \ -p "$CYLC_TASK_CYCLE_POINT" -n "$CYLC_TASK_NAME" \ --set '[environment]horse=dorothy' \ --set 'post-script=echo "$horse"' """ script = """ cylc broadcast "$CYLC_WORKFLOW_ID" --display --format json > out.json cylc broadcast "$CYLC_WORKFLOW_ID" --display --format raw > raw1.stdout # Test deprecated option: cylc broadcast "$CYLC_WORKFLOW_ID" --display --raw 1> raw2.stdout 2> raw2.stderr """ __EOF__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" --no-detach FOO_WORK_DIR="${WORKFLOW_RUN_DIR}/work/1/foo" TEST_NAME="${TEST_NAME_BASE}-cmp-json" cmp_json "$TEST_NAME" "${FOO_WORK_DIR}/out.json" << '__EOF__' { "1": { "foo": { "environment": { "horse": "dorothy" }, "post-script": "echo \"$horse\"" } } } __EOF__ cmp_ok "${FOO_WORK_DIR}/raw1.stdout" << '__EOF__' {'1': {'foo': {'environment': {'horse': 'dorothy'}, 'post-script': 'echo "$horse"'}}} __EOF__ cmp_ok "${FOO_WORK_DIR}/raw2.stdout" "${FOO_WORK_DIR}/raw1.stdout" cmp_ok "${FOO_WORK_DIR}/raw2.stderr" << __EOF__ DEPRECATED: the --raw option will be removed at Cylc 8.9; use --format=raw instead. __EOF__ purge cylc-flow-8.7.0/tests/functional/broadcast/05-bad-point/0000775000175000017500000000000015264420307023161 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/05-bad-point/flow.cylc0000664000175000017500000000113315264420307025002 0ustar alastairalastair[meta] title=broadcast bad point description=Test broadcast to an invalid cycle point fails. # And see github #1415 - it did cause the scheduler to abort. [scheduler] [[events]] abort on stall timeout = True stall timeout=PT1M [scheduling] initial cycle point = 20150808 final cycle point = 20150808 [[graph]] P1M = broadcast [runtime] [[broadcast]] script=""" # Broadcast to an integer point, not valid for this workflow; and # fail if the broadcast succeeds (it should fail). ! cylc broadcast -s 'title=foo' -p '1' "${CYLC_WORKFLOW_ID}" """ cylc-flow-8.7.0/tests/functional/broadcast/test_header0000777000175000017500000000000015264420307027377 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/08-space/0000775000175000017500000000000015264420307022402 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/08-space/reference.log0000664000175000017500000000025115264420307025041 0ustar alastairalastairInitial point: 20200202T0000Z Final point: 20200202T0000Z 20200202T0000Z/broadcast -triggered off [] 20200202T0000Z/test-env -triggered off ['20200202T0000Z/broadcast'] cylc-flow-8.7.0/tests/functional/broadcast/08-space/flow.cylc0000664000175000017500000000112315264420307024222 0ustar alastairalastair[meta] title=broadcast section-space-key description=Test broadcast set section-space-key syntax [scheduler] UTC mode = True [[events]] abort on stall timeout = True stall timeout=PT1M [scheduling] initial cycle point = 20200202 final cycle point = 20200202 [[graph]] P1M = "broadcast => test-env" [runtime] [[broadcast]] script=""" cylc broadcast -s '[environment] FOO=${FOO:-foo}' -n 'test-env' "${CYLC_WORKFLOW_ID}" """ [[test-env]] script=""" test "${FOO}" = 'foo' """ [[[environment]]] FOO=bar cylc-flow-8.7.0/tests/functional/broadcast/11-file-2.t0000775000175000017500000000172115264420307022550 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test broadcast settings in 2 files . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/broadcast/00-simple/0000775000175000017500000000000015264420307022570 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/00-simple/expected-prep.out0000664000175000017500000000256515264420307026076 0ustar alastairalastairBroadcast set: + [*/root] [environment]BCAST=ROOT Broadcast set: + [20100808T00/foo] [environment]BCAST=FOO Broadcast set: + [*/bar] [environment]BCAST=BAR Broadcast set: + [20100809T00/baz] [environment]BCAST=BAZ Broadcast set: + [20100809T00/qux] [environment]BCAST=QUX Broadcast cancelled: - [20100809T00/qux] [environment]BCAST=QUX Broadcast set: + [*/wibble] [environment]BCAST=WIBBLE Broadcast cancelled: - [*/wibble] [environment]BCAST=WIBBLE Broadcast set: + [*/ENS] [environment]BCAST=ENS Broadcast set: + [*/ENS1] [environment]BCAST=ENS1 Broadcast set: + [20100809T00/m2] [environment]BCAST=M2 Broadcast set: + [*/m7] [environment]BCAST=M7 Broadcast set: + [*/m8] [environment]BCAST=M8 Broadcast set: + [*/m9] [environment]BCAST=M9 * |-ENS | `-environment | `-BCAST ENS |-ENS1 | `-environment | `-BCAST ENS1 |-bar | `-environment | `-BCAST BAR |-m7 | `-environment | `-BCAST M7 |-m8 | `-environment | `-BCAST M8 |-m9 | `-environment | `-BCAST M9 `-root `-environment `-BCAST ROOT 20100808T00 `-foo `-environment `-BCAST FOO 20100809T00 |-baz | `-environment | `-BCAST BAZ `-m2 `-environment `-BCAST M2 cylc-flow-8.7.0/tests/functional/broadcast/00-simple/broadcast.ref0000664000175000017500000000177415264420307025241 0ustar alastairalastair20100808T00 bar BCAST is BAR 20100808T00 baz BCAST is ROOT 20100808T00 check BCAST is ROOT 20100808T00 foo BCAST is FOO 20100808T00 m1 BCAST is ENS1 20100808T00 m2 BCAST is ENS1 20100808T00 m3 BCAST is ENS1 20100808T00 m4 BCAST is ENS1 20100808T00 m5 BCAST is ENS1 20100808T00 m6 BCAST is ENS1 20100808T00 m7 BCAST is M7 20100808T00 m8 BCAST is M8 20100808T00 m9 BCAST is M9 20100808T00 n1 BCAST is ENS 20100808T00 o1 BCAST is ENS 20100808T00 qux BCAST is ROOT 20100808T00 wibble BCAST is ROOT 20100808T00 wobble BCAST is ROOT 20100809T00 bar BCAST is BAR 20100809T00 baz BCAST is BAZ 20100809T00 foo BCAST is ROOT 20100809T00 m1 BCAST is ENS1 20100809T00 m2 BCAST is M2 20100809T00 m3 BCAST is ENS1 20100809T00 m4 BCAST is ENS1 20100809T00 m5 BCAST is ENS1 20100809T00 m6 BCAST is ENS1 20100809T00 m7 BCAST is M7 20100809T00 m8 BCAST is M8 20100809T00 m9 BCAST is M9 20100809T00 n1 BCAST is ENS 20100809T00 o1 BCAST is ENS 20100809T00 qux BCAST is ROOT 20100809T00 wibble BCAST is ROOT 20100809T00 wobble BCAST is ROOT cylc-flow-8.7.0/tests/functional/broadcast/00-simple/reference.log0000664000175000017500000000670715264420307025243 0ustar alastairalastairInitial point: 20100808T00 Final point: 20100809T00 20100808T00/prep -triggered off [] 20100808T00/foo -triggered off ['20100808T00/prep'] 20100808T00/check -triggered off ['20100808T00/prep'] 20100808T00/bar -triggered off ['20100808T00/foo'] 20100808T00/qux -triggered off ['20100808T00/foo'] 20100808T00/baz -triggered off ['20100808T00/foo'] 20100808T00/wibble -triggered off ['20100808T00/foo'] 20100809T00/foo -triggered off [] 20100808T00/m1 -triggered off ['20100808T00/bar', '20100808T00/baz', '20100808T00/qux', '20100808T00/wibble'] 20100808T00/m2 -triggered off ['20100808T00/bar', '20100808T00/baz', '20100808T00/qux', '20100808T00/wibble'] 20100808T00/m3 -triggered off ['20100808T00/bar', '20100808T00/baz', '20100808T00/qux', '20100808T00/wibble'] 20100808T00/m4 -triggered off ['20100808T00/bar', '20100808T00/baz', '20100808T00/qux', '20100808T00/wibble'] 20100808T00/m5 -triggered off ['20100808T00/bar', '20100808T00/baz', '20100808T00/qux', '20100808T00/wibble'] 20100808T00/m6 -triggered off ['20100808T00/bar', '20100808T00/baz', '20100808T00/qux', '20100808T00/wibble'] 20100808T00/m7 -triggered off ['20100808T00/bar', '20100808T00/baz', '20100808T00/qux', '20100808T00/wibble'] 20100808T00/m8 -triggered off ['20100808T00/bar', '20100808T00/baz', '20100808T00/qux', '20100808T00/wibble'] 20100808T00/m9 -triggered off ['20100808T00/bar', '20100808T00/baz', '20100808T00/qux', '20100808T00/wibble'] 20100808T00/n1 -triggered off ['20100808T00/bar', '20100808T00/baz', '20100808T00/qux', '20100808T00/wibble'] 20100808T00/o1 -triggered off ['20100808T00/bar', '20100808T00/baz', '20100808T00/qux', '20100808T00/wibble'] 20100809T00/bar -triggered off ['20100809T00/foo'] 20100809T00/qux -triggered off ['20100809T00/foo'] 20100809T00/baz -triggered off ['20100809T00/foo'] 20100809T00/wibble -triggered off ['20100809T00/foo'] 20100808T00/wobble -triggered off ['20100808T00/m1', '20100808T00/m2', '20100808T00/m3', '20100808T00/m4', '20100808T00/m5', '20100808T00/m6', '20100808T00/m7', '20100808T00/m8', '20100808T00/m9', '20100808T00/n1', '20100808T00/o1'] 20100809T00/m2 -triggered off ['20100809T00/bar', '20100809T00/baz', '20100809T00/qux', '20100809T00/wibble'] 20100809T00/m8 -triggered off ['20100809T00/bar', '20100809T00/baz', '20100809T00/qux', '20100809T00/wibble'] 20100809T00/m5 -triggered off ['20100809T00/bar', '20100809T00/baz', '20100809T00/qux', '20100809T00/wibble'] 20100809T00/m4 -triggered off ['20100809T00/bar', '20100809T00/baz', '20100809T00/qux', '20100809T00/wibble'] 20100809T00/m7 -triggered off ['20100809T00/bar', '20100809T00/baz', '20100809T00/qux', '20100809T00/wibble'] 20100809T00/m6 -triggered off ['20100809T00/bar', '20100809T00/baz', '20100809T00/qux', '20100809T00/wibble'] 20100809T00/m1 -triggered off ['20100809T00/bar', '20100809T00/baz', '20100809T00/qux', '20100809T00/wibble'] 20100809T00/n1 -triggered off ['20100809T00/bar', '20100809T00/baz', '20100809T00/qux', '20100809T00/wibble'] 20100809T00/o1 -triggered off ['20100809T00/bar', '20100809T00/baz', '20100809T00/qux', '20100809T00/wibble'] 20100809T00/m3 -triggered off ['20100809T00/bar', '20100809T00/baz', '20100809T00/qux', '20100809T00/wibble'] 20100809T00/m9 -triggered off ['20100809T00/bar', '20100809T00/baz', '20100809T00/qux', '20100809T00/wibble'] 20100809T00/wobble -triggered off ['20100809T00/m1', '20100809T00/m2', '20100809T00/m3', '20100809T00/m4', '20100809T00/m5', '20100809T00/m6', '20100809T00/m7', '20100809T00/m8', '20100809T00/m9', '20100809T00/n1', '20100809T00/o1'] cylc-flow-8.7.0/tests/functional/broadcast/00-simple/expected-prep.err0000664000175000017500000000071015264420307026045 0ustar alastairalastairERROR: No broadcast to cancel/clear for these options: --cancel=[environment]BCAST --namespace=m4 --point=20100809T00 ERROR: No broadcast to cancel/clear for these options: --cancel=[environment]BCAST --namespace=m5 --point=* ERROR: No broadcast to cancel/clear for these options: --namespace=m6 ERROR: No broadcast to cancel/clear for these options: --namespace=m7 ERROR: No broadcast to cancel/clear for these options: --namespace=ENS3 cylc-flow-8.7.0/tests/functional/broadcast/00-simple/flow.cylc0000664000175000017500000001236415264420307024421 0ustar alastairalastair[meta] title = "test workflow for broadcast functionality" description = """ The first task broadcasts an environment variable "BCAST" to various cycles and namespaces. Then each task writes its point, name, and value of BCAST to a log for comparison with the expected result. """ [scheduler] cycle point format = %Y%m%dT%H allow implicit tasks = True [scheduling] initial cycle point = 20100808T00 final cycle point = 20100809T00 # Ensure the first cycle finishes before the second (last) one, so that the # broadcast to foo in the first cycle expires before shutdown: runahead limit = P0 [[graph]] R1 = "prep => check & foo" T00 = """ foo => bar & baz & qux & wibble => ENS? ENS:finish-all => wobble """ [runtime] [[root]] pre-script = "echo $CYLC_TASK_CYCLE_POINT $CYLC_TASK_NAME BCAST is $BCAST | tee -a $BCASTLOG" script = "true" # no sleep [[[environment]]] BCAST = ${BCAST:-(not set)} BCASTLOG = ${CYLC_WORKFLOW_SHARE_DIR}/broadcast.log PREPLOG = ${CYLC_WORKFLOW_SHARE_DIR}/prep [[prep]] pre-script = "rm -f $BCASTLOG $PREPLOG" script = """ set +x { # broadcast to all cycles and namespaces: cylc broadcast -s "[environment]BCAST = ROOT" $CYLC_WORKFLOW_ID # broadcast to 20100808T00/foo: cylc broadcast -p 20100808T00 -n foo -s "[environment]BCAST = FOO" $CYLC_WORKFLOW_ID # broadcast to bar at all cycles: cylc broadcast -n bar -s "[environment]BCAST = BAR" $CYLC_WORKFLOW_ID # broadcast to baz at 20100809T00: cylc broadcast -n baz -p 20100809T00 -s "[environment]BCAST = BAZ" $CYLC_WORKFLOW_ID # broadcast to qux at 20100809T00, then cancel it: cylc broadcast -n qux -p 20100809T00 -s "[environment]BCAST = QUX" $CYLC_WORKFLOW_ID cylc broadcast -n qux -p 20100809T00 --cancel "[environment]BCAST" $CYLC_WORKFLOW_ID # broadcast to wibble at all cycles, then clear it: cylc broadcast -n wibble -s "[environment]BCAST = WIBBLE" $CYLC_WORKFLOW_ID cylc broadcast -n wibble --clear $CYLC_WORKFLOW_ID # broadcast to all members of ENS, all cycles: cylc broadcast -n ENS -s "[environment]BCAST = ENS" $CYLC_WORKFLOW_ID # broadcast to all members of ENS1, all cycles: cylc broadcast -n ENS1 -s "[environment]BCAST = ENS1" $CYLC_WORKFLOW_ID # broadcast to a single member m2 of ENS1, in 20100809T00: cylc broadcast -n m2 -p 20100809T00 -s "[environment]BCAST = M2" $CYLC_WORKFLOW_ID # cancel broadcast to m4 of ENS1, in 20100809T00 (will not work): ! cylc broadcast -n m4 -p 20100809T00 --cancel "[environment]BCAST" $CYLC_WORKFLOW_ID # cancel broadcast to m5 of ENS1 at all cycles (will not work): ! cylc broadcast -n m5 --cancel "[environment]BCAST" $CYLC_WORKFLOW_ID # clear broadcast to m6 of ENS1 at all cycles (will not work): ! cylc broadcast -n m6 --clear $CYLC_WORKFLOW_ID # clear, then reset, broadcast to m7 of ENS1 at all cycles: ! cylc broadcast -n m7 --clear $CYLC_WORKFLOW_ID cylc broadcast -n m7 -s "[environment]BCAST = M7" $CYLC_WORKFLOW_ID # reset broadcast to m8 of ENS1 at 20100809T00 cylc broadcast -n m8 -s "[environment]BCAST = M8" $CYLC_WORKFLOW_ID # reset broadcast to m9 of ENS1 at all cycles cylc broadcast -n m9 -s "[environment]BCAST = M9" $CYLC_WORKFLOW_ID # clear broadcast for ENS3 (will not work): ! cylc broadcast -n ENS3 --clear $CYLC_WORKFLOW_ID } 1>${PREPLOG}.out 2>${PREPLOG}.err """ [[check]] # Check that the broadcasts performed by the previous task were # recorded properly by the scheduler (doing this in another task # gives time for the datastore to update broadcast data). script = """ # list the result to prep task stdout: cylc broadcast --display $CYLC_WORKFLOW_ID \ 1>>${PREPLOG}.out 2>>${PREPLOG}.err set -x sed -i '/DEBUG -/d' ${PREPLOG}.out sed -i '/\(DEBUG\|WARNING\|ERROR\) -/d' ${PREPLOG}.err # workaround for platforms affected by https://github.com/cylc/cylc-flow/issues/3585 sed -i '/BASH_XTRACEFD/d' ${PREPLOG}.err diff -u "${CYLC_WORKFLOW_RUN_DIR}/expected-prep.out" ${PREPLOG}.out diff -u "${CYLC_WORKFLOW_RUN_DIR}/expected-prep.err" ${PREPLOG}.err """ [[ENS]] [[ENS1]] inherit = ENS [[m1,m2,m3,m4,m5,m6,m7,m8,m9]] inherit = ENS1 [[ENS2]] inherit = ENS [[n1]] inherit = ENS2 [[ENS3]] inherit = ENS [[o1]] inherit = ENS3 [[wobble]] script = """ if [[ "${CYLC_TASK_CYCLE_POINT}" == "20100809T00" ]]; then sleep 5 fi """ cylc-flow-8.7.0/tests/functional/broadcast/02-inherit.t0000775000175000017500000000173415264420307023140 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test broadcasts, with overriding inheritance. . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/broadcast/14-broadcast-checkpoint/0000775000175000017500000000000015264420307025373 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/broadcast/14-broadcast-checkpoint/flow.cylc0000664000175000017500000000064415264420307027222 0ustar alastairalastair[scheduler] UTC mode = True [scheduling] [[graph]] R1 = "t1 => t2" [runtime] [[t1]] script = """ cylc broadcast -s "[environment]VERSE = the quick brown fox" "${CYLC_WORKFLOW_ID}" cylc checkpoint "${CYLC_WORKFLOW_ID}" test1 """ [[t2]] script = """ cylc broadcast -s "[environment]PHRASE = the quick brown fox" "${CYLC_WORKFLOW_ID}" cylc checkpoint "${CYLC_WORKFLOW_ID}" test2 """ cylc-flow-8.7.0/tests/functional/broadcast/07-timeout.t0000775000175000017500000000172015264420307023164 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test broadcasts a timeout setting . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/broadcast/13-file-cancel.t0000775000175000017500000000173415264420307023642 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test broadcast cancel with settings in a file . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/broadcast/10-file-1.t0000775000175000017500000000172015264420307022545 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test broadcast settings in a file . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/cylc-subscribe/0000775000175000017500000000000015264420307022031 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-subscribe/01-subscribe.t0000664000175000017500000000350015264420307024413 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test `cylc subscribe`. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 6 #------------------------------------------------------------------------------- init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduling] [[graph]] R1 = foo => bar => wiq => qux [runtime] [[foo, bar, wiq, qux]] script = sleep 2 __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-subscribe-1" run_ok "${TEST_NAME}" cylc subscribe --once --topics="workflow" "${WORKFLOW_NAME}" grep_ok "lastUpdated" "${TEST_NAME}.stdout" TEST_NAME="${TEST_NAME_BASE}-subscribe-2" run_ok "${TEST_NAME}" cylc subscribe --once --topics="workflow" "${WORKFLOW_NAME}" grep_ok "lastUpdated" "${TEST_NAME}.stdout" cylc stop --kill --max-polls=20 --interval=1 "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/cylc-subscribe/test_header0000777000175000017500000000000015264420307030346 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/platforms/0000775000175000017500000000000015264420307021127 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/platforms/03-platform-db/0000775000175000017500000000000015264420307023556 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/platforms/03-platform-db/flow.cylc0000664000175000017500000000034415264420307025402 0ustar alastairalastair[scheduling] [[graph]] R1 = frozen & disney [runtime] [[frozen]] script = "echo let it go" platform = elsa [[disney]] script = "echo cant hold it back anymore" platform = olaf cylc-flow-8.7.0/tests/functional/platforms/08-warn-if-regex-matches-localhost.t0000775000175000017500000000365015264420307027633 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check that ``[platforms][localhost]`` is only set automatically if it # not set in ``global.cylc``. . "$(dirname "$0")/test_header" set_test_number 4 # shellcheck disable=SC2016 create_test_global_config '' ' [platforms] [[localh...]] # This should not override `localh...` in this one case, because the # localhost default pins it to the top of the list. [[localhost]] [[[meta]]] foo = "foo" ' make_rnd_workflow cat > "${RND_WORKFLOW_SOURCE}/flow.cylc" <<__HEREDOC__ [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = foo __HEREDOC__ ERR_STR='cannot be defined using a regular expression' TEST_NAME="${TEST_NAME_BASE}-validate" run_fail "${TEST_NAME}" cylc validate "${RND_WORKFLOW_SOURCE}" grep_ok "${ERR_STR}" \ "${TEST_NAME}.stderr" -F TEST_NAME="${TEST_NAME_BASE}-cylc-install" run_fail "${TEST_NAME}" cylc install \ "${RND_WORKFLOW_SOURCE}" \ --workflow-name "${RND_WORKFLOW_NAME}" grep_ok "${ERR_STR}" \ "${TEST_NAME}.stderr" -F purge_rnd_workflow exit cylc-flow-8.7.0/tests/functional/platforms/06-host-to-platform-upgrade-dummy-mode.t0000664000175000017500000000277215264420307030466 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check that setting the platform to localhost for dummy mode doesn't # cause conflicts with Cylc 7 settings # TODO Remove test at Cylc 8.x. . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # Ensure that you can validate workflow run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}-run" \ cylc play "${WORKFLOW_NAME}" --no-detach --mode=dummy # Check that the upgradeable config has been run on a sensible host. grep_ok \ "(dummy job succeed)"\ "${WORKFLOW_RUN_DIR}/log/job/1/upgradeable_cylc7_settings/NN/job.out" purge exit cylc-flow-8.7.0/tests/functional/platforms/14-trigger-paused-subshell/0000775000175000017500000000000015264420307026112 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/platforms/14-trigger-paused-subshell/flow.cylc0000664000175000017500000000061015264420307027732 0ustar alastairalastair#!jinja2 [scheduling] [[graph]] R1 = foo:never # Allows triggering repeatedly without workflow shutting down [runtime] [[foo]] platform = $("${CYLC_WORKFLOW_RUN_DIR}/toggle_platform.sh" {{ CYLC_TEST_PLATFORM }}) # Check what host we have truly run on: script = cylc message -- "$(hostname)" [[[outputs]]] never = gonna_give_u_up cylc-flow-8.7.0/tests/functional/platforms/14-trigger-paused-subshell/toggle_platform.sh0000775000175000017500000000242115264420307031635 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Script that outputs the current platform written in the hall file and changes # it to the other one. This ensures the platform subshell result will alternate # each time it is called. remote_platform=$1 hall_file="${CYLC_WORKFLOW_RUN_DIR}/pretend_hall_info" if [[ ! -f "${hall_file}" ]]; then current=localhost else current=$(cat "$hall_file") fi echo "$current" if [[ "$current" == localhost ]]; then echo "$remote_platform" > "$hall_file" else echo localhost > "$hall_file" fi cylc-flow-8.7.0/tests/functional/platforms/14-trigger-paused-subshell.t0000664000175000017500000000527415264420307026307 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Triggering a task with a subshell platform setting while the workflow is # paused should only evaluate the subshell once. # Here we have a subshell command that alternates between two platforms on each # call, to check the platform does not change during a manual trigger. # https://github.com/cylc/cylc-flow/issues/6994 export REQUIRE_PLATFORM='loc:remote' . "$(dirname "$0")/test_header" set_test_number 11 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" local_host_name=$(hostname) remote_host_name=$(cylc config -i "[platforms][${CYLC_TEST_PLATFORM}]hosts") remote_host_name=$(ssh -oStrictHostKeyChecking=no "$remote_host_name" hostname) workflow_log="${WORKFLOW_RUN_DIR}/log/scheduler/log" # shellcheck disable=SC2034 # LOG_SCAN_GREP_OPTS is not unused LOG_SCAN_GREP_OPTS="-E" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" cylc play "${WORKFLOW_NAME}" --pause poll_grep_workflow_log "1/foo:waiting" cylc trigger "${WORKFLOW_NAME}//1/foo" log_scan "log-grep-01" "$workflow_log" 10 2 \ "\[1/foo/01:preparing\] submitted to localhost" \ "\[1/foo/01:.*\] \(received\)${local_host_name}" \ "\[1/foo/01:.*\] => succeeded" cylc trigger "${WORKFLOW_NAME}//1/foo" log_scan "log-grep-02" "$workflow_log" 10 2 \ "\[1/foo/02:preparing\] submitted to ${CYLC_TEST_PLATFORM}" \ "\[1/foo/02:.*\] \((received|polled)\)${remote_host_name}" \ "\[1/foo/02:.*\] => succeeded" cylc trigger "${WORKFLOW_NAME}//1/foo" log_scan "log-grep-03" "$workflow_log" 10 2 \ "\[1/foo/03:preparing\] submitted to localhost" \ "\[1/foo/03:.*\] \(received\)${local_host_name}" \ "\[1/foo/03:.*\] => succeeded" cylc stop "${WORKFLOW_NAME}" --now --now # Check DB as well: sqlite3 "${WORKFLOW_RUN_DIR}/.service/db" \ "SELECT submit_num, platform_name FROM task_jobs" > task_jobs.out cmp_ok task_jobs.out <<__EOF__ 1|localhost 2|${CYLC_TEST_PLATFORM} 3|localhost __EOF__ poll_workflow_stopped purge cylc-flow-8.7.0/tests/functional/platforms/10-do-not-host-check-platforms.t0000775000175000017500000000333115264420307026770 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check that platform names are not treated as host names. E.g. a platform # name starting with "localhost" should not be treated as localhost. # https://github.com/cylc/cylc-flow/issues/5342 . "$(dirname "$0")/test_header" set_test_number 2 # shellcheck disable=SC2016 create_test_global_config '' ' [platforms] [[localhost_spice]] hosts = unreachable ' make_rnd_workflow cat > "${RND_WORKFLOW_SOURCE}/flow.cylc" <<__HEREDOC__ [scheduler] [[events]] stall timeout = PT0S [scheduling] [[graph]] R1 = foo [runtime] [[foo]] platform = localhost_spice __HEREDOC__ ERR_STR='Unable to find contactable host for platform: localhost_spice' TEST_NAME="${TEST_NAME_BASE}-vip-workflow" run_fail "${TEST_NAME}" cylc vip "${RND_WORKFLOW_SOURCE}" --no-detach grep_ok "${ERR_STR}" \ "${TEST_NAME}.stderr" -F purge_rnd_workflow exit cylc-flow-8.7.0/tests/functional/platforms/02-host-to-platform-upgrade/0000775000175000017500000000000015264420307026212 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/platforms/02-host-to-platform-upgrade/reference.log0000664000175000017500000000045215264420307030654 0ustar alastairalastairInitial point: 1 Final point: 1 1/fin -triggered off ['1/no_settings', '1/t1', '1/t2'] 2/fin -triggered off ['2/no_settings', '2/t1', '2/t2'] 1/no_settings -triggered off [] 2/no_settings -triggered off [] 1/t1 -triggered off [] 2/t1 -triggered off [] 1/t2 -triggered off [] 2/t2 -triggered off [] cylc-flow-8.7.0/tests/functional/platforms/02-host-to-platform-upgrade/flow.cylc0000664000175000017500000000130515264420307030034 0ustar alastairalastair#!jinja2 [scheduler] UTC mode = True [scheduling] cycling mode = integer initial cycle point = 1 final cycle point = 2 # Note: we check the upgrade persists over > 1 cycle # https://github.com/cylc/cylc-flow/issues/4167 [[graph]] P1 = """ no_settings & t1 & t2 => fin """ [runtime] [[root]] script = true [[no_settings, fin]] [[t1]] script = test {{CYLC_TEST_HOST_FQDN}} == "$(hostname -f)" [[[remote]]] host = {{CYLC_TEST_HOST}} [[T2]] script = test {{CYLC_TEST_HOST_FQDN}} == "$(hostname -f)" [[[remote]]] host = {{CYLC_TEST_HOST}} [[t2]] inherit = T2 cylc-flow-8.7.0/tests/functional/platforms/02-host-to-platform-upgrade/bad/0000775000175000017500000000000015264420307026740 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/platforms/02-host-to-platform-upgrade/bad/flow.cylc0000664000175000017500000000053115264420307030562 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [scheduling] [[graph]] R1 = """ just_wrong """ [runtime] [[root]] script = true [[just_wrong]] platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[[remote]]] host = 'parasite' [[[job]]] batch system = 'loaf' cylc-flow-8.7.0/tests/functional/platforms/01-platform-basic.t0000775000175000017500000000265115264420307024444 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test validating platforms in workflow. . "$(dirname "$0")/test_header" set_test_number 1 TEST_NAME="${TEST_NAME_BASE}-val" create_test_global_config "" " [platforms] [[localhost, lewis]] hosts = localhost install target = localhost " cat >'flow.cylc' <<'__FLOW_CONFIG__' [meta] title = "Test validation of simple multiple inheritance" description = """Bug identified at 5.1.1-314-g4960684.""" [scheduling] [[graph]] R1 = """foo""" [runtime] [[foo]] platform=lewis __FLOW_CONFIG__ run_ok "${TEST_NAME}" cylc validate . cylc-flow-8.7.0/tests/functional/platforms/03-platform-db.t0000664000175000017500000000330215264420307023741 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check db stores correct platform export REQUIRE_PLATFORM='loc:remote comms:?(tcp|ssh)' . "$(dirname "$0")/test_header" set_test_number 3 create_test_global_config '' " [platforms] [[elsa]] hosts = ${CYLC_TEST_HOST} install target = ${CYLC_TEST_INSTALL_TARGET} [[olaf]] hosts = ${CYLC_TEST_HOST} install target = ${CYLC_TEST_INSTALL_TARGET} " install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" DB_FILE="${WORKFLOW_RUN_DIR}/log/db" NAME='select-name-platform.out' sqlite3 "${DB_FILE}" 'SELECT name, platform_name FROM task_jobs ORDER BY name' \ >"${NAME}" cmp_ok "${NAME}" <<__SELECT__ disney|olaf frozen|elsa __SELECT__ purge exit cylc-flow-8.7.0/tests/functional/platforms/07-localhost-set-if-not-in-globalcfg/0000775000175000017500000000000015264420307027650 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/platforms/07-localhost-set-if-not-in-globalcfg/flow.cylc0000664000175000017500000000014515264420307031473 0ustar alastairalastair[meta] [scheduling] [[graph]] R1 = """foo""" [runtime] [[foo]] script = true cylc-flow-8.7.0/tests/functional/platforms/12-ping-pong/0000775000175000017500000000000015264420307023245 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/platforms/12-ping-pong/flow.cylc0000664000175000017500000000172115264420307025071 0ustar alastairalastair#!jinja2 [scheduler] [[events]] stall timeout = PT0S [scheduling] cycling mode = integer final cycle point = 2 [[graph]] P1 = remote_task[-P1] => toggler => remote_task [runtime] [[remote_task]] platform = $(cat ${CYLC_WORKFLOW_RUN_DIR}/pretend-hall-info) [[toggler]] # Toggle the platform between localhost and the remote host # using the content of a file, ${CYLC_WORKFLOW_RUN_DIR}/pretend-hall-info. script = """ if (( $CYLC_TASK_CYCLE_POINT % 2 == 1 )); then echo ${REMOTE_PLATFORM} > ${CYLC_WORKFLOW_RUN_DIR}/pretend-hall-info cylc message -- "changing platform to ${REMOTE_PLATFORM}" else echo "localhost" > ${CYLC_WORKFLOW_RUN_DIR}/pretend-hall-info cylc message -- "changing platform to localhost" fi """ [[[environment]]] REMOTE_PLATFORM = {{ CYLC_TEST_PLATFORM }} cylc-flow-8.7.0/tests/functional/platforms/05-host-to-platform-upgrade-fail/0000775000175000017500000000000015264420307027126 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/platforms/05-host-to-platform-upgrade-fail/flow.cylc0000664000175000017500000000103015264420307030743 0ustar alastairalastair[meta] description = """ The task settings for not_upgradable_cylc7_settings should not match any platform: The workflow should fail as a result. """ [scheduler] UTC mode = True [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] [[graph]] R1 = """ not_upgradable_cylc7_settings """ [runtime] [[not_upgradable_cylc7_settings]] script = True [[[remote]]] host = parasite [[[job]]] batch system = 'loaf' cylc-flow-8.7.0/tests/functional/platforms/12-ping-pong.t0000664000175000017500000000320015264420307023425 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #----------------------------------------------------------------------------- # If a task has a platform set using a subshell this should be evaluated # every time the task is run. # https://github.com/cylc/cylc-flow/issues/6808 export REQUIRE_PLATFORM='loc:remote' . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play "${WORKFLOW_NAME}" --debug --no-detach named_grep_ok \ "${TEST_NAME_BASE}-remote_task-submits-to-${CYLC_TEST_PLATFORM}" \ "\[1/remote_task/01:preparing\] submitted to ${CYLC_TEST_PLATFORM}" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" named_grep_ok \ "${TEST_NAME_BASE}-remote_task-submits-to-localhost" \ "\[2/remote_task/01:preparing\] submitted to localhost" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" purge cylc-flow-8.7.0/tests/functional/platforms/02-host-to-platform-upgrade.t0000664000175000017500000000516315264420307026404 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check that platform upgraders work sensibly. # The following scenarios should be covered: # - Task with no settings # - Task with a host setting that should match the test platform # Note: We cannot run this test with job runners that may be configured to # submit the job to a compute node (e.g. pbs, slurm, etc) as we are unable # to check the hostname export REQUIRE_PLATFORM='loc:remote runner:?(background|at)' . "$(dirname "$0")/test_header" set_test_number 7 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # Ensure that a mix of syntax will fail. run_fail "${TEST_NAME_BASE}-validate-fail" \ cylc validate "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/bad" # and fail in this specific manner: grep_ok "cannot be used with" "${TEST_NAME_BASE}-validate-fail.stderr" # Ensure that you can validate workflow run_ok "${TEST_NAME_BASE}-run" \ cylc validate "${WORKFLOW_NAME}" \ -s "CYLC_TEST_HOST='${CYLC_TEST_HOST}'" \ -s CYLC_TEST_HOST_FQDN="'$(ssh "$CYLC_TEST_HOST" hostname -f)'" # Check that the cfgspec/workflow.py has issued a warning about upgrades. grep_ok "\[t1\]\[remote\]host = ${CYLC_TEST_HOST}"\ "${TEST_NAME_BASE}-run.stderr" # the namespace with the host setting will be logged not the task that # inherits from it (because it happens in the cfgspec not the config) grep_ok "\[T2\]\[remote\]host = ${CYLC_TEST_HOST}"\ "${TEST_NAME_BASE}-run.stderr" # Run the workflow workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test \ -s CYLC_TEST_HOST="'$CYLC_TEST_HOST'" \ -s CYLC_TEST_HOST_FQDN="'$(ssh "$CYLC_TEST_HOST" hostname -f)'" \ "${WORKFLOW_NAME}" grep "host=" "${WORKFLOW_RUN_DIR}/log/scheduler/log" > hosts.log grep_ok "\[2/t2.*\].*host=${CYLC_TEST_HOST}" hosts.log purge exit cylc-flow-8.7.0/tests/functional/platforms/05-host-to-platform-upgrade-fail.t0000664000175000017500000000363215264420307027317 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check that platform upgraders fail if no platform can be found which # matches host settings. export REQUIRE_PLATFORM='loc:remote' . "$(dirname "$0")/test_header" set_test_number 4 create_test_global_config '' " [platforms] [[${CYLC_TEST_PLATFORM}]] retrieve job logs = True " install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # Both of these cases should validate ok. run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" \ -s "CYLC_TEST_HOST='${CYLC_TEST_HOST}'" # Check that the cfgspec/workflow.py has issued a warning about upgrades. grep_ok "\[not_upgradable_cylc7_settings\]\[remote\]host = parasite"\ "${TEST_NAME_BASE}-validate.stderr" # Run the workflow workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach \ -s "CYLC_TEST_HOST='${CYLC_TEST_HOST}'" "${WORKFLOW_NAME}" # Check that the workflow failed because no matching platform could be found. grep_ok "\[jobs-submit err\] No platform found matching your task"\ "${WORKFLOW_RUN_DIR}/log/scheduler/log" purge exit cylc-flow-8.7.0/tests/functional/platforms/09-default-directives.t0000775000175000017500000000431715264420307025335 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Platforms can have default directives and these are overridden as expected. export REQUIRE_PLATFORM="runner:slurm" . "$(dirname "$0")/test_header" set_test_number 5 create_test_global_config "" " [platforms] [[no_default, default_only, overridden, neither]] hosts = localhost job runner = slurm [[default_only, overridden]] [[[directives]]] --wurble=foo " init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONF__' [scheduler] [[events]] stall timeout = PT0S [scheduling] [[graph]] R1 = lewis & morse & barnaby & cadfael [runtime] [[lewis]] platform = no_default [[[directives]]] --wurble=bar [[morse]] platform = default_only [[barnaby]] platform = overridden [[[directives]]] --wurble=qux [[cadfael]] platform = neither __FLOW_CONF__ run_fail "${TEST_NAME_BASE}-play" \ cylc play "${WORKFLOW_NAME}" --no-detach LOG_DIR="${RUN_DIR}/${WORKFLOW_NAME}/log/job/1/" named_grep_ok "${TEST_NAME_BASE}-no-default" "--wurble=bar" "${LOG_DIR}/lewis/NN/job" named_grep_ok "${TEST_NAME_BASE}-default-only" "--wurble=foo" "${LOG_DIR}/morse/NN/job" named_grep_ok "${TEST_NAME_BASE}-overridden" "--wurble=qux" "${LOG_DIR}/barnaby/NN/job" grep_fail "--wurble" "${LOG_DIR}/cadfael/NN/job" purge exit 0 cylc-flow-8.7.0/tests/functional/platforms/11-platform-gone-on-restart.t0000664000175000017500000000417415264420307026407 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # If a platform is deleted from global config and restart cannot find an # approproiate match, don't keep going. # 1. Run a workflow which stops leaving a job running on a platform. # 2. Delete the platform from global.cylc # 3. Attempt to restart. # 4. Check that restart fails in the desired manner. . "$(dirname "$0")/test_header" set_test_number 3 create_test_global_config "" " [platforms] [[myplatform]] hosts = localhost install target = localhost " init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONF__' [scheduling] initial cycle point = 2934 [[graph]] R1 = foo => bar [runtime] [[foo]] script = """ cylc stop ${CYLC_WORKFLOW_ID} --now --now """ platform = myplatform [[bar]] script = true # only runs on restart __FLOW_CONF__ run_ok "${TEST_NAME_BASE}-play" \ cylc play "${WORKFLOW_NAME}" --no-detach # Wait for workflow to stop, then wreck the global config: create_test_global_config "" " " # Test that restart fails: run_fail "${TEST_NAME_BASE}-restart" \ cylc play "${WORKFLOW_NAME}" --no-detach named_grep_ok \ "${TEST_NAME_BASE}-cannot-restart" \ "platforms are not defined in the global.cylc" \ "${RUN_DIR}/${WORKFLOW_NAME}/log/scheduler/log" purge cylc-flow-8.7.0/tests/functional/platforms/04-host-to-platform-upgrade-fail-inherit/0000775000175000017500000000000015264420307030565 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/platforms/04-host-to-platform-upgrade-fail-inherit/flow.cylc0000664000175000017500000000050615264420307032411 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [scheduling] [[graph]] R1 = """ non-valid-child """ [runtime] [[root]] script = true [[VALID_PARENT]] [[[remote]]] host = parasite [[non-valid-child]] inherit = VALID_PARENT platform = _wibble cylc-flow-8.7.0/tests/functional/platforms/06-host-to-platform-upgrade-dummy-mode/0000775000175000017500000000000015264420307030271 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/platforms/06-host-to-platform-upgrade-dummy-mode/flow.cylc0000664000175000017500000000056115264420307032116 0ustar alastairalastair#!jinja2 [scheduler] UTC mode = True [scheduling] [[graph]] R1 = """ upgradeable_cylc7_settings """ [runtime] [[upgradeable_cylc7_settings]] script = echo "In Dummy Mode this shouldn't be written to job.out" [[[remote]]] host = localhost [[[job]]] batch system = background batch submit command template = timeout 10 cylc-flow-8.7.0/tests/functional/platforms/13-ping-pong-host/0000775000175000017500000000000015264420307024221 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/platforms/13-ping-pong-host/flow.cylc0000664000175000017500000000201715264420307026044 0ustar alastairalastair#!jinja2 [scheduler] [[events]] stall timeout = PT0S [scheduling] cycling mode = integer final cycle point = 2 [[graph]] P1 = remote_task[-P1] => toggler => remote_task [runtime] [[remote_task]] [[[remote]]] host = $(cat ${CYLC_WORKFLOW_RUN_DIR}/pretend-hall-info) [[toggler]] script = """ # Toggle the platform between localhost and the remote host # using the content of a file, ${CYLC_WORKFLOW_RUN_DIR}/pretend-hall-info. # between localhost and the remote. if (( $CYLC_TASK_CYCLE_POINT % 2 == 1 )); then echo ${REMOTE_HOST} > ${CYLC_WORKFLOW_RUN_DIR}/pretend-hall-info cylc message -- "changing platform to ${REMOTE_HOST}" else echo "localhost" > ${CYLC_WORKFLOW_RUN_DIR}/pretend-hall-info cylc message -- "changing platform to localhost" fi """ [[[environment]]] REMOTE_HOST = {{ CYLC_TEST_HOST }} cylc-flow-8.7.0/tests/functional/platforms/test_header0000777000175000017500000000000015264420307027444 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/platforms/13-ping-pong-host.t0000664000175000017500000000307615264420307024414 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #----------------------------------------------------------------------------- # If a task has [remote]host=$(subshell) this should be evaluated # every time the task is run. # https://github.com/cylc/cylc-flow/issues/6808 export REQUIRE_PLATFORM='loc:remote runner:background' . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play "${WORKFLOW_NAME}" --debug --no-detach named_grep_ok "${TEST_NAME_BASE}-remote" \ "\[1/remote_task/01:preparing\] submitted to ${CYLC_TEST_PLATFORM}" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" named_grep_ok "${TEST_NAME_BASE}-local" \ "\[2/remote_task/01:preparing\] submitted to localhost" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" purge cylc-flow-8.7.0/tests/functional/platforms/07-localhost-set-if-not-in-globalcfg.t0000775000175000017500000000301115264420307030033 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check that ``[platforms][localhost]`` is only set automatically if it # not set in ``global.cylc``. export REQUIRE_PLATFORM='runner:at' . "$(dirname "$0")/test_header" set_test_number 3 create_test_global_config "" " [platforms] [[localhost, nine_and_three_quarters]] hosts = localhost job runner = at " install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" # Run the workflow workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" grep_ok "Job runner: at" "${WORKFLOW_RUN_DIR}/log/job/1/foo/NN/job" purge exit cylc-flow-8.7.0/tests/functional/platforms/04-host-to-platform-upgrade-fail-inherit.t0000664000175000017500000000270215264420307030753 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Parent and child tasks are both valid, before inheritance calculated. # Child function not valid after inheritance. # Check for task failure at job-submit. . "$(dirname "$0")/test_header" set_test_number 2 create_test_global_config '' " # non-existent platform [platforms] [[_wibble]] " install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # Run the workflow workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --no-detach "${WORKFLOW_NAME}" grep_ok "Task 'non-valid-child' has the following deprecated '\[runtime\]' setting(s)" \ "${TEST_NAME_BASE}-run.stderr" purge exit cylc-flow-8.7.0/tests/functional/rnd/0000775000175000017500000000000015264420307017703 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/rnd/04-conditions.t0000664000175000017500000000254115264420307022464 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- # check `cylc help license` run_ok "${TEST_NAME_BASE}" cylc help license # remove trailing newlines sed -i -e :a -e '/^\n*$/{$d;N;};/\n$/ba' "${TEST_NAME_BASE}.stdout" # check `cylc help license` output matches the COPYING file cmp_ok "${CYLC_REPO_DIR}/COPYING" "${TEST_NAME_BASE}.stdout" cylc-flow-8.7.0/tests/functional/rnd/03-check-versions.t0000664000175000017500000000336415264420307023241 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test cylc check-versions # WARNING: This is testing your remote platform setup, if you do not # have the exactly same version of Cylc installed locally and # remotely the test will fail. export REQUIRE_PLATFORM='loc:remote fs:indep' . "$(dirname "$0")/test_header" set_test_number 3 #------------------------------------------------------------------------------- init_workflow "${TEST_NAME_BASE}" <<__FLOW__ [scheduler] [[events]] abort on stall timeout = True stall timeout=PT30S [scheduling] [[graph]] R1 = foo [runtime] [[foo]] platform = $CYLC_TEST_PLATFORM __FLOW__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-check" run_ok "${TEST_NAME}" cylc check-versions "${WORKFLOW_NAME}" contains_ok "${TEST_NAME}.stdout" <<__HERE__ $CYLC_TEST_PLATFORM: $(cylc version) __HERE__ purge cylc-flow-8.7.0/tests/functional/rnd/00-run-funtional-tests.t0000664000175000017500000000567315264420307024261 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test the run-functional-tests --chunk option . "$(dirname "$0")/test_header" set_test_number 16 #------------------------------------------------------------------------------- CTB="${CYLC_REPO_DIR}/etc/bin/run-functional-tests" unset CHUNK # ensure that 'tests/f' is used as the default test base TEST_NAME="${TEST_NAME_BASE}-base" run_ok "${TEST_NAME}-1" "$CTB" --dry run_ok "${TEST_NAME}-2" "$CTB" --dry tests/functional sort -o "${TEST_NAME}-1.stdout" "${TEST_NAME}-1.stdout" sort -o "${TEST_NAME}-2.stdout" "${TEST_NAME}-2.stdout" cmp_ok "${TEST_NAME}-1.stdout" "${TEST_NAME}-2.stdout" TEST_NAME="${TEST_NAME_BASE}-chunk-base" run_ok "${TEST_NAME}-1" env CHUNK="1/4" "$CTB" --dry run_ok "${TEST_NAME}-2" env CHUNK="1/4" "$CTB" --dry tests/functional sort -o "${TEST_NAME}-1.stdout" "${TEST_NAME}-1.stdout" sort -o "${TEST_NAME}-2.stdout" "${TEST_NAME}-2.stdout" cmp_ok "${TEST_NAME}-1.stdout" "${TEST_NAME}-2.stdout" # ensure that mixing test bases works correctly TEST_NAME="${TEST_NAME_BASE}-testbase" run_ok "${TEST_NAME}-1" "$CTB" --dry tests/f run_ok "${TEST_NAME}-2" "$CTB" --dry tests/k run_ok "${TEST_NAME}-3" "$CTB" --dry tests/f tests/k cat "${TEST_NAME}-2.stdout" >> "${TEST_NAME}-1.stdout" sort -o "${TEST_NAME}-1.stdout" "${TEST_NAME}-1.stdout" sort -o "${TEST_NAME}-3.stdout" "${TEST_NAME}-3.stdout" cmp_ok "${TEST_NAME}-1.stdout" "${TEST_NAME}-3.stdout" # ensure that the whole is equal to the sum of its parts N_CHUNKS=4 DRY_TEST_NAME="${TEST_NAME_BASE}-all" run_ok "${DRY_TEST_NAME}" "${CTB}" --dry 'tests/f' 'tests/k' # list tests for each chunk (from prove not run-functional-tests) for i_chunk in $(seq "${N_CHUNKS}"); do TEST_NAME="${TEST_NAME_BASE}-chunk_n-${i_chunk}" run_ok "${TEST_NAME}" env CHUNK="${i_chunk}/${N_CHUNKS}" "${CTB}" --dry 'tests/f' 'tests/k' cat "${TEST_NAME}.stdout" >>'chunks.out' done # sort files ($CYLC_REPO_DIR/etc/bin/run-functional-tests uses --shuffle) sort -o "${DRY_TEST_NAME}.stdout" "${DRY_TEST_NAME}.stdout" sort -o 'chunks.out' 'chunks.out' # compare test plan for the full and chunked versions cmp_ok "${DRY_TEST_NAME}.stdout" 'chunks.out' exit cylc-flow-8.7.0/tests/functional/rnd/07-completion-server.t0000664000175000017500000000361315264420307023774 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . . "$(dirname "$0")/test_header" set_test_number 6 # NOTE: The completion server is heavily unit-tested # Run a couple of quick functional tests to make sure the CLI interface / reading # from stdin is working correctly. # $ cylc t # tui # trigger TEST_NAME="${TEST_NAME_BASE}-t" run_ok "${TEST_NAME}" cylc completion-server --once <<< 'cylc|t' grep_ok trigger "${TEST_NAME}.stdout" grep_ok tui "${TEST_NAME}.stdout" # $ cylc trigg # trigger TEST_NAME="${TEST_NAME_BASE}-trigg" run_ok "${TEST_NAME}" cylc completion-server --once <<< 'cylc|trigg' cmp_ok "${TEST_NAME}.stdout" << __HERE__ trigger __HERE__ # Make sure the server exits timeout when trying to read from stdin # (Note the completion server exits 0 on timeout) TEST_NAME="${TEST_NAME_BASE}-timeout" # shellcheck disable=SC2162 if ! read -t 0 -N 0; then # If stdin is not a terminal or contains readable data this test will # fail. run_ok "${TEST_NAME}" timeout 5 cylc completion-server --timeout=1 else # This should happen in non-interactive environments e.g. CI skip 1 'Test requires an stdin stream (cannot be faked)' fi exit cylc-flow-8.7.0/tests/functional/rnd/08-cylc-list.t0000664000175000017500000000330015264420307022214 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Functional test of the cylc-list command # (see integration tests for more comprehensive tests) . "$(dirname "$0")/test_header" set_test_number 4 cat > flow.cylc <<__FLOW__ [scheduling] [[graph]] R1 = c [runtime] [[A]] [[[meta]]] title = Aaa [[B]] inherit = A [[[meta]]] title = Bbb [[c]] inherit = B [[[meta]]] title = Ccc __FLOW__ # test signals on a detached scheduler TEST_NAME="${TEST_NAME_BASE}-list" run_ok "$TEST_NAME" cylc list '.' --all-namespaces --with-titles cmp_ok "${TEST_NAME}.stdout" << __HERE__ A Aaa B Bbb c Ccc root __HERE__ TEST_NAME="${TEST_NAME_BASE}-tree" run_ok "$TEST_NAME" cylc list '.' --tree --with-titles cmp_ok "${TEST_NAME}.stdout" << '__HERE__' root `-A `-B `-c Ccc __HERE__ cylc-flow-8.7.0/tests/functional/rnd/02-lib-python-in-job/0000775000175000017500000000000015264420307023363 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/rnd/02-lib-python-in-job/lib/0000775000175000017500000000000015264420307024131 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/rnd/02-lib-python-in-job/lib/python/0000775000175000017500000000000015264420307025452 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/rnd/02-lib-python-in-job/lib/python/pub/0000775000175000017500000000000015264420307026240 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/rnd/02-lib-python-in-job/lib/python/pub/beer.py0000664000175000017500000000156315264420307027534 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . BOTTLES = [99] def drink(): BOTTLES[0] -= 1 return f'{BOTTLES[0]} bottles of beer on the wall.' cylc-flow-8.7.0/tests/functional/rnd/02-lib-python-in-job/flow.cylc0000664000175000017500000000106615264420307025211 0ustar alastairalastair[scheduler] [[events]] [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = """ # check PYTHONPATH is set correctly grep -q "${CYLC_WORKFLOW_RUN_DIR}/lib/python" <<< "${PYTHONPATH}" # run a toy example python3 -c ' from pub import beer assert beer.drink() == "98 bottles of beer on the wall." assert beer.drink() == "97 bottles of beer on the wall." assert beer.drink() == "96 bottles of beer on the wall." ' """ cylc-flow-8.7.0/tests/functional/rnd/06-unicode.t0000664000175000017500000000301615264420307021741 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Ensure that UnicodeDecodeError's are caught and handled elegantly # See: https://github.com/cylc/cylc-flow/pull/4947 . "$(dirname "$0")/test_header" if [[ -n ${CI:-} ]]; then # test requires a real terminal to write to skip_all fi set_test_number 4 # this command should work where UTF-8 is supported run_ok "${TEST_NAME_BASE}-good" env LANG=en_GB.UTF-8 cylc scan --help # but fail where it is not run_fail "${TEST_NAME_BASE}-bad" env LANG=en_GB cylc scan --help # we should raise a sensible error message grep_ok \ 'A UTF-8 compatible terminal is required for this command' \ "${TEST_NAME_BASE}-bad.stderr" # and provide some helpful advice grep_ok \ 'LANG=C.UTF-8 cylc scan --help' \ "${TEST_NAME_BASE}-bad.stderr" exit cylc-flow-8.7.0/tests/functional/rnd/01-signals.t0000664000175000017500000000377715264420307021764 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test scheduler signal handling # # See https://github.com/cylc/cylc-flow/issues/6438 . "$(dirname "$0")/test_header" set_test_number 6 init_workflow "${TEST_NAME_BASE}" <<__FLOW__ [scheduling] [[graph]] R1 = foo [runtime] [[foo]] __FLOW__ # test signals on a detached scheduler TEST_NAME="${TEST_NAME_BASE}-detatch" cylc play "${WORKFLOW_NAME}" --pause poll_workflow_running PID="$(sed -n 's/CYLC_WORKFLOW_PID=//p' "$HOME/cylc-run/$WORKFLOW_NAME/.service/contact")" kill -s SIGINT "$PID" poll_workflow_stopped log_scan "${TEST_NAME}" "$(cylc cat-log -m p "${WORKFLOW_NAME}")" 10 1 \ 'Signal SIGINT received' \ 'Workflow shutting down - REQUEST(NOW)' \ 'DONE' # test signals on a non-detached scheduler TEST_NAME="${TEST_NAME_BASE}-no-detach" cylc play "${WORKFLOW_NAME}" --pause --no-detach 2>/dev/null & poll_workflow_running PID="$(sed -n 's/CYLC_WORKFLOW_PID=//p' "$HOME/cylc-run/$WORKFLOW_NAME/.service/contact")" kill -s SIGTERM "$PID" poll_workflow_stopped log_scan "${TEST_NAME}" "$(cylc cat-log -m p "${WORKFLOW_NAME}")" 10 1 \ 'Signal SIGTERM received' \ 'Workflow shutting down - REQUEST(NOW)' \ 'DONE' purge cylc-flow-8.7.0/tests/functional/rnd/05-main-loop.t0000664000175000017500000000433515264420307022212 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test the development main-loop plugins to ensure they can run to completion . "$(dirname "$0")/test_header" expected_log_files=( # these are the files they should produce cylc.flow.main_loop.log_data_store.json cylc.flow.main_loop.log_data_store.pdf cylc.flow.main_loop.log_db.sql cylc.flow.main_loop.log_main_loop.json cylc.flow.main_loop.log_main_loop.pdf cylc.flow.main_loop.log_memory.json cylc.flow.main_loop.log_memory.pdf ) set_test_number $(( 1 + ${#expected_log_files[@]} )) init_workflow "${TEST_NAME_BASE}" <<__FLOW_CYLC__ [scheduler] # make sure periodic plugins actually run [[main loop]] [[[log data store]]] interval = PT1S [[[log main loop]]] interval = PT1S [[[log memory]]] interval = PT1S [scheduling] [[graph]] R1 = a [runtime] [[a]] script = sleep 5 __FLOW_CYLC__ # run a workflow with all the development main-loop plugins turned on run_ok "${TEST_NAME_BASE}-run" \ cylc play "${WORKFLOW_NAME}" \ --no-detach \ --debug \ --main-loop 'log data store' \ --main-loop 'log db' \ --main-loop 'log main loop' \ --main-loop 'log memory' # check the expected files are generated for log_file in "${expected_log_files[@]}"; do file_path="${HOME}/cylc-run/${WORKFLOW_NAME}/${log_file}" run_ok "${TEST_NAME_BASE}.${log_file}" \ stat "${file_path}" done purge cylc-flow-8.7.0/tests/functional/rnd/02-lib-python-in-job.t0000664000175000017500000000314615264420307023554 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test ${CYLC_WORKFLOW_DIR} in task PYTHONPATH. . "$(dirname "$0")/test_header" set_test_number 2 #------------------------------------------------------------------------------- CHOSEN_WORKFLOW="$(basename "$0" | sed 's/\..*//')" install_workflow "${TEST_NAME_BASE}" "$CHOSEN_WORKFLOW" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --no-detach --abort-if-any-task-fails "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/rnd/test_header0000777000175000017500000000000015264420307026220 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/directives/0000775000175000017500000000000015264420307021261 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/directives/03-pbs.t0000777000175000017500000000000015264420307025345 200-loadleveler.tustar alastairalastaircylc-flow-8.7.0/tests/functional/directives/loadleveler/0000775000175000017500000000000015264420307023557 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/directives/loadleveler/reference.log0000664000175000017500000000017415264420307026222 0ustar alastairalastairInitial point: 1 Final point: 1 1/rem1 -triggered off [ 1/rem2 -triggered off ['1/rem1' 1/killer -triggered 0off ['1/rem2'] cylc-flow-8.7.0/tests/functional/directives/loadleveler/flow.cylc0000664000175000017500000000126015264420307025401 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] expected task failures = rem2.1 [scheduling] [[graph]] R1 = """ rem1 => rem2 rem2:start => killer => !rem2 """ [runtime] [[LLSETTINGS]] platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[[directives]]] class = serial job_type = serial notification = error wall_clock_limit = '120,60' [[rem1]] inherit = LLSETTINGS script = "sleep 10; true" [[rem2]] inherit = LLSETTINGS script = "sleep 30" [[killer]] script = cylc kill "$CYLC_WORKFLOW_ID//*/rem2"; sleep 10 cylc-flow-8.7.0/tests/functional/directives/01-at/0000775000175000017500000000000015264420307022103 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/directives/01-at/reference.log0000664000175000017500000000017515264420307024547 0ustar alastairalastairInitial point: 1 Final point: 1 1/rem1 -triggered off [] 1/rem2 -triggered off ['1/rem1'] 1/killer -triggered off ['1/rem2'] cylc-flow-8.7.0/tests/functional/directives/01-at/flow.cylc0000664000175000017500000000077615264420307023740 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] expected task failures = 1/rem2 [scheduling] [[graph]] R1 = """ rem1 => rem2 rem2:start => killer => !rem2 """ [runtime] [[ATSETTINGS]] platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[rem1]] inherit = ATSETTINGS script = "sleep 10; true" [[rem2]] inherit = ATSETTINGS script = "sleep 30" [[killer]] script = cylc kill "$CYLC_WORKFLOW_ID//*/rem2"; sleep 10 cylc-flow-8.7.0/tests/functional/directives/00-loadleveler.t0000664000175000017500000000237715264420307024172 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test job runner directives # (the job runner is given by filename, e.g. 02-slurm.t) JOB_RUNNER="${0##*\/??-}" JOB_RUNNER_NAME="${JOB_RUNNER%%.t}" export REQUIRE_PLATFORM="runner:${JOB_RUNNER_NAME} comms:tcp" . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 reftest "${TEST_NAME_BASE}" "${JOB_RUNNER_NAME}" purge exit cylc-flow-8.7.0/tests/functional/directives/01-at.t0000664000175000017500000000175615264420307022301 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test at submission export REQUIRE_PLATFORM='runner:at comms:tcp' . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/directives/slurm/0000775000175000017500000000000015264420307022423 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/directives/slurm/reference.log0000664000175000017500000000017515264420307025067 0ustar alastairalastairInitial point: 1 Final point: 1 1/rem1 -triggered off [] 1/rem2 -triggered off ['1/rem1'] 1/killer -triggered off ['1/rem2'] cylc-flow-8.7.0/tests/functional/directives/slurm/flow.cylc0000664000175000017500000000107615264420307024252 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] expected task failures = 1/rem2 [scheduling] [[graph]] R1 = """ rem1 => rem2 rem2:start => killer => !rem2 """ [runtime] [[SLURM_SETTINGS]] platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[[directives]]] --time = 02:00 [[rem1]] inherit = SLURM_SETTINGS script = "sleep 10; true" [[rem2]] inherit = SLURM_SETTINGS script = "sleep 30" [[killer]] script = cylc kill "$CYLC_WORKFLOW_ID//*/rem2"; sleep 10 cylc-flow-8.7.0/tests/functional/directives/pbs/0000775000175000017500000000000015264420307022045 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/directives/pbs/reference.log0000664000175000017500000000017515264420307024511 0ustar alastairalastairInitial point: 1 Final point: 1 1/rem1 -triggered off [] 1/rem2 -triggered off ['1/rem1'] 1/killer -triggered off ['1/rem2'] cylc-flow-8.7.0/tests/functional/directives/pbs/flow.cylc0000664000175000017500000000113315264420307023666 0ustar alastairalastair#!Jinja2 [scheduler] [[events]] expected task failures = rem2.1 [scheduling] [[graph]] R1 = """ rem1 => rem2 rem2:start => killer => !rem2 """ [runtime] [[PBS_SETTINGS]] platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[[directives]]] -l walltime=00:60:00 -l cput=00:02:00 [[rem1]] inherit = PBS_SETTINGS script = "sleep 10; true" [[rem2]] inherit = PBS_SETTINGS script = "sleep 30" [[killer]] script = cylc kill "$CYLC_WORKFLOW_ID//*/rem2"; sleep 10 cylc-flow-8.7.0/tests/functional/directives/test_header0000777000175000017500000000000015264420307027576 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/directives/02-slurm.t0000777000175000017500000000000015264420307025722 200-loadleveler.tustar alastairalastaircylc-flow-8.7.0/tests/functional/intelligent-host-selection/0000775000175000017500000000000015264420307024374 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intelligent-host-selection/01-periodic-clear-badhosts/0000775000175000017500000000000015264420307031301 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intelligent-host-selection/01-periodic-clear-badhosts/flow.cylc0000664000175000017500000000043715264420307033130 0ustar alastairalastair[scheduler] [[events]] stall timeout = PT0S abort on stall timeout = True [scheduling] cycling mode = integer initial cycle point = 1 [[graph]] R1 = a [runtime] [[a]] platform = fake-platform submission retry delays = 1*PT1S cylc-flow-8.7.0/tests/functional/intelligent-host-selection/01-periodic-clear-badhosts/bin/0000775000175000017500000000000015264420307032051 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intelligent-host-selection/01-periodic-clear-badhosts/bin/mock-ssh0000775000175000017500000000076415264420307033532 0ustar alastairalastair#!/usr/bin/env bash shift # the first argument to SSH is the host we are connecting to (ignore it) COUNT_FILE="$(dirname "$0")/count" echo 'x' >> "$COUNT_FILE" if [[ $(wc -l "$COUNT_FILE" | cut -d ' ' -f 1) -eq 1 ]]; then # the first time we make it look like an SSH failure exit 255 else # from then on we make it look like SSH is working fine # do the bare minimum to make it look like remote-init worked echo 'KEYSTARTxxxxKEYEND' echo 'REMOTE INIT DONE' exit 0 fi cylc-flow-8.7.0/tests/functional/intelligent-host-selection/01-periodic-clear-badhosts.t0000664000175000017500000000611515264420307031471 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test mainloop plugin periodically clears badhosts: # * simulate remote-init failure due to SSH issues # * ensure that "reset bad hosts" allows this task to auto "submit retry" # once the bad host is cleared . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # Install the fake background job runner. cp -r "${TEST_SOURCE_DIR}/lib" "${WORKFLOW_RUN_DIR}" create_test_global_config '' " [scheduler] [[main loop]] [[[reset bad hosts]]] interval = PT1S [platforms] [[fake-platform]] hosts = localhost # we set the install target to make it look like a remote platform # (and so trigger remote-init) install target = fake-install-target # we botch the SSH command so we can simulate SSH failure ssh command = $HOME/cylc-run/$WORKFLOW_NAME/bin/mock-ssh " #------------------------------------------------------------------------------- run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play \ --debug \ --no-detach \ --abort-if-any-task-fails \ "${WORKFLOW_NAME}" # scrape platform events from the log sed -n \ 's/.* - \(platform: .*\)/\1/p' \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" \ > platform-log # check this matches expectations # we would expect: # * the task will attempt to remote-init # * this will fail (because we made it fail) # * the task will retry (because of the retry delays) # * the task will attempt to remote-init again # * the remote init will succeed this time # * the task will attempt file-installation # * file installation will fail because the install target is incorrect cmp_ok platform-log <<__HERE__ platform: fake-platform - remote init (on localhost) platform: fake-platform - initialisation did not complete platform: fake-platform - remote init (on localhost) platform: fake-platform - remote file install (on localhost) platform: fake-platform - initialisation did not complete platform: fake-platform - remote tidy (on localhost) __HERE__ purge exit 0 cylc-flow-8.7.0/tests/functional/intelligent-host-selection/04-kill.t0000664000175000017500000000601615264420307025740 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test job kill will retry on a different host if there is a connection failure export REQUIRE_PLATFORM='loc:remote fs:indep comms:tcp runner:background' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 6 create_test_global_config "" " [scheduler] [[main loop]] [[[reset bad hosts]]] # Set the auto clearance of badhosts to be << small time so that # kill will need to retry, despite 'unreachable_host' being # identified as unreachable by job submission. interval = PT5S [platforms] [[goodhostplatform]] $(cylc config -i "[platforms][$CYLC_TEST_PLATFORM]") [[goodhostplatform]] hosts = ${CYLC_TEST_HOST} install target = ${CYLC_TEST_HOST} [[mixedhostplatform]] $(cylc config -i "[platforms][$CYLC_TEST_PLATFORM]") [[mixedhostplatform]] # Use a fake background job runner to get around the # single host restriction. job runner = my_background hosts = unreachable_host, ${CYLC_TEST_HOST} install target = ${CYLC_TEST_HOST} [[[selection]]] method = 'definition order' " #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # Install the fake background job runner. cp -r "${TEST_SOURCE_DIR}/lib" "${WORKFLOW_RUN_DIR}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach \ "${WORKFLOW_NAME}" # job kill for mixedhosttask should have attempted on both hosts grep_workflow_log_ok "${TEST_NAME_BASE}-kill-failed" \ 'jobs-kill for mixedhostplatform on unreachable_host' # fail grep_workflow_log_ok "${TEST_NAME_BASE}-kill-retried" \ "jobs-kill for mixedhostplatform on $CYLC_TEST_HOST" # retry # both job kills should succeed grep_workflow_log_ok "${TEST_NAME_BASE}-kill-succeeded-goodhosttask" \ '1/goodhosttask/01.* job killed' grep_workflow_log_ok "${TEST_NAME_BASE}-kill-succeeded-mixedhosttask" \ '1/mixedhosttask/01.* job killed' purge exit 0 cylc-flow-8.7.0/tests/functional/intelligent-host-selection/06-from-platform-group-fails.t0000664000175000017500000000613715264420307032026 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that Cylc fails sensibly when a platform group with no # accessible hosts is selected. # n.b. We don't care about definition order in this test because all # hosts and platforms fail. . "$(dirname "$0")/test_header" set_test_number 12 #------------------------------------------------------------------------------- # Uses a fake background job runner to get around the single host restriction. create_test_global_config "" " [platforms] [[badhostplatform1]] job runner = my_background hosts = bad_host1, bad_host2 [[badhostplatform2]] job runner = my_background hosts = bad_host3, bad_host4 [platform groups] [[badplatformgroup]] platforms = badhostplatform1, badhostplatform2 " install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # Install the fake background job runner. cp -r "${TEST_SOURCE_DIR}/lib" "${WORKFLOW_RUN_DIR}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" logfile="${WORKFLOW_RUN_DIR}/log/scheduler/log" # Check workflow fails for the reason we want it to fail named_grep_ok \ "${TEST_NAME_BASE}-workflow-stalled-with-bad-submit-failed" \ "1/bad did not complete the required outputs" \ "$logfile" # Look for message indicating that remote init has failed on each bad_host # on every bad platform. platform='badhostplatform1' for host in {1..2}; do host="bad_host${host}" log_scan \ "${TEST_NAME_BASE}-remote-init-fail-${host}" \ "${logfile}" 1 0 \ "platform: ${platform} - remote init (on ${host})" \ "platform: ${platform} - Could not connect to ${host}." done platform='badhostplatform2' for host in {3..4}; do host="bad_host${host}" log_scan \ "${TEST_NAME_BASE}-remote-init-fail-${host}" \ "${logfile}" 1 0 \ "platform: ${platform} - remote init (on ${host})" \ "platform: ${platform} - Could not connect to ${host}." done # Look for message indicating that remote init has failed. named_grep_ok \ "${TEST_NAME_BASE}-initialisation-did-not-complete" \ "platform: badhostplatform. - initialisation did not complete (no hosts were reachable)" \ "${logfile}" purge exit 0 cylc-flow-8.7.0/tests/functional/intelligent-host-selection/lib/0000775000175000017500000000000015264420307025142 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intelligent-host-selection/lib/python/0000775000175000017500000000000015264420307026463 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intelligent-host-selection/lib/python/my_background.py0000664000175000017500000000171115264420307031661 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cylc.flow.job_runner_handlers.background import BgCommandHandler class MyBgCommandHandler(BgCommandHandler): pass JOB_RUNNER_HANDLER = MyBgCommandHandler() cylc-flow-8.7.0/tests/functional/intelligent-host-selection/00-mixedhost/0000775000175000017500000000000015264420307026615 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intelligent-host-selection/00-mixedhost/flow.cylc0000664000175000017500000000072415264420307030443 0ustar alastairalastair [meta] title = "Try out scenarios for intelligent host selection." description = """ Tasks - goodhost: a control to check that everything works - mixedhost contains some hosts that will and won't fail """ [scheduling] initial cycle point = 1 [[graph]] R1 = mixedhosttask & goodhosttask [runtime] [[root]] script = true [[goodhosttask]] platform = goodhostplatform [[mixedhosttask]] platform = mixedhostplatform cylc-flow-8.7.0/tests/functional/intelligent-host-selection/06-from-platform-group-fails/0000775000175000017500000000000015264420307031632 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intelligent-host-selection/06-from-platform-group-fails/flow.cylc0000664000175000017500000000064715264420307033464 0ustar alastairalastair[meta] title = "Try out scenarios for intelligent host selection." description = """ Tasks ===== Bad --- Fails on all hosts on all platforms """ [scheduler] [[events]] stall timeout = PT0S [scheduling] initial cycle point = 1 [[graph]] R1 = bad [runtime] [[root]] script = true [[bad]] platform = badplatformgroup cylc-flow-8.7.0/tests/functional/intelligent-host-selection/00-mixedhost.t0000664000175000017500000000475215264420307027012 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test remote initialisation - where a task has a platform with an unreachable # host. export REQUIRE_PLATFORM='loc:remote fs:indep comms:tcp' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 4 # Uses a fake background job runner to get around the single host restriction. create_test_global_config "" " [platforms] [[goodhostplatform]] hosts = ${CYLC_TEST_HOST} install target = ${CYLC_TEST_INSTALL_TARGET} retrieve job logs = True [[mixedhostplatform]] job runner = my_background hosts = unreachable_host, ${CYLC_TEST_HOST} install target = ${CYLC_TEST_INSTALL_TARGET} retrieve job logs = True [[[selection]]] method = 'definition order' " #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # Install the fake background job runner. cp -r "${TEST_SOURCE_DIR}/lib" "${WORKFLOW_RUN_DIR}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" # Run a bunch of tests on the workflow logs to ensure that warning messages # produced by Intelligent Host Selection Logic have happened. named_grep_ok "${TEST_NAME_BASE}-unreachable-host-warning" \ 'Could not connect to unreachable_host' \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" # Ensure that retrying in this context doesn't increment try number: grep_fail "1/mixedhosttask/02" "${WORKFLOW_RUN_DIR}/log/scheduler/log" purge exit 0 cylc-flow-8.7.0/tests/functional/intelligent-host-selection/07-cylc7-badhost.t0000664000175000017500000000361115264420307027451 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test task does not run on localhost if Cylc 7 syntax # [runtime][][remote]host is unreachable - # https://github.com/cylc/cylc-flow/issues/4569 . "$(dirname "$0")/test_header" set_test_number 4 # Host name picked for unlikelihood of matching any real host BAD_HOST="f65b965bb914" create_test_global_config "" " [platforms] [[badhostplatform]] hosts = ${BAD_HOST} " init_workflow "${TEST_NAME_BASE}" << __FLOW__ [scheduler] [[events]] stall timeout = PT0M abort on stall timeout = True [scheduling] cycling mode = integer [[graph]] R1 = sattler [runtime] [[sattler]] [[[remote]]] host = ${BAD_HOST} __FLOW__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --no-detach "${WORKFLOW_NAME}" grep_workflow_log_ok "${TEST_NAME_BASE}-grep-1" \ "platform: badhostplatform - initialisation did not complete" grep_workflow_log_ok "${TEST_NAME_BASE}-grep-2" "CRITICAL - Workflow stalled" purge cylc-flow-8.7.0/tests/functional/intelligent-host-selection/02-badhosts/0000775000175000017500000000000015264420307026422 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intelligent-host-selection/02-badhosts/flow.cylc0000664000175000017500000000155015264420307030246 0ustar alastairalastair [meta] title = "Try out scenarios for intelligent host selection." description = """ Tasks: - goodhost: a control to check that everything works - badhost is always going to fail - mixedhost contains some hosts that will and won't fail """ [scheduler] [[events]] abort on stall timeout = true stall timeout = PT0S [scheduling] cycling mode = integer initial cycle point = 1 [[graph]] # Run good and mixed as controls R1 = """ badhosttask:submit-fail? => goodhosttask & mixedhosttask mixedhosttask:submit-fail? # permit mixedhosttask to submit-fail """ [runtime] [[root]] script = true [[badhosttask]] platform = badhostplatform [[goodhosttask]] platform = goodhostplatform [[mixedhosttask]] platform = mixedhostplatform cylc-flow-8.7.0/tests/functional/intelligent-host-selection/05-from-platform-group.t0000664000175000017500000000676115264420307030734 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that Cylc Can select a host from a platform group # Failing if there is no good host _any_ platform # Succeeding if there is no bad host on any platform in the group export REQUIRE_PLATFORM='loc:remote fs:indep comms:tcp' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 11 # Uses a fake background job runner to get around the single host restriction. create_test_global_config "" " [platforms] [[${CYLC_TEST_PLATFORM}]] # mixed host platform job runner = my_background hosts = unreachable_host, ${CYLC_TEST_HOST} [[[selection]]] method = 'definition order' [[badhostplatform]] job runner = my_background hosts = bad_host1, bad_host2 [[[selection]]] method = 'definition order' [platform groups] [[mixedplatformgroup]] platforms = badhostplatform, ${CYLC_TEST_PLATFORM} [[[selection]]] method = definition order [[goodplatformgroup]] platforms = ${CYLC_TEST_PLATFORM} [[[selection]]] method = definition order " #------------------------------------------------------------------------------- # Uncomment to print config for manual testing of workflow. # cylc config -i '[platforms]' >&2 # cylc config -i '[platform groups]' >&2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # Install the fake background job runner. cp -r "${TEST_SOURCE_DIR}/lib" "${WORKFLOW_RUN_DIR}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" --reference-test # should try remote-init on bad_host{1,2} then fail log_scan \ "${TEST_NAME_BASE}-badhostplatformgroup" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" 1 0 \ 'platform: badhostplatform - remote init (on bad_host1)' \ 'platform: badhostplatform - Could not connect to bad_host1.' \ 'platform: badhostplatform - remote init (on bad_host2)' \ 'platform: badhostplatform - Could not connect to bad_host2.' \ # should try remote-init on unreachable_host, then $CYLC_TEST_HOST then pass log_scan \ "${TEST_NAME_BASE}-goodplatformgroup" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" 1 0 \ "platform: ${CYLC_TEST_PLATFORM} - remote init (on unreachable_host)" \ "platform: ${CYLC_TEST_PLATFORM} - Could not connect to unreachable_host." \ "platform: ${CYLC_TEST_PLATFORM} - remote init (on ${CYLC_TEST_HOST})" \ "platform: ${CYLC_TEST_PLATFORM} - remote file install (on ${CYLC_TEST_HOST})" \ "\[1/ugly/01:preparing\] => submitted" purge exit 0 cylc-flow-8.7.0/tests/functional/intelligent-host-selection/04-kill/0000775000175000017500000000000015264420307025550 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intelligent-host-selection/04-kill/flow.cylc0000664000175000017500000000217615264420307027401 0ustar alastairalastair[meta] title = "Try out scenarios for intelligent host selection." description = """ Tasks - goodhost: a control to check that everything works - mixedhost contains some hosts that will and won't fail """ [scheduler] [[events]] expected task failures = 1/goodhosttask, 1/mixedhosttask [scheduling] cycling mode = integer initial cycle point = 1 [[graph]] # Run good and mixed as controls R1 = """ goodhosttask:start => stop_g mixedhosttask:start => stop_m """ [runtime] [[goodhosttask]] script = sleep 60 platform = goodhostplatform [[mixedhosttask]] script = sleep 60 platform = mixedhostplatform [[mystop]] script=""" sleep 5 # Give the badhosts list time to empty cylc kill "$CYLC_WORKFLOW_ID//$TASK" cylc stop $CYLC_WORKFLOW_ID """ [[stop_g]] inherit = mystop [[[environment]]] TASK = 1/goodhosttask [[stop_m]] inherit = mystop [[[environment]]] TASK = 1/mixedhosttask cylc-flow-8.7.0/tests/functional/intelligent-host-selection/03-polling.t0000664000175000017500000000615215264420307026451 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test intelligent host selection for job polling. # # Set the periodic clearance of unreachable hosts by # `[scheduler][main loop][reset bad hosts]interval = PT1S` so that between # Submission of a job and execution polling the list of bad hosts will have # cleared. Having cleared bad hosts we can then test that polling goes through # the host selection process. export REQUIRE_PLATFORM='loc:remote fs:indep' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 4 # Uses a fake background job runner to get around the single host restriction. create_test_global_config "" " [scheduler] [[main loop]] [[[reset bad hosts]]] interval = PT1S [platforms] [[goodhostplatform]] hosts = ${CYLC_TEST_HOST} install target = ${CYLC_TEST_INSTALL_TARGET} retrieve job logs = True communication method = poll execution polling intervals = 10*PT3S submission polling intervals = PT0S, 5*PT2S [[mixedhostplatform]] job runner = my_background hosts = unreachable_host, ${CYLC_TEST_HOST} install target = ${CYLC_TEST_INSTALL_TARGET} retrieve job logs = True communication method = poll execution polling intervals = 10*PT3S submission polling intervals = PT0S, 5*PT2S [[[selection]]] method = 'definition order' " #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # Install the fake background job runner. cp -r "${TEST_SOURCE_DIR}/lib" "${WORKFLOW_RUN_DIR}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach \ "${WORKFLOW_NAME}" LOGFILE="${WORKFLOW_RUN_DIR}/log/scheduler/log" # Check that when a task fail badhosts associated with that task's platform # are removed from the badhosts set. named_grep_ok \ "${TEST_NAME_BASE}-job-poll-fails" \ "Could not connect to unreachable_host" \ "${LOGFILE}" named_grep_ok \ "${TEST_NAME_BASE}-job-poll-retries-and-succeeds" \ "\[jobs-poll out\] \[TASK JOB SUMMARY\].*1/mixedhosttask/01" \ "${LOGFILE}" purge exit 0 cylc-flow-8.7.0/tests/functional/intelligent-host-selection/03-polling/0000775000175000017500000000000015264420307026260 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intelligent-host-selection/03-polling/flow.cylc0000664000175000017500000000250115264420307030101 0ustar alastairalastair[meta] title = "Try out scenarios for intelligent host selection." description = """ Runs two long running tasks on a known good platform and on a platform with some unreachable hosts. Once each has started trigger a task which kills each to test the execution polling. """ [scheduling] cycling mode = integer initial cycle point = 1 [[graph]] R1 = """ goodhosttask:start => stop_g mixedhosttask:start => stop_m goodhosttask:fail & mixedhosttask:fail """ [runtime] [[goodhosttask]] # Sleep for a long time, expect to be killed by stop_g script=sleep 120 & echo $! >file; wait platform = goodhostplatform [[mixedhosttask]] # Sleep for a long time, expect to be killed by stop_m script=sleep 120 & echo $! >file; wait platform = mixedhostplatform [[stop_g]] # Kill goodhosttask when polling confirms it's started. script=""" sleep 5 # Give the badhosts list time to empty cylc kill "$CYLC_WORKFLOW_ID//1/goodhosttask" || true """ [[stop_m]] # Kill mixedhosttask when polling confirms it's started. script=""" sleep 5 # Give the badhosts list time to empty cylc kill "$CYLC_WORKFLOW_ID//1/mixedhosttask" || true """ cylc-flow-8.7.0/tests/functional/intelligent-host-selection/test_header0000777000175000017500000000000015264420307032711 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/intelligent-host-selection/02-badhosts.t0000664000175000017500000000630615264420307026614 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test remote initialisation still fails where a task has a platform # with only unreachable hosts. # n.b. Hosts picked for unlikelyhood of names matching any real host. export REQUIRE_PLATFORM='loc:remote fs:indep comms:tcp' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 7 # Uses a fake background job runner to get around the single host restriction. create_test_global_config "" " [platforms] [[badhostplatform]] job runner = my_background hosts = e9755ca30f5, 3c0b4799402 install target = ${CYLC_TEST_INSTALL_TARGET} retrieve job logs = True [[[selection]]] method = definition order [[goodhostplatform]] hosts = ${CYLC_TEST_HOST} install target = ${CYLC_TEST_INSTALL_TARGET} retrieve job logs = True [[mixedhostplatform]] job runner = my_background hosts = unreachable_host, ${CYLC_TEST_HOST} install target = ${CYLC_TEST_INSTALL_TARGET} retrieve job logs = True [[[selection]]] method = 'definition order' " #------------------------------------------------------------------------------- # Uncomment to print config for manual testing of workflow. # cylc config -i '[platforms]' >&2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # Install the fake background job runner. cp -r "${TEST_SOURCE_DIR}/lib" "${WORKFLOW_RUN_DIR}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" workflow_log="${WORKFLOW_RUN_DIR}/log/scheduler/log" # Check that badhosttask has submit failed, but not good or mixed named_grep_ok "${TEST_NAME_BASE}-badhost-task-submit-failed" \ "1/badhosttask.* submit-failed" "${workflow_log}" named_grep_ok "${TEST_NAME_BASE}-goodhost-suceeded" \ "1/mixedhosttask.* succeeded" "${workflow_log}" named_grep_ok "${TEST_NAME_BASE}-mixedhost-task-suceeded" \ "1/goodhosttask.* succeeded" "${workflow_log}" # Check for remote-init fail reason given in scheduler and job activity logs job_activity_log="${WORKFLOW_RUN_DIR}/log/job/1/badhosttask/01/job-activity.log" msg="initialisation did not complete (no hosts were reachable)" grep_ok "$msg" "$workflow_log" grep_ok "$msg" "$job_activity_log" purge cylc-flow-8.7.0/tests/functional/intelligent-host-selection/05-from-platform-group/0000775000175000017500000000000015264420307030535 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/intelligent-host-selection/05-from-platform-group/reference.log0000664000175000017500000000013215264420307033172 0ustar alastairalastairInitial point: 1 Final point: 1 1/good -triggered off [] 1/ugly -triggered off ['1/good'] cylc-flow-8.7.0/tests/functional/intelligent-host-selection/05-from-platform-group/flow.cylc0000664000175000017500000000125615264420307032364 0ustar alastairalastair#!Jinja2 [meta] title = "Try out scenarios for intelligent host selection." description = """ Tasks ===== Good ---- Should pass without problems. Ugly ---- - Fails entirely on a duff platform. - Fails on the first host of a mixed platform. - Succeeds on the second host of the second platform. """ [scheduler] [[events]] # abort on stalled = true [scheduling] initial cycle point = 1 [[graph]] R1 = good => ugly [runtime] [[root]] script = true [[good]] platform = goodplatformgroup [[ugly]] platform = mixedplatformgroup cylc-flow-8.7.0/tests/functional/n-window/0000775000175000017500000000000015264420307020662 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/n-window/02-big-window/0000775000175000017500000000000015264420307023147 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/n-window/02-big-window/flow.cylc0000664000175000017500000000176415264420307025002 0ustar alastairalastair[scheduler] allow implicit tasks = True [[events]] inactivity timeout = PT1M abort on inactivity timeout = True [scheduling] [[graph]] R1 = """ a => b => c => d => e => f => g => h b => i => j => f """ [runtime] [[a]] script = """ set +e read -r -d '' gqlDoc <<_DOC_ {"request_string": " mutation { setGraphWindowExtent ( workflows: [\"${CYLC_WORKFLOW_ID}\"], nEdgeDistance: 5) { result } }", "variables": null} _DOC_ echo "${gqlDoc}" cylc client "$CYLC_WORKFLOW_ID" graphql < <(echo ${gqlDoc}) 2>/dev/null set -e """ [[c]] script = """ cylc show "$CYLC_WORKFLOW_ID//1/a" >> $CYLC_WORKFLOW_RUN_DIR/show-a.txt cylc show "$CYLC_WORKFLOW_ID//1/j" >> $CYLC_WORKFLOW_RUN_DIR/show-j.txt cylc show "$CYLC_WORKFLOW_ID//1/h" >> $CYLC_WORKFLOW_RUN_DIR/show-h.txt """ [[i]] script = """ # Slow 2nd branch down sleep 5 """ [[f]] script = """ # test re-trigger of old point cylc trigger "$CYLC_WORKFLOW_ID//1/b" """ cylc-flow-8.7.0/tests/functional/n-window/01-past-present-future.t0000664000175000017500000000420015264420307025216 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test window size using graphql and cylc-show for all tasks. . "$(dirname "$0")/test_header" set_test_number 7 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-run" # 'a => b => c => d => e', 'a' sets window size to 2, 'c' uses cylc show on all. workflow_run_ok "${TEST_NAME}" cylc play --no-detach --debug "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-show-a.past" contains_ok "$WORKFLOW_RUN_DIR/show-a.txt" <<__END__ state: succeeded prerequisites: (None) __END__ TEST_NAME="${TEST_NAME_BASE}-show-b.past" contains_ok "$WORKFLOW_RUN_DIR/show-b.txt" <<__END__ state: succeeded prerequisites: (n/a for past tasks) __END__ TEST_NAME="${TEST_NAME_BASE}-show-c.present" contains_ok "${WORKFLOW_RUN_DIR}/show-c.txt" <<__END__ prerequisites: ('⨯': not satisfied) ✓ 1/b succeeded __END__ TEST_NAME="${TEST_NAME_BASE}-show-d.future" contains_ok "${WORKFLOW_RUN_DIR}/show-d.txt" <<__END__ state: waiting prerequisites: ('⨯': not satisfied) ⨯ 1/c succeeded __END__ TEST_NAME="${TEST_NAME_BASE}-show-e.future" contains_ok "${WORKFLOW_RUN_DIR}/show-e.txt" <<__END__ state: waiting prerequisites: ('⨯': not satisfied) ⨯ 1/d succeeded __END__ purge cylc-flow-8.7.0/tests/functional/n-window/01-past-present-future/0000775000175000017500000000000015264420307025035 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/n-window/01-past-present-future/flow.cylc0000664000175000017500000000166515264420307026670 0ustar alastairalastair[scheduler] allow implicit tasks = True [[events]] inactivity timeout = PT1M abort on inactivity timeout = True [scheduling] [[graph]] R1 = """ a => b => c => d => e """ [runtime] [[a]] script = """ set +e read -r -d '' gqlDoc <<_DOC_ {"request_string": " mutation { setGraphWindowExtent ( workflows: [\"${CYLC_WORKFLOW_ID}\"], nEdgeDistance: 2) { result } }", "variables": null} _DOC_ echo "${gqlDoc}" cylc client "$CYLC_WORKFLOW_ID" graphql < <(echo ${gqlDoc}) 2>/dev/null set -e """ [[c]] script = """ cylc show "$CYLC_WORKFLOW_ID//1/a" >> $CYLC_WORKFLOW_RUN_DIR/show-a.txt cylc show "$CYLC_WORKFLOW_ID//1/b" >> $CYLC_WORKFLOW_RUN_DIR/show-b.txt cylc show "$CYLC_WORKFLOW_ID//1/c" >> $CYLC_WORKFLOW_RUN_DIR/show-c.txt cylc show "$CYLC_WORKFLOW_ID//1/d" >> $CYLC_WORKFLOW_RUN_DIR/show-d.txt cylc show "$CYLC_WORKFLOW_ID//1/e" >> $CYLC_WORKFLOW_RUN_DIR/show-e.txt """ cylc-flow-8.7.0/tests/functional/n-window/test_header0000777000175000017500000000000015264420307027177 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/n-window/02-big-window.t0000664000175000017500000000364115264420307023340 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test large window size using graphql and find tasks in window. # This is helpful with coverage by using most the no-rewalk mechanics. . "$(dirname "$0")/test_header" set_test_number 5 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-run" # 'a => b => . . . f => g => h', 'a' sets window size to 5, # 'b => i => j => f', 'c' finds 'a', 'j', 'h' workflow_run_ok "${TEST_NAME}" cylc play --no-detach --debug "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-show-a.past" contains_ok "$WORKFLOW_RUN_DIR/show-a.txt" <<__END__ state: succeeded prerequisites: (None) __END__ TEST_NAME="${TEST_NAME_BASE}-show-j.parallel" contains_ok "${WORKFLOW_RUN_DIR}/show-j.txt" <<__END__ state: waiting prerequisites: ('⨯': not satisfied) ⨯ 1/i succeeded __END__ TEST_NAME="${TEST_NAME_BASE}-show-h.future" contains_ok "${WORKFLOW_RUN_DIR}/show-h.txt" <<__END__ state: waiting prerequisites: ('⨯': not satisfied) ⨯ 1/g succeeded __END__ purge cylc-flow-8.7.0/tests/functional/restart/0000775000175000017500000000000015264420307020604 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/37-auto-restart-delay.t0000664000175000017500000000711615264420307024753 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- export REQUIRE_PLATFORM='loc:remote fs:shared runner:background' . "$(dirname "$0")/test_header" set_test_number 6 #------------------------------------------------------------------------------- time_gt () { python3 -c " import sys from metomi.isodatetime.parsers import TimePointParser parser = TimePointParser() sys.exit(not parser.parse('$1') > parser.parse('$2')) " } BASE_GLOBAL_CONFIG=" [scheduler] [[main loop]] plugins = health check, auto restart [[[auto restart]]] interval = PT5S [[events]] abort on inactivity timeout = True abort on stall timeout = True inactivity timeout = PT2M stall timeout = PT2M " #------------------------------------------------------------------------------- # Test the delayed restart feature init_workflow "${TEST_NAME_BASE}" <<< ' [scheduler] UTC mode = True [scheduling] initial cycle point = 2000 [[graph]] P1D = foo [runtime] [[foo]] ' MAX_RESTART_DELAY=30 create_test_global_config '' " ${BASE_GLOBAL_CONFIG} [scheduler] auto restart delay = PT${MAX_RESTART_DELAY}S [[run hosts]] available = localhost " # Run workflow. cylc play "${WORKFLOW_NAME}" --pause poll_workflow_running # Condemn host - trigger stop-restart. create_test_global_config '' " ${BASE_GLOBAL_CONFIG} [scheduler] auto restart delay = PT20S [[run hosts]] available = ${CYLC_TEST_HOST} condemned = $(hostname) " # Check stop-restart working. FILE=$(cylc cat-log "${WORKFLOW_NAME}" -m p |xargs readlink -f) log_scan "${TEST_NAME_BASE}-auto-restart" "${FILE}" 60 1 \ 'The Cylc workflow host will soon become un-available' \ 'Workflow will restart in' \ "Attempting to restart on \"${CYLC_TEST_HOST}\"" \ "Workflow now running on \"${CYLC_TEST_HOST}\"" # Extract scheduled restart time from the log. TIMES=$(grep --color=never 'Workflow will restart in' "${FILE}" | \ sed 's/.*will restart in \(.*\)s (at \(.*\))/\1|\2/') RESTART_DELAY=$(cut -d '|' -f 1 <<< "${TIMES}") RESTART_SCHEDULED_TIME=$(cut -d '|' -f 2 <<< "${TIMES}") # Extract actual restart time from the log. RESTART_TIME=$(grep --color=never 'Attempting to restart' "${FILE}" | \ sed 's/\(.*\) INFO.*Attempting to restart.*/\1/') # Check the restart delay is correct. TEST_NAME="${TEST_NAME_BASE}-restart-delay" if [[ "${RESTART_DELAY}" -lt "${MAX_RESTART_DELAY}" && "${RESTART_DELAY}" -gt 0 ]] then ok "${TEST_NAME}" else fail "${TEST_NAME}" fi # Check the actual restart time is after the scheduled restart time. TEST_NAME="${TEST_NAME_BASE}-restart-time" if time_gt "${RESTART_TIME}" "${RESTART_SCHEDULED_TIME}"; then ok "${TEST_NAME}" else fail "${TEST_NAME}" fi cylc stop "${WORKFLOW_NAME}" --now --now 2>/dev/null purge cylc-flow-8.7.0/tests/functional/restart/26-remote-kill/0000775000175000017500000000000015264420307023255 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/26-remote-kill/flow.cylc0000664000175000017500000000071415264420307025102 0ustar alastairalastair#!jinja2 [scheduler] UTC mode = True cycle point format = %Y [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = P6M [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = sleep 300 platform = {{environ["CYLC_TEST_PLATFORM"]}} [[[events]]] started handlers = cylc stop --now --now '%(workflow)s' cylc-flow-8.7.0/tests/functional/restart/32-reload-runahead-no-stop-point.t0000775000175000017500000000235715264420307027004 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test cylc/cylc-flow#2722 . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --no-detach "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play --no-detach "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/restart/53-task-prerequisites.t0000664000175000017500000000415515264420307025067 0ustar alastairalastair#!/bin/bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test custom task prerequisites and the task_prerequisites DB table work as # expected . "$(dirname "$0")/test_header" set_test_number 7 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_fail "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" --stopcp=2 --no-detach DB_FILE="${WORKFLOW_RUN_DIR}/log/db" # Check task_prerequisites table: TEST_NAME="${TEST_NAME_BASE}-db-task-prereq" QUERY='SELECT * FROM task_prerequisites ORDER BY cycle, name, prereq_cycle;' run_ok "$TEST_NAME" sqlite3 "$DB_FILE" "$QUERY" cmp_ok "${TEST_NAME}.stdout" << '__EOF__' 2|bar|[1]|foo|1|succeeded|0 2|bar|[1]|apollo|2|The Eagle has landed|satisfied naturally 3|bar|[1]|foo|2|succeeded|satisfied naturally 3|bar|[1]|apollo|3|The Eagle has landed|0 __EOF__ workflow_run_fail "${TEST_NAME_BASE}-restart" cylc play "${WORKFLOW_NAME}" --stopcp=3 --no-detach # Check 2/bar is still waiting (i.e. prereqs not satisfied): TEST_NAME="${TEST_NAME_BASE}-db-task-pool" QUERY='SELECT cycle, name, status FROM task_pool ORDER BY cycle, name;' run_ok "$TEST_NAME" sqlite3 "$DB_FILE" "$QUERY" cmp_ok "${TEST_NAME}.stdout" << '__EOF__' 1|foo|failed 2|bar|waiting 4|apollo|waiting 4|bar|waiting 4|foo|waiting __EOF__ purge exit cylc-flow-8.7.0/tests/functional/restart/13-bad-job-host.t0000775000175000017500000000366315264420307023476 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restarting a workflow when the host of a submitted or running job is not # available. https://github.com/cylc/cylc-flow/issues/1327 export REQUIRE_PLATFORM='loc:remote comms:tcp' . "$(dirname "$0")/test_header" set_test_number 4 install_workflow "${TEST_NAME_BASE}" bad-job-host #------------------------------------------------------------------------------- run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --debug --no-detach --abort-if-any-task-fails "${WORKFLOW_NAME}" # Modify DB with garbage host CYLC_WORKFLOW_RUN_DIR="$RUN_DIR/${WORKFLOW_NAME}" for DB_NAME in 'log/db' '.service/db'; do sqlite3 "${CYLC_WORKFLOW_RUN_DIR}/${DB_NAME}" \ "UPDATE task_jobs SET platform_name='garbage' WHERE name=='t-remote';" done workflow_run_fail "${TEST_NAME_BASE}-restart" cylc play --debug --no-detach --abort-if-any-task-fails "${WORKFLOW_NAME}" grep_ok PlatformLookupError "${CYLC_WORKFLOW_RUN_DIR}/log/scheduler/log" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/restart/bad-state/0000775000175000017500000000000015264420307022450 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/bad-state/flow.cylc0000664000175000017500000000012015264420307024264 0ustar alastairalastair[scheduling] [[graph]] R1 = "foo => bar" [runtime] [[foo, bar]] cylc-flow-8.7.0/tests/functional/restart/01-broadcast.t0000775000175000017500000000600615264420307023156 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restarting a simple workflow with a broadcast if [[ -z ${TEST_DIR:-} ]]; then . "$(dirname "$0")/test_header" fi #------------------------------------------------------------------------------- set_test_number 8 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" 'broadcast' cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "${WORKFLOW_RUN_DIR}/" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" cmp_ok "${TEST_NAME}.stderr" <'/dev/null' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach --abort-if-any-task-fails "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-restart-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach --abort-if-any-task-fails "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- grep_ok "send_a_broadcast_task|20130923T0000Z|1|1|succeeded" \ "${WORKFLOW_RUN_DIR}/pre-restart-db" contains_ok "${WORKFLOW_RUN_DIR}/post-restart-db" <<'__DB_DUMP__' send_a_broadcast_task|20130923T0000Z|1|1|succeeded shutdown|20130923T0000Z|1|1|succeeded __DB_DUMP__ "${WORKFLOW_RUN_DIR}/bin/ctb-select-task-states" "${WORKFLOW_RUN_DIR}" \ > "${TEST_DIR}/db" contains_ok "${TEST_DIR}/db" <<'__DB_DUMP__' broadcast_task|20130923T0000Z|1|1|succeeded finish|20130923T0000Z|1|1|succeeded output_states|20130923T0000Z|1|1|succeeded send_a_broadcast_task|20130923T0000Z|1|1|succeeded shutdown|20130923T0000Z|1|1|succeeded __DB_DUMP__ sqlite3 "${WORKFLOW_RUN_DIR}/log/db" ' SELECT point,namespace,key,value FROM broadcast_states ORDER BY point,namespace,key' >'select-broadcast-states.out' cmp_ok 'select-broadcast-states.out' \ <<<"20130923T0000Z|broadcast_task|[environment]MY_VALUE|'something'" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/restart/16-template-vars/0000775000175000017500000000000015264420307023614 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/16-template-vars/reference.log0000664000175000017500000000020415264420307026251 0ustar alastairalastairInitial point: 20160101T0000Z Final point: 20200101T0000Z 20190101T0000Z/foo -triggered off [] 20200101T0000Z/foo -triggered off [] cylc-flow-8.7.0/tests/functional/restart/16-template-vars/flow.cylc0000664000175000017500000000032615264420307025440 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [scheduling] initial cycle point = 2016 final cycle point = {{FINAL_CYCLE_POINT}} [[graph]] P1Y = foo [runtime] [[foo]] script = {{COMMAND}} cylc-flow-8.7.0/tests/functional/restart/27-broadcast-timeout/0000775000175000017500000000000015264420307024460 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/27-broadcast-timeout/reference.log0000664000175000017500000000007715264420307027125 0ustar alastairalastairInitial point: 1 Final point: 1 1/bar -triggered off ['1/foo'] cylc-flow-8.7.0/tests/functional/restart/27-broadcast-timeout/flow.cylc0000664000175000017500000000045115264420307026303 0ustar alastairalastair[scheduler] UTC mode = True cycle point format = %Y [scheduling] [[graph]] R1 = foo => bar [runtime] [[foo]] script=""" cylc broadcast "${CYLC_WORKFLOW_ID}" --set='[events]submission timeout=PT1M' cylc stop "${CYLC_WORKFLOW_ID}" """ [[bar]] script=true cylc-flow-8.7.0/tests/functional/restart/deleted-logs/0000775000175000017500000000000015264420307023154 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/deleted-logs/flow.cylc0000664000175000017500000000047015264420307025000 0ustar alastairalastair[scheduler] [[events]] abort on stall timeout = True stall timeout = PT20S [scheduling] [[graph]] R1 = one => two [runtime] [[one]] script = """ # Tell the workflow to stop after I've finished. cylc stop $CYLC_WORKFLOW_ID sleep 10""" [[two]] script = true cylc-flow-8.7.0/tests/functional/restart/waiting/0000775000175000017500000000000015264420307022246 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/waiting/flow.cylc0000664000175000017500000000106315264420307024071 0ustar alastairalastair#!jinja2 {%- set TEST_DIR = environ['TEST_DIR'] %} [scheduler] UTC mode = True [[events]] abort on stall timeout = True stall timeout = PT3M [scheduling] initial cycle point = 20130923T00 final cycle point = 20130923T00 [[graph]] R1 = """ shutdown => output_states output_states => waiting_task => finish """ [runtime] [[waiting_task]] script = true [[[meta]]] description = "Waiting task (runs after restart)" {% include 'flow-runtime-restart.cylc' %} cylc-flow-8.7.0/tests/functional/restart/waiting/bin/0000775000175000017500000000000015264420307023016 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/waiting/bin/ctb-select-task-states0000775000175000017500000000257115264420307027237 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- set -eu CYLC_WORKFLOW_RUN_DIR="$1" CYLC_TASK_NAME="${2:-}" sqlite3 "${CYLC_WORKFLOW_RUN_DIR}/log/db" " SELECT task_states.name, task_states.cycle, task_states.submit_num, task_jobs.try_num, task_states.status FROM task_states LEFT OUTER JOIN task_jobs ON task_states.name == task_jobs.name AND task_states.cycle == task_jobs.cycle AND task_states.submit_num == task_jobs.submit_num WHERE task_states.name != '${CYLC_TASK_NAME}' ORDER BY task_states.name, task_states.cycle ;" cylc-flow-8.7.0/tests/functional/restart/03-retrying.t0000775000175000017500000000615715264420307023070 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restarting with a task waiting to retry (was retrying state). . "$(dirname "$0")/test_header" set_test_number 8 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT3M [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = """ cylc__job__wait_cylc_message_started if ((CYLC_TASK_TRY_NUMBER < 3)); then exit 1 elif ((CYLC_TASK_TRY_NUMBER == 3)); then cylc stop "${CYLC_WORKFLOW_ID}" exit 1 fi """ [[[job]]] execution retry delays = 3*PT0S __FLOW_CONFIG__ #------------------------------------------------------------------------------- run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" sqlite3 "${WORKFLOW_RUN_DIR}/log/db" 'SELECT cycle, name, status FROM task_pool' >'sqlite3.out' cmp_ok 'sqlite3.out' <<'__DB_DUMP__' 1|t1|waiting __DB_DUMP__ workflow_run_ok "${TEST_NAME_BASE}-restart-pause" \ cylc play --debug --pause "${WORKFLOW_NAME}" # query jobs TEST_NAME="${TEST_NAME_BASE}-jobs-query" read -r -d '' jobsQuery <<_args_ { "request_string": " query { jobs (sort: {keys: [\"submitNum\"]}) { state submitNum } }", "variables": null } _args_ run_graphql_ok "${TEST_NAME}" "${WORKFLOW_NAME}" "${jobsQuery}" cmp_json "${TEST_NAME}-out" "${TEST_NAME_BASE}-jobs-query.stdout" << __HERE__ { "jobs": [ { "state": "failed", "submitNum": 1 }, { "state": "failed", "submitNum": 2 }, { "state": "failed", "submitNum": 3 } ] } __HERE__ # stop workflow cylc stop --max-polls=10 --interval=2 "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-restart-resume" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" sqlite3 "${WORKFLOW_RUN_DIR}/log/db" 'SELECT * FROM task_pool' >'sqlite3.out' cmp_ok 'sqlite3.out' shutdown shutdown => output_states output_states => finish """ [runtime] [[failed_task]] script = """ exit 1 """ [[[meta]]] description = "Failed task (runs before restart)" {% include 'flow-runtime-restart.cylc' %} cylc-flow-8.7.0/tests/functional/restart/failed/bin/0000775000175000017500000000000015264420307022600 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/failed/bin/ctb-select-task-states0000775000175000017500000000257115264420307027021 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- set -eu CYLC_WORKFLOW_RUN_DIR="$1" CYLC_TASK_NAME="${2:-}" sqlite3 "${CYLC_WORKFLOW_RUN_DIR}/log/db" " SELECT task_states.name, task_states.cycle, task_states.submit_num, task_jobs.try_num, task_states.status FROM task_states LEFT OUTER JOIN task_jobs ON task_states.name == task_jobs.name AND task_states.cycle == task_jobs.cycle AND task_states.submit_num == task_jobs.submit_num WHERE task_states.name != '${CYLC_TASK_NAME}' ORDER BY task_states.name, task_states.cycle ;" cylc-flow-8.7.0/tests/functional/restart/28-execution-timeout.t0000775000175000017500000000312315264420307024711 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restart with running task with execution timeout. . "$(dirname "$0")/test_header" set_test_number 4 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" --no-detach workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play "${WORKFLOW_NAME}" --no-detach --reference-test contains_ok "${WORKFLOW_RUN_DIR}/log/job/1/foo/NN/job-activity.log" <<'__LOG__' [(('event-handler-00', 'execution timeout'), 1) cmd] echo 1/foo 'execution timeout' [(('event-handler-00', 'execution timeout'), 1) ret_code] 0 [(('event-handler-00', 'execution timeout'), 1) out] 1/foo execution timeout __LOG__ purge exit cylc-flow-8.7.0/tests/functional/restart/58-waiting-manual-triggered.t0000664000175000017500000000355315264420307026120 0ustar alastairalastair#!/bin/bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that a task manually triggered just before shutdown will run on restart. . "$(dirname "$0")/test_header" set_test_number 6 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --no-detach "${WORKFLOW_NAME}" DB_FILE="${WORKFLOW_RUN_DIR}/log/db" # It should have shut down with 2/foo waiting with the is_manual_submit flag on. TEST_NAME="${TEST_NAME_BASE}-db-task-states" QUERY="SELECT status, is_manual_submit FROM task_states WHERE cycle IS 2 AND flow_nums is '[1]';" run_ok "$TEST_NAME" sqlite3 "$DB_FILE" "$QUERY" cmp_ok "${TEST_NAME}.stdout" << '__EOF__' waiting|1 __EOF__ # It should restart and shut down normally, not stall with 2/foo waiting on 1/foo. workflow_run_ok "${TEST_NAME_BASE}-restart" cylc play --no-detach "${WORKFLOW_NAME}" # Check that 2/foo job 02 did run before shutdown. grep_workflow_log_ok "${TEST_NAME_BASE}-grep" "\[2\/foo\/02:running\] => succeeded" purge exit cylc-flow-8.7.0/tests/functional/restart/58-waiting-manual-triggered/0000775000175000017500000000000015264420307025725 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/58-waiting-manual-triggered/flow.cylc0000664000175000017500000000135615264420307027555 0ustar alastairalastair[scheduler] [[events]] stall timeout = PT0S abort on stall timeout = True [scheduling] cycling mode = integer runahead limit = P1 final cycle point = 3 [[graph]] P1 = foo[-P1] => foo [runtime] [[foo]] script = """ if (( CYLC_TASK_CYCLE_POINT == 3 )); then # Order a normal shutdown: no more job submissions, and shut # down after active jobs (i.e. this one) finish. cylc stop "$CYLC_WORKFLOW_ID" # Force-trigger 2/foo before shutdown. On restart it should be # in the waiting state with the force-triggered flag set. cylc trigger "${CYLC_WORKFLOW_ID}//2/foo" fi """ cylc-flow-8.7.0/tests/functional/restart/18-template-vars-override.t0000775000175000017500000000300415264420307025620 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test automatic custom template variables (with override) on restart. . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" \ --set='FINAL_CYCLE_POINT="2020"' --set='COMMAND="true"' workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play "${WORKFLOW_NAME}" \ --set='FINAL_CYCLE_POINT="2020"' --set='COMMAND="true"' \ --stopcp=2018 --debug --no-detach workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play "${WORKFLOW_NAME}" --debug --no-detach --reference-test \ --set='FINAL_CYCLE_POINT="2022"' purge exit cylc-flow-8.7.0/tests/functional/restart/35-auto-restart-recovery.t0000664000175000017500000000531715264420307025512 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- export REQUIRE_PLATFORM='loc:remote fs:shared runner:background' . "$(dirname "$0")/test_header" set_test_number 10 #------------------------------------------------------------------------------- # test the failure recovery mechanism BASE_GLOBAL_CONFIG=" [scheduler] [[main loop]] plugins = health check, auto restart [[[auto restart]]] interval = PT15S [[events]] abort on inactivity timeout = True abort on stall timeout = True inactivity timeout = PT2M stall timeout = PT2M [scheduler] [[run hosts]] available = localhost, ${CYLC_TEST_HOST}" TEST_NAME="${TEST_NAME_BASE}" init_workflow "${TEST_NAME}" <<< ' [scheduling] [[graph]] R1 = foo [runtime] [[foo]] ' create_test_global_config '' "${BASE_GLOBAL_CONFIG}" run_ok "${TEST_NAME}-workflow-start" \ cylc play "${WORKFLOW_NAME}" --host=localhost --pause poll_workflow_running # corrupt workflow rm "${WORKFLOW_RUN_DIR}/flow.cylc" # condemn localhost create_test_global_config '' " ${BASE_GLOBAL_CONFIG} [scheduler] [[run hosts]] condemned = $(hostname) " FILE=$(cylc cat-log "${WORKFLOW_NAME}" -m p |xargs readlink -f) log_scan "${TEST_NAME}-shutdown" "${FILE}" 20 1 \ 'The Cylc workflow host will soon become un-available' \ 'Workflow shutting down - REQUEST(NOW-NOW)' \ "Attempting to restart on \"${CYLC_TEST_HOST}\"" \ 'Could not restart workflow will retry in 5s' \ "Attempting to restart on \"${CYLC_TEST_HOST}\"" \ 'Could not restart workflow will retry in 5s' \ "Attempting to restart on \"${CYLC_TEST_HOST}\"" \ 'Could not restart workflow will retry in 5s' \ 'Workflow unable to automatically restart after 3 tries' # stop workflow - workflow should already by stopped but just to be safe cylc stop --max-polls=10 --interval=2 --kill "${WORKFLOW_NAME}" 2>'/dev/null' purge cylc-flow-8.7.0/tests/functional/restart/50-two-flows.t0000664000175000017500000000237715264420307023165 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restart with two active flows present. . "$(dirname "$0")/test_header" set_test_number 5 # first run reference test install_and_validate reftest_run # restart reference test mv "${WORKFLOW_RUN_DIR}/reference.restart.log" "${WORKFLOW_RUN_DIR}/reference.log" reftest_run grep_workflow_log_ok flow-1 "flow: 1 (original flow from 1)" grep_workflow_log_ok flow-2 "flow: 2 (cheese wizard)" purge cylc-flow-8.7.0/tests/functional/restart/02-failed.t0000775000175000017500000000520615264420307022442 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restarting a simple workflow with a failed task if [[ -z ${TEST_DIR:-} ]]; then . "$(dirname "$0")/test_header" fi #------------------------------------------------------------------------------- set_test_number 7 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" 'failed' cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "${RUN_DIR}/${WORKFLOW_NAME}/" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" cmp_ok "${TEST_NAME}.stderr" <'/dev/null' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-restart-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- grep_ok "failed_task|20130923T0000Z|1|1|failed" \ "${RUN_DIR}/${WORKFLOW_NAME}/pre-restart-db" contains_ok "${RUN_DIR}/${WORKFLOW_NAME}/post-restart-db" <<'__DB_DUMP__' failed_task|20130923T0000Z|1|1|failed shutdown|20130923T0000Z|1|1|succeeded __DB_DUMP__ "${TEST_SOURCE_DIR}/bin/ctb-select-task-states" "${WORKFLOW_RUN_DIR}" \ > "${RUN_DIR}/${WORKFLOW_NAME}/db" contains_ok "${RUN_DIR}/${WORKFLOW_NAME}/db" <<'__DB_DUMP__' failed_task|20130923T0000Z|1|1|failed finish|20130923T0000Z|1|1|succeeded output_states|20130923T0000Z|1|1|succeeded shutdown|20130923T0000Z|1|1|succeeded __DB_DUMP__ #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/restart/04-running.t0000775000175000017500000000450015264420307022674 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restarting a simple workflow with a task still running (orphaned) . "$(dirname "$0")/test_header" set_test_number 5 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT3M [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = """ cylc__job__wait_cylc_message_started cylc__job__poll_grep_workflow_log -F '[1/t1/01:submitted] => running' cylc stop --now "${CYLC_WORKFLOW_ID}" cylc__job__poll_grep_workflow_log -F 'Run: (re)start number=2' # Should be good to send succeeded message at this point """ __FLOW_CONFIG__ #------------------------------------------------------------------------------- run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" sqlite3 "${WORKFLOW_RUN_DIR}/log/db" 'SELECT cycle, name, status FROM task_pool' >'sqlite3.out' cmp_ok 'sqlite3.out' <<'__DB_DUMP__' 1|t1|running __DB_DUMP__ workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" sqlite3 "${WORKFLOW_RUN_DIR}/log/db" 'SELECT * FROM task_pool' >'sqlite3.out' cmp_ok 'sqlite3.out' <'/dev/null' #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/restart/23-hold-retry.t0000775000175000017500000000347115264420307023314 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restart with held task waiting to retry . "$(dirname "$0")/test_header" set_test_number 5 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ 'SELECT cycle, name, status FROM task_pool ORDER BY cycle, name' >'task-pool.out' cmp_ok 'task-pool.out' <<__OUT__ 1|t1|waiting __OUT__ # restart cylc play "${WORKFLOW_NAME}" --debug --no-detach 1>'out' 2>&1 & WORKFLOW_PID=$! poll_grep_workflow_log -F 'INFO - + 1/t1 waiting (held)' run_ok "${TEST_NAME_BASE}-release" cylc release "${WORKFLOW_NAME}//1/t1" poll_grep_workflow_log -F 'INFO - DONE' if ! wait "${WORKFLOW_PID}"; then cat 'out' >&2 fi sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ 'SELECT * FROM task_pool ORDER BY cycle, name' >'task-pool.out' cmp_ok 'task-pool.out' shutdown submit_failed_task:submitted? => error shutdown => output_states output_states => remove => finish """ [runtime] [[submit_failed_task]] script = """ exit 1 # Should not submit, so this shouldn't run! """ platform = {{ environ['CYLC_TEST_PLATFORM'] }} [[[meta]]] description = "Submit-failed task (runs before restart)" [[remove]] script = """ cylc remove "${CYLC_WORKFLOW_ID}//${CYLC_TASK_CYCLE_POINT}/submit_failed_task" """ [[error]] script = """ cylc message -- "ERROR:this-task-should-have-submit-failed" exit 1 """ {% include 'flow-runtime-restart.cylc' %} cylc-flow-8.7.0/tests/functional/restart/submit-failed/bin/0000775000175000017500000000000015264420307024101 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/submit-failed/bin/ctb-select-task-states0000775000175000017500000000257115264420307030322 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- set -eu CYLC_WORKFLOW_RUN_DIR="$1" CYLC_TASK_NAME="${2:-}" sqlite3 "${CYLC_WORKFLOW_RUN_DIR}/log/db" " SELECT task_states.name, task_states.cycle, task_states.submit_num, task_jobs.try_num, task_states.status FROM task_states LEFT OUTER JOIN task_jobs ON task_states.name == task_jobs.name AND task_states.cycle == task_jobs.cycle AND task_states.submit_num == task_jobs.submit_num WHERE task_states.name != '${CYLC_TASK_NAME}' ORDER BY task_states.name, task_states.cycle ;" cylc-flow-8.7.0/tests/functional/restart/08-stop-after-cycle-point.t0000664000175000017500000001114215264420307025525 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check that workflow does not run beyond stopcp whether set in flow.cylc or # on the command line. . "$(dirname "$0")/test_header" dumpdbtables() { sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ "SELECT value FROM workflow_params WHERE key=='stopcp';" > db_stopcp.out sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ 'SELECT cycle, name, status FROM task_pool ORDER BY cycle, name;' > db_taskpool.out.out } set_test_number 29 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" # Set the config stop point (accessed via Jinja2) export CFG_STOPCP CFG_STOPCP="1970" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" # Check config stop point works workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --no-detach "${WORKFLOW_NAME}" grep_workflow_log_ok "${TEST_NAME_BASE}-log-grep" "Stop point: 1970" dumpdbtables # Task 1971/hello (after stop point) should be spawned but not submitted cmp_ok db_taskpool.out.out << '__OUT__' 1971|hello|waiting __OUT__ # Check that the config stop point does not get stored in DB cmp_ok db_stopcp.out <<< '' CFG_STOPCP="1972" # Check that changing config stop point has worked workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play --no-detach "${WORKFLOW_NAME}" \ -s 'MANUAL_SHUTDOWN="1971"' grep_workflow_log_ok "${TEST_NAME_BASE}-log-grep" "Stop point: 1972" dumpdbtables # Note we have manually stopped at 1971 before the stop point at 1972: # 1972: waiting # 1973: waiting and runahead limited cmp_ok db_taskpool.out.out << '__OUT__' 1972|hello|waiting 1973|hello|waiting __OUT__ cmp_ok db_stopcp.out <<< '' # Check that the command line stop point works workflow_run_ok "${TEST_NAME_BASE}-restart-cli-stopcp" \ cylc play --no-detach --stopcp=1974 "${WORKFLOW_NAME}" \ -s 'MANUAL_SHUTDOWN="1973"' grep_workflow_log_ok "${TEST_NAME_BASE}-log-grep" "Stop point: 1974" dumpdbtables # Note we have manually stopped before the stop point cmp_ok db_taskpool.out.out << '__OUT__' 1974|hello|waiting 1975|hello|waiting __OUT__ # Check CLI stop point is stored in DB cmp_ok db_stopcp.out <<< '1974' # Check that the stop point stored in DB works on restart workflow_run_ok "${TEST_NAME_BASE}-restart-db-stopcp" \ cylc play --no-detach "${WORKFLOW_NAME}" grep_workflow_log_ok "${TEST_NAME_BASE}-log-grep" "Stop point: 1974" dumpdbtables cmp_ok db_taskpool.out.out <<'__OUT__' 1975|hello|waiting __OUT__ # Stop point should be removed from DB once reached cmp_ok db_stopcp.out <<< '' # Restart again with new CLI stop point workflow_run_ok "${TEST_NAME_BASE}-restart-cli-stopcp-2" \ cylc play --no-detach --stopcp=1978 "${WORKFLOW_NAME}" \ -s 'MANUAL_SHUTDOWN="1975"' grep_workflow_log_ok "${TEST_NAME_BASE}-log-grep" "Stop point: 1978" dumpdbtables # Note we have manually stopped before the stop point cmp_ok db_taskpool.out.out << '__OUT__' 1976|hello|waiting 1977|hello|waiting __OUT__ cmp_ok db_stopcp.out <<< '1978' CFG_STOPCP="1979" # Check that --stopcp=reload takes value from flow.cylc on restart workflow_run_ok "${TEST_NAME_BASE}-restart-cli-stopcp-reload" \ cylc play --no-detach --stopcp=reload "${WORKFLOW_NAME}" \ -s 'MANUAL_SHUTDOWN="1976"' grep_workflow_log_ok "${TEST_NAME_BASE}-log-grep" "Stop point: 1979" dumpdbtables cmp_ok db_taskpool.out.out << '__OUT__' 1977|hello|waiting 1978|hello|waiting __OUT__ # Stop point should be removed from DB if --stopcp=reload used cmp_ok db_stopcp.out <<< '' CFG_STOPCP="1971" # Check that workflow stops immediately if restarted past stopcp workflow_run_ok "${TEST_NAME_BASE}-restart-past-stopcp" \ cylc play --no-detach "${WORKFLOW_NAME}" grep_workflow_log_ok "${TEST_NAME_BASE}-log-grep" "Stop point: 1971" dumpdbtables cmp_ok db_taskpool.out.out << '__OUT__' 1977|hello|waiting 1978|hello|waiting __OUT__ cmp_ok db_stopcp.out <<< '' purge cylc-flow-8.7.0/tests/functional/restart/06-succeeded.t0000775000175000017500000000514515264420307023150 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restarting a simple workflow with a succeeded task if [[ -z ${TEST_DIR:-} ]]; then . "$(dirname "$0")/test_header" fi #------------------------------------------------------------------------------- set_test_number 7 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" 'succeeded' cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "${WORKFLOW_RUN_DIR}/" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" cmp_ok "${TEST_NAME}.stderr" <'/dev/null' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME=${TEST_NAME_BASE}-restart-run workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- grep_ok "succeeded_task|20130923T0000Z|1|1|succeeded" \ "${WORKFLOW_RUN_DIR}/pre-restart-db" contains_ok "${WORKFLOW_RUN_DIR}/post-restart-db" <<'__DB_DUMP__' shutdown|20130923T0000Z|1|1|succeeded succeeded_task|20130923T0000Z|1|1|succeeded __DB_DUMP__ "${WORKFLOW_RUN_DIR}/bin/ctb-select-task-states" "${WORKFLOW_RUN_DIR}" \ > "${TEST_DIR}/db" contains_ok "${TEST_DIR}/db" <<'__DB_DUMP__' finish|20130923T0000Z|1|1|succeeded output_states|20130923T0000Z|1|1|succeeded shutdown|20130923T0000Z|1|1|succeeded succeeded_task|20130923T0000Z|1|1|succeeded __DB_DUMP__ #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/restart/succeeded/0000775000175000017500000000000015264420307022530 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/succeeded/flow.cylc0000664000175000017500000000111715264420307024353 0ustar alastairalastair#!jinja2 {%- set TEST_DIR = environ['TEST_DIR'] %} [scheduler] UTC mode = True [[events]] abort on stall timeout = True stall timeout = PT3M [scheduling] initial cycle point = 20130923T00 final cycle point = 20130923T00 [[graph]] R1 = """ succeeded_task => shutdown shutdown => output_states output_states => finish """ [runtime] [[succeeded_task]] script = true [[[meta]]] description = "Succeeded task (runs before restart)" {% include 'flow-runtime-restart.cylc' %} cylc-flow-8.7.0/tests/functional/restart/succeeded/bin/0000775000175000017500000000000015264420307023300 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/succeeded/bin/ctb-select-task-states0000775000175000017500000000257115264420307027521 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- set -eu CYLC_WORKFLOW_RUN_DIR="$1" CYLC_TASK_NAME="${2:-}" sqlite3 "${CYLC_WORKFLOW_RUN_DIR}/log/db" " SELECT task_states.name, task_states.cycle, task_states.submit_num, task_jobs.try_num, task_states.status FROM task_states LEFT OUTER JOIN task_jobs ON task_states.name == task_jobs.name AND task_states.cycle == task_jobs.cycle AND task_states.submit_num == task_jobs.submit_num WHERE task_states.name != '${CYLC_TASK_NAME}' ORDER BY task_states.name, task_states.cycle ;" cylc-flow-8.7.0/tests/functional/restart/00-pre-initial.t0000775000175000017500000000330015264420307023422 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restarting a workflow with pre-initial cycle dependencies . "$(dirname "$0")/test_header" set_test_number 5 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --debug --no-detach "${WORKFLOW_NAME}" sqlite3 "${RUN_DIR}/${WORKFLOW_NAME}/log/db" \ 'SELECT name, cycle, status FROM task_pool ORDER BY name, cycle' \ >'mid-state' cmp_ok 'mid-state' <<"__OUT__" p1|20100808T0000Z|running __OUT__ workflow_run_ok "${TEST_NAME_BASE}-restart" cylc play --debug --no-detach "${WORKFLOW_NAME}" sqlite3 "${RUN_DIR}/${WORKFLOW_NAME}/log/db" \ 'SELECT name, cycle, status FROM task_states ORDER BY name, cycle' \ >'final-state' contains_ok 'final-state' "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/ref-state" purge exit cylc-flow-8.7.0/tests/functional/restart/lib/0000775000175000017500000000000015264420307021352 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/lib/flow-runtime-restart.cylc0000664000175000017500000000165315264420307026345 0ustar alastairalastair [[OUTPUT]] script = """ cylc__job__wait_cylc_message_started # Handle db locks. for i in {0..10}; do ctb-select-task-states \ "${CYLC_WORKFLOW_RUN_DIR}" "${CYLC_TASK_NAME}" \ > "${CYLC_WORKFLOW_RUN_DIR}/$OUTPUT_SUFFIX-db" && break sleep 1 done """ [[shutdown]] inherit = OUTPUT post-script = cylc shutdown "${CYLC_WORKFLOW_ID}" [[[meta]]] description = "Force a shutdown of the workflow" [[[environment]]] OUTPUT_SUFFIX=pre-restart [[output_states]] inherit = OUTPUT pre-script = cylc__job__wait_cylc_message_started [[[meta]]] description = "Wait for the restart to complete, then output states" [[[environment]]] OUTPUT_SUFFIX=post-restart [[finish]] script = true cylc-flow-8.7.0/tests/functional/restart/57-ghost-job.t0000664000175000017500000000634215264420307023123 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restarting when there is a "ghost job" in the DB (has a submit_time in # task_jobs table but no submit_exit_time or run_time) . "$(dirname "$0")/test_header" set_test_number 9 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" mkdir "${WORKFLOW_RUN_DIR}/.service" DB_PATH="${WORKFLOW_RUN_DIR}/.service/db" sqlite3 "$DB_PATH" < db.sqlite3 query_db_task_jobs() { TEST_NAME="$1" sqlite3 "$DB_PATH" \ 'SELECT cycle, name, submit_num FROM task_jobs' > "${TEST_NAME}.stdout" } get_submit_time() { sqlite3 "$DB_PATH" 'SELECT time_submit FROM task_jobs' } # cylc client input stored as var: read -r -d '' gqlQuery << _args_ { "request_string": " query { workflows(ids: [\"${WORKFLOW_NAME}\"]) { jobs { cyclePoint, name, submitNum } } }" } _args_ # ----------------------------------------------------------------------------- run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-restart" cylc play --upgrade -vv "${WORKFLOW_NAME}" --pause # There will be 1 ghost job in DB: TEST_NAME="${TEST_NAME_BASE}-db-query-1" query_db_task_jobs "$TEST_NAME" cmp_ok "${TEST_NAME}.stdout" << EOF 1|foo|1 EOF # Get submit time for that job: orig_submit_time="$(get_submit_time)" # Ghost job should not be in data store: TEST_NAME="${TEST_NAME_BASE}-datastore-query-1" run_graphql_ok "$TEST_NAME" "$WORKFLOW_NAME" "$gqlQuery" cmp_json "${TEST_NAME}-cmp" "${TEST_NAME}.stdout" << EOF { "workflows": [{ "jobs": [] }] } EOF workflow_run_ok "${TEST_NAME_BASE}-resume" cylc play --upgrade "${WORKFLOW_NAME}" poll_workflow_stopped # Job should have been replaced in DB with same submit num: TEST_NAME="${TEST_NAME_BASE}-db-query-2" query_db_task_jobs "$TEST_NAME" cmp_ok "${TEST_NAME}.stdout" << EOF 1|foo|1 EOF # Submit time should be different however: TEST_NAME="${TEST_NAME_BASE}-submit-time-cmp" if [[ "$(get_submit_time)" != "$orig_submit_time" ]]; then ok "$TEST_NAME" else fail "$TEST_NAME" \ "Expected time_submit in task_jobs table to be different from original" fi # Job should be in data store: TEST_NAME="${TEST_NAME_BASE}-datastore-query-2" cmp_json "${TEST_NAME}-cmp" "${WORKFLOW_RUN_DIR}/work/1/foo/gqlResponse.json" << EOF { "workflows": [{ "jobs": [{ "cyclePoint": "1", "name": "foo", "submitNum": 1 }] }] } EOF purge cylc-flow-8.7.0/tests/functional/restart/18-template-vars-override/0000775000175000017500000000000015264420307025433 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/18-template-vars-override/reference.log0000664000175000017500000000031615264420307030074 0ustar alastairalastairInitial point: 20160101T0000Z Final point: 20220101T0000Z 20190101T0000Z/foo -triggered off [] 20200101T0000Z/foo -triggered off [] 20210101T0000Z/foo -triggered off [] 20220101T0000Z/foo -triggered off [] cylc-flow-8.7.0/tests/functional/restart/18-template-vars-override/flow.cylc0000664000175000017500000000032515264420307027256 0ustar alastairalastair#!Jinja2 [scheduler] UTC mode = True [scheduling] initial cycle point = 2016 final cycle point = {{FINAL_CYCLE_POINT}} [[graph]] P1Y = foo [runtime] [[foo]] script = {{COMMAND}} cylc-flow-8.7.0/tests/functional/restart/25-hold-workflow/0000775000175000017500000000000015264420307023626 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/25-hold-workflow/flow.cylc0000664000175000017500000000104215264420307025446 0ustar alastairalastair[scheduler] UTC mode=True cycle point format = %Y [[events]] abort on inactivity timeout = True inactivity timeout = P2M [scheduling] initial cycle point = 2016 final cycle point = 2017 [[graph]] P1Y = t1[-P1Y] => t1 => t2 [runtime] [[t1]] script = """ if [[ "${CYLC_TASK_CYCLE_POINT}" == '2016' ]]; then cylc hold --after=1900 "${CYLC_WORKFLOW_ID}" cylc stop "${CYLC_WORKFLOW_ID}" fi """ [[t2]] script = true cylc-flow-8.7.0/tests/functional/restart/22-hold.t0000775000175000017500000000413515264420307022146 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restart with held (waiting) task . "$(dirname "$0")/test_header" set_test_number 8 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" --debug --no-detach # Check task pool sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ 'SELECT cycle, name, status FROM task_pool WHERE is_held==1 ORDER BY cycle, name' \ >'task_pool-1.out' cmp_ok 'task_pool-1.out' << __EOF__ 2016|t2|waiting __EOF__ # Check tasks_to_hold table sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ 'SELECT cycle, name FROM tasks_to_hold ORDER BY cycle, name' \ > 'tasks_to_hold-1.out' cmp_ok 'tasks_to_hold-1.out' << __EOF__ 2016|t2 2017|t2 __EOF__ workflow_run_ok "${TEST_NAME_BASE}-restart" cylc play "${WORKFLOW_NAME}" --debug --no-detach grep_ok 'INFO - + 2016/t2 waiting (held)' "${WORKFLOW_RUN_DIR}/log/scheduler/log" # Check task pool sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ 'SELECT * FROM task_pool' >'task_pool-end.out' cmp_ok 'task_pool-end.out' 'tasks_to_hold-end.out' cmp_ok 'tasks_to_hold-end.out' < /dev/null purge exit cylc-flow-8.7.0/tests/functional/restart/16-template-vars.t0000775000175000017500000000271515264420307024011 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test automatic custom template variables on restart. . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" \ --set='FINAL_CYCLE_POINT="2020"' --set='COMMAND="true"' workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play "${WORKFLOW_NAME}" \ --set='FINAL_CYCLE_POINT="2020"' --set='COMMAND="true"' \ --stopcp=2018 --debug --no-detach workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play "${WORKFLOW_NAME}" --debug --no-detach --reference-test purge exit cylc-flow-8.7.0/tests/functional/restart/26-remote-kill.t0000775000175000017500000000340515264420307023447 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test stop with a remote running task, restart, kill the task. export REQUIRE_PLATFORM='loc:remote' . "$(dirname "$0")/test_header" set_test_number 5 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" --debug --no-detach sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ "SELECT status FROM task_pool WHERE cycle=='1' AND NAME=='t1'" \ >'t1-status.out' cmp_ok 't1-status.out' <<<'running' run_ok "${TEST_NAME_BASE}-restart" cylc play "${WORKFLOW_NAME}" # Ensure workflow has started poll_workflow_running cylc kill "${WORKFLOW_NAME}//1/t1" # Ensure workflow has completed poll_workflow_stopped sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ "SELECT status FROM task_pool WHERE cycle=='1' AND NAME=='t1'" \ >'t1-status.out' cmp_ok 't1-status.out' <<<'failed' purge exit cylc-flow-8.7.0/tests/functional/restart/45-stop-task.t0000664000175000017500000000504515264420307023150 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restart with stop task . "$(dirname "$0")/test_header" dumpdbtables() { sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ "SELECT value FROM workflow_params WHERE key=='stop_task';" >'stoptask.out' sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ 'SELECT cycle, name, status FROM task_pool ORDER BY cycle, name;' >'taskpool.out' } set_test_number 10 # Event should look like this: # Start workflow # At 1/t1, set stop task to 1/t5 # At 1/t2, stop workflow at 1/t2 # Restart # Workflow runs to stop task 1/t5, reset stop task. # Restart # Workflow stops normally at 1/t8 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [task parameters] i = 1..8 [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = P1M [scheduling] [[graph]] R1 = t => t [runtime] [[t]] script = true [[t]] script = cylc stop "${CYLC_WORKFLOW_ID}//1/t_i5" [[t]] script = cylc stop "${CYLC_WORKFLOW_ID}" __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" --no-detach dumpdbtables cmp_ok 'stoptask.out' <<<'1/t_i5' cmp_ok 'taskpool.out' <<'__OUT__' 1|t_i3|waiting __OUT__ workflow_run_ok "${TEST_NAME_BASE}-restart-1" \ cylc play "${WORKFLOW_NAME}" --no-detach dumpdbtables cmp_ok 'stoptask.out' <<<'' cmp_ok 'taskpool.out' <<'__OUT__' 1|t_i6|waiting __OUT__ workflow_run_ok "${TEST_NAME_BASE}-restart-2" \ cylc play "${WORKFLOW_NAME}" --no-detach dumpdbtables cmp_ok 'stoptask.out' <<<'' cmp_ok 'taskpool.out' <'/dev/null' purge exit cylc-flow-8.7.0/tests/functional/restart/34-auto-restart-basic.t0000664000175000017500000000735715264420307024742 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- export REQUIRE_PLATFORM='loc:remote fs:shared runner:background' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 10 if ${CYLC_TEST_DEBUG:-false}; then ERR=2; else ERR=1; fi #------------------------------------------------------------------------------- # run through the shutdown - restart procedure BASE_GLOBAL_CONFIG=" [scheduler] [[main loop]] plugins = health check, auto restart [[[auto restart]]] interval = PT2S [[events]] abort on inactivity timeout = True abort on stall timeout = True inactivity timeout = PT1M stall timeout = PT1M [scheduler] [[run hosts]] available = localhost, ${CYLC_TEST_HOST}" TEST_NAME="${TEST_NAME_BASE}" init_workflow "${TEST_NAME}" - <<'__FLOW_CONFIG__' [task parameters] foo = 1..25 [scheduling] [[graph]] R1 = "task => task" [runtime] [[task]] [[task_26]] __FLOW_CONFIG__ # run workflow on localhost normally create_test_global_config '' "${BASE_GLOBAL_CONFIG}" run_ok "${TEST_NAME}-workflow-start" cylc play "${WORKFLOW_NAME}" --host=localhost -s 'FOO="foo"' -v cylc workflow-state "${WORKFLOW_NAME}//1/task_foo01:succeeded" --interval=1 --max-polls=20 >& $ERR # condemn localhost create_test_global_config '' " ${BASE_GLOBAL_CONFIG} [scheduler] [[run hosts]] condemned = $(hostname) " # test shutdown procedure - scan the first log file FILE=$(cylc cat-log "${WORKFLOW_NAME}" -m p |xargs readlink -f) log_scan "${TEST_NAME}-shutdown-log-scan" "${FILE}" 20 1 \ 'The Cylc workflow host will soon become un-available' \ 'Workflow shutting down - REQUEST(NOW-NOW)' \ "Attempting to restart on \"${CYLC_TEST_HOST}\"" \ "Workflow now running on \"${CYLC_TEST_HOST}\"" LATEST_TASK=$(cylc workflow-state --old-format "${WORKFLOW_NAME}//*/*:succeeded" \ | cut -d ',' -f 1 | sort | tail -n 1 | sed 's/task_foo//') # test restart procedure - scan the second log file created on restart poll_workflow_restart FILE=$(cylc cat-log "${WORKFLOW_NAME}" -m p |xargs readlink -f) log_scan "${TEST_NAME}-restart-log-scan" "${FILE}" 20 1 \ "Scheduler: url=tcp://$(get_fqdn "${CYLC_TEST_HOST}")" run_ok "${TEST_NAME}-restart-success" \ cylc workflow-state "${WORKFLOW_NAME}//1/$(printf 'task_foo%02d' $(( LATEST_TASK + 3 ))):succeeded" \ --interval=1 --max-polls=60 # check the command the workflow has been restarted with run_ok "${TEST_NAME}-contact" cylc get-contact "${WORKFLOW_NAME}" grep_ok "cylc play ${WORKFLOW_NAME} -v --host=${CYLC_TEST_HOST} --host=localhost" \ "${TEST_NAME}-contact.stdout" # stop workflow cylc stop "${WORKFLOW_NAME}" --kill --max-polls=10 --interval=2 2>'/dev/null' # Check correct number of logs ls "${WORKFLOW_RUN_DIR}/log/scheduler/" > ls_logs.out cmp_ok ls_logs.out << __EOF__ 01-start-01.log 02-restart-02.log log __EOF__ purge cylc-flow-8.7.0/tests/functional/restart/57-ghost-job/0000775000175000017500000000000015264420307022731 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/57-ghost-job/flow.cylc0000664000175000017500000000123015264420307024550 0ustar alastairalastair[scheduling] [[graph]] R1 = foo [runtime] [[foo]] # This task checks for its job in the data store and outputs it to file script = """ set +e # (read exits 1 for some reason) read -r -d '' gqlQuery << _args_ { "request_string": " query { workflows(ids: [\"${CYLC_WORKFLOW_ID}\"]) { jobs { cyclePoint, name, submitNum } } }" } _args_ set -e cylc client "$CYLC_WORKFLOW_ID" graphql < <(echo $gqlQuery) > gqlResponse.json """ cylc-flow-8.7.0/tests/functional/restart/57-ghost-job/db.sqlite30000664000175000017500000000666715264420307024643 0ustar alastairalastair-- DB with a "ghost job" in the task_jobs table that has a time_submit but no time_submit_exit or time_run PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE absolute_outputs(cycle TEXT, name TEXT, output TEXT); CREATE TABLE broadcast_events(time TEXT, change TEXT, point TEXT, namespace TEXT, key TEXT, value TEXT); CREATE TABLE broadcast_states(point TEXT, namespace TEXT, key TEXT, value TEXT, PRIMARY KEY(point, namespace, key)); CREATE TABLE inheritance(namespace TEXT, inheritance TEXT, PRIMARY KEY(namespace)); INSERT INTO inheritance VALUES('root','["root"]'); INSERT INTO inheritance VALUES('foo','["foo", "root"]'); CREATE TABLE task_action_timers(cycle TEXT, name TEXT, ctx_key TEXT, ctx TEXT, delays TEXT, num INTEGER, delay TEXT, timeout TEXT, PRIMARY KEY(cycle, name, ctx_key)); INSERT INTO task_action_timers VALUES('1','foo','["try_timers", "submission-retry"]','null','[]',0,NULL,NULL); INSERT INTO task_action_timers VALUES('1','foo','["try_timers", "execution-retry"]','null','[]',0,NULL,NULL); CREATE TABLE task_events(name TEXT, cycle TEXT, time TEXT, submit_num INTEGER, event TEXT, message TEXT); CREATE TABLE task_jobs(cycle TEXT, name TEXT, submit_num INTEGER, is_manual_submit INTEGER, try_num INTEGER, time_submit TEXT, time_submit_exit TEXT, submit_status INTEGER, time_run TEXT, time_run_exit TEXT, run_signal TEXT, run_status INTEGER, platform_name TEXT, job_runner_name TEXT, job_id TEXT, PRIMARY KEY(cycle, name, submit_num)); INSERT INTO task_jobs VALUES('1','foo',1,0,1,'2022-07-25T16:18:23+01:00',NULL,NULL,NULL,NULL,NULL,NULL,'localhost','background',NULL); CREATE TABLE task_late_flags(cycle TEXT, name TEXT, value INTEGER, PRIMARY KEY(cycle, name)); CREATE TABLE task_outputs(cycle TEXT, name TEXT, flow_nums TEXT, outputs TEXT, PRIMARY KEY(cycle, name, flow_nums)); INSERT INTO task_outputs VALUES('1','foo','[1]','[]'); CREATE TABLE task_pool(cycle TEXT, name TEXT, flow_nums TEXT, status TEXT, is_held INTEGER, PRIMARY KEY(cycle, name, flow_nums)); INSERT INTO task_pool VALUES('1','foo','[1]','preparing',0); CREATE TABLE task_prerequisites(cycle TEXT, name TEXT, flow_nums TEXT, prereq_name TEXT, prereq_cycle TEXT, prereq_output TEXT, satisfied TEXT, PRIMARY KEY(cycle, name, flow_nums, prereq_name, prereq_cycle, prereq_output)); CREATE TABLE task_states(name TEXT, cycle TEXT, flow_nums TEXT, time_created TEXT, time_updated TEXT, submit_num INTEGER, status TEXT, flow_wait INTEGER, is_manual_submit INTEGER, PRIMARY KEY(name, cycle, flow_nums)); INSERT INTO task_states VALUES('foo','1','[1]','2022-07-25T16:18:23+01:00','2022-07-25T16:18:23+01:00',1,'preparing',NULL, '0'); CREATE TABLE task_timeout_timers(cycle TEXT, name TEXT, timeout REAL, PRIMARY KEY(cycle, name)); CREATE TABLE tasks_to_hold(name TEXT, cycle TEXT); CREATE TABLE workflow_flows(flow_num INTEGER, start_time TEXT, description TEXT, PRIMARY KEY(flow_num)); INSERT INTO workflow_flows VALUES(1,'2022-07-25 16:18:23','original flow from 1'); CREATE TABLE workflow_params(key TEXT, value TEXT, PRIMARY KEY(key)); INSERT INTO workflow_params VALUES('uuid_str','4972bc10-a016-46b0-b313-b10f3cb63bf5'); INSERT INTO workflow_params VALUES('cylc_version','8.0.3.dev'); INSERT INTO workflow_params VALUES('UTC_mode','0'); INSERT INTO workflow_params VALUES('n_restart','0'); INSERT INTO workflow_params VALUES('cycle_point_tz','Z'); CREATE TABLE workflow_template_vars(key TEXT, value TEXT, PRIMARY KEY(key)); CREATE TABLE xtriggers(signature TEXT, results TEXT, PRIMARY KEY(signature)); COMMIT; cylc-flow-8.7.0/tests/functional/restart/53-task-prerequisites/0000775000175000017500000000000015264420307024675 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/53-task-prerequisites/flow.cylc0000664000175000017500000000105215264420307026516 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] cycling mode = integer final cycle point = 5 [[graph]] P1 = """ foo foo[-P1] & apollo:landed => bar """ [runtime] [[foo]] script = """ [[ "$CYLC_TASK_JOB" != '1/foo/01' ]] """ [[apollo]] script = cylc message -- "The Eagle has landed" [[[outputs]]] landed = The Eagle has landed cylc-flow-8.7.0/tests/functional/restart/52-cycle-point-time-zone.t0000664000175000017500000000406115264420307025351 0ustar alastairalastair#!/bin/bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test saving and loading of cycle point time zone to/from database on a run # followed by a restart. Important for restarting a workflow after a system # time zone change. . "$(dirname "$0")/test_header" set_test_number 6 init_workflow "${TEST_NAME_BASE}" << '__FLOW__' #!jinja2 [scheduler] cycle point time zone = {{ CPTZ }} UTC mode = False allow implicit tasks = True [scheduling] initial cycle point = now [[graph]] R1 = foo __FLOW__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" -s "CPTZ='Z'" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" --pause -s "CPTZ='+0100'" poll_workflow_running cylc stop "${WORKFLOW_NAME}" poll_workflow_stopped sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ "SELECT * FROM workflow_params WHERE key=='cycle_point_tz';" > 'dump.out' cmp_ok 'dump.out' <<< 'cycle_point_tz|+0100' # Simulate DST change export TZ=UTC workflow_run_ok "${TEST_NAME_BASE}-restart" cylc play "${WORKFLOW_NAME}" --pause poll_workflow_running cylc stop "${WORKFLOW_NAME}" log_scan "${TEST_NAME_BASE}-log-scan" "${WORKFLOW_RUN_DIR}/log/scheduler/log" 1 0 \ 'LOADING saved workflow parameters' \ '+ cycle point time zone = +0100' purge cylc-flow-8.7.0/tests/functional/restart/38-auto-restart-stopping.t0000664000175000017500000000460115264420307025515 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- export REQUIRE_PLATFORM='loc:remote fs:shared runner:background' . "$(dirname "$0")/test_header" set_test_number 3 if ${CYLC_TEST_DEBUG:-false}; then ERR=2; else ERR=1; fi #------------------------------------------------------------------------------- # ensure that workflows don't get auto stop-restarted if they are already stopping BASE_GLOBAL_CONFIG=" [scheduler] [[main loop]] plugins = health check, auto restart [[[auto restart]]] interval = PT1S [[events]] abort on inactivity timeout = True abort on stall timeout = True inactivity timeout = PT1M stall timeout = PT1M [[run hosts]] available = localhost, ${CYLC_TEST_HOST}" TEST_NAME="${TEST_NAME_BASE}" init_workflow "${TEST_NAME}" - <<'__FLOW_CONFIG__' [scheduling] [[graph]] R1 = foo => bar [runtime] [[foo]] script = cylc stop "${CYLC_WORKFLOW_ID}"; sleep 15 [[bar]] __FLOW_CONFIG__ create_test_global_config '' " ${BASE_GLOBAL_CONFIG} " run_ok "${TEST_NAME}-workflow-start" cylc play "${WORKFLOW_NAME}" --host=localhost run_ok "wait-for-task-foo-to-start" \ cylc workflow-state "${WORKFLOW_NAME}//1/foo:started" --triggers --interval=1 --max-polls=20 >& $ERR # condemn localhost create_test_global_config '' " ${BASE_GLOBAL_CONFIG} [scheduler] [[run hosts]] condemned = $(hostname) " # wait for workflow to die of natural causes poll_workflow_stopped grep_ok 'Workflow shutting down - REQUEST(CLEAN)' \ "$(cylc cat-log "${WORKFLOW_NAME}" -m p)" purge exit cylc-flow-8.7.0/tests/functional/restart/51-final-point-reload.t0000664000175000017500000000556415264420307024712 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restart with reloaded final point . "$(dirname "$0")/test_header" dumpdbtables() { sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ "SELECT * FROM workflow_params WHERE key=='fcp';" >'fcp.out' sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ "SELECT cycle, name, status FROM task_pool ORDER BY cycle, name;" >'taskpool.out' } set_test_number 13 # Event should look like this: # Start workflow with final point = 2018 # Request workflow stop while at 2015 # Restart # Workflow runs to final cycle point == 2018 # Restart # Workflow stop immediately # Restart, reload final cycle point # Workflow runs to final cycle point == 2020 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduler] UTC mode=True cycle point format = %Y [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = P1M [scheduling] initial cycle point = 2015 final cycle point = 2018 [[graph]] P1Y = t1[-P1Y] => t1 [runtime] [[t1]] script = """ case "${CYLC_TASK_CYCLE_POINT}" in 2015) cylc stop "${CYLC_WORKFLOW_ID}" :;; esac """ __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play "${WORKFLOW_NAME}" --no-detach --fcp=2017 dumpdbtables cmp_ok 'fcp.out' <<<'fcp|2017' cmp_ok 'taskpool.out' <<'__OUT__' 2016|t1|waiting __OUT__ workflow_run_ok "${TEST_NAME_BASE}-restart-1" \ cylc play "${WORKFLOW_NAME}" --no-detach dumpdbtables cmp_ok 'fcp.out' <<<'fcp|2017' cmp_ok 'taskpool.out' <'/dev/null' workflow_run_ok "${TEST_NAME_BASE}-restart-2" \ cylc play "${WORKFLOW_NAME}" --no-detach dumpdbtables cmp_ok 'fcp.out' <<<'fcp|2017' cmp_ok 'taskpool.out' <'/dev/null' workflow_run_ok "${TEST_NAME_BASE}-restart-3" \ cylc play "${WORKFLOW_NAME}" --no-detach --fcp=reload dumpdbtables cmp_ok 'fcp.out' <<<'fcp|' cmp_ok 'taskpool.out' <'/dev/null' purge cylc-flow-8.7.0/tests/functional/restart/56-db-saves-type.t0000775000175000017500000000433615264420307023715 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restarting a simple workflow with a task still running (orphaned) . "$(dirname "$0")/test_header" set_test_number 3 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' #!Jinja2 {{ assert(a_str == 'foo', 'variable "a_str" was not a string') }} {{ assert(an_int == 24, 'variable "an_int" was not an int') }} {{ assert(a_float == 1.1111, 'variable "a_float" was not a float') }} {{ assert(complex == {'foo': [True, 42, 'string']}, 'Complex variable did not validate')}} [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = foo __FLOW_CONFIG__ #------------------------------------------------------------------------------- run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" \ --set "a_str='foo'" --set "an_int=24" --set "a_float=1.1111" --set "complex={'foo': [True, 42, 'string']}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug "${WORKFLOW_NAME}" --pause\ --set "a_str='foo'" --set "an_int=24" --set "a_float=1.1111" --set "complex={'foo': [True, 42, 'string']}" sqlite3 "${WORKFLOW_RUN_DIR}/log/db" 'SELECT * FROM workflow_template_vars' >'sqlite3.out' cylc stop --max-polls=10 --interval=2 "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/restart/05-submit-failed.t0000775000175000017500000000536515264420307023754 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restarting a simple workflow with a submit-failed task export REQUIRE_PLATFORM='runner:at' if [[ -z ${TEST_DIR:-} ]]; then . "$(dirname "$0")/test_header" fi #------------------------------------------------------------------------------- set_test_number 7 #------------------------------------------------------------------------------- create_test_global_config " [platforms] [[$CYLC_TEST_PLATFORM]] job runner = at job runner command template = at oh-no " #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" 'submit-failed' cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "$RUN_DIR/${WORKFLOW_NAME}/" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" cmp_ok "${TEST_NAME}.stderr" <'/dev/null' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-restart" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- grep_ok "submit_failed_task|20130923T0000Z|1|1|submit-failed" \ "${WORKFLOW_RUN_DIR}/pre-restart-db" contains_ok "${WORKFLOW_RUN_DIR}/post-restart-db" <<'__DB_DUMP__' submit_failed_task|20130923T0000Z|1|1|submit-failed __DB_DUMP__ "${WORKFLOW_RUN_DIR}/bin/ctb-select-task-states" "${WORKFLOW_RUN_DIR}" \ > "${TEST_DIR}/db" contains_ok "${TEST_DIR}/db" <<'__DB_DUMP__' submit_failed_task|20130923T0000Z|1|1|submit-failed __DB_DUMP__ #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/restart/17-template-vars-file0000777000175000017500000000000015264420307027403 216-template-varsustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/30-outputs/0000775000175000017500000000000015264420307022547 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/30-outputs/reference.log0000664000175000017500000000007515264420307025212 0ustar alastairalastairInitial point: 1 Final point: 1 1/t3 -triggered off ['1/t1'] cylc-flow-8.7.0/tests/functional/restart/30-outputs/flow.cylc0000664000175000017500000000073215264420307024374 0ustar alastairalastair# only the t1:hello output is generated [scheduler] [[events]] abort on inactivity timeout = True abort on stall timeout = True stall timeout = PT0S inactivity timeout = PT1M [scheduling] [[graph]] R1 = """ t1:hello => t2 t1:greet? => t3 """ [runtime] [[t1]] script = cylc message -- 'hi there' [[[outputs]]] hello = "hi there" greet = greeting [[t2, t3]] script = true cylc-flow-8.7.0/tests/functional/restart/07-waiting.t0000775000175000017500000000501215264420307022660 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restarting a simple workflow with a waiting task # TODO: this test is not very meaningful in SoD if [[ -z ${TEST_DIR:-} ]]; then . "$(dirname "$0")/test_header" fi #------------------------------------------------------------------------------- set_test_number 6 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" 'waiting' cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "$RUN_DIR/${WORKFLOW_NAME}/" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" cmp_ok "${TEST_NAME}.stderr" <'/dev/null' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-restart-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- contains_ok "$WORKFLOW_RUN_DIR/post-restart-db" <<'__DB_DUMP__' shutdown|20130923T0000Z|1|1|succeeded __DB_DUMP__ "${WORKFLOW_RUN_DIR}/bin/ctb-select-task-states" "${WORKFLOW_RUN_DIR}" \ > "${TEST_DIR}/db" contains_ok "${TEST_DIR}/db" <<'__DB_DUMP__' finish|20130923T0000Z|1|1|succeeded output_states|20130923T0000Z|1|1|succeeded shutdown|20130923T0000Z|1|1|succeeded waiting_task|20130923T0000Z|1|1|succeeded __DB_DUMP__ #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/restart/10-pre-initial-2.t0000775000175000017500000000324115264420307023566 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restarting a workflow with pre-initial cycle dependencies and no # initial cycle point in workflow definition, ref. github #957. . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" 'pre-init-2' #------------------------------------------------------------------------------- ICP='20100808T00' run_ok "${TEST_NAME_BASE}-validate" \ cylc validate --initial-cycle-point="${ICP}" "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach \ --initial-cycle-point="${ICP}" \ --stop-cycle-point="${ICP}" \ "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/restart/33-simulation.t0000664000175000017500000000400515264420307023377 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test https://github.com/cylc/cylc-flow/issues/2788 . "$(dirname "$0")/test_header" set_test_number 5 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduler] cycle point format = %Y [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] initial cycle point = 2018 [[graph]] P1Y = t1 [runtime] [[t1]] script = true __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --no-detach --stopcp=2019 --mode=simulation --abort-if-any-task-fails "${WORKFLOW_NAME}" # Force a waiting task into a running task run_ok "${TEST_NAME_BASE}-sql1" sqlite3 "${HOME}/cylc-run/${WORKFLOW_NAME}/.service/db" \ "UPDATE task_states SET status='running' WHERE name=='t1' AND cycle=='2019'" run_ok "${TEST_NAME_BASE}-sql1" sqlite3 "${HOME}/cylc-run/${WORKFLOW_NAME}/.service/db" \ "UPDATE task_pool SET status='running' WHERE name=='t1' AND cycle=='2019'" workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play --debug --no-detach --fcp=2020 --mode=simulation --abort-if-any-task-fails "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/restart/pre-init-2/0000775000175000017500000000000015264420307022472 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/pre-init-2/reference.log0000664000175000017500000000033615264420307025135 0ustar alastairalastairInitial point: 20100808T00 Final point: 20100810T00 20100809T00/bar -triggered off ['20100808T00/foo'] 20100809T00/foo -triggered off [] 20100810T00/foo -triggered off [] 20100810T00/bar -triggered off ['20100809T00/foo'] cylc-flow-8.7.0/tests/functional/restart/pre-init-2/flow.cylc0000664000175000017500000000122615264420307024316 0ustar alastairalastair# Test restart with an inter-cycle trigger and no initial cycle point in # the workflow definition. See github #957. # INSTRUCTIONS: # 1) first run to set up the restart: # cylc play --fcp=20100808T00 WORKFLOW 20100808T00 # 2) the restart as reference test: # cylc play --debug --reference-test WORKFLOW [scheduler] cycle point format = %Y%m%dT%H [scheduling] # DO NOT PUT AN INITIAL CYCLE TIME HERE, # use the command line as described above final cycle point = 20100810T00 [[graph]] T00 = """ foo[-P1D] => bar foo """ [runtime] [[root]] script = true [[foo]] [[bar]] cylc-flow-8.7.0/tests/functional/restart/bin/0000775000175000017500000000000015264420307021354 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/bin/ctb-select-task-states0000775000175000017500000000257115264420307025575 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- set -eu CYLC_WORKFLOW_RUN_DIR="$1" CYLC_TASK_NAME="${2:-}" sqlite3 "${CYLC_WORKFLOW_RUN_DIR}/log/db" " SELECT task_states.name, task_states.cycle, task_states.submit_num, task_jobs.try_num, task_states.status FROM task_states LEFT OUTER JOIN task_jobs ON task_states.name == task_jobs.name AND task_states.cycle == task_jobs.cycle AND task_states.submit_num == task_jobs.submit_num WHERE task_states.name != '${CYLC_TASK_NAME}' ORDER BY task_states.name, task_states.cycle ;" cylc-flow-8.7.0/tests/functional/restart/bin/shutdown_this_workflow_hook0000775000175000017500000000011215264420307027150 0ustar alastairalastair#!/usr/bin/env bash cylc shutdown --kill --max-polls=30 --interval=2 "$2" cylc-flow-8.7.0/tests/functional/restart/27-broadcast-timeout.t0000775000175000017500000000271615264420307024656 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restart with broadcast to "[events]submission timeout". . "$(dirname "$0")/test_header" set_test_number 4 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" --debug --no-detach sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ 'SELECT * FROM broadcast_states' >'sqlite3.out' cmp_ok 'sqlite3.out' <<<'*|root|[events]submission timeout|PT1M' workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play "${WORKFLOW_NAME}" --debug --no-detach --reference-test purge exit cylc-flow-8.7.0/tests/functional/restart/59-retart-timeout.t0000775000175000017500000000421615264420307024217 0ustar alastairalastair#!/bin/bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # GitHub 5231: Test that a finished workflow waits on a timeout if restarted. . "$(dirname "$0")/test_header" set_test_number 8 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" TEST_NAME="${TEST_NAME_BASE}-val" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" # Run to completion. TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --no-detach "${WORKFLOW_NAME}" # Restart completed workflow: it should stall on a restart timer. TEST_NAME="${TEST_NAME_BASE}-restart" run_ok "${TEST_NAME}" cylc play "${WORKFLOW_NAME}" # Search log for restart timer. TEST_NAME="${TEST_NAME_BASE}-grep1" grep_workflow_log_ok "${TEST_NAME}" "restart timer starts NOW" # Check that it has not shut down automatically. TEST_NAME="${TEST_NAME_BASE}-grep2" grep_fail "Workflow shutting down" "${WORKFLOW_RUN_DIR}/log/scheduler/log" # Retriggering the task should stop the timer, and shut down as complete again. TEST_NAME="${TEST_NAME_BASE}-trigger" run_ok "${TEST_NAME}" cylc trigger "${WORKFLOW_NAME}//1/foo" poll_grep_workflow_log "Workflow shutting down - AUTOMATIC" TEST_NAME="${TEST_NAME_BASE}-grep3" grep_workflow_log_ok "${TEST_NAME}" "restart timer stopped" # It should not be running now. TEST_NAME="${TEST_NAME_BASE}-ping" run_fail "${TEST_NAME}" cylc ping "${WORKFLOW_NAME}" purge cylc-flow-8.7.0/tests/functional/restart/17-template-vars-file.t0000775000175000017500000000300415264420307024717 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test automatic custom template variables (from file) on restart. . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" cat >'template-vars.list' <<'__LIST__' COMMAND="true" FINAL_CYCLE_POINT="2020" __LIST__ run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" --set-file="${PWD}/template-vars.list" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play "${WORKFLOW_NAME}" --set-file="${PWD}/template-vars.list" \ --stopcp=2018 --debug --no-detach workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play "${WORKFLOW_NAME}" --debug --no-detach --reference-test purge exit cylc-flow-8.7.0/tests/functional/restart/44-reinvoke.t0000775000175000017500000000476715264420307023061 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- export REQUIRE_PLATFORM='loc:remote fs:shared runner:background' . "$(dirname "$0")/test_header" set_test_number 3 #------------------------------------------------------------------------------- # The Cylc CLI can "infer" run numbers e.g. `cylc play foo` might run `foo/run1`. # This makes monitoring difficult because you can't see what workflow run a Cylc # process is running so when/if we re-invoke the `cylc play` command on a remote # host we use the full ID (including the run number) not the abbreviated version. # install a workflow *with* the standard run numbers echo ' [scheduler] allow implicit tasks = True [scheduling] [[graph]] R1 = one ' > flow.cylc init_workflow "${TEST_NAME_BASE}" flow.cylc true WORKFLOW_RUN_DIR="$WORKFLOW_RUN_DIR/run1" # allow poll_workflow to work # get the workflow to startup on the test platform create_test_global_config '' " [scheduler] [[run hosts]] available = ${CYLC_TEST_HOST} " # run the workflow workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --pause "${WORKFLOW_NAME}" poll_workflow_running # extract the command being run from the contact file run_ok "${TEST_NAME_BASE}-cmd" \ cylc get-workflow-contact "${WORKFLOW_NAME}" sed -n -i 's/CYLC_WORKFLOW_COMMAND=.*cylc //p' "${TEST_NAME_BASE}-cmd.stdout" # ensure the whole workflow ID is present in the command (including the run number) CMD="^play --pause ${WORKFLOW_NAME}/run1 --host=localhost --color=never$" if grep "${CMD}" "${TEST_NAME_BASE}-cmd.stdout"; then ok "${TEST_NAME_BASE}-re-invoked-id" else fail "${TEST_NAME_BASE}-re-invoked-id" fi # shutdown cylc stop "${WORKFLOW_NAME}" poll_workflow_stopped purge exit cylc-flow-8.7.0/tests/functional/restart/42-auto-restart-ping-pong.t0000664000175000017500000000703715264420307025551 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # play a game of Cylc workflow ping pong bouncing a workflow back and forth between # two servers by condemning them in turn in order to see if anything breaks export REQUIRE_PLATFORM='loc:remote fs:shared runner:background' . "$(dirname "$0")/test_header" export CLOWNS="${CYLC_TEST_HOST}" export JOKERS="${HOSTNAME}" BASE_GLOBAL_CONFIG=' [scheduler] [[main loop]] plugins = health check, auto restart [[[auto restart]]] interval = PT5S [[events]] abort on inactivity timeout = True abort on stall timeout = True inactivity timeout = PT2M stall timeout = PT2M ' init_workflow "${TEST_NAME_BASE}" <<< ' [scheduling] initial cycle point = 2000 final cycle point = 9999 # test cylc/cylc-flow/issues/2799 [[graph]] P1Y = foo[-P1Y] => foo [runtime] [[foo]] script = sleep 5 ' cd "${WORKFLOW_RUN_DIR}" || exit 1 stuck_in_the_middle() { # swap the condemned host forcing the workflow to jump ship local temp="${JOKERS}" JOKERS="${CLOWNS}"; CLOWNS="${temp}" create_test_global_config '' " ${BASE_GLOBAL_CONFIG} [scheduler] [[run hosts]] available = ${JOKERS}, ${CLOWNS} condemned = ${CLOWNS} " } kill_workflow() { cylc stop --now --now --max-polls=10 --interval=2 "${WORKFLOW_NAME}" 2>'/dev/null' purge } log_scan2() { NO_TESTS="$(( NO_TESTS - $# + 4 ))" if ! log_scan "$@"; then skip $NO_TESTS # skip remaining tests kill_workflow exit 1 fi } EARS=5 # number of times to bounce the workflow between hosts NO_TESTS="$(( EARS * 5 + 1 ))" set_test_number "${NO_TESTS}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" # run the workflow stuck_in_the_middle cylc play "${WORKFLOW_NAME}" --host="${JOKERS}" --abort-if-any-task-fails poll_workflow_running sleep 1 # get the log file FILE=$(cylc cat-log "${WORKFLOW_NAME}" -m p |xargs readlink -f) #------------------------------------------------------------------------------- for ear in $(seq 1 "${EARS}"); do stuck_in_the_middle # swap the condemned host # test the shutdown procedure log_scan2 "${TEST_NAME_BASE}-${ear}-stop" "${FILE}" 40 1 \ 'The Cylc workflow host will soon become un-available' \ 'Workflow shutting down - REQUEST(NOW-NOW)' \ "Attempting to restart on \"${JOKERS}\""\ "Workflow now running on \"${JOKERS}\"" poll_workflow_restart # test the restart procedure FILE=$(cylc cat-log "${WORKFLOW_NAME}" -m p |xargs readlink -f) log_scan2 "${TEST_NAME_BASE}-${ear}-restart" "${FILE}" 20 1 \ "Scheduler: url=tcp://$(get_fqdn "${JOKERS}")" sleep 2 done kill_workflow exit cylc-flow-8.7.0/tests/functional/restart/broadcast/0000775000175000017500000000000015264420307022546 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/broadcast/flow.cylc0000664000175000017500000000234015264420307024370 0ustar alastairalastair#!jinja2 {%- set TEST_DIR = environ['TEST_DIR'] %} [scheduler] UTC mode = True [[events]] abort on stall timeout = True stall timeout = PT1M [scheduling] initial cycle point = 20130923T00 final cycle point = 20130923T00 [[graph]] R1 = """ send_a_broadcast_task => shutdown shutdown => output_states output_states => broadcast_task broadcast_task => finish """ [runtime] [[send_a_broadcast_task]] script = """ cylc broadcast -n broadcast_task -p $CYLC_TASK_CYCLE_POINT -s "[environment]MY_VALUE='something'" $CYLC_WORKFLOW_ID cylc broadcast -d $CYLC_WORKFLOW_ID """ [[[meta]]] description = "Broadcast setup task" [[broadcast_task]] script = """ if [[ "$MY_VALUE" != "something" ]]; then echo "[FAIL] MY_VALUE ($MY_VALUE) not set correctly by broadcast" >&2 exit 1 fi echo "[PASS] MY_VALUE=$MY_VALUE" """ [[[meta]]] description = "Broadcast-recipient task (runs after restart)" [[[environment]]] MY_VALUE=nothing {% include 'flow-runtime-restart.cylc' %} cylc-flow-8.7.0/tests/functional/restart/broadcast/bin/0000775000175000017500000000000015264420307023316 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/broadcast/bin/ctb-select-task-states0000775000175000017500000000257115264420307027537 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- set -eu CYLC_WORKFLOW_RUN_DIR="$1" CYLC_TASK_NAME="${2:-}" sqlite3 "${CYLC_WORKFLOW_RUN_DIR}/log/db" " SELECT task_states.name, task_states.cycle, task_states.submit_num, task_jobs.try_num, task_states.status FROM task_states LEFT OUTER JOIN task_jobs ON task_states.name == task_jobs.name AND task_states.cycle == task_jobs.cycle AND task_states.submit_num == task_jobs.submit_num WHERE task_states.name != '${CYLC_TASK_NAME}' ORDER BY task_states.name, task_states.cycle ;" cylc-flow-8.7.0/tests/functional/restart/55-stalled.t0000664000175000017500000000355315264420307022656 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test restarting a stalled workflow: stall timer should reset. . "$(dirname "$0")/test_header" grep_for_stall() { SUFFIX=$1 grep_workflow_log_ok "grep-${SUFFIX}-1" 'Workflow stalled' grep_workflow_log_ok "grep-${SUFFIX}-2" 'PT2S stall timer starts NOW' grep_workflow_log_ok "grep-${SUFFIX}-3" 'stall timer timed out after PT2S' grep_workflow_log_ok "grep-${SUFFIX}-4" 'Workflow shutting down - "abort on stall timeout" is set' } set_test_number 12 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduler] [[events]] stall timeout = PT2S abort on stall timeout = True [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = false __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_fail "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" --no-detach grep_for_stall run # Rinse and repeat workflow_run_fail "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" --no-detach grep_workflow_log_ok grep-restart 'Run: \(re\)start number=2, log rollover=1' -E grep_for_stall rstart purge cylc-flow-8.7.0/tests/functional/restart/23-hold-retry/0000775000175000017500000000000015264420307023117 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/23-hold-retry/flow.cylc0000664000175000017500000000111015264420307024733 0ustar alastairalastair[scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = P3M [scheduling] [[graph]] R1 = t1 [runtime] [[t1]] script = """ if ((CYLC_TASK_TRY_NUMBER == 1)); then cylc stop "${CYLC_WORKFLOW_ID}" cylc kill "${CYLC_WORKFLOW_ID}//${CYLC_TASK_ID}" sleep 120 # Should not matter because the job will be killed fi """ [[[job]]] execution retry delays = PT0S cylc-flow-8.7.0/tests/functional/restart/bad-job-host/0000775000175000017500000000000015264420307023055 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/bad-job-host/flow.cylc0000664000175000017500000000221115264420307024674 0ustar alastairalastair#!jinja2 [scheduler] [[events]] workflow timeout = PT2M [scheduling] [[graph]] R1 = """ t-remote:start => t-shutdown t-shutdown => t-remote-2 t-shutdown => t-check-log """ [runtime] [[t-remote]] script = """ # Hang the task on a remote host for up to 30 seconds touch 'file' timeout 30 bash -c 'while [[ -e 'file' ]]; do sleep 1; done' || true """ platform = {{environ['CYLC_TEST_PLATFORM']}} [[t-shutdown]] script = """ # Shutdown and wait cylc shutdown --now --max-polls=10 --interval=1 "${CYLC_WORKFLOW_ID}" """ [[t-remote-2]] script = """ # Release t-remote-1 rm -f "${CYLC_WORKFLOW_WORK_DIR}/${CYLC_TASK_CYCLE_POINT}/t-remote/file" """ platform = {{environ['CYLC_TEST_PLATFORM']}} [[t-check-log]] script = """ grep -q 'ERROR - garbage: initialisation did not complete' \ "${CYLC_WORKFLOW_LOG_DIR}/log" cylc shutdown --now "${CYLC_WORKFLOW_ID}" """ cylc-flow-8.7.0/tests/functional/restart/08-stop-after-cycle-point/0000775000175000017500000000000015264420307025341 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/08-stop-after-cycle-point/reference.log0000664000175000017500000000026215264420307030002 0ustar alastairalastairnitial point: 19700101T0000Z inal point: 19700101T0300Z 19700101T0000Z/hello -triggered off ['19691231T2300Z/hello'] 19700101T0100Z/hello -triggered off ['19700101T0000Z/hello'] cylc-flow-8.7.0/tests/functional/restart/08-stop-after-cycle-point/flow.cylc0000664000175000017500000000150615264420307027166 0ustar alastairalastair#!jinja2 [meta] title = Test: flow.cylc[scheduling]stop after cycle point description = """ Test: 1. That [scheduling]stop after cycle point works 2. That cylc play --stop-cycle-point overrides the config. """ [scheduler] UTC mode = True cycle point format = %Y [[events]] # prevent workflow hanging if restarted with nothing more to do restart timeout = PT0S [scheduling] runahead limit = P0 initial cycle point = 1970 final cycle point = 1980 stop after cycle point = {{ environ['CFG_STOPCP'] }} [[graph]] P1Y = hello [runtime] [[hello]] {% if MANUAL_SHUTDOWN is defined %} script = """ if [[ "$CYLC_TASK_CYCLE_POINT" == {{ MANUAL_SHUTDOWN }} ]]; then cylc stop "$CYLC_WORKFLOW_ID" fi """ {% endif %} cylc-flow-8.7.0/tests/functional/restart/43-auto-restart-force-override-normal.t0000664000175000017500000000650015264420307030047 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Check that "Force Mode" can override a scheduler "Normal Mode" restart. export REQUIRE_PLATFORM='loc:remote fs:shared runner:background' . "$(dirname "$0")/test_header" # shellcheck disable=SC2153 export CYLC_TEST_HOST_2="${CYLC_TEST_HOST}" export CYLC_TEST_HOST_1="${HOSTNAME}" #------------------------------------------------------------------------------- BASE_GLOBAL_CONFIG=' [scheduler] [[main loop]] plugins = health check, auto restart [[[auto restart]]] interval = PT5S [[events]] abort on inactivity timeout = True abort on stall timeout = True inactivity timeout = PT2M stall timeout = PT2M ' init_workflow "${TEST_NAME_BASE}" <<< ' [scheduler] [[events]] [scheduling] initial cycle point = 2000 [[graph]] P1Y = foo[-P1Y] => foo [runtime] [[foo]] ' create_test_global_config '' " ${BASE_GLOBAL_CONFIG} [scheduler] [[run hosts]] available = ${CYLC_TEST_HOST_1}, ${CYLC_TEST_HOST_2} # ensure the workflow can start if a host is condemned # in force mode see #6623 condemned = ${CYLC_TEST_HOST_2}! " set_test_number 8 #------------------------------------------------------------------------------- run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" # run workflow cylc play "${WORKFLOW_NAME}" --abort-if-any-task-fails poll_workflow_running sleep 1 FILE=$(cylc cat-log "${WORKFLOW_NAME}" -m p |xargs readlink -f) # condemn the host, the workflow will schedule restart in PT60S create_test_global_config '' " ${BASE_GLOBAL_CONFIG} [scheduler] auto restart delay = -PT60S # results in +PT60S delay [[run hosts]] available = ${CYLC_TEST_HOST_1}, ${CYLC_TEST_HOST_2} condemned = ${CYLC_TEST_HOST_1} " log_scan "${TEST_NAME_BASE}-stop" "${FILE}" 40 1 \ 'The Cylc workflow host will soon become un-available' \ 'Workflow will restart in 60s' # condemn the host in "Force Mode", this should cancel the scheduled restart create_test_global_config '' " ${BASE_GLOBAL_CONFIG} [scheduler] [[run hosts]] condemned = ${CYLC_TEST_HOST_1}! " log_scan "${TEST_NAME_BASE}-stop" "${FILE}" 40 1 \ 'This workflow will be shutdown as the workflow host is' \ 'When another workflow host becomes available the workflow can' \ 'Scheduled automatic restart canceled' \ 'Workflow shutting down - REQUEST(NOW)' \ 'DONE' cylc stop --now --now--max-polls=20 --interval=2 "${WORKFLOW_NAME}" 2>'/dev/null' purge exit cylc-flow-8.7.0/tests/functional/restart/test_header0000777000175000017500000000000015264420307027121 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/28-execution-timeout/0000775000175000017500000000000015264420307024522 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/28-execution-timeout/reference.log0000664000175000017500000000007715264420307027167 0ustar alastairalastairInitial point: 1 Final point: 1 1/bar -triggered off ['1/foo'] cylc-flow-8.7.0/tests/functional/restart/28-execution-timeout/flow.cylc0000664000175000017500000000066515264420307026354 0ustar alastairalastair[scheduler] UTC mode = True cycle point format = %Y [scheduling] [[graph]] R1 = foo => bar [runtime] [[foo]] script=""" cylc__job__wait_cylc_message_started sleep 1 cylc stop --now --now "${CYLC_WORKFLOW_ID}" sleep 60 """ [[[events]]] execution timeout = PT10S handlers = echo %(id)s %(event)s handler events = execution timeout [[bar]] script=true cylc-flow-8.7.0/tests/functional/restart/12-deleted-logs.t0000775000175000017500000000413315264420307023565 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restarting a workflow after its log directory tree has been removed. # (the workflow run tree should be created if necessary on restart). . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" 'deleted-logs' #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-restart" if [[ -z "${WORKFLOW_NAME}" ]]; then # For safety, abort before attempting log tree removal. echo "ERROR: \${WORKFLOW_NAME} does not exist" >&2 exit 1 fi rm -r "${WORKFLOW_RUN_DIR}/log" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/restart/59-retart-timeout/0000775000175000017500000000000015264420307024024 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/59-retart-timeout/flow.cylc0000664000175000017500000000030315264420307025643 0ustar alastairalastair[scheduler] [[events]] inactivity timeout = PT1M abort on inactivity timeout = True [scheduling] [[graph]] R1 = "foo" [runtime] [[foo]] script = "true" cylc-flow-8.7.0/tests/functional/restart/00-pre-initial/0000775000175000017500000000000015264420307023236 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/00-pre-initial/flow.cylc0000664000175000017500000000120615264420307025060 0ustar alastairalastair#!jinja2 [scheduler] UTC mode = True [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT3M [scheduling] initial cycle point = 20100808T00 final cycle point = 20100808T06 [[graph]] R1 = "p1 => p2" T00 = """ p2 => foo foo[-P1D] => foo => bar """ [runtime] [[p1]] script = """ cylc__job__wait_cylc_message_started sleep 1 cylc shutdown --now --max-polls=30 --interval=1 "${CYLC_WORKFLOW_ID}" """ [[foo, p2]] script = true [[bar]] script = true cylc-flow-8.7.0/tests/functional/restart/00-pre-initial/ref-state0000664000175000017500000000016215264420307025052 0ustar alastairalastairbar|20100808T0000Z|succeeded foo|20100808T0000Z|succeeded p1|20100808T0000Z|succeeded p2|20100808T0000Z|succeeded cylc-flow-8.7.0/tests/functional/restart/20-event-retry/0000775000175000017500000000000015264420307023307 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/20-event-retry/flow.cylc0000664000175000017500000000036415264420307025135 0ustar alastairalastair[scheduling] [[graph]] R1 = t01 [runtime] [[t01]] script = true [[[events]]] handlers = my-handler "%(workflow)s" handler events = succeeded handler retry delays = PT0S, PT10S cylc-flow-8.7.0/tests/functional/restart/20-event-retry/bin/0000775000175000017500000000000015264420307024057 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/20-event-retry/bin/my-handler0000775000175000017500000000045215264420307026046 0ustar alastairalastair#!/usr/bin/env bash set -euo pipefail WORKFLOW="$1" OUT_FILE="$CYLC_WORKFLOW_RUN_DIR/file" if grep -q -F '1' "${OUT_FILE}" 2>'/dev/null'; then echo '2' >>"${OUT_FILE}" cylc stop "${WORKFLOW}" exit 0 else echo '1' >"${OUT_FILE}" cylc stop --now --now "${WORKFLOW}" exit 1 fi cylc-flow-8.7.0/tests/functional/restart/32-reload-runahead-no-stop-point/0000775000175000017500000000000015264420307026605 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/32-reload-runahead-no-stop-point/reference.log0000664000175000017500000000022215264420307031242 0ustar alastairalastair2017-07-19T14:54:42+01 INFO - Initial point: 2018 2018/t1 -triggered off [] 2018/t2 -triggered off ['2018/t1'] 2018/t3 -triggered off ['2018/t2'] cylc-flow-8.7.0/tests/functional/restart/32-reload-runahead-no-stop-point/flow.cylc0000664000175000017500000000075215264420307030434 0ustar alastairalastair[scheduler] cycle point format = %Y [scheduling] initial cycle point = 2018 runahead limit = P1 [[graph]] P1Y = t1[-P3Y] => t1 => t2 => t3 [runtime] [[t1]] script = """ cylc__job__wait_cylc_message_started cylc stop --now "${CYLC_WORKFLOW_ID}" """ [[t2]] script = """ cylc__job__wait_cylc_message_started cylc reload "${CYLC_WORKFLOW_ID}" """ [[t3]] script = true cylc-flow-8.7.0/tests/functional/restart/30-outputs.t0000775000175000017500000000334715264420307022746 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test task outputs status is retained on restart # TODO SoD: this is no longer a restart test (the original was based on # stalling with a task waiting on the other output) # Now it tests that the right thing happens with mutually exclusive outputs. . "$(dirname "$0")/test_header" set_test_number 4 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --no-detach "${WORKFLOW_NAME}" sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ "SELECT outputs FROM task_outputs WHERE name IS 't1'" >'sqlite3.out' cmp_json 'sqlite3.out' 'sqlite3.out' <<<'{"submitted": "submitted", "started": "started", "succeeded": "succeeded", "hello": "hi there"}' sqlite3 "${WORKFLOW_RUN_DIR}/log/db" 'SELECT * FROM task_pool' >'task-pool.out' cmp_ok 'task-pool.out' <'/dev/null' purge exit cylc-flow-8.7.0/tests/functional/restart/41-auto-restart-local-jobs.t0000664000175000017500000001027615264420307025676 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- export REQUIRE_PLATFORM='loc:remote fs:shared runner:background' . "$(dirname "$0")/test_header" # shellcheck disable=SC2153 export CYLC_TEST_HOST2="${CYLC_TEST_HOST}" export CYLC_TEST_HOST1="${HOSTNAME}" if ${CYLC_TEST_DEBUG:-false}; then ERR=2; else ERR=1; fi set_test_number 13 #------------------------------------------------------------------------------- BASE_GLOBAL_CONFIG=" [scheduler] [[main loop]] plugins = health check, auto restart [[[auto restart]]] interval = PT2S [[events]] abort on inactivity timeout = True abort on stall timeout = True inactivity timeout = PT2M stall timeout = PT2M " init_workflow "${TEST_NAME_BASE}" <<< ' [scheduling] [[graph]] R1 = foo => bar [runtime] [[foo]] # we will trigger bar manually later script = sleep 15; false [[bar]] script = sleep 60 ' cd "${WORKFLOW_RUN_DIR}" || exit 1 create_test_global_config '' " ${BASE_GLOBAL_CONFIG} [scheduler] [[run hosts]] available = ${CYLC_TEST_HOST1} " cylc play "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- # auto stop-restart - normal mode: # ensure the workflow WAITS for local jobs to complete before restarting TEST_NAME="${TEST_NAME_BASE}-normal-mode" cylc workflow-state "${WORKFLOW_NAME}//1/foo:started" \ --triggers --interval=1 --max-polls=20 >& $ERR create_test_global_config '' " ${BASE_GLOBAL_CONFIG} [scheduler] [[run hosts]] available = ${CYLC_TEST_HOST1}, ${CYLC_TEST_HOST2} condemned = ${CYLC_TEST_HOST1} " UUID="$(get_workflow_uuid)" LOG_FILE="$(cylc cat-log "${WORKFLOW_NAME}" -m p |xargs readlink -f)" log_scan "${TEST_NAME}-stop-log-scan" "${LOG_FILE}" 40 1 \ 'The Cylc workflow host will soon become un-available' \ 'Waiting for jobs running on localhost to complete' \ 'Waiting for jobs running on localhost to complete' \ 'Workflow shutting down - REQUEST(NOW-NOW)' \ "Attempting to restart on \"${CYLC_TEST_HOST2}\"" # we shouldn't have any orphaned tasks because we should # have waited for them to complete grep_fail 'orphaned task' "$LOG_FILE" run_ok "${TEST_NAME_BASE}-restart" poll_workflow_restart 10 "$UUID" log_scan "${TEST_NAME}-restart-log-scan" "$LOG_FILE" 20 1 \ "Workflow now running on \"${CYLC_TEST_HOST2}\"" #------------------------------------------------------------------------------- # auto stop-restart - force mode: # ensure the workflow DOESN'T WAIT for local jobs to complete before stopping TEST_NAME="${TEST_NAME_BASE}-force-mode" cylc trigger "${WORKFLOW_NAME}//1/bar" cylc workflow-state "${WORKFLOW_NAME}//1/bar:started" \ --triggers --interval=1 --max-polls=20 >& $ERR create_test_global_config '' " ${BASE_GLOBAL_CONFIG} [scheduler] [[run hosts]] available = ${CYLC_TEST_HOST1}, ${CYLC_TEST_HOST2} condemned = ${CYLC_TEST_HOST2}! " LOG_FILE="$(cylc cat-log "${WORKFLOW_NAME}" -m p |xargs readlink -f)" log_scan "${TEST_NAME}-stop-log-scan" "${LOG_FILE}" 40 1 \ 'The Cylc workflow host will soon become un-available' \ 'This workflow will be shutdown as the workflow host is unable to continue' \ 'Workflow shutting down - REQUEST(NOW)' \ 'Orphaned tasks:' \ '* 1/bar (running)' cylc stop "${WORKFLOW_NAME}" --now --now 2>/dev/null || true poll_workflow_stopped sleep 1 purge exit cylc-flow-8.7.0/tests/functional/restart/22-hold/0000775000175000017500000000000015264420307021753 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/22-hold/flow.cylc0000664000175000017500000000164215264420307023601 0ustar alastairalastair# Designed to shutdown with a held task in the pool. [scheduler] UTC mode=True cycle point format = %Y [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = P1M [scheduling] initial cycle point = 2016 final cycle point = 2017 [[graph]] R1/2016 = t1 & fast => t2 R1/2017 = t1[2016] => t1 => t2 [runtime] [[t1]] script = """ if [[ "${CYLC_TASK_CYCLE_POINT}" == '2016' ]]; then cylc__job__poll_grep_workflow_log -E '2016/t2.* added to the n=0 window' cylc hold "${CYLC_WORKFLOW_ID}//" //2016/t2 //2017/t2 cylc stop "${CYLC_WORKFLOW_ID}" else cylc release "${CYLC_WORKFLOW_ID}//" //2016/t2 //2017/t2 fi """ [[fast]] script = true [[t2]] script = sleep 2 cylc-flow-8.7.0/tests/functional/restart/09-reload.t0000775000175000017500000000352415264420307022474 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restarting after a reload has been done (ref bug fix 64e3b040). . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" reload #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-restart" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/restart/reload/0000775000175000017500000000000015264420307022052 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/reload/reference.log0000664000175000017500000000034515264420307024515 0ustar alastairalastairInitial point: 20100808T0000Z Final point: 20100809T0000Z 20100808T0000Z/bar -triggered off ['20100808T0000Z/foo'] 20100809T0000Z/foo -triggered off ['20100808T0000Z/foo'] 20100809T0000Z/bar -triggered off ['20100809T0000Z/foo'] cylc-flow-8.7.0/tests/functional/restart/reload/flow.cylc0000664000175000017500000000135715264420307023703 0ustar alastairalastair[meta] title = "Workflow to test that restart after reload works." description = """First task reloads and stops the workflow, which should run to completion on restarting.""" # The restart can be run as a reference test. [scheduler] UTC mode = True [scheduling] initial cycle point = 20100808T00 final cycle point = 20100809T00 #[[special tasks]] # sequential = foo [[graph]] T00 = "foo[-P1D] => foo => bar" [runtime] [[bar]] script = true [[foo]] script = """ if [[ "$CYLC_TASK_CYCLE_POINT" == "$CYLC_WORKFLOW_INITIAL_CYCLE_POINT" ]]; then cylc reload "${CYLC_WORKFLOW_ID}" cylc__job__poll_grep_workflow_log -F 'Reload completed' cylc stop "${CYLC_WORKFLOW_ID}" fi""" cylc-flow-8.7.0/tests/functional/restart/25-hold-workflow.t0000775000175000017500000000417215264420307024022 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test restart with held tasks . "$(dirname "$0")/test_header" set_test_number 5 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play "${WORKFLOW_NAME}" --debug --no-detach --abort-if-any-task-fails T1_2016_PID="$(sed -n 's/CYLC_JOB_PID=//p' "${WORKFLOW_RUN_DIR}/log/job/2016/t1/01/job.status")" poll_pid_done "${T1_2016_PID}" sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ 'SELECT cycle, name, status FROM task_pool WHERE is_held IS 1 ORDER by cycle, name' >'workflow-stopped.out' cmp_ok 'workflow-stopped.out' <<'__OUT__' 2016|t2|waiting 2017|t1|waiting __OUT__ # Restart cylc play "${WORKFLOW_NAME}" --debug --no-detach 1>"${TEST_NAME_BASE}-restart.out" 2>&1 & CYLC_RESTART_PID=$! poll_workflow_running cylc release "${WORKFLOW_NAME}//2016/t2" poll_grep 'CYLC_JOB_EXIT' "${WORKFLOW_RUN_DIR}/log/job/2016/t2/01/job.status" cylc release --all "${WORKFLOW_NAME}" cylc poll "${WORKFLOW_NAME}//*" # Ensure workflow has completed run_ok "${TEST_NAME_BASE}-restart" wait "${CYLC_RESTART_PID}" sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ 'SELECT cycle, name, status FROM task_pool ORDER BY cycle, name' >'task-pool.out' cmp_ok 'task-pool.out' < /dev/null purge exit cylc-flow-8.7.0/tests/functional/restart/50-two-flows/0000775000175000017500000000000015264420307022767 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/restart/50-two-flows/reference.log0000664000175000017500000000020215264420307025422 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off ['1/a'] 1/c -triggered off ['1/b'] 1/a -triggered off [] cylc-flow-8.7.0/tests/functional/restart/50-two-flows/flow.cylc0000664000175000017500000000157515264420307024622 0ustar alastairalastair# A workflow that triggers a new flow in the graph and then shuts down # so that we can restart with two active flows present. [scheduler] [[events]] inactivity timeout = PT1M abort on inactivity timeout = True [scheduling] [[graph]] R1 = "a => b => c => d" [runtime] [[root]] pre-script = sleep 2 [[a]] script = """ if ((CYLC_TASK_FLOW_NUMBERS == 2)); then cylc__job__poll_grep_workflow_log -E "/c.* succeeded" fi """ [[b, d]] [[c]] script = """ if ((CYLC_TASK_FLOW_NUMBERS == 1)); then cylc trigger --flow=new --meta="cheese wizard" \ "$CYLC_WORKFLOW_ID//1/a" cylc__job__poll_grep_workflow_log -E "\[1/a/02\(flows=2\):submitted\] => running" cylc stop $CYLC_WORKFLOW_ID fi """ cylc-flow-8.7.0/tests/functional/restart/50-two-flows/reference.restart.log0000664000175000017500000000021415264420307027110 0ustar alastairalastairInitial point: 1 Final point: 1 1/d -triggered off ['1/c'] 1/b -triggered off ['1/a'] 1/c -triggered off ['1/b'] 1/d -triggered off ['1/c'] cylc-flow-8.7.0/tests/functional/restart/20-event-retry.t0000775000175000017500000000332315264420307023500 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test re-run event handler on restart. . "$(dirname "$0")/test_header" set_test_number 7 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" WORKFLOWD="$RUN_DIR/${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" --debug --no-detach sqlite3 "${WORKFLOWD}/log/db" \ "SELECT COUNT(*) FROM task_action_timers WHERE ctx_key GLOB '*event-handler-00*'" \ >"${TEST_NAME_BASE}-db-n-entries" cmp_ok "${TEST_NAME_BASE}-db-n-entries" <<<'1' workflow_run_ok "${TEST_NAME_BASE}-restart" cylc play "${WORKFLOW_NAME}" --debug --no-detach cmp_ok "${WORKFLOWD}/file" <<'__TEXT__' 1 2 __TEXT__ grep_ok 'LOADING task action timers' "${WORKFLOWD}/log/scheduler/log" grep_ok "+ 1/t01 \[\['event-handler-00', 'succeeded'\], 1\]" "${WORKFLOWD}/log/scheduler/log" purge exit cylc-flow-8.7.0/tests/functional/triggering/0000775000175000017500000000000015264420307021261 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/21-expire.t0000664000175000017500000000170515264420307023165 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test expire triggering . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/triggering/19-and-suicide/0000775000175000017500000000000015264420307023675 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/19-and-suicide/reference.log0000664000175000017500000000015315264420307026335 0ustar alastairalastairInitial point: 1 Final point: 1 1/t0 -triggered off [] 1/t1 -triggered off [] 1/t2 -triggered off ['1/t0'] cylc-flow-8.7.0/tests/functional/triggering/19-and-suicide/flow.cylc0000664000175000017500000000114615264420307025522 0ustar alastairalastair# This is an explicit test of suicide triggers. # Under SoD it isn't really a useful test. [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S expected task failures = 1/t1 [scheduling] [[graph]] R1 = """ t0:fail? & t1:fail? => !t2 t0? | t1? => t2 """ [runtime] [[t0]] # https://github.com/cylc/cylc-flow/issues/2655 # "1/t2" should not suicide on "1/t1:failed" script = cylc__job__poll_grep_workflow_log -E '1/t1.* failed' [[t1]] script = false [[t2]] script = true cylc-flow-8.7.0/tests/functional/triggering/04-fam-fail-all.t0000664000175000017500000000172515264420307024116 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test correct expansion of FAM:fail-all . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/triggering/05-fam-finish-all/0000775000175000017500000000000015264420307024272 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/05-fam-finish-all/reference.log0000664000175000017500000000021515264420307026731 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/c -triggered off [] 1/b -triggered off [] 1/foo -triggered off ['1/a', '1/b', '1/c'] cylc-flow-8.7.0/tests/functional/triggering/05-fam-finish-all/flow.cylc0000664000175000017500000000046315264420307026120 0ustar alastairalastair[scheduler] [[events]] expected task failures = 1/a, 1/c [scheduling] [[graph]] R1 = "FAM:finish-all => foo" [runtime] [[FAM]] script = "false" [[a,c]] inherit = FAM [[b]] inherit = FAM script = "true" [[foo]] script = "true" cylc-flow-8.7.0/tests/functional/triggering/00a-recovery/0000775000175000017500000000000015264420307023475 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/00a-recovery/reference.log0000664000175000017500000000076615264420307026147 0ustar alastairalastairInitial point: 20110101T0000Z Final point: 20110101T1200Z 20110101T0000Z/pre -triggered off [] 20110101T1200Z/pre -triggered off [] 20110101T0000Z/model -triggered off ['20110101T0000Z/pre'] 20110101T1200Z/model -triggered off ['20110101T1200Z/pre'] 20110101T0000Z/post -triggered off ['20110101T0000Z/model'] 20110101T1200Z/diagnose -triggered off ['20110101T1200Z/model'] 20110101T1200Z/recover -triggered off ['20110101T1200Z/diagnose'] 20110101T1200Z/post -triggered off ['20110101T1200Z/recover'] cylc-flow-8.7.0/tests/functional/triggering/00a-recovery/flow.cylc0000664000175000017500000000240115264420307025315 0ustar alastairalastair# Spawn on Demand variant of 00-recovery # No need for suicide triggers. [meta] title = "automated failure recovery example" description = """ Model task failure triggers diagnosis and recovery tasks, which otherwise take themselves out of the workflow if model succeeds. Model post processing triggers off model or recovery tasks. """ [scheduler] UTC mode = True allow implicit tasks = True [[events]] expected task failures = 20110101T1200Z/model [scheduling] initial cycle point = 20110101T00 final cycle point = 20110101T12 [[graph]] T00,T12 = """ pre:finish => model? # finish trigger model:fail? => diagnose => recover # fail trigger model:succeed? | recover => post # conditional and explicit success """ [runtime] [[root]] script = "true" # fast [[model]] script = """ echo Hello from $CYLC_TASK_ID if [[ $(cylc cycletime --print-hour) == 12 ]]; then echo "FAILING NOW!" false else echo "Succeeded" true fi """ [[[meta]]] description = A task that succeeds at 0 UTC and fails at 12 UTC cylc-flow-8.7.0/tests/functional/triggering/02-fam-start-all.t0000664000175000017500000000172615264420307024337 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test correct expansion of FAM:start-all . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/triggering/11-start.t0000664000175000017500000000170415264420307023024 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test start triggering . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/triggering/11-start/0000775000175000017500000000000015264420307022635 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/11-start/reference.log0000664000175000017500000000012715264420307025276 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/bar -triggered off ['1/foo'] cylc-flow-8.7.0/tests/functional/triggering/11-start/flow.cylc0000664000175000017500000000015515264420307024461 0ustar alastairalastair[scheduling] [[graph]] R1 = "foo:start => bar" [runtime] [[foo,bar]] script = "true" cylc-flow-8.7.0/tests/functional/triggering/02-fam-start-all/0000775000175000017500000000000015264420307024144 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/02-fam-start-all/reference.log0000664000175000017500000000021515264420307026603 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/c -triggered off [] 1/b -triggered off [] 1/foo -triggered off ['1/a', '1/b', '1/c'] cylc-flow-8.7.0/tests/functional/triggering/02-fam-start-all/flow.cylc0000664000175000017500000000032715264420307025771 0ustar alastairalastair[scheduling] [[graph]] R1 = """FAM:start-all => foo""" [runtime] [[FAM]] script = cylc__job__wait_cylc_message_started [[a,b,c]] inherit = FAM [[foo]] script = "true" cylc-flow-8.7.0/tests/functional/triggering/14-submit-fail/0000775000175000017500000000000015264420307023717 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/14-submit-fail/reference.log0000664000175000017500000000012715264420307026360 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/bar -triggered off ['1/foo'] cylc-flow-8.7.0/tests/functional/triggering/14-submit-fail/flow.cylc0000664000175000017500000000044315264420307025543 0ustar alastairalastair[scheduler] [[events]] expected task failures = 1/foo [scheduling] [[graph]] R1 = """ foo:submit-fail? => bar bar => !foo """ [runtime] [[foo]] script = true platform = idontexist [[bar]] script = true cylc-flow-8.7.0/tests/functional/triggering/10-finish/0000775000175000017500000000000015264420307022757 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/10-finish/reference.log0000664000175000017500000000021615264420307025417 0ustar alastairalastairInitial point: 1 Final point: 1 1/baz -triggered off [] 1/foo -triggered off [] 1/bar -triggered off ['1/foo'] 1/qux -triggered off ['1/baz'] cylc-flow-8.7.0/tests/functional/triggering/10-finish/flow.cylc0000664000175000017500000000037615264420307024610 0ustar alastairalastair[scheduler] [[events]] expected task failures = 1/foo [scheduling] [[graph]] R1 = """foo:finish => bar baz:finish => qux""" [runtime] [[foo]] script = false [[bar,baz,qux]] script = "true" cylc-flow-8.7.0/tests/functional/triggering/16-fam-expansion.t0000664000175000017500000000405515264420307024443 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test correct expansion of (FOO:finish-all & FOO:fail-any) . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 3 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" fam-expansion SHOW_OUT="$PWD/show.out" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" run_ok "${TEST_NAME}" cylc validate --set="SHOW_OUT='$SHOW_OUT'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" \ cylc play --debug --no-detach --set="SHOW_OUT='$SHOW_OUT'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- contains_ok "$SHOW_OUT" <<'__SHOW_DUMP__' ✓ (((1 | 0) & (3 | 2) & (5 | 4)) & (0 | 2 | 4)) ✓ 0 = 1/foo1 failed ⨯ 1 = 1/foo1 succeeded ✓ 2 = 1/foo2 failed ⨯ 3 = 1/foo2 succeeded ✓ 4 = 1/foo3 failed ⨯ 5 = 1/foo3 succeeded __SHOW_DUMP__ #------------------------------------------------------------------------------- purge cylc-flow-8.7.0/tests/functional/triggering/20-and-outputs-suicide.t0000664000175000017500000000261015264420307025572 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "and" outputs from same task triggering suicide. . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" DBFILE="$RUN_DIR/${WORKFLOW_NAME}/log/db" sqlite3 "${DBFILE}" 'SELECT cycle, name, status FROM task_pool ORDER BY cycle, name;' \ >'sqlite3.out' cmp_ok 'sqlite3.out' <'/dev/null' purge exit cylc-flow-8.7.0/tests/functional/triggering/13-submit/0000775000175000017500000000000015264420307023005 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/13-submit/reference.log0000664000175000017500000000012715264420307025446 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/bar -triggered off ['1/foo'] cylc-flow-8.7.0/tests/functional/triggering/13-submit/flow.cylc0000664000175000017500000000015615264420307024632 0ustar alastairalastair[scheduling] [[graph]] R1 = "foo:submit => bar" [runtime] [[foo,bar]] script = "true" cylc-flow-8.7.0/tests/functional/triggering/00-recovery.t0000664000175000017500000000172615264420307023527 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test automated failure recovery example . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/triggering/05-fam-finish-all.t0000664000175000017500000000173115264420307024461 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test correct expansion of 'FAM:finish-all' . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/triggering/01-or-conditional/0000775000175000017500000000000015264420307024420 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/01-or-conditional/reference.log0000664000175000017500000000023515264420307027061 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off [] 1/c -triggered off ['1/a'] 1/d -triggered off ['1/a'] 1/e -triggered off ['1/d'] cylc-flow-8.7.0/tests/functional/triggering/01-or-conditional/flow.cylc0000664000175000017500000000047415264420307026250 0ustar alastairalastair[scheduler] [[events]] expected task failures = 1/b, 1/c [scheduling] [[graph]] R1 = """ a | b? => c? & d c? | d => e b:failed? => !b c:failed? => !c """ [runtime] [[b,c]] script = false [[a,d, e]] script = true cylc-flow-8.7.0/tests/functional/triggering/00a-recovery.t0000664000175000017500000000172615264420307023670 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test automated failure recovery example . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/triggering/08-fam-finish-any.t0000664000175000017500000000175615264420307024512 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test correct expansion of 'FAM:finish-any' . "$(dirname "$0")/test_header" skip_macos_gh_actions set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/triggering/17-suicide-multi/0000775000175000017500000000000015264420307024263 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/17-suicide-multi/reference.log0000664000175000017500000000053115264420307026723 0ustar alastairalastairInitial point: 1 Final point: 3 1/showdown -triggered off ['0/fin'] 1/ugly -triggered off ['1/showdown'] 1/fin -triggered off ['1/ugly'] 2/showdown -triggered off ['1/fin'] 2/bad -triggered off ['2/showdown'] 2/fin -triggered off ['2/bad'] 3/showdown -triggered off ['2/fin'] 3/good -triggered off ['3/showdown'] 3/fin -triggered off ['3/good'] cylc-flow-8.7.0/tests/functional/triggering/17-suicide-multi/flow.cylc0000664000175000017500000000227415264420307026113 0ustar alastairalastair[scheduler] allow implicit tasks = True [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] cycling mode = integer initial cycle point = 1 final cycle point = 3 [[graph]] P1 = """ fin[-P1] => showdown showdown:good? => good showdown:bad? => bad showdown:ugly? => ugly showdown:good? | showdown:bad? => ! ugly showdown:good? | showdown:ugly? => ! bad showdown:bad? | showdown:ugly? => ! good good | bad | ugly => fin """ [runtime] [[root]] script = true [[showdown]] script = """ if ! (( ${CYLC_TASK_CYCLE_POINT} % 3 )); then cylc message -- "${CYLC_WORKFLOW_ID}" "${CYLC_TASK_JOB}" 'The-Good' elif ! (( ( ${CYLC_TASK_CYCLE_POINT} + 1 ) % 3 )); then cylc message -- "${CYLC_WORKFLOW_ID}" "${CYLC_TASK_JOB}" 'The-Bad' else cylc message -- "${CYLC_WORKFLOW_ID}" "${CYLC_TASK_JOB}" 'The-Ugly' fi """ [[[outputs]]] good = 'The-Good' bad = 'The-Bad' ugly = 'The-Ugly' cylc-flow-8.7.0/tests/functional/triggering/19-and-suicide.t0000664000175000017500000000265115264420307024066 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "and" outputs from 2 tasks triggering suicide. # https://github.com/cylc/cylc-flow/issues/2655 . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" DBFILE="$RUN_DIR/${WORKFLOW_NAME}/log/db" sqlite3 "${DBFILE}" 'SELECT cycle, name, status FROM task_pool ORDER BY name;' >'sqlite3.out' cmp_ok 'sqlite3.out' <'/dev/null' purge exit cylc-flow-8.7.0/tests/functional/triggering/12-succeed/0000775000175000017500000000000015264420307023114 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/12-succeed/reference.log0000664000175000017500000000012715264420307025555 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/bar -triggered off ['1/foo'] cylc-flow-8.7.0/tests/functional/triggering/12-succeed/flow.cylc0000664000175000017500000000015715264420307024742 0ustar alastairalastair[scheduling] [[graph]] R1 = "foo:succeed => bar" [runtime] [[foo,bar]] script = "true" cylc-flow-8.7.0/tests/functional/triggering/21-expire/0000775000175000017500000000000015264420307022775 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/21-expire/reference.log0000664000175000017500000000041615264420307025437 0ustar alastairalastair19990101T0000Z/bar2 -triggered off [] in flow 1 19990101T0000Z/baz -triggered off ['19990101T0000Z/foo1', '19990101T0000Z/foo2'] in flow 1 19990101T0000Z/qux -triggered off ['19990101T0000Z/bar1'] in flow 1 19990101T0000Z/y -triggered off ['19990101T0000Z/x'] in flow 1 cylc-flow-8.7.0/tests/functional/triggering/21-expire/flow.cylc0000664000175000017500000000074615264420307024627 0ustar alastairalastair[scheduling] initial cycle point = 1999 [[special tasks]] clock-expire = foo1(PT0S), foo2(PT0S), bar1(PT0S), x(PT0S) [[graph]] # Expire: foo1, foo2, bar1, x # Run: y, bar2, baz, qux R1 = """ x? FOO? BAR? x:expire? => y FOO:expire-all? => baz BAR:expire-any? => qux """ [runtime] [[FOO, BAR]] [[foo1, foo2]] inherit = FOO [[bar1, bar2]] inherit = BAR [[x, y, baz, qux]] cylc-flow-8.7.0/tests/functional/triggering/13-submit.t0000664000175000017500000000170515264420307023175 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test submit triggering . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/triggering/12-succeed.t0000664000175000017500000000170615264420307023305 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test succeed triggering . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/triggering/08-fam-finish-any/0000775000175000017500000000000015264420307024314 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/08-fam-finish-any/reference.log0000664000175000017500000000017715264420307026762 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/c -triggered off [] 1/b -triggered off [] 1/foo -triggered off ['1/b'] cylc-flow-8.7.0/tests/functional/triggering/08-fam-finish-any/flow.cylc0000664000175000017500000000065715264420307026147 0ustar alastairalastair[scheduling] [[graph]] R1 = """FAM:finish-any => foo""" [runtime] [[root]] script = true [[FAM]] [[a]] inherit = FAM script = """ cylc__job__poll_grep_workflow_log -E "1/b.*succeeded" """ [[b]] inherit = FAM [[c]] inherit = FAM script = """ cylc__job__poll_grep_workflow_log -E "1/b.*succeeded" """ [[foo]] cylc-flow-8.7.0/tests/functional/triggering/10-finish.t0000664000175000017500000000170515264420307023147 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test finish triggering . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/triggering/17-suicide-multi.t0000664000175000017500000000261115264420307024450 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "or" outputs from same task triggering suicide triggering . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" DBFILE="$RUN_DIR/${WORKFLOW_NAME}/log/db" sqlite3 "${DBFILE}" 'SELECT cycle, name, status FROM task_pool ORDER BY cycle, name;' \ >'sqlite3.out' cmp_ok 'sqlite3.out' <'/dev/null' purge exit cylc-flow-8.7.0/tests/functional/triggering/00-recovery/0000775000175000017500000000000015264420307023334 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/00-recovery/reference.log0000664000175000017500000000076615264420307026006 0ustar alastairalastairInitial point: 20110101T0000Z Final point: 20110101T1200Z 20110101T0000Z/pre -triggered off [] 20110101T1200Z/pre -triggered off [] 20110101T0000Z/model -triggered off ['20110101T0000Z/pre'] 20110101T1200Z/model -triggered off ['20110101T1200Z/pre'] 20110101T0000Z/post -triggered off ['20110101T0000Z/model'] 20110101T1200Z/diagnose -triggered off ['20110101T1200Z/model'] 20110101T1200Z/recover -triggered off ['20110101T1200Z/diagnose'] 20110101T1200Z/post -triggered off ['20110101T1200Z/recover'] cylc-flow-8.7.0/tests/functional/triggering/00-recovery/flow.cylc0000664000175000017500000000252015264420307025156 0ustar alastairalastair[meta] title = "automated failure recovery example" description = """ Model task failure triggers diagnosis and recovery tasks, which otherwise take themselves out of the workflow if model succeeds. Model post processing triggers off model or recovery tasks. """ [scheduler] UTC mode = True allow implicit tasks = True [[events]] expected task failures = 20110101T1200Z/model [scheduling] initial cycle point = 20110101T00 final cycle point = 20110101T12 [[graph]] T00,T12 = """ pre:finish => model? # finish trigger model:fail? => diagnose => recover # fail trigger model? => !diagnose & !recover # explicit success trigger model:succeed? | recover => post # conditional and explicit success post => !model # removes failed model to allow workflow to auto finish """ [runtime] [[root]] script = "true" # fast [[model]] script = """ echo Hello from $CYLC_TASK_ID if [[ $(cylc cycletime --print-hour) == 12 ]]; then echo "FAILING NOW!" false else echo "Succeeded" true fi """ [[[meta]]] description = A task that succeeds at 0 UTC and fails at 12 UTC cylc-flow-8.7.0/tests/functional/triggering/06-fam-succeed-any/0000775000175000017500000000000015264420307024445 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/06-fam-succeed-any/reference.log0000664000175000017500000000024715264420307027111 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/c -triggered off [] 1/b -triggered off [] 1/foo -triggered off ['1/b'] 1/handled -triggered off ['1/a', '1/c'] cylc-flow-8.7.0/tests/functional/triggering/06-fam-succeed-any/flow.cylc0000664000175000017500000000060115264420307026265 0ustar alastairalastair[scheduler] [[events]] expected task failures = 1/a, 1/c [scheduling] [[graph]] R1 = """ FAM:succeed-any? => foo a:fail? & c:fail? => handled """ [runtime] [[FAM]] script = "false" [[a,c]] inherit = FAM [[b]] inherit = FAM script = "true" [[foo, handled]] script = "true" cylc-flow-8.7.0/tests/functional/triggering/04-fam-fail-all/0000775000175000017500000000000015264420307023724 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/04-fam-fail-all/reference.log0000664000175000017500000000021515264420307026363 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/c -triggered off [] 1/b -triggered off [] 1/foo -triggered off ['1/a', '1/b', '1/c'] cylc-flow-8.7.0/tests/functional/triggering/04-fam-fail-all/flow.cylc0000664000175000017500000000040015264420307025541 0ustar alastairalastair[scheduler] [[events]] expected task failures = 1/a, 1/b, 1/c [scheduling] [[graph]] R1 = "FAM:fail-all => foo" [runtime] [[FAM]] script = "false" [[a,b,c]] inherit = FAM [[foo]] script = "true" cylc-flow-8.7.0/tests/functional/triggering/07-fam-fail-any.t0000664000175000017500000000172515264420307024140 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test correct expansion of FAM:fail-any . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/triggering/test_header0000777000175000017500000000000015264420307027576 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/01-or-conditional.t0000664000175000017500000000170415264420307024607 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "or" conditionals . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/triggering/06-fam-succeed-any.t0000664000175000017500000000173015264420307024633 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test correct expansion of FAM:succeed-any . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/triggering/fam-expansion/0000775000175000017500000000000015264420307024026 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/fam-expansion/flow.cylc0000664000175000017500000000041015264420307025644 0ustar alastairalastair#!Jinja2 [scheduling] [[graph]] R1 = "(FOO:finish-all & FOO:fail-any?) => bar" [runtime] [[FOO]] script = false [[foo1,foo2,foo3]] inherit = FOO [[bar]] script = cylc show "${CYLC_WORKFLOW_ID}//1/bar" > {{SHOW_OUT}} cylc-flow-8.7.0/tests/functional/triggering/03-fam-succeed-all/0000775000175000017500000000000015264420307024423 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/03-fam-succeed-all/reference.log0000664000175000017500000000021515264420307027062 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/c -triggered off [] 1/b -triggered off [] 1/foo -triggered off ['1/a', '1/b', '1/c'] cylc-flow-8.7.0/tests/functional/triggering/03-fam-succeed-all/flow.cylc0000664000175000017500000000027315264420307026250 0ustar alastairalastair[scheduling] [[graph]] R1 = """FAM:succeed-all => foo""" [runtime] [[FAM]] script = "true" [[a,b,c]] inherit = FAM [[foo]] script = "true" cylc-flow-8.7.0/tests/functional/triggering/14-submit-fail.t0000664000175000017500000000201115264420307024076 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test submit-fail triggering . "$(dirname "$0")/test_header" set_test_number 2 create_test_global_config '' ' [platforms] [[idontexist]] ' reftest exit cylc-flow-8.7.0/tests/functional/triggering/07-fam-fail-any/0000775000175000017500000000000015264420307023746 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/07-fam-fail-any/reference.log0000664000175000017500000000017715264420307026414 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/c -triggered off [] 1/b -triggered off [] 1/foo -triggered off ['1/b'] cylc-flow-8.7.0/tests/functional/triggering/07-fam-fail-any/flow.cylc0000664000175000017500000000045515264420307025575 0ustar alastairalastair[scheduler] [[events]] expected task failures = 1/b [scheduling] [[graph]] R1 = "FAM:fail-any? => foo" [runtime] [[FAM]] script = "true" [[a,c]] inherit = FAM [[b]] inherit = FAM script = "false" [[foo]] script = "true" cylc-flow-8.7.0/tests/functional/triggering/20-and-outputs-suicide/0000775000175000017500000000000015264420307025406 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/triggering/20-and-outputs-suicide/reference.log0000664000175000017500000000074415264420307030054 0ustar alastairalastairInitial point: 1 Final point: 3 1/showdown -triggered off ['0/fin'] 1/bad -triggered off ['1/showdown'] 1/ugly -triggered off ['1/showdown'] 1/fin -triggered off ['1/bad', '1/ugly'] 2/showdown -triggered off ['1/fin'] 2/good -triggered off ['2/showdown'] 2/ugly -triggered off ['2/showdown'] 2/fin -triggered off ['2/good', '2/ugly'] 3/showdown -triggered off ['2/fin'] 3/bad -triggered off ['3/showdown'] 3/good -triggered off ['3/showdown'] 3/fin -triggered off ['3/bad', '3/good'] cylc-flow-8.7.0/tests/functional/triggering/20-and-outputs-suicide/flow.cylc0000664000175000017500000000242715264420307027236 0ustar alastairalastair[scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] cycling mode = integer initial cycle point = 1 final cycle point = 3 [[graph]] P1 = """ fin[-P1] => showdown showdown:good? => good & ! bad & ! ugly showdown:bad? => bad & ! good & ! ugly showdown:ugly? => ugly & ! good & ! bad # Note: The above implies: # showdown:good => good # showdown:bad => bad # showdown:ugly => ugly # showdown:good & showdown:bad => ! ugly # showdown:good & showdown:ugly => ! bad # showdown:bad & showdown:ugly => ! good (good & bad) | (bad & ugly) | (ugly & good) => fin """ [runtime] [[showdown]] script = """ if ((${CYLC_TASK_CYCLE_POINT} == 1)); then cylc message -- "${CYLC_WORKFLOW_ID}" "${CYLC_TASK_JOB}" 'bad' 'ugly' elif ((${CYLC_TASK_CYCLE_POINT} == 2)); then cylc message -- "${CYLC_WORKFLOW_ID}" "${CYLC_TASK_JOB}" 'good' 'ugly' else cylc message -- "${CYLC_WORKFLOW_ID}" "${CYLC_TASK_JOB}" 'good' 'bad' fi """ [[[outputs]]] good = good bad = bad ugly = ugly [[good, bad, ugly, fin]] script = true cylc-flow-8.7.0/tests/functional/triggering/03-fam-succeed-all.t0000664000175000017500000000173015264420307024611 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test correct expansion of FAM:succeed-all . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/README.md0000664000175000017500000002116015264420307020377 0ustar alastairalastair# Functional Tests This directory contains Cylc functional tests. ## How To Run These Tests ```console # run all tests in this directory $ etc/bin/run-functional-tests tests/f # run a specified test $ etc/bin/run-functional-tests tests/f/cylc-cat-log/00-local.t # run a specified test in debug mode $ etc/bin/run-functional-tests -v tests/f/cylc-cat-log/00-local.t # run tests with 5x parallelism $ etc/bin/run-functional-tests -j 5 tests/f # split the tests into 4 "chunks" and run the first chunk $ CHUNK='1/4' etc/bin/run-functional-tests tests/f # measure code coverage # (coverage files are not automatically transferred from remote platforms) $ export CYLC_COVERAGE=1 $ etc/bin/run-functional-tests tests/f $ coverage combine ``` ## What Are Functional Tests? These tests ensure end-to-end functionality is as expected. With Cylc this typically involves building workflow configurations which cause many parts of the system (and other systems) to be activated. This includes interaction with other systems (e.g. batch schedulers/job runners), command line interfaces / outputs, etc. ## How Are Tests Implemented Tests are written in files with the `.t` extension. These are Bash scripts (despite the file extension). The tests are run with a tool called `prove` which is invoked via the `etc/bin/run-functional-tests` command. `prove` is typically installed as part of the `perl` package on most Linux distributions. Each test file starts by sourcing the file `lib/bash/test_header`, this contains various Bash functions which provide assertion functions, e.g. `run_ok` (which asserts that the provided command succeeds). Assertions run a command, then write either `ok` or `not ok` to stdout, prove then scrapes this output to determine test outcome. Each assertion (or subtest) needs a name, this is usually the first argument to the function. You should use the prefix `$TEST_NAME_BASE`, e.g, this assertion tests that the command `sleep 10` succeeds `run_ok "${TEST_NAME_BASE}-sleep" sleep 10`. Note, some assertions infer the test name for you from the arguments. For the list of available assertions, and usage info, see the comments at the top of `lib/bash/test_header`. Each test file needs to declare the number of subtests within it, this is done with the function `set_test_number` which must be run at the start of the test. Many tests install workflows for testing. These are installed into cylc-run as usual under the prefix `cylctb`. ## recording tests that fail When running many tests you can record the tests that fail using `--state=save`. You can then re-run the only the failed tests using `--state=failed`. For more details see `man prove`. ## Debugging When a test fails, it may print some directories to the screen which contain the stderr of the failed tests, or the workflow logs. Tips: * Run the test using the `-v` argument, this will reveal which subtest(s) failed. * Some tests reveal extra debug info when you set `CYLC_TEST_DEBUG=true`. * If the test fails, any workflows installed into the `~/cylc-run/cylctb???` directory will not be deleted, you can inspect the workflow logs there. * In GitHub actions, the `~/cylc-run` directory is uploaded as an "artifact" where it can be downloaded for debugging. * Any text that gets written to stdout will be swallowed by the test framework, redirect to stderr in order to use `echo` statements for debugging, e.g, `echo "DEBUG: platform=$CYLC_PLATFORM_NAME" >&2`. ### Python Debuggers "Normal" Python debuggers (e.g, `pdb`) will not work from within the functional test framework because the Python code maybe run non-interactively. However, you can use remote debuggers (e.g. [`remote_pdb`](https://pypi.org/project/remote-pdb/)). Remote PDB is probably the simplest remote debugger, here's a simple example to get started: Open two terminal tabs and run this code in both of them: ```shell PYTHONBREAKPOINT=remote_pdb.set_trace REMOTE_PDB_HOST=0.0.0.0 REMOTE_PDB_PORT=${-4444} export PYTHONBREAKPOINT REMOTE_PDB_HOST REMOTE_PDB_PORT ``` Run the test in the first tab, then run this command in the second: ```shell nc -C "$REMOTE_PDB_HOST" "$REMOTE_PDB_PORT" ``` ### Traps for the unwary #### `grep_ok` vs `comp_ok` Tests that use `comp_ok` generally compare `${TEST_NAME}.stdout` or `${TEST_NAME}.stderr` against either a reference or against `/dev/null`. They expect the entire output to be **exactly** the same as the reference and are therefore very unforgiving. `grep_ok` is much less sensitive only requiring the reference output to be present **somewhere** in the test output. ### Further Reading #### Heredocs If you see code that looks like this: ```bash cat >'hello.py' <<'__HELLO_PY__' print("Hello World") __HELLO_PY__ ``` You are looking at an "heredoc" and you may wish to read about heredocs: [A modern looking bloggy guide](https://linuxize.com/post/bash-heredoc/) [A web 1.0 manual](http://tldp.org/LDP/abs/html/here-docs.html) ## How To Run "Non-Generic" Tests? Some tests require job runners (e.g. at, slurm, pbs), remote platforms or specific configurations. To run these tests you must configure remote platforms, your options are: 1. Use a swarm of docker containers (currently does not cover all platform types) $ etc/bin/swarm configure $ etc/bin/swarm build $ etc/bin/swarm run 2. Configure your own remote platforms in your Cylc config. See the next section for the platform matrix and naming conventions. Once you have defined your remote platforms provide them with the `-p` arg: ```console # run ONLY tests compatible with the _remote_background_indep_tcp platform $ etc/bin/run-functional-tests -p _remote_background_indep_tcp tests/f # run tests on the first compatible platform with the "at" job runner $ etc/bin/run-functional-tests -p '_*at*' tests/f # run tests on the first compatible platform configured $ etc/bin/run-functional-tests -p '*' tests/f ``` ## Test Platform Names Each platform is named using this convention: ____ `loc` "submission locality" - `{local, remote}`: Where do jobs get submitted? * Locally (on the scheduler host). * Remotely (on a remote host). `runner` "job runner" - `{background, at, slurm, ...}`: The name of the job runner the container is configured to use. `fs` "filesystem" - `{indep, shared}`: What is the relationship between the filesystem in the container to the filesystem on the host? * Independent (indep) - it is a completely different filesystem. * Shared (shared) - the cylc-run directory is shared. warning: For shared filesystems cylc-run is shared, however, the absolute path to cylc-run on the host system may be different to that on the container, use ~/cylc-run for safety. `comms` "task communication method" - `{tcp, ssh, poll}` The task communication method to use. Define any test platforms in your global config e.g: ``` # ~/.cylc/global.cylc [platforms] [[_remote_background_indep_tcp]] hosts = my_remote_host ``` ## Test Global Config Cylc supports a `global-tests.cylc` file which can be used to define some top-level configurations to run tests with. Do not use this file to define test platforms, put them in your regular global config where they can also be used for interactive work. ## How To Configure "Non-Generic" Tests? By default tests require the platform `_local_background_indep_tcp`. If you want your test to run on any other platform export an environment variable called `REQUIRE_PLATFORM` *before* the `test_header` is sourced. This variable should define a test's requirements of a platform e.g: ```bash # require a remote platform with a shared filesystem and tcp task comms export REQUIRE_PLATFORM='loc:remote fs:shared comms:tcp' ``` Bash extglob pattern matching can be provided (https://www.gnu.org/software/bash/manual/html_node/Pattern-Matching.html) e.g: ```bash # run for all job runners with tcp comms on remote platforms export REQUIRE_PLATFORM='loc:remote runner:* comms:tcp' # run for tcp and ssh comms on remote platform export REQUIRE_PLATFORM='loc:remote comms:?(tcp|ssh)' ``` If a field is not provided it defaults to `*` with the exception of `loc` which defaults to `local` i.e: ```bash # run for all job runners on local platforms export REQUIRE_PLATFORM='runner:*' # run for all job runners on remote platforms export REQUIRE_PLATFORM='loc:remote runner:*' ``` ## Guidelines Don't write functional tests when you can write integration tests: * If a test requires a workflow to be put in an "exotic" state, consider if this can be achieved artificially. * If a test can be broken down into smaller more targeted sub-tests then integration testing might be more appropriate. cylc-flow-8.7.0/tests/functional/logging/0000775000175000017500000000000015264420307020546 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/logging/02-duplicates.t0000664000175000017500000000310315264420307023304 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 4 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --no-detach "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play --no-detach "${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}-check" \ test -e "${WORKFLOW_RUN_DIR}/work/2/pub/log-duplication" run_fail "${TEST_NAME_BASE}-check" \ test -s "${WORKFLOW_RUN_DIR}/work/2/pub/log-duplication" #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/logging/02-duplicates/0000775000175000017500000000000015264420307023122 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/logging/02-duplicates/flow.cylc0000664000175000017500000000176515264420307024756 0ustar alastairalastair[scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT3M [scheduling] cycling mode = integer initial cycle point = 1 [[graph]] R1/1 = """ foo:fail? => bar foo? & bar => restart """ R1/2 = """ restart[-P1] => foo? foo:fail? => bar foo? & bar => pub """ [runtime] [[foo]] script = false [[bar]] script = """ cylc set --output=succeeded \ "${CYLC_WORKFLOW_ID}//${CYLC_TASK_CYCLE_POINT}/foo" """ [[restart]] script = """ cylc stop "${CYLC_WORKFLOW_ID}" """ [[pub]] script = """ # Extract timestamp lines from logs for file in $(find "${CYLC_WORKFLOW_RUN_DIR}/log/schedulerr/" -name '*.*'); do grep '.*-.*-.*' "${file}" | sort -u || true done | sort | uniq -d > 'log-duplication' """ cylc-flow-8.7.0/tests/functional/logging/03-roll.t0000775000175000017500000000351115264420307022126 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test log rolling. . "$(dirname "$0")/test_header" set_test_number 11 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S [scheduling] cycling mode = integer initial cycle point = 1 final cycle point = 20 [[graph]] P1 = t1 & t2 & t3 [runtime] [[t1, t2, t3]] script = true __FLOW_CONFIG__ create_test_global_config '' ' [scheduler] [[logging]] rolling archive length = 8 maximum size in bytes = 2048 ' run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" FILES="$(ls "${HOME}/cylc-run/${WORKFLOW_NAME}/log/scheduler/"*.log)" run_ok "${TEST_NAME_BASE}-n-logs" test 8 -eq "$(wc -l <<<"${FILES}")" for FILE in ${FILES}; do run_ok "${TEST_NAME_BASE}-log-size" test "$(stat -c'%s' "${FILE}")" -le 2048 done purge exit cylc-flow-8.7.0/tests/functional/logging/test_header0000777000175000017500000000000015264420307027063 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/logging/05-validation-detach.t0000664000175000017500000000246115264420307024540 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # ---------------------------------------------------------------------------- # Test that validation errors on play are logged before daemonisation . "$(dirname "$0")/test_header" set_test_number 3 init_workflow "${TEST_NAME_BASE}" <<'__FLOW__' [scheduler] horse = dorothy __FLOW__ run_fail "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_fail "$TEST_NAME" cylc play "${WORKFLOW_NAME}" grep_ok "IllegalItemError: [scheduler]horse" "${TEST_NAME}.stderr" -F purge cylc-flow-8.7.0/tests/functional/logging/01-basic.t0000664000175000017500000000501115264420307022227 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test start/restart/reload config logs are created correctly . "$(dirname "$0")/test_header" set_test_number 7 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduler] [[events]] abort on stall timeout = true stall timeout = PT0S abort on inactivity timeout = true inactivity timeout = PT1M [scheduling] [[graph]] R1 = reloader1 => stopper => reloader2 [runtime] [[reloader1, reloader2]] script = """ cylc reload "${CYLC_WORKFLOW_ID}" # wait for the command to complete cylc__job__poll_grep_workflow_log 'Reload completed' """ [[stopper]] script = cylc stop --now --now "${CYLC_WORKFLOW_ID}" __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --no-detach "${WORKFLOW_NAME}" # Check scheduler logs. ls "${WORKFLOW_RUN_DIR}/log/scheduler/" > schd_1.out cmp_ok schd_1.out << __EOF__ 01-start-01.log log __EOF__ # Check config logs. ls "${WORKFLOW_RUN_DIR}/log/config/" > conf_1.out cmp_ok conf_1.out << __EOF__ 01-start-01.cylc 02-reload-01.cylc flow-processed.cylc __EOF__ mv "$WORKFLOW_RUN_DIR/cylc.flow.main_loop.log_db.sql" "$WORKFLOW_RUN_DIR/01.cylc.flow.main_loop.log_db.sql" workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --no-detach "${WORKFLOW_NAME}" ls "${WORKFLOW_RUN_DIR}/log/scheduler/" > schd_2.out cmp_ok schd_2.out << __EOF__ 01-start-01.log 02-restart-02.log log __EOF__ ls "${WORKFLOW_RUN_DIR}/log/config/" > conf_2.out cmp_ok conf_2.out << __EOF__ 01-start-01.cylc 02-reload-01.cylc 03-restart-02.cylc 04-reload-02.cylc flow-processed.cylc __EOF__ purge cylc-flow-8.7.0/tests/functional/logging/04-dev_mode.t0000664000175000017500000000314415264420307022740 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test log dev-mode. . "$(dirname "$0")/test_header" set_test_number 5 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduling] cycling mode = integer initial cycle point = 1 final cycle point = 1 [[graph]] P1 = t1 [runtime] [[t1]] script = true __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-validate-plain" \ cylc validate "${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}-validate-vvv" \ cylc validate --timestamp -vvv "${WORKFLOW_NAME}" grep_ok " DEBUG - \[config:.*\]" "${TEST_NAME_BASE}-validate-vvv.stderr" run_ok "${TEST_NAME_BASE}-validate-vvv--no-timestamp" \ cylc validate -vvv "${WORKFLOW_NAME}" grep_ok "^DEBUG - \[config:.*\]" "${TEST_NAME_BASE}-validate-vvv--no-timestamp.stderr" purge exit cylc-flow-8.7.0/tests/functional/flow-triggers/0000775000175000017500000000000015264420307021713 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/flow-triggers/08-none-past-switch/0000775000175000017500000000000015264420307025343 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/flow-triggers/08-none-past-switch/reference.log0000664000175000017500000000025515264420307030006 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] in flow 1 1/b -triggered off ['1/a'] in flow 1 1/c -triggered off ['1/b'] in flow 1 1/a -triggered off [] in flow none cylc-flow-8.7.0/tests/functional/flow-triggers/08-none-past-switch/flow.cylc0000664000175000017500000000242215264420307027166 0ustar alastairalastair# a:x? => b => c # a:y? => d => e # # Start with the "x" pathway completed and re-trigger "a". # The second run of "a" yields the output "y". # # flow:1 # a(succeeded:x) # b(succeeded) # c(succeeded) # flow: # a(running) # # by: # cylc trigger --flow=none a # # Result: # Task "a" is re-run as a no-flow; so the other branch does not get run. # # flow:1 # a#1 (the naturally triggered run) # b # c # flow:none # a#2 (the manually triggered run) [scheduler] allow implicit tasks = True [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT1M [scheduling] [[graph]] R1 = """ a:x? => b => c a:y? => d => e """ [runtime] [[a]] script = """ if (( CYLC_TASK_SUBMIT_NUMBER == 1 )); then cylc message x else cylc message y fi """ [[[outputs]]] x = x y = y [[c]] script = """ cylc trigger --flow=none ${CYLC_WORKFLOW_ID}//1/a cylc__job__poll_grep_workflow_log -E '1/a/02\(flows=none\):submitted.*started' """ cylc-flow-8.7.0/tests/functional/flow-triggers/01-all-future.t0000664000175000017500000000341215264420307024376 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" set_test_number 7 install_and_validate "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" true DB="${WORKFLOW_RUN_DIR}/runN/log/db" reftest_run TEST_NAME="${TEST_NAME_BASE}-order-no-wait" QUERY="SELECT name,flow_nums,flow_wait FROM task_states\ WHERE name IN ('a','d','c','e') ORDER BY time_created;" run_ok "${TEST_NAME}" sqlite3 "${DB}" "$QUERY" # If not waiting, e should come before c. cmp_ok "${TEST_NAME}.stdout" <<\__END__ a|[1]|0 d|[1]|0 e|[1]|0 c|[1]|0 __END__ export REFTEST_OPTS=--set="WAIT=1" install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" true reftest_run TEST_NAME="${TEST_NAME_BASE}-order-wait" # Note flow_wait is updated to False once used. QUERY="SELECT name,flow_nums,flow_wait FROM task_states ORDER BY time_created" run_ok "${TEST_NAME}" sqlite3 "${DB}" "$QUERY" cmp_ok "${TEST_NAME}.stdout" <<\__END__ a|[1]|0 d|[1]|0 b|[1]|0 c|[1]|0 e|[1]|0 f|[1]|0 __END__ purge cylc-flow-8.7.0/tests/functional/flow-triggers/12-all-future-multi/0000775000175000017500000000000015264420307025343 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/flow-triggers/12-all-future-multi/reference.log0000664000175000017500000000065015264420307030005 0ustar alastairalastairInitial point: 1 Final point: 1 # the first flow 1/a -triggered off ['0/a'] in flow 1 2/a -triggered off ['1/a'] in flow 1 3/a -triggered off ['2/a'] in flow 1 4/a -triggered off ['3/a'] in flow 1 # the second flow 3/a -triggered off ['2/a'] in flow 2 4/a -triggered off ['3/a'] in flow 2 # the joined flow 5/a -triggered off ['4/a'] in flow 1,2 6/a -triggered off ['5/a'] in flow 1,2 7/a -triggered off ['6/a'] in flow 1,2 cylc-flow-8.7.0/tests/functional/flow-triggers/12-all-future-multi/flow.cylc0000664000175000017500000000230415264420307027165 0ustar alastairalastair# flow:1 # 1/a(running) # flow:2 # 3/a(running) # flow:1,2 # 5/a(running) # # Result: # The task 5/a is triggered in both flows so joins the two. # # flow:1 # 1/a # 2/a # 3/a # 4/a # flow:2 # 3/a # 4/a # flow:1,2 # 5/a # 6/a # 7/a [scheduler] allow implicit tasks = True [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT1M [scheduling] cycling mode = integer initial cycle point = 1 final cycle point = 7 [[graph]] P1 = a[-P1] => a [runtime] [[a]] script = """ if (( CYLC_TASK_CYCLE_POINT == 1 && CYLC_TASK_SUBMIT_NUMBER == 1 )); then # trigger 3/a in a new flow cylc trigger --flow=new ${CYLC_WORKFLOW_ID}//3/a cylc__job__poll_grep_workflow_log -E '3/a.*=> running' # trigger 5/a in all flows cylc trigger ${CYLC_WORKFLOW_ID}//5/a cylc__job__poll_grep_workflow_log -E '5/a.*=> running' fi """ cylc-flow-8.7.0/tests/functional/flow-triggers/06-new-past-switch/0000775000175000017500000000000015264420307025173 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/flow-triggers/06-new-past-switch/reference.log0000664000175000017500000000036415264420307027637 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] in flow 1 1/b -triggered off ['1/a'] in flow 1 1/c -triggered off ['1/b'] in flow 1 1/a -triggered off [] in flow 2 1/d -triggered off ['1/a'] in flow 2 1/e -triggered off ['1/d'] in flow 2 cylc-flow-8.7.0/tests/functional/flow-triggers/06-new-past-switch/flow.cylc0000664000175000017500000000244415264420307027022 0ustar alastairalastair# a:x? => b => c # a:y? => d => e # # Start with the "x" pathway completed and re-trigger "a". # The second run of "a" yields the output "y". # # flow:1 # a(succeeded:x) # b(succeeded) # c(succeeded) # flow: # a(running) # # by: # cylc trigger --flow=new a # # Result: # Task "a" is re-run, the workflow runs on from there following the "y" pathway. # # flow:1 # a (the naturally triggered run) # b # c # flow:2 # a (the manually triggered run) # d # e [scheduler] allow implicit tasks = True [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT1M [scheduling] [[graph]] R1 = """ a:x? => b => c a:y? => d => e """ [runtime] [[a]] script = """ if (( CYLC_TASK_SUBMIT_NUMBER == 1 )); then cylc message x else cylc message y fi """ [[[outputs]]] x = x y = y [[c]] script = """ cylc trigger --flow=new ${CYLC_WORKFLOW_ID}//1/a cylc__job__poll_grep_workflow_log -E '1/a/02\(flows=2\):submitted.*started' """ cylc-flow-8.7.0/tests/functional/flow-triggers/10-specific-flow/0000775000175000017500000000000015264420307024663 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/flow-triggers/10-specific-flow/reference.log0000664000175000017500000000105015264420307027320 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] in flow 1 1/trigger-happy -triggered off [] in flow 1 1/f -triggered off ['1/e'] in flow 2 1/b -triggered off ['1/a'] in flow 1 1/c -triggered off ['1/b'] in flow 1 1/d -triggered off ['1/c'] in flow 1 1/e -triggered off ['1/d'] in flow 1 1/b -triggered off ['1/a'] in flow 2 1/f -triggered off ['1/e'] in flow 1 1/c -triggered off ['1/b'] in flow 2 1/g -triggered off ['1/f'] in flow 1 1/d -triggered off ['1/c'] in flow 2 1/e -triggered off ['1/d'] in flow 2 1/g -triggered off ['1/f'] in flow 2 cylc-flow-8.7.0/tests/functional/flow-triggers/10-specific-flow/flow.cylc0000664000175000017500000000125015264420307026504 0ustar alastairalastair# Check targeting a specific flow works, with trigger --wait. # At start-up, trigger f with --wait for flow 2 # Then when flow 1 reaches d, trigger flow 2 at b. # Flow 1 should overrun the manually triggered f. # Flow 2 should skip over it to g. [scheduling] [[graph]] R1 = """ trigger-happy a => b => c => d => e => f => g """ [runtime] [[a, b, c, d, e, f,g]] [[trigger-happy]] script = """ cylc trigger --flow=2 --wait ${CYLC_WORKFLOW_ID}//1/f """ [[d]] script = """ if [[ "$CYLC_TASK_SUBMIT_NUMBER" == "1" ]]; then cylc trigger --flow=2 ${CYLC_WORKFLOW_ID}//1/b fi """ cylc-flow-8.7.0/tests/functional/flow-triggers/06-new-past-switch.t0000664000175000017500000000165515264420307025367 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/flow-triggers/03-new-past/0000775000175000017500000000000015264420307023671 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/flow-triggers/03-new-past/reference.log0000664000175000017500000000047615264420307026341 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] in flow 1 1/b -triggered off ['1/a'] in flow 1 1/c -triggered off ['1/b'] in flow 1 1/a -triggered off [] in flow 2 1/d -triggered off ['1/c'] in flow 1 1/b -triggered off ['1/a'] in flow 2 1/c -triggered off ['1/b'] in flow 2 1/d -triggered off ['1/c'] in flow 2 cylc-flow-8.7.0/tests/functional/flow-triggers/03-new-past/flow.cylc0000664000175000017500000000156415264420307025522 0ustar alastairalastair# flow:1 # a(succeeded) # b(succeeded) # c(running) # flow: # a(running) # # by: # cylc trigger --flow=new a # # Result: # A new flow is started which overruns the previous flow. # # flow:1 # a # b # c # d # flow: 2 # a # b # c # d [scheduler] allow implicit tasks = True [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT1M [scheduling] [[graph]] R1 = "a => b => c => d" [runtime] [[d]] script = """ if (( $CYLC_TASK_SUBMIT_NUMBER == 1 )); then cylc trigger --flow=new ${CYLC_WORKFLOW_ID}//1/a cylc__job__poll_grep_workflow_log -E '1/a/02\(flows=2\):submitted.*started' fi """ cylc-flow-8.7.0/tests/functional/flow-triggers/02-none-future.t0000664000175000017500000000165515264420307024575 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/flow-triggers/00-new-future.t0000664000175000017500000000165415264420307024424 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/flow-triggers/05-none-past/0000775000175000017500000000000015264420307024041 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/flow-triggers/05-none-past/reference.log0000664000175000017500000000032215264420307026477 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] in flow 1 1/b -triggered off ['1/a'] in flow 1 1/c -triggered off ['1/b'] in flow 1 1/a -triggered off [] in flow none 1/d -triggered off ['1/c'] in flow 1 cylc-flow-8.7.0/tests/functional/flow-triggers/05-none-past/flow.cylc0000664000175000017500000000155015264420307025665 0ustar alastairalastair# flow:1 # a(succeeded) # b(succeeded) # c(running) # flow: # a(running) # # by: # cylc trigger --flow=none a # # Result: # The triggered task belongs to no flow; it will run but not flow on. # # flow:1 # a # b # c # d # flow:none # a [scheduler] allow implicit tasks = True [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT1M [scheduling] [[graph]] R1 = "a => b => c => d" [runtime] [[d]] script = """ if (( $CYLC_TASK_SUBMIT_NUMBER == 1 )); then cylc trigger --flow=none ${CYLC_WORKFLOW_ID}//1/a cylc__job__poll_grep_workflow_log -E '1/a/02\(flows=none\):submitted.*started' fi """ cylc-flow-8.7.0/tests/functional/flow-triggers/12-all-future-multi.t0000664000175000017500000000165315264420307025535 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/flow-triggers/03-new-past.t0000664000175000017500000000165515264420307024065 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/flow-triggers/11-wait-merge/0000775000175000017500000000000015264420307024173 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/flow-triggers/11-wait-merge/reference.log0000664000175000017500000000043515264420307026636 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] in flow 1 1/b -triggered off ['1/a'] in flow 1 1/a -triggered off [] in flow 2 1/c -triggered off ['1/b'] in flow 2 1/x -triggered off ['1/c'] in flow 1,2 1/d -triggered off ['1/c'] in flow 1,2 1/b -triggered off ['1/a'] in flow 2 cylc-flow-8.7.0/tests/functional/flow-triggers/11-wait-merge/flow.cylc0000664000175000017500000000223515264420307026020 0ustar alastairalastair# Task b(1) triggers: # - a(2), which succeeds after b(1) finishes (to avoid merge) # - c(2) --wait, which succeeds after merge with b(1) # Then b waits for c:x before succeeding. # When flow 2 reaches c, it should # - spawn onward to d(1,2) # - retroactively spawn on c:x => x(1,2) [scheduling] [[graph]] R1 = """ a => b => c => d c:x => x """ [runtime] [[a]] script = """ if ((CYLC_TASK_SUBMIT_NUMBER == 2)); then cylc__job__poll_grep_workflow_log "\[1/d/01(flows=1,2):submitted] (received)started" fi """ [[b]] script = """ if ((CYLC_TASK_SUBMIT_NUMBER == 1)); then cylc trigger --flow=new ${CYLC_WORKFLOW_ID}//1/a cylc trigger --flow=2 --wait ${CYLC_WORKFLOW_ID}//1/c cylc__job__poll_grep_workflow_log "\[1/c/01(flows=2):running] (received)x" fi """ [[c]] script = """ cylc__job__wait_cylc_message_started cylc message x if ((CYLC_TASK_SUBMIT_NUMBER == 1)); then cylc__job__poll_grep_workflow_log "merged" fi """ [[[outputs]]] x = x [[d, x]] cylc-flow-8.7.0/tests/functional/flow-triggers/13-noflow-nomerge.t0000664000175000017500000000334415264420307025263 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" set_test_number 7 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" DB="${WORKFLOW_RUN_DIR}/log/db" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" poll_grep_workflow_log "Workflow stalled" run_ok "${TEST_NAME_BASE}-trigger" cylc trigger --flow=none "${WORKFLOW_NAME}//1/a" poll_grep_workflow_log -E "1/a/02\(flows=none\):running.*=> succeeded" cylc stop --now --now --max-polls=5 --interval=2 "$WORKFLOW_NAME" TEST_NAME="${TEST_NAME_BASE}-count" QUERY="SELECT COUNT(*) FROM task_states WHERE name=='a'" run_ok "${TEST_NAME}" sqlite3 "${DB}" "$QUERY" cmp_ok "${TEST_NAME}.stdout" <<__END__ 2 __END__ QUERY="SELECT COUNT(*) FROM task_states WHERE name=='b'" run_ok "${TEST_NAME}" sqlite3 "${DB}" "$QUERY" cmp_ok "${TEST_NAME}.stdout" <<__END__ 1 __END__ purge cylc-flow-8.7.0/tests/functional/flow-triggers/00-new-future/0000775000175000017500000000000015264420307024231 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/flow-triggers/00-new-future/reference.log0000664000175000017500000000055015264420307026672 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] in flow 1 1/d -triggered off ['1/c'] in flow 2 1/e -triggered off ['1/d'] in flow 2 1/b -triggered off ['1/a'] in flow 1 1/f -triggered off ['1/e'] in flow 2 1/c -triggered off ['1/b'] in flow 1 1/d -triggered off ['1/c'] in flow 1 1/e -triggered off ['1/d'] in flow 1 1/f -triggered off ['1/e'] in flow 1 cylc-flow-8.7.0/tests/functional/flow-triggers/00-new-future/flow.cylc0000664000175000017500000000135515264420307026060 0ustar alastairalastair# flow:1 # a(running) # flow: # d(running) # by: # cylc trigger --flow=new d # # Result: # The two flows never meet in the pool so never merge. # # flow:1 # a # b # c # d # e # f # flow:2 # d # e # f [scheduler] allow implicit tasks = True [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT1M [scheduling] [[graph]] R1 = "a => b => c => d => e => f" [runtime] [[a]] script = """ cylc trigger --flow=new ${CYLC_WORKFLOW_ID}//1/d cylc__job__poll_grep_workflow_log -E '1/d/01.*started' """ cylc-flow-8.7.0/tests/functional/flow-triggers/11-wait-merge.t0000664000175000017500000000363215264420307024364 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" skip_macos_gh_actions set_test_number 4 install_and_validate "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" true DB="${WORKFLOW_RUN_DIR}/runN/log/db" reftest_run TEST_NAME="${TEST_NAME_BASE}-order-no-wait" QUERY="SELECT cycle, name,flow_nums,outputs FROM task_outputs;" run_ok "${TEST_NAME}" sqlite3 "${DB}" "$QUERY" cmp_ok "${TEST_NAME}.stdout" <<\__END__ 1|a|[1]|{"submitted": "submitted", "started": "started", "succeeded": "succeeded"} 1|b|[1]|{"submitted": "submitted", "started": "started", "succeeded": "succeeded"} 1|a|[2]|{"submitted": "submitted", "started": "started", "succeeded": "succeeded"} 1|c|[2]|{"submitted": "submitted", "started": "started", "x": "x"} 1|c|[1, 2]|{"submitted": "submitted", "started": "started", "succeeded": "succeeded", "x": "x"} 1|x|[1, 2]|{"submitted": "submitted", "started": "started", "succeeded": "succeeded"} 1|d|[1, 2]|{"submitted": "submitted", "started": "started", "succeeded": "succeeded"} 1|b|[2]|{"submitted": "submitted", "started": "started", "succeeded": "succeeded"} __END__ purge cylc-flow-8.7.0/tests/functional/flow-triggers/07-all-past-switch/0000775000175000017500000000000015264420307025153 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/flow-triggers/07-all-past-switch/flow.cylc0000664000175000017500000000322315264420307026776 0ustar alastairalastair#!Jinja2 # a:x? => b => c # a:y? => d => e # # Start with the "x" pathway completed and re-trigger "a". # The second run of "a" yields the output "y". # # flow:1 # a(succeeded:x) # b(succeeded) # c(succeeded) # flow: # a(running) # # by: # cylc trigger [--wait] a # # Result: # Task "a" is re-run, the workflow runs on from there following the "y" pathway. # (UNLESS --wait, in which case the y pathway doesn't run before shutdown). # # flow:1 # a#2 (the naturally triggered run) # a#2 (the manually triggered run) # b (the naturally triggered "x" pathway) # c (the naturally triggered "x" pathway) # d (the manually triggered "y" pathway) # e (the manually triggered "y" pathway) {% if WAIT is defined %} {% set OPT = "--wait" %} {% else %} {% set OPT = "" %} {% endif %} [scheduler] allow implicit tasks = True [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT1M [scheduling] [[graph]] R1 = """ a:x? => b => c a:y? => d => e """ [runtime] [[a]] script = """ if (( CYLC_TASK_SUBMIT_NUMBER == 1 )); then cylc message x else cylc message y fi """ [[[outputs]]] x = x y = y [[c]] script = """ if (( CYLC_TASK_SUBMIT_NUMBER == 1 )); then cylc trigger {{OPT}} ${CYLC_WORKFLOW_ID}//1/a cylc__job__poll_grep_workflow_log -E '1/a/02:running.*succeeded' fi """ cylc-flow-8.7.0/tests/functional/flow-triggers/07-all-past-switch/reflog-no-wait.log0000664000175000017500000000036415264420307030513 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] in flow 1 1/b -triggered off ['1/a'] in flow 1 1/c -triggered off ['1/b'] in flow 1 1/a -triggered off [] in flow 1 1/d -triggered off ['1/a'] in flow 1 1/e -triggered off ['1/d'] in flow 1 cylc-flow-8.7.0/tests/functional/flow-triggers/07-all-past-switch/reflog-wait.log0000664000175000017500000000025215264420307030075 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] in flow 1 1/a -triggered off [] in flow 1 1/b -triggered off ['1/a'] in flow 1 1/c -triggered off ['1/b'] in flow 1 cylc-flow-8.7.0/tests/functional/flow-triggers/01-all-future/0000775000175000017500000000000015264420307024211 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/flow-triggers/01-all-future/reference.log0000664000175000017500000000037115264420307026653 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] in flow 1 1/b -triggered off ['1/a'] in flow 1 1/c -triggered off ['1/b'] in flow 1 1/d -triggered off ['1/c'] in flow 1 1/e -triggered off ['1/d'] in flow 1 1/f -triggered off ['1/e'] in flow 1 cylc-flow-8.7.0/tests/functional/flow-triggers/01-all-future/flow.cylc0000664000175000017500000000156115264420307026037 0ustar alastairalastair#!Jinja2 # flow:1 # a(running) # flow: # d(running) # by: # cylc trigger d # # Result: # The two flows never meet in the pool so never merge. But the triggered # task belongs to the original flow so it does not run again. # # flow:1 # a # b # c # d # e # f {% if WAIT is defined %} {% set OPT = "--wait" %} {% else %} {% set OPT = "" %} {% endif %} [scheduler] allow implicit tasks = True [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT1M [scheduling] [[graph]] R1 = "a => b => c => d => e => f" [runtime] [[a]] script = """ cylc trigger {{OPT}} ${CYLC_WORKFLOW_ID}//1/d cylc__job__poll_grep_workflow_log -E '1/d.*succeeded' """ cylc-flow-8.7.0/tests/functional/flow-triggers/test_header0000777000175000017500000000000015264420307030230 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/flow-triggers/04-all-past.t0000664000175000017500000000355415264420307024045 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" set_test_number 7 install_and_validate "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" true DB="${WORKFLOW_RUN_DIR}/runN/log/db" reftest_run # NOTE task_states table only keeps the final submit number of a task for each flow TEST_NAME="${TEST_NAME_BASE}-order-no-wait" QUERY="SELECT name,submit_num,flow_wait FROM task_states WHERE flow_nums is '[1]' ORDER BY time_updated;" # (Ordering by time_updated, 'c' comes before 'a' job:02) run_ok "${TEST_NAME}" sqlite3 "${DB}" "$QUERY" cmp_ok "${TEST_NAME}.stdout" <<\__END__ b|1|0 c|1|0 a|2|0 d|1|0 __END__ export REFTEST_OPTS=--set="WAIT=1" install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" true reftest_run TEST_NAME="${TEST_NAME_BASE}-order-wait" QUERY="SELECT name,submit_num,flow_wait FROM task_states WHERE flow_nums is '[1]' ORDER BY time_updated" # (Ordering by time_updated, 'c' comes before 'a' job:02) run_ok "${TEST_NAME}" sqlite3 "${DB}" "$QUERY" cmp_ok "${TEST_NAME}.stdout" <<\__END__ b|1|0 c|1|0 a|2|1 d|1|0 __END__ purge cylc-flow-8.7.0/tests/functional/flow-triggers/13-noflow-nomerge/0000775000175000017500000000000015264420307025072 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/flow-triggers/13-noflow-nomerge/flow.cylc0000664000175000017500000000016615264420307026720 0ustar alastairalastair[scheduling] [[graph]] R1 = "a => b" [runtime] [[a]] [[b]] script = false # die as incomplete cylc-flow-8.7.0/tests/functional/flow-triggers/08-none-past-switch.t0000664000175000017500000000165515264420307025537 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/flow-triggers/07-all-past-switch.t0000664000175000017500000000416515264420307025346 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" set_test_number 7 install_and_validate "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" true DB="${WORKFLOW_RUN_DIR}/runN/log/db" cp "${WORKFLOW_RUN_DIR}/runN/reflog-no-wait.log" "${WORKFLOW_RUN_DIR}/runN/reference.log" reftest_run # NOTE task_states table only keeps the final submit number of a task for each flow TEST_NAME="${TEST_NAME_BASE}-order-no-wait" QUERY="SELECT name,submit_num,flow_wait FROM task_states WHERE flow_nums is '[1]' ORDER BY time_updated;" # Ordering by time_updated a(job:02) comes before c, which triggers it and # waits for it to finish. run_ok "${TEST_NAME}" sqlite3 "${DB}" "$QUERY" cmp_ok "${TEST_NAME}.stdout" <<\__END__ b|1|0 a|2|0 c|1|0 d|1|0 e|1|0 __END__ export REFTEST_OPTS=--set="WAIT=1" install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" true cp "${WORKFLOW_RUN_DIR}/runN/reflog-wait.log" "${WORKFLOW_RUN_DIR}/runN/reference.log" reftest_run TEST_NAME="${TEST_NAME_BASE}-order-wait" QUERY="SELECT name,submit_num,flow_wait FROM task_states WHERE flow_nums is '[1]' ORDER BY time_updated" # Ordering by time_updated a(job:02) comes before c, which triggers it and # waits for it to finish. run_ok "${TEST_NAME}" sqlite3 "${DB}" "$QUERY" cmp_ok "${TEST_NAME}.stdout" <<\__END__ b|1|0 a|2|1 c|1|0 __END__ purge cylc-flow-8.7.0/tests/functional/flow-triggers/05-none-past.t0000664000175000017500000000165515264420307024235 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/flow-triggers/04-all-past/0000775000175000017500000000000015264420307023651 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/flow-triggers/04-all-past/reference.log0000664000175000017500000000031715264420307026313 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] in flow 1 1/b -triggered off ['1/a'] in flow 1 1/c -triggered off ['1/b'] in flow 1 1/a -triggered off [] in flow 1 1/d -triggered off ['1/c'] in flow 1 cylc-flow-8.7.0/tests/functional/flow-triggers/04-all-past/flow.cylc0000664000175000017500000000165515264420307025503 0ustar alastairalastair#!Jinja2 # flow:1 # a(succeeded) # b(succeeded) # c(running) # flow: # a(running) # # by: # cylc trigger a # # Result: # The triggered task belongs to flow 1; it will run but not flow on. # # flow:1 # a #1 # a #2 # b # c # d {% if WAIT is defined %} {% set OPT = "--wait" %} {% else %} {% set OPT = "" %} {% endif %} [scheduler] allow implicit tasks = True [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT1M [scheduling] [[graph]] R1 = "a => b => c => d" [runtime] [[d]] script = """ if (( $CYLC_TASK_SUBMIT_NUMBER == 1 )); then cylc trigger {{OPT}} ${CYLC_WORKFLOW_ID}//1/a cylc__job__poll_grep_workflow_log -E '1/a/02:running.*succeeded' fi """ cylc-flow-8.7.0/tests/functional/flow-triggers/10-specific-flow.t0000664000175000017500000000174315264420307025055 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test targeting a specific flow, with trigger --wait. . "$(dirname "$0")/test_header" set_test_number 2 reftest exit cylc-flow-8.7.0/tests/functional/flow-triggers/02-none-future/0000775000175000017500000000000015264420307024401 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/flow-triggers/02-none-future/reference.log0000664000175000017500000000044115264420307027041 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] in flow 1 1/d -triggered off ['1/c'] in flow none 1/b -triggered off ['1/a'] in flow 1 1/c -triggered off ['1/b'] in flow 1 1/d -triggered off ['1/c'] in flow 1 1/e -triggered off ['1/d'] in flow 1 1/f -triggered off ['1/e'] in flow 1 cylc-flow-8.7.0/tests/functional/flow-triggers/02-none-future/flow.cylc0000664000175000017500000000133515264420307026226 0ustar alastairalastair# flow:1 # a(running) # flow 2: # d(running) # # by: # cylc trigger --flow=none d # # Result: # d runs once as a no-flow, and once in flow 1. # # flow:1 # a # b # c # d # e # f # flow:none # d [scheduler] allow implicit tasks = True [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT1M [scheduling] [[graph]] R1 = "a => b => c => d => e => f" [runtime] [[a]] script = """ cylc trigger --flow=none ${CYLC_WORKFLOW_ID}//1/d cylc__job__poll_grep_workflow_log -E '1/d.*started' """ cylc-flow-8.7.0/tests/functional/cylc-graph-diff/0000775000175000017500000000000015264420307022057 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-graph-diff/00-simple.t0000664000175000017500000001103415264420307023751 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test cylc graph --diff for two workflows. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 15 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}-control" "${TEST_NAME_BASE}-control" CONTROL_WORKFLOW_NAME="${WORKFLOW_NAME}" install_workflow "${TEST_NAME_BASE}-diffs" "${TEST_NAME_BASE}-diffs" DIFF_WORKFLOW_NAME="${WORKFLOW_NAME}" install_workflow "${TEST_NAME_BASE}-same" "${TEST_NAME_BASE}-same" SAME_WORKFLOW_NAME="${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate-diffs" run_ok "${TEST_NAME}" cylc validate "${DIFF_WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate-same" run_ok "${TEST_NAME}" cylc validate "${SAME_WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate-new" run_ok "${TEST_NAME}" cylc validate "${CONTROL_WORKFLOW_NAME}" #------------------------------------------------------------------------------- #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-bad-workflow-name" run_fail "${TEST_NAME}" \ cylc graph "${DIFF_WORKFLOW_NAME}" --diff "${CONTROL_WORKFLOW_NAME}.bad" cmp_ok "${TEST_NAME}.stdout" foo T00 = foo => bar & baz [runtime] [[FOO]] [[foo]] inherit = None, FOO cylc-flow-8.7.0/tests/functional/cylc-graph-diff/00-simple-diffs/0000775000175000017500000000000015264420307024656 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-graph-diff/00-simple-diffs/flow.cylc0000664000175000017500000000040415264420307026477 0ustar alastairalastair[scheduler] UTC mode = True allow implicit tasks = True [scheduling] initial cycle point = 20140808T00 [[graph]] R1 = cold_foo => foo T00 = foo => bar => baz [runtime] [[FOO]] [[foo,bar,baz]] inherit = None, FOO cylc-flow-8.7.0/tests/functional/cylc-graph-diff/00-simple-same0000777000175000017500000000000015264420307027535 200-simple-controlustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-graph-diff/test_header0000777000175000017500000000000015264420307030374 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-graph-diff/01-icp.t0000775000175000017500000000410315264420307023236 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test for "cylc graph-diff WORKFLOW1 WORKFLOW2 -- ICP". . "$(dirname "$0")/test_header" set_test_number 3 init_workflow "${TEST_NAME_BASE}-1" <<'__FLOW_CONFIG__' [scheduler] UTC mode = True [scheduling] [[graph]] R1 = foo => bar [runtime] [[foo, bar]] script = true __FLOW_CONFIG__ # shellcheck disable=SC2153 WORKFLOW_NAME1="${WORKFLOW_NAME}" init_workflow "${TEST_NAME_BASE}-2" <<'__FLOW_CONFIG__' [scheduler] UTC mode = True [scheduling] [[graph]] R1 = food => barley [runtime] [[food, barley]] script = true __FLOW_CONFIG__ # shellcheck disable=SC2153 WORKFLOW_NAME2="${WORKFLOW_NAME}" run_fail "${TEST_NAME_BASE}" \ cylc graph "${WORKFLOW_NAME1}" --diff "${WORKFLOW_NAME2}" --icp='20200101T0000Z' contains_ok "${TEST_NAME_BASE}.stdout" <<__OUT__ -edge "20200101T0000Z/foo" "20200101T0000Z/bar" +edge "20200101T0000Z/food" "20200101T0000Z/barley" graph -node "20200101T0000Z/bar" "bar\n20200101T0000Z" -node "20200101T0000Z/foo" "foo\n20200101T0000Z" +node "20200101T0000Z/barley" "barley\n20200101T0000Z" +node "20200101T0000Z/food" "food\n20200101T0000Z" __OUT__ cmp_ok "${TEST_NAME_BASE}.stderr" <'/dev/null' purge "${WORKFLOW_NAME1}" purge "${WORKFLOW_NAME2}" exit cylc-flow-8.7.0/tests/functional/deprecations/0000775000175000017500000000000015264420307021600 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/deprecations/01-cylc8-basic.t0000775000175000017500000000313115264420307024305 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test all current non-silent workflow obsoletions and deprecations. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 #------------------------------------------------------------------------------- install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-val" run_ok "${TEST_NAME}" cylc validate -v "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-cmp" cylc validate "${WORKFLOW_NAME}" 2> 'val.out' cmp_ok val.out "$TEST_SOURCE_DIR/${TEST_NAME_BASE}/validation.stderr" purge cylc-flow-8.7.0/tests/functional/deprecations/01-cylc8-basic/0000775000175000017500000000000015264420307024117 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/deprecations/01-cylc8-basic/flow.cylc0000664000175000017500000000415515264420307025747 0ustar alastairalastair# Test automatic deprecation and deletion of config items as specified # in cylc/flow/cfgspec/workflow.py. [cylc] log resolved dependencies = abort if any task fails = authentication = required run mode = force run mode = task event mail interval = disable automatic shutdown = [[environment]] darmok = [[events]] mail to = mail from = mail smtp = mail footer = timeout = abort on timeout = timeout handler = inactivity = inactivity handler = aborted handler = stalled handler = startup handler = shutdown handler = abort on stalled = abort on inactivity = abort if timeout handler fails = abort if stalled handler fails = abort if inactivity handler fails = abort if startup handler fails = abort if shutdown handler fails = handler events = stalled [[reference test]] allow task failures = live mode suite timeout = dummy mode suite timeout = dummy-local mode suite timeout = simulation mode suite timeout = required run mode = suite shutdown event handler = [[simulation]] disable suite event handlers = [[parameters]] [[parameter templates]] [scheduling] max active cycle points = 2 hold after point = initial cycle point = 20150808T00 final cycle point = 20150808T00 [[dependencies]] [[[P1D]]] graph = foo => cat & dog [runtime] [[foo, cat, dog]] extra log files = [[[job]]] shell = fish execution polling intervals = execution retry delays = execution time limit = submission polling intervals = submission retry delays = [[[events]]] mail from = mail to = mail smtp = mail retry delays = [[[suite state polling]]] interval = PT10S message = "pork scratchings" [[[parameter environment templates]]] FOO = telemachus cylc-flow-8.7.0/tests/functional/deprecations/01-cylc8-basic/validation.stderr0000664000175000017500000001751115264420307027503 0ustar alastairalastairWARNING - Obsolete config items were automatically deleted. Please check your workflow and remove them permanently. WARNING - Deprecated config items were automatically upgraded. Please alter your workflow to use the new syntax. WARNING - * (8.0.0) [cylc]force run mode - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc][authentication] - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc]log resolved dependencies - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc]required run mode - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [runtime][foo, cat, dog][events]mail retry delays - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [runtime][foo, cat, dog]extra log files - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [runtime][foo, cat, dog][job]shell - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc]abort if any task fails - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc]disable automatic shutdown - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc][environment] - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc][reference test] - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc][simulation]disable suite event handlers - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc][simulation] - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc]task event mail interval -> [cylc][mail]task event batch interval - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [runtime][foo, cat, dog][suite state polling] -> [runtime][foo, cat, dog][workflow state polling] - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][parameters] -> [task parameters] - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][parameter templates] -> [task parameters][templates] - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][events]mail to -> [cylc][mail]to - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][events]mail from -> [cylc][mail]from - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][events]mail footer -> [cylc][mail]footer - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [runtime][foo, cat, dog][events]mail to -> [runtime][foo, cat, dog][mail]to - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [runtime][foo, cat, dog][events]mail from -> [runtime][foo, cat, dog][mail]from - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][events]mail smtp - DELETED (OBSOLETE) - use "global.cylc[scheduler][mail]smtp" instead - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [runtime][foo, cat, dog][events]mail smtp - DELETED (OBSOLETE) - use "global.cylc[scheduler][mail]smtp" instead - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [scheduling]max active cycle points -> [scheduling]runahead limit - "2" -> "P1" - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [scheduling]hold after point -> [scheduling]hold after cycle point - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [runtime][foo, cat, dog][job]execution polling intervals -> [runtime][foo, cat, dog]execution polling intervals - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [runtime][foo, cat, dog][job]execution retry delays -> [runtime][foo, cat, dog]execution retry delays - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [runtime][foo, cat, dog][job]execution time limit -> [runtime][foo, cat, dog]execution time limit - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [runtime][foo, cat, dog][job]submission polling intervals -> [runtime][foo, cat, dog]submission polling intervals - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [runtime][foo, cat, dog][job]submission retry delays -> [runtime][foo, cat, dog]submission retry delays - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][events]timeout -> [cylc][events]stall timeout - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][events]abort on timeout -> [cylc][events]abort on stall timeout - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][events]inactivity -> [cylc][events]inactivity timeout - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][events]abort on inactivity -> [cylc][events]abort on inactivity timeout - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][events]startup handler -> [cylc][events]startup handlers - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][events]shutdown handler -> [cylc][events]shutdown handlers - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][events]timeout handler -> [cylc][events]stall timeout handlers - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][events]stalled handler -> [cylc][events]stall handlers - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][events]aborted handler -> [cylc][events]abort handlers - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][events]inactivity handler -> [cylc][events]inactivity timeout handlers - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - * (8.0.0) [cylc][events]abort on stalled - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc][events]abort if startup handler fails - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc][events]abort if shutdown handler fails - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc][events]abort if timeout handler fails - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc][events]abort if inactivity handler fails - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc][events]abort if stalled handler fails - DELETED (OBSOLETE) - this will fail validation in Cylc 8.9 WARNING - * (8.0.0) [cylc] -> [scheduler] - value unchanged - support for the old item will be removed at Cylc 8.9 WARNING - graph items were automatically upgraded in "workflow definition" - support for the old item will be removed at Cylc 8.9: * (8.0.0) [scheduling][dependencies][X]graph -> [scheduling][graph]X - for X in: P1D WARNING - Deprecated config items were automatically upgraded. Please alter your workflow to use the new syntax. WARNING - * (8.0.0) [runtime][foo, cat, dog][parameter environment templates] contents prepended to [runtime][foo, cat, dog][environment] - support for parameter environment templates will be removed at Cylc 8.9 WARNING - Deprecated config items were automatically upgraded. Please alter your workflow to use the new syntax. * (8.0.0) [scheduler][events][handler events] stalled -> stall - support for the old item will be removed at Cylc 8.9 cylc-flow-8.7.0/tests/functional/deprecations/02-overwrite/0000775000175000017500000000000015264420307024045 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/deprecations/02-overwrite/flow.cylc0000664000175000017500000000047315264420307025674 0ustar alastairalastair# Test automatic deprecation and deletion of config items as specified # in lib/cylc/cfgspec/workflow.py. [scheduling] initial cycle point = 20150808T00 # Deprecated: [[dependencies]] [[[P1D]]] graph = bar => horse # New in Cylc 8: [[graph]] P1D = foo => cat & dog cylc-flow-8.7.0/tests/functional/deprecations/02-overwrite.t0000664000175000017500000000267215264420307024241 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that upgraded deprecations don't overwrite existing items . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 1 #------------------------------------------------------------------------------- install_workflow "$TEST_NAME_BASE" "$TEST_NAME_BASE" #------------------------------------------------------------------------------- cylc validate "$WORKFLOW_NAME" 2> val.out grep_ok "UpgradeError" "val.out" #------------------------------------------------------------------------------- purge "$WORKFLOW_NAME" cylc-flow-8.7.0/tests/functional/deprecations/test_header0000777000175000017500000000000015264420307030115 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/test_functions/0000775000175000017500000000000015264420307022167 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/test_functions/00-grep_fail.t0000664000175000017500000000272315264420307024525 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test that grep_fail fails if test is not there: . "$(dirname "$0")/test_header" set_test_number 3 echo "The finger writes, and having writ moves on." > test_file.txt # It passes if file exists and search term not there: grep_fail 'foo' 'test_file.txt' # It fails if the file exists and the search term is there: grep_fail 'writ' 'test_file.txt' | tee "out1" > /dev/null grep_ok 'not ok 2 - test_file.txt-grep-fail' 'out1' # It fails if the file does not exist: grep_fail "writ" "test_file2.txt" | tee "out2" > /dev/null grep_ok 'not ok 3 - test_file2.txt-grep-fail-file to be grepped does not exist.' 'out2' cylc-flow-8.7.0/tests/functional/test_functions/test_header0000777000175000017500000000000015264420307030504 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/authentication/0000775000175000017500000000000015264420307022137 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/authentication/basic/0000775000175000017500000000000015264420307023220 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/authentication/basic/flow.cylc0000664000175000017500000000113615264420307025044 0ustar alastairalastair[meta] title = Authentication test workflow. description = """ Stalls when the first task fails. Here we test out a multi-line description! """ custom_metadata = something_custom another_metadata = 1 [scheduler] [[events]] stall timeout = PT1M abort on stall timeout = True [scheduling] cycling mode = integer initial cycle point = 1 final cycle point = 2 [[graph]] R/1 = foo => bar R/2 = bar[-P1] => pub [runtime] [[foo]] script = false [[bar]] script = true [[pub]] script = true cylc-flow-8.7.0/tests/functional/authentication/00-shared-fs/0000775000175000017500000000000015264420307024230 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/authentication/00-shared-fs/reference.log0000664000175000017500000000020215264420307026663 0ustar alastairalastairInitial point: 19700101T0000Z Final point: 19700101T0000Z 19700101T0000Z/t1 -triggered off [] 19700101T0000Z/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/authentication/00-shared-fs/flow.cylc0000664000175000017500000000052615264420307026056 0ustar alastairalastair#!jinja2 [scheduler] UTC mode=True [[events]] abort on inactivity timeout = True inactivity timeout = P1M expected task failures = 19700101T0000Z/t1 [scheduling] initial cycle point=1970 final cycle point=1970 [[graph]] P1Y=t1 [runtime] [[t1]] script=printenv CYLC_TEST_VAR_FOO cylc-flow-8.7.0/tests/functional/authentication/01-remote-workflow-same-name/0000775000175000017500000000000015264420307027361 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/authentication/01-remote-workflow-same-name/reference.log0000664000175000017500000000013615264420307032022 0ustar alastairalastairInitial point: 19700101T0000Z Final point: 19700101T0000Z 19700101T0000Z/t1 -triggered off [] cylc-flow-8.7.0/tests/functional/authentication/01-remote-workflow-same-name/flow.cylc0000664000175000017500000000036315264420307031206 0ustar alastairalastair#!jinja2 [scheduler] UTC mode = True [scheduling] initial cycle point = 1970 final cycle point = 1970 [[graph]] P1Y = t1 [runtime] [[t1]] script = true platform = {{ environ['CYLC_TEST_PLATFORM'] }} cylc-flow-8.7.0/tests/functional/authentication/01-remote-workflow-same-name.t0000775000175000017500000000375415264420307027562 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test communication from a remote host (non-shared file system) when it has # a workflow with the same name registered, but not running. (Obviously, it will # be very confused if it is running under its ~/cylc-run/WORKFLOW as well.) export REQUIRE_PLATFORM='loc:remote fs:indep' . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" SSH_OPTS='-oBatchMode=yes -oConnectTimeout=5' # shellcheck disable=SC2029,SC2086 #ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" mkdir -p "cylctb-cylc-source/${WORKFLOW_NAME}" SRC_DIR="$(ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" mktemp -d)" # shellcheck disable=SC2086 scp ${SSH_OPTS} -pqr "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/"* \ "${CYLC_TEST_HOST}:${SRC_DIR}" # shellcheck disable=SC2086 run_ok "${TEST_NAME_BASE}-install" \ ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" bash -lc \ "CYLC_VERSION=$(cylc version) cylc install ${SRC_DIR} --workflow-name=${WORKFLOW_NAME} --no-run-name" workflow_run_ok "${TEST_NAME_BASE}" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/authentication/test_header0000777000175000017500000000000015264420307030454 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/authentication/02-workflow2-stop-workflow1.t0000775000175000017500000000445515264420307027426 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test calling "cylc shutdown workflow1" from workflow2. # See https://github.com/cylc/cylc-flow/issues/1843 . "$(dirname "$0")/test_header" set_test_number 1 RUND="$RUN_DIR" NAME1="${CYLC_TEST_REG_BASE}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BASE}-1" NAME2="${CYLC_TEST_REG_BASE}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BASE}-2" WORKFLOW1_RUND="${RUND}/${NAME1}" RND_WORKFLOW_NAME=x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6) RND_WORKFLOW_SOURCE="$PWD/${RND_WORKFLOW_NAME}" mkdir -p "${RND_WORKFLOW_SOURCE}" cp -p "${TEST_SOURCE_DIR}/basic/flow.cylc" "${RND_WORKFLOW_SOURCE}" cylc install "${RND_WORKFLOW_SOURCE}" --workflow-name="${NAME1}" --no-run-name RND_WORKFLOW_NAME2=x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6) RND_WORKFLOW_SOURCE2="$PWD/${RND_WORKFLOW_NAME2}" mkdir -p "${RND_WORKFLOW_SOURCE2}" cat >"${RND_WORKFLOW_SOURCE2}/flow.cylc" <<__FLOW_CONFIG__ [scheduler] [[events]] [scheduling] [[graph]] R1=t1 [runtime] [[t1]] script=cylc shutdown "${NAME1}" __FLOW_CONFIG__ cylc install "${RND_WORKFLOW_SOURCE2}" --workflow-name="${NAME2}" --no-run-name cylc play --no-detach "${NAME1}" 1>'1.out' 2>&1 & WORKFLOW_RUN_DIR="${WORKFLOW1_RUND}" poll_workflow_running run_ok "${TEST_NAME_BASE}" cylc play --no-detach --abort-if-any-task-fails "${NAME2}" cylc shutdown "${NAME1}" --max-polls=20 --interval=1 1>'/dev/null' 2>&1 || true purge "${NAME1}" purge "${NAME2}" rm -rf "${RND_WORKFLOW_SOURCE}" rm -rf "${RND_WORKFLOW_SOURCE2}" exit cylc-flow-8.7.0/tests/functional/authentication/00-shared-fs.t0000775000175000017500000000504115264420307024420 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test sending commands to a workflow on a host with shared file system with # current host. export REQUIRE_PLATFORM='loc:remote fs:shared' . "$(dirname "$0")/test_header" set_test_number 4 # "install_workflow" does not work here because it installs workflows on the TMPDIR, # which is often on local file systems. We need to ensure that the workflow # definition directory is on a shared file system. WORKFLOW_NAME="${CYLC_TEST_REG_BASE}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BASE}" WORKFLOW_RUN_DIR="$RUN_DIR/${WORKFLOW_NAME}" mkdir -p "$(dirname "${WORKFLOW_RUN_DIR}")" cp -r "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}" "${WORKFLOW_RUN_DIR}" cylc install --workflow-name="${WORKFLOW_NAME}" --no-run-name 2>'/dev/null' run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" cylc play --debug --no-detach --reference-test \ --host="${CYLC_TEST_HOST}" "${WORKFLOW_NAME}" 1>'out' 2>&1 & WORKFLOW_PID="$!" # Poll for job to fail WORKFLOW_LOG="${WORKFLOW_RUN_DIR}/log/scheduler/log" # Note: double poll existence of workflow log on workflow host and then localhost to # avoid any issues with unstable mounting of the shared file system. poll ssh -oBatchMode=yes -n "${CYLC_TEST_HOST}" test -e "${WORKFLOW_LOG}" poll_grep_workflow_log -E '19700101T0000Z/t1/01:submitted.* => running' poll_grep_workflow_log -E '19700101T0000Z/t1/01:running.* => failed' run_ok "${TEST_NAME_BASE}-broadcast" \ cylc broadcast -n 't1' -s '[environment]CYLC_TEST_VAR_FOO=foo' "${WORKFLOW_NAME}" run_ok "${TEST_NAME_BASE}-trigger" \ cylc trigger "${WORKFLOW_NAME}//19700101T0000Z/t1" if wait "${WORKFLOW_PID}"; then ok "${TEST_NAME_BASE}-run" else fail "${TEST_NAME_BASE}-run" cat 'out' >&2 fi purge exit cylc-flow-8.7.0/tests/functional/retries/0000775000175000017500000000000015264420307020575 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/retries/02-xtriggers/0000775000175000017500000000000015264420307023032 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/retries/02-xtriggers/reference.log0000664000175000017500000000030415264420307025470 0ustar alastairalastairInitial point: 1 Final point: 1 1/retry -triggered off [] 1/retry -triggered off [] 1/retry -triggered off [] 1/retry -triggered off [] 1/retry -triggered off [] 1/test -triggered off ['1/retry'] cylc-flow-8.7.0/tests/functional/retries/02-xtriggers/flow.cylc0000664000175000017500000000277115264420307024664 0ustar alastairalastair[scheduling] [[graph]] R1 = retry => test [runtime] [[retry]] # capture task info - incl xtriggers pre-script = """ cylc show "${CYLC_WORKFLOW_ID}" \ "${CYLC_TASK_NAME}.${CYLC_TASK_CYCLE_POINT}" \ > "${CYLC_TASK_LOG_ROOT}-show" """ # fail four times then pass script = test "${CYLC_TASK_SUBMIT_NUMBER}" -ge 5 # stagger retries every two seconds [[[job]]] execution retry delays = 5*PT3S [[test]] script = """ cylc cat-log "${CYLC_WORKFLOW_ID}" > log # get a list of the times cylc says tasks will retry after mapfile -t RETRY_TIMES \ < <(sed -n 's/.*retrying.*after \(.*\)).*/\1/p' log) # get a list of the times when the xtriggers actually succeeded mapfile -t XTRIGGER_TIMES \ < <(sed -n 's/\(.*\) INFO.*xtrigger succeeded.*/\1/p' log) test "${#RETRY_TIMES[@]}" -eq 4 test "${#XTRIGGER_TIMES[@]}" -eq 4 # make sure that tasks retried when they said they would for N in $(seq 0 3); do INTERVAL="$(isodatetime \ "${RETRY_TIMES[$N]}" "${XTRIGGER_TIMES[$N]}" --as-total s)" echo "RETRY=${RETRY_TIMES[$N]}" echo "XTRIGGER=${XTRIGGER_TIMES[$N]}" echo "INTERVAL=${INTERVAL}" python3 -c "assert ${INTERVAL} >= 0.0" done """ cylc-flow-8.7.0/tests/functional/retries/submission/0000775000175000017500000000000015264420307022770 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/retries/submission/reference.log0000664000175000017500000000020015264420307025421 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/foo -triggered off [] 1/foo -triggered off [] 1/foo -triggered off [] cylc-flow-8.7.0/tests/functional/retries/submission/flow.cylc0000664000175000017500000000065715264420307024623 0ustar alastairalastair[scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT3M expected task failures = 1/foo [scheduling] [[graph]] R1 = "foo:submit-fail? => !foo" [runtime] [[foo]] script = true platform = nonsense-platform [[[job]]] submission retry delays = PT0S, PT0S, PT0S cylc-flow-8.7.0/tests/functional/retries/01-submission-retry.t0000664000175000017500000000342015264420307024535 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test execution retries are working . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" 'submission' create_test_global_config "" " [platforms] [[nonsense-platform]] hosts = notahost " #------------------------------------------------------------------------------- run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- if ! command -v 'sqlite3' >'/dev/null'; then skip 1 'sqlite3 not installed?' else sqlite3 \ "$RUN_DIR/${WORKFLOW_NAME}/log/db" \ 'SELECT try_num, submit_num FROM task_jobs' >'select.out' cmp_ok 'select.out' <<'__OUT__' 1|1 1|2 1|3 1|4 __OUT__ fi #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/retries/execution/0000775000175000017500000000000015264420307022600 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/retries/execution/reference.log0000664000175000017500000000020015264420307025231 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/foo -triggered off [] 1/foo -triggered off [] 1/foo -triggered off [] cylc-flow-8.7.0/tests/functional/retries/execution/flow.cylc0000664000175000017500000000054015264420307024422 0ustar alastairalastair[scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT3M [scheduling] [[graph]] R1 = foo [runtime] [[foo]] script = test "${CYLC_TASK_TRY_NUMBER}" '-ge' '4' [[[job]]] execution retry delays = 3*PT0S cylc-flow-8.7.0/tests/functional/retries/00-execution-retry.t0000664000175000017500000000314615264420307024351 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test execution retries are working . "$(dirname "$0")/test_header" set_test_number 3 install_workflow "${TEST_NAME_BASE}" 'execution' #------------------------------------------------------------------------------- run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" if command -v 'sqlite3' >'/dev/null'; then sqlite3 \ "$RUN_DIR/${WORKFLOW_NAME}/log/db" \ 'SELECT try_num, submit_num FROM task_jobs' >'select.out' cmp_ok 'select.out' <<'__OUT__' 1|1 2|2 3|3 4|4 __OUT__ else skip 1 "sqlite3 not installed?" fi #------------------------------------------------------------------------------- purge exit cylc-flow-8.7.0/tests/functional/retries/02-xtriggers.t0000664000175000017500000000172115264420307023220 0ustar alastairalastair#!/bin/bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) 2008-2019 Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test kill running jobs only . "$(dirname "$0")/test_header" set_test_number 2 reftest purge exit cylc-flow-8.7.0/tests/functional/retries/03-upgrade/0000775000175000017500000000000015264420307022444 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/retries/03-upgrade/flow.cylc0000664000175000017500000000066515264420307024276 0ustar alastairalastair[scheduler] [[events]] abort on inactivity timeout = True abort on stall timeout = True inactivity timeout = PT1M stall timeout = PT1M [scheduling] [[dependencies]] graph = """ a => b => c """ [runtime] [[b]] # fail four times then pass script = test "$CYLC_TASK_SUBMIT_NUMBER" -ge 3; [[[job]]] execution retry delays = 2*PT2S cylc-flow-8.7.0/tests/functional/retries/test_header0000777000175000017500000000000015264420307027112 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/clock-expire/0000775000175000017500000000000015264420307021505 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/clock-expire/00-basic.t0000664000175000017500000000242215264420307023170 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Validate and run the clock-expire test workflow . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" CPTZ=$(date '+%z') run_ok "${TEST_NAME_BASE}-validate" cylc validate -s "CPTZ='${CPTZ}'" "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --abort-if-any-task-fails -s "CPTZ='${CPTZ}'" "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/clock-expire/00-basic/0000775000175000017500000000000015264420307023003 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/clock-expire/00-basic/flow.cylc0000664000175000017500000000236115264420307024630 0ustar alastairalastair#!Jinja2 [meta] title = task expire example workflow description = """ Skip a daily post-processing workflow if the 'copy' task has expired.""" [scheduler] cycle point format = %Y-%m-%dT%H allow implicit tasks = True cycle point time zone = {{CPTZ}} [[events]] abort on stall timeout = True stall timeout = PT1M [scheduling] initial cycle point = now final cycle point = +P3D [[special tasks]] clock-expire = copy(-P1DT1H) # NOTE this would normally be copy(P1D) i.e. expire if more than 1 day # behind the wall clock, but here we have to start from 'now' in order # to stay near the wall clock, so expire the task if more than 1 day # behind "now + 1 day". This makes the first two 'copy' tasks expire. [[graph]] P1D = """ model[-P1D] => model => copy? => proc copy:expired? => !proc """ [runtime] [[root]] script = true [[copy]] script = """ # Abort if I run in either of the first two cycle points. test "${CYLC_TASK_CYCLE_POINT}" != "${CYLC_WORKFLOW_INITIAL_CYCLE_POINT}" P2D="$(cylc cyclepoint --offset='P1D' "${CYLC_WORKFLOW_INITIAL_CYCLE_POINT}")" test "${CYLC_TASK_CYCLE_POINT}" != "${P2D}" """ cylc-flow-8.7.0/tests/functional/clock-expire/test_header0000777000175000017500000000000015264420307030022 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/run_modes/0000775000175000017500000000000015264420307021113 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/run_modes/06-run-mode-overrides/0000775000175000017500000000000015264420307025064 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/run_modes/06-run-mode-overrides/flow.cylc0000664000175000017500000000132315264420307026706 0ustar alastairalastair#!Jinja2 [scheduler] cycle point format = %Y [scheduling] initial cycle point = 1000 final cycle point = 1001 [[graph]] R1/1000 = default_ & live_ & skip_ => end R1/1001 = end[-P1Y] => broadcaster => default_ [runtime] [[root]] script = echo "=== this task ran in live mode ===" [[[simulation]]] default run length = PT0S [[default_, end]] [[live_]] run mode = live [[skip_]] run mode = skip [[broadcaster]] script = """ cylc broadcast "${CYLC_WORKFLOW_ID}" \ --name default_ \ --point 1001 \ --set='run mode="{{changemode | default("skip")}}"' """ cylc-flow-8.7.0/tests/functional/run_modes/02-dummy-message-outputs/0000775000175000017500000000000015264420307025630 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/run_modes/02-dummy-message-outputs/reference.log0000664000175000017500000000012715264420307030271 0ustar alastairalastairInitial point: 1 Final point: 1 1/foo -triggered off [] 1/bar -triggered off ['1/foo'] cylc-flow-8.7.0/tests/functional/run_modes/02-dummy-message-outputs/flow.cylc0000664000175000017500000000101415264420307027447 0ustar alastairalastair[meta] title = A workflow that should only succeed in dummy mode. [scheduler] [[events]] abort on stall timeout = True stall timeout = PT0S abort on inactivity timeout = True inactivity timeout = PT3M [scheduling] [[graph]] R1 = foo:x => bar [runtime] [[root]] [[[simulation]]] default run length = PT0S time limit buffer = PT1M [[bar]] [[foo]] script = false [[[outputs]]] x = "the quick brown fox" cylc-flow-8.7.0/tests/functional/run_modes/05-sim-trigger.t0000664000175000017500000000321715264420307023756 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test that we can re-trigger a task in sim mode . "$(dirname "$0")/test_header" set_test_number 5 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-start" \ cylc play "${WORKFLOW_NAME}" --mode=simulation SCHD_LOG="${WORKFLOW_RUN_DIR}/log/scheduler/log" # Wait for stall, then check for first task failure: poll_grep_workflow_log 'stall timer starts' grep_ok '\[1/fail_fail_fail/01:running\] => failed' "${SCHD_LOG}" # Trigger task again, and check that it too failed: cylc trigger "${WORKFLOW_NAME}//1/fail_fail_fail" poll_grep_workflow_log -E \ '1/fail_fail_fail/02.* did not complete the required outputs' grep_ok '\[1/fail_fail_fail/02:running\] => failed' "${SCHD_LOG}" run_ok "stop" cylc stop --max-polls=10 --interval=1 "${WORKFLOW_NAME}" purge cylc-flow-8.7.0/tests/functional/run_modes/02-dummy-message-outputs.t0000664000175000017500000000220315264420307026012 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test that message outputs are faked in dummy mode. . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --no-detach --mode=dummy --reference-test "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/run_modes/05-sim-trigger/0000775000175000017500000000000015264420307023566 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/run_modes/05-sim-trigger/reference.log0000664000175000017500000000017015264420307026225 0ustar alastairalastair23590101T0000Z/get_observations -triggered off [] in flow 1 23590101T0000Z/get_observations -triggered off [] in flow 1 cylc-flow-8.7.0/tests/functional/run_modes/05-sim-trigger/flow.cylc0000664000175000017500000000035715264420307025416 0ustar alastairalastair[scheduling] [[graph]] R1 = fail_fail_fail => bar [runtime] [[root, bar]] [[[simulation]]] default run length = PT0S [[fail_fail_fail]] [[[simulation]]] fail cycle points = all cylc-flow-8.7.0/tests/functional/run_modes/04-simulation-runtime.t0000664000175000017500000000461615264420307025375 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test that we can broadcast an alteration to simulation mode. . "$(dirname "$0")/test_header" skip_macos_gh_actions set_test_number 7 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-start" \ cylc play "${WORKFLOW_NAME}" --mode=simulation --pause SCHD_LOG="${WORKFLOW_RUN_DIR}/log/scheduler/log" # This broadcast will ensure that second_task will not finish # in time for the workflow timeout, unless the broadcast # cancelletion works correctly: run_ok "second-task-broadcast-too-long" \ cylc broadcast "${WORKFLOW_NAME}" \ -n second_task \ -s 'execution time limit = P1D' poll_grep 'Broadcast set:' "${SCHD_LOG}" # Test cancelling broadcast changes the task config back. run_ok "cancel-second-task-broadcast" \ cylc broadcast "${WORKFLOW_NAME}" \ -n second_task\ --clear poll_grep 'Broadcast cancelled:' "${SCHD_LOG}" # If we speed up the simulated first_task task we # can make it finish before workflow timeout # (neither change will do this on its own): run_ok "first-task-speed-up-broadcast" \ cylc broadcast "${WORKFLOW_NAME}" \ -n first_task \ -s '[simulation]speedup factor = 60' \ -s 'execution time limit = PT60S' workflow_run_ok "${TEST_NAME_BASE}-unpause" \ cylc play "${WORKFLOW_NAME}" # Wait for the workflow to finish (it wasn't run in no-detach mode): poll_workflow_stopped # If we hadn't changed the speedup factor using broadcast # The workflow timeout would have been hit: grep_fail "WARNING - Orphaned tasks" "${SCHD_LOG}" purge cylc-flow-8.7.0/tests/functional/run_modes/01-dummy/0000775000175000017500000000000015264420307022464 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/run_modes/01-dummy/reference.log0000664000175000017500000000015415264420307025125 0ustar alastairalastairInitial point: 1 Final point: 1 1/a -triggered off [] 1/b -triggered off ['1/a'] 1/c -triggered off ['1/b'] cylc-flow-8.7.0/tests/functional/run_modes/01-dummy/flow.cylc0000664000175000017500000000043315264420307024307 0ustar alastairalastair[scheduler] [[events]] expected task failures = 1/a, 1/b, 1/c [scheduling] [[graph]] R1 = """a:fail? => b? b:fail? => c? c:fail? => !a & !b & !c""" [runtime] [[a, b, c]] [[[simulation]]] fail cycle points = all cylc-flow-8.7.0/tests/functional/run_modes/01-dummy.t0000664000175000017500000000174115264420307022654 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test dummy mode . "$(dirname "$0")/test_header" set_test_number 2 export REFTEST_OPTS="--mode=dummy" reftest exit cylc-flow-8.7.0/tests/functional/run_modes/06-run-mode-overrides.t0000664000175000017500000000342015264420307025250 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Testing Skip mode functionality. . "$(dirname "$0")/test_header" set_test_number 6 # Install and run the workflow in live mode (default). # Check that tasks with run mode unset and run mode = live # leave log files, and that skip mode tasks don't. install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-play" \ cylc play "${WORKFLOW_NAME}" \ --no-detach JOB_LOGS="${WORKFLOW_RUN_DIR}/log/job/1000" run_fail "${TEST_NAME_BASE}-config run mode=skip" ls "${JOB_LOGS}/skip_" for MODE in default live; do named_grep_ok \ "${TEST_NAME_BASE}-mode=${MODE}" \ "===.*===" \ "${JOB_LOGS}/${MODE}_/NN/job.out" done # After broadcasting a change in run_mode to task default_ it now runs # in skip mode and fails to produce a log file: JOB_LOGS="${WORKFLOW_RUN_DIR}/log/job/1001" run_fail "${TEST_NAME_BASE}-broadcast-run-mode=skip" ls "${JOB_LOGS}/default_/" purge exit 0 cylc-flow-8.7.0/tests/functional/run_modes/test_header0000777000175000017500000000000015264420307027430 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/run_modes/03-simulation/0000775000175000017500000000000015264420307023517 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/run_modes/03-simulation/reference.log0000664000175000017500000000017015264420307026156 0ustar alastairalastair23590101T0000Z/get_observations -triggered off [] in flow 1 23590101T0000Z/get_observations -triggered off [] in flow 1 cylc-flow-8.7.0/tests/functional/run_modes/03-simulation/flow.cylc0000664000175000017500000000050015264420307025335 0ustar alastairalastair[scheduler] [[events]] workflow timeout = PT30S [scheduling] initial cycle point = 2359 [[graph]] R1 = get_observations [runtime] [[get_observations]] execution retry delays = PT2S [[[simulation]]] fail cycle points = all fail try 1 only = True cylc-flow-8.7.0/tests/functional/run_modes/04-simulation-runtime/0000775000175000017500000000000015264420307025201 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/run_modes/04-simulation-runtime/reference.log0000664000175000017500000000017015264420307027640 0ustar alastairalastair23590101T0000Z/get_observations -triggered off [] in flow 1 23590101T0000Z/get_observations -triggered off [] in flow 1 cylc-flow-8.7.0/tests/functional/run_modes/04-simulation-runtime/flow.cylc0000664000175000017500000000065615264420307027033 0ustar alastairalastair[scheduler] [[events]] workflow timeout = PT30S [scheduling] initial cycle point = 2359 [[graph]] R1 = first_task & second_task [runtime] [[first_task]] execution time limit = P2D execution retry delays = PT10M [[[simulation]]] speedup factor = 1 [[second_task]] execution time limit = PT1S [[[simulation]]] speedup factor = 1 cylc-flow-8.7.0/tests/functional/run_modes/03-simulation.t0000664000175000017500000000234615264420307023711 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Test that simulation mode runs, and reruns a failed task successfully # when execution retry delays is configured. . "$(dirname "$0")/test_header" set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play \ --no-detach \ --mode=simulation \ --reference-test "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/cylc-message/0000775000175000017500000000000015264420307021474 5ustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-message/00-ssh.t0000775000175000017500000000261115264420307022676 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- # Test "cylc message" in SSH mode, test needs to have compatible version # installed on the remote host. export REQUIRE_PLATFORM='loc:remote comms:ssh' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 2 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${WORKFLOW_NAME}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach --reference-test "${WORKFLOW_NAME}" purge exit cylc-flow-8.7.0/tests/functional/cylc-message/test_header0000777000175000017500000000000015264420307030011 2../lib/bash/test_headerustar alastairalastaircylc-flow-8.7.0/tests/functional/cylc-message/01-newline.t0000775000175000017500000000406715264420307023552 0ustar alastairalastair#!/usr/bin/env bash # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------ # Test "cylc message" with multi-line messages. The RE to strip 'at => `). ## __cylc-8.1.0 (Released 2023-01-16)__ ### Breaking Changes * Workflows started with Cylc 8.0 which contain multiple "flows" cannot be restarted with Cylc 8.1 due to database changes. ### Enhancements [#5229](https://github.com/cylc/cylc-flow/pull/5229) - - Added a single command to validate a previously run workflow against changes to its source and reinstall a workflow. - Allows Cylc commands (including validate, list, view, config, and graph) to load template variables configured by `cylc install` and `cylc play`. [#5121](https://github.com/cylc/cylc-flow/pull/5121) - Added a single command to validate, install and play a workflow. [#5184](https://github.com/cylc/cylc-flow/pull/5184) - Scan for active runs of the same workflow at install time. [#5084](https://github.com/cylc/cylc-flow/pull/5084) - Assign the most recent previous flow numbers to tasks triggered when no flows are present (e.g. on restarting a finished workflow). [#5032](https://github.com/cylc/cylc-flow/pull/5032) - Set a default limit of 100 for the "default" queue. [#5055](https://github.com/cylc/cylc-flow/pull/5055) and [#5086](https://github.com/cylc/cylc-flow/pull/5086) - Upgrades to `cylc lint` - Allow users to ignore Cylc Lint issues using `--ignore `. - Allow settings for `cylc lint` to be recorded in a pyproject.toml file. - Allow files to be excluded from `cylc lint` checks. [#5081](https://github.com/cylc/cylc-flow/pull/5081) - Reduced amount that gets logged at "INFO" level in scheduler logs. [#5259](https://github.com/cylc/cylc-flow/pull/5259) - Add flow_nums to task_jobs table in the workflow database. ### Fixes [#5286](https://github.com/cylc/cylc-flow/pull/5286) - Fix bug where `[scheduling][special tasks]clock-trigger` would skip execution retry delays. [#5292](https://github.com/cylc/cylc-flow/pull/5292) - Fix an issue where polling could be repeated if the job's platform was not available. ## __cylc-8.0.4 (Released 2022-12-14)__ Maintenance release. ### Fixes [##5205](https://github.com/cylc/cylc-flow/pull/#5205) - Fix bug which caused orphaned running tasks to silently skip remote file installation at scheduler restart. [#5224](https://github.com/cylc/cylc-flow/pull/5225) - workflow installation: disallow reserved names only in the top level source directory. [#5211](https://github.com/cylc/cylc-flow/pull/5211) - Provide better explanation of failure if `icp = next (T-02, T-32)` when list should be semicolon separated. [#5196](https://github.com/cylc/cylc-flow/pull/5196) - Replace traceback with warning, for scan errors where workflow is stopped. [#5199](https://github.com/cylc/cylc-flow/pull/5199) - Fix a problem with the consolidation tutorial. [#5195](https://github.com/cylc/cylc-flow/pull/5195) - Fix issue where workflows can fail to shutdown due to unavailable remote platforms and make job log retrieval more robust. ## __cylc-8.0.3 (Released 2022-10-17)__ Maintenance release. ### Fixes [#5192](https://github.com/cylc/cylc-flow/pull/5192) - Recompute runahead limit after use of `cylc remove`. [#5188](https://github.com/cylc/cylc-flow/pull/5188) - Fix task state selectors in `cylc trigger` and other commands. [#5125](https://github.com/cylc/cylc-flow/pull/5125) - Allow rose-suite.conf changes to be considered by ``cylc reinstall``. [#5023](https://github.com/cylc/cylc-flow/pull/5023), [#5187](https://github.com/cylc/cylc-flow/pull/5187) - tasks force-triggered after a shutdown was ordered should submit to run immediately on restart. [#5137](https://github.com/cylc/cylc-flow/pull/5137) - Install the `ana/` directory to remote platforms by default. [#5146](https://github.com/cylc/cylc-flow/pull/5146) - no-flow tasks should not retrigger incomplete children. [#5104](https://github.com/cylc/cylc-flow/pull/5104) - Fix retriggering of failed tasks after a reload. [#5139](https://github.com/cylc/cylc-flow/pull/5139) - Fix bug where `cylc install` could hang if there was a large uncommitted diff in the source dir (for git/svn repos). [#5131](https://github.com/cylc/cylc-flow/pull/5131) - Infer workflow run number for `workflow_state` xtrigger. ## __cylc-8.0.2 (Released 2022-09-12)__ Maintenance release. ### Fixes [#5115](https://github.com/cylc/cylc-flow/pull/5115) - Updates rsync commands to make them compatible with latest rsync releases. [#5119](https://github.com/cylc/cylc-flow/pull/5119) - Fix formatting of deprecation warnings at validation. [#5067](https://github.com/cylc/cylc-flow/pull/5067) - Datastore fix for taskdefs removed before restart. [#5066](https://github.com/cylc/cylc-flow/pull/5066) - Fix bug where .cylcignore only found if `cylc install` is run in source directory. [#5091](https://github.com/cylc/cylc-flow/pull/5091) - Fix problems with tutorial workflows. [#5098](https://github.com/cylc/cylc-flow/pull/5098) - Fix bug where final task status updates were not being sent to UI before shutdown. [#5114](https://github.com/cylc/cylc-flow/pull/5114) - Fix bug where validation errors during workflow startup were not printed to stderr before daemonisation. [#5110](https://github.com/cylc/cylc-flow/pull/5110) - Fix bug where reloading a stalled workflow would cause it stall again. ## __cylc-8.0.1 (Released 2022-08-16)__ Maintenance release. ### Fixes [#5025](https://github.com/cylc/cylc-flow/pull/5025) - Fix a bug where polling causes a failed task to be shown as submitted when the workflow is reloaded. [#5045](https://github.com/cylc/cylc-flow/pull/5045) - Fix issue where unsatisfied xtriggers could be wiped on reload. [#5031](https://github.com/cylc/cylc-flow/pull/5031) - Fix bug where specifying multiple datetime offsets (e.g. `final cycle point = +P1M-P1D`) would not obey the given order. [#5033](https://github.com/cylc/cylc-flow/pull/5033) - Running `cylc clean` on a top level dir containing run dir(s) will now remove that top level dir in addition to the run(s) (if there is nothing else inside it). [#5007](https://github.com/cylc/cylc-flow/pull/5007) - Fix for `cylc broadcast` cycle point validation in the UI. [#5037](https://github.com/cylc/cylc-flow/pull/5037) - Fix bug where the workflow restart number would get wiped on reload. [#5049](https://github.com/cylc/cylc-flow/pull/5049) - Fix several small bugs related to auto restart. [#5062](https://github.com/cylc/cylc-flow/pull/5062) - Fix bug where preparing tasks could sometimes get orphaned when an auto restart occurred. ## __cylc-8.0.0 (Released 2022-07-28)__ Cylc 8 production-ready release. ### Major Changes * Python 2 -> 3. * Internal communications converted from HTTPS to ZMQ (TCP). * PyGTK GUIs replaced by: * Terminal user interface (TUI) included in cylc-flow. * Web user interface provided by the cylc-uiserver package. * A new scheduling algorithm with support for branched workflows. * Command line changes: * `cylc run` -> `cylc play` * `cylc restart` -> `cylc play` * `rose suite-run` -> `cylc install; cylc play ` * The core package containing Cylc scheduler program has been renamed cylc-flow. * Cylc review has been removed, the Cylc 7 version remains Cylc 8 compatible. * [New documentation](https://cylc.github.io/cylc-doc/stable). See the [migration guide](https://cylc.github.io/cylc-doc/stable/html/7-to-8/index.html) for a full list of changes. ### Enhancements [#4964](https://github.com/cylc/cylc-flow/pull/4964) - `cylc reinstall` now displays the changes it would make when run interactively and has improved help / documentation. [#4836](https://github.com/cylc/cylc-flow/pull/4836) - The log directory has been tidied. Workflow logs are now found in `log/scheduler` rather than `log/workflow`, filenames now include `start`/`restart`. Other minor directory changes. Remote file installation logs are now per install target. [#4938](https://github.com/cylc/cylc-flow/pull/4938) - Detect bad Platforms config: background and at job runners should have a single host. [#4877](https://github.com/cylc/cylc-flow/pull/4877) - Upgrade the version of Jinja2 used by Cylc from 2.11 to 3.0. [#4896](https://github.com/cylc/cylc-flow/pull/4896) - Allow the setting of default job runner directives for platforms. [#4900](https://github.com/cylc/cylc-flow/pull/4900) - Added a command to assist with upgrading Cylc 7 workflows to Cylc 8: Try `cylc lint `. [#5009](https://github.com/cylc/cylc-flow/pull/5009) - Added new job environment variable `$CYLC_WORKFLOW_NAME_BASE` as the basename of `$CYLC_WORKFLOW_NAME`. [#4993](https://github.com/cylc/cylc-flow/pull/4993) - Remove the few remaining uses of a configured text editor (via `cylc view` and `cylc cat-log` options). The primary uses of it (`cylc trigger --edit` and `cylc edit` in Cylc 7) have already been removed from Cylc 8. ### Fixes [#5011](https://github.com/cylc/cylc-flow/pull/5011) - Removes preparing jobs appearing in UI, and reuse submit number on restart for preparing tasks. [#5008](https://github.com/cylc/cylc-flow/pull/5008) - Autospawn absolute-triggered tasks exactly the same way as parentless tasks. [#4984](https://github.com/cylc/cylc-flow/pull/4984) - Fixes an issue with `cylc reload` which could cause preparing tasks to become stuck. [#4976](https://github.com/cylc/cylc-flow/pull/4976) - Fix bug causing tasks to be stuck in UI due to discontinued graph of optional outputs. [#4975](https://github.com/cylc/cylc-flow/pull/4975) - Fix selection of platforms from `[job]` and `[remote]` configs. [#4948](https://github.com/cylc/cylc-flow/pull/4948) - Fix lack of errors/warnings for deprecated `[runtime][][remote]retrieve job logs *` settings. [#4970](https://github.com/cylc/cylc-flow/pull/4970) - Fix handling of suicide triggers in back-compat mode. [#4887](https://github.com/cylc/cylc-flow/pull/4887) - Disallow relative paths in `global.cylc[install]source dirs`. [#4906](https://github.com/cylc/cylc-flow/pull/4906) - Fix delayed spawning of parentless tasks that do have parents in a previous cycle point. - Make integer-interval runahead limits consistent with time-interval limits: `P0` means just the runahead base point; `P1` the base point and the point (i.e. one cycle interval), and so on. [#4936](https://github.com/cylc/cylc-flow/pull/4936) - Fix incorrect error messages when workflow CLI commands fail. [#4941](https://github.com/cylc/cylc-flow/pull/4941) - Fix job state for platform submit-failures. [#4931](https://github.com/cylc/cylc-flow/pull/4931) - Fix cylc install for installing workflows from multi-level directories. [#4926](https://github.com/cylc/cylc-flow/pull/4926) - Fix a docstring formatting problem presenting in the UI mutation flow argument info. [#4891](https://github.com/cylc/cylc-flow/pull/4891) - Fix bug that could cause past jobs to be omitted in the UI. [#4860](https://github.com/cylc/cylc-flow/pull/4860) - Workflow validation now fails if [owner setting](https://cylc.github.io/cylc-doc/stable/html/reference/config/workflow.html#flow.cylc[runtime][%3Cnamespace%3E][remote]owner) is used, as that setting no longer has any effect. [#4978](https://github.com/cylc/cylc-flow/pull/4978) - `cylc clean`: fix occasional failure to clean on remote hosts due to leftover contact file. [#4889](https://github.com/cylc/cylc-flow/pull/4889) - `cylc clean`: don't prompt if no matching workflows. [#4890](https://github.com/cylc/cylc-flow/pull/4890) - `cylc install`: don't overwrite symlink dir targets if they were not cleaned properly before. [#4881](https://github.com/cylc/cylc-flow/pull/4881) - Fix bug where commands targeting a specific cycle point would not work if using an abbreviated cycle point format. ## Older Releases * [Cylc 7 changelog](https://github.com/cylc/cylc-flow/blob/7.8.x/CHANGES.md) * [Cylc 8 pre-release changelog](https://github.com/cylc/cylc-flow/blob/8.0.0/CHANGES.md) cylc-flow-8.7.0/pytest.ini0000664000175000017500000000263215264420307015650 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . [pytest] addopts = --verbose --doctest-modules # default to running tests in one process -n=1 # group tests by module or class --dist=loadscope # ignore files which cause issues with test collection --ignore=cylc/flow/data_messages_pb2.py --ignore=cylc/flow/parsec/example # disable pytest-tornasync because it conflicts with pytest-asyncio's auto mode -p no:tornado -m "not linkcheck" verbosity_assertions = 2 testpaths = cylc/flow/ tests/unit/ tests/integration/ tests/conftest.py doctest_optionflags = NORMALIZE_WHITESPACE ELLIPSIS asyncio_mode = auto markers= linkcheck: Test links cylc-flow-8.7.0/.mailmap0000664000175000017500000001452115264420307015240 0ustar alastairalastair# FORMAT: # Omit commit-name or commit-email if same as proper Alex Reinecke P. A. Reinecke Alexander Paulsell AlexanderPaulsell Ben Fitzpatrick benfitzpatrick Bruno Kinoshita Bruno Kinoshita Bruno Kinoshita David Matthews dpmatthews David Sutherland David Sutherland sutherlander Declan Valters Declan Valters Hilary Oliver Hilary Oliver Hilary Oliver Hilary Oliver Hilary Oliver Hilary Oliver Hilary James Oliver Ivor Blockley Jonathan Thomas Jonathan Thomas Jonny Williams williamsjh Kerry Day Kerry Day kaday Lois Huggett lhuggett Luis Kornblueh Martin Dix MartinDix Martin Ryan Martin Ryan Matt Shin Matt Shin Matt Shin matthew.shin Matt Shin matthewrmshin Matt Shin matthewrmshin Mel Hall <37735232+datamel@users.noreply.github.com> Mike Taves Oliver Sanders Oliver sanders Prasanna Challuri challurip Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> Rosalyn Hatcher RosalynHatcher Sadie Bartholomew <30274190+sadielbartholomew@users.noreply.github.com> Sadie Bartholomew Sadie L. Bartholomew <30274190+sadielbartholomew@users.noreply.github.com> Sadie Bartholomew sadielbartholomew Thomas Coleman ColemanTom Thomas Coleman Tom Coleman Thomas Coleman ColemanTom <15375218+ColemanTom@users.noreply.github.com> Thomas Coleman Tom Coleman <15375218+ColemanTom@users.noreply.github.com> Tim Pillinger wxtim Tim Pillinger <26465611+wxtim@users.noreply.github.com> Tim Whitcomb Tim Whitcomb trwhitcomb Tomek Trzeciak Tomek Trzeciak TomekTrzeciak Utheri Wagura Utheri Wagura <36386988+uwagura@users.noreply.github.com> github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> github-actions[bot] GitHub Action Diquan Jabbour <165976689+Diquan-BOM@users.noreply.github.com> Maxime Rio Christopher Bennett christopher.bennett Samuel Denton samuel-denton Scott Owen James Scott-Owen-James James Frost cylc-flow-8.7.0/.codecov.yml0000664000175000017500000000144315264420307016041 0ustar alastairalastair# Codecov settings # After modifying this file, it might be worth to validate it with: # `curl --data-binary @.codecov.yml https://codecov.io/validate` codecov: notify: require_ci_to_pass: yes # define the colour bar limits here coverage: precision: 2 round: down range: "75...100" # diff type status: project: default: # commits below this threshold will be marked as failed target: '92%' # how much we allow the coverage to drop threshold: '2%' patch: default: target: '97%' threshold: '5%' # files to ignore ignore: - "tests/**" - "ws_messages_pb2.py" - "cylc/flow/network/graphql_subscribe.py" flag_management: default_rules: carryforward: true # turn off comments to pull requests comment: false cylc-flow-8.7.0/changes.d/0000775000175000017500000000000015264420307015446 5ustar alastairalastaircylc-flow-8.7.0/changes.d/changelog-template.jinja0000664000175000017500000000046115264420307022224 0ustar alastairalastair{% if sections[""] %} {% for category, val in definitions.items() if category in sections[""] %} ### {{ definitions[category]['name'] }} {% for text, pulls in sections[""][category].items() %} {{ pulls|join(', ') }} - {{ text }} {% endfor %} {% endfor %} {% else %} No significant changes. {% endif %} cylc-flow-8.7.0/tox.ini0000664000175000017500000000216015264420307015126 0ustar alastairalastair[flake8] extend-select= ; Use `raise from` inside `except` for error handling ; https://docs.python.org/3/tutorial/errors.html#exception-chaining B904 ; Use Counter() instead of defaultdict(int) to avoid excessive memory use: B910 ignore= ; module level import not at top of file E402, ; line break before binary operator W503, ; line break after binary operator W504 ; "experimental" SIM9xx rules (flake8-simplify) SIM9 ; explicitly concatenated strings (flake8-implicit-str-concat) ISC003 ; not all exception subclass __init__ args passed to super __init__ ; see https://github.com/cylc/cylc-flow/issues/7059 B042 per-file-ignores= ; TYPE_CHECKING block suggestions ; B904 needs explicitly excepting because it's added in extend-select tests/*: A, B, B904, C, E731, TC, ISC, N, SIM ; for clarity we don't merge 'with Conf():' context trees tests/unit/parsec/*: B, B904, C, SIM, TC exclude= build, dist, .git, __pycache__, .tox, **graphql_subscribe.py, **data_messages_pb2.py paths = ./cylc/flow ./tests cylc-flow-8.7.0/.codacy.yml0000664000175000017500000000010515264420307015653 0ustar alastairalastairexclude_paths: - 'etc/**' - 'tests/**' - 'cylc/flow/**_pb2.py' cylc-flow-8.7.0/doc/0000775000175000017500000000000015264420307014361 5ustar alastairalastaircylc-flow-8.7.0/doc/README.md0000664000175000017500000000010615264420307015635 0ustar alastairalastairThe Cylc User Guide has been moved to the `cylc/cylc-doc` repository. cylc-flow-8.7.0/.coveragerc0000664000175000017500000000221115264420307015731 0ustar alastairalastair# This is the Coverage.py configuration file. This is used by CI when running # the tests and collecting coverage [run] branch = True cover_pylib = False concurrency = thread, multiprocessing data_file = .coverage disable_warnings = trace-changed module-not-python module-not-imported no-data-collected module-not-measured omit = tests/* */cylc/flow/*_pb2.py cylc/flow/etc/* parallel = True source = ./cylc # https://github.com/coveragepy/coveragepy/issues/2082: core = ctrace [report] exclude_lines = pragma: no cover # Don't complain if tests don't hit defensive assertion code: raise NotImplementedError return NotImplemented # Ignore type checking code: if (typing\.)?TYPE_CHECKING: @overload( |$) # Don't complain about ellipsis (exception classes, typing overloads etc): \.\.\. # Ignore abstract methods @(abc\.)?abstractmethod fail_under=0 ignore_errors = False omit = tests/* */cylc/flow/*_pb2.py cylc/flow/etc/* precision = 2 show_missing = False skip_covered = False sort = Name [html] directory = htmlcov [xml] output = coverage.xml package_depth = 99 cylc-flow-8.7.0/MANIFEST.in0000664000175000017500000000007615264420307015355 0ustar alastairalastairinclude cylc/flow/py.typed recursive-include cylc/flow/etc/ * cylc-flow-8.7.0/README.md0000664000175000017500000001165715264420307015105 0ustar alastairalastair

Cylc (pronounced silk) is a general purpose workflow engine that also manages cycling systems very efficiently. It is used in production weather, climate, and environmental forecasting on HPC, but is not specialized to those domains. ### Quick Start [Installation](https://cylc.github.io/cylc-doc/stable/html/installation.html) | [Documentation](https://cylc.github.io/cylc-doc/stable/html/index.html) ```bash # install cylc conda install cylc-flow # extract an example to run cylc get-resources examples/2-integer-cycling # install and run it cylc vip integer-cycling # vip = validate, install and play # watch it run cylc tui integer-cycling ``` ### The Cylc Ecosystem - [cylc-flow](https://github.com/cylc/cylc-flow) - The core Cylc Scheduler for defining and running workflows. - [cylc-uiserver](https://github.com/cylc/cylc-uiserver) - The web-based Cylc graphical user interface for monitoring and controlling workflows. - [cylc-rose](https://github.com/cylc/cylc-rose) - Provides integration with [Rose](http://metomi.github.io/rose/). ### Migrating From Cylc 7 [Migration Guide](https://cylc.github.io/cylc-doc/stable/html/7-to-8/index.html) | [Migration Support](https://cylc.discourse.group/c/cylc/7-to-8/13) Quick summary of major changes: * Python 2 -> 3. * Internal communications converted from HTTPS to ZMQ (TCP). * PyGTK GUIs replaced by: * Terminal user interface (TUI) included in cylc-flow. * Web user interface provided by the cylc-uiserver package. * A new scheduling algorithm with support for branched workflows. * Command line changes: * `cylc run ` -> `cylc play ` * `cylc restart ` -> `cylc play ` * `rose suite-run` -> `cylc install; cylc play ` * The core package containing Cylc scheduler program has been renamed cylc-flow. * Cylc Review has been moved to [cylc-uiserver](https://github.com/cylc/cylc-uiserver). ### Citations & Publications [![DOI](https://zenodo.org/badge/1836229.svg)](https://zenodo.org/badge/latestdoi/1836229) [![JOSS](http://joss.theoj.org/papers/10.21105/joss.00737/status.svg)](https://doi.org/10.21105/joss.00737) [![CISE](https://img.shields.io/website/https/ieeexplore.ieee.org/document/8675433.svg?color=orange&label=CISE&up_message=10.1109%2FMCSE.2019.2906593)](https://ieeexplore.ieee.org/document/8675433) ### Copyright and Terms of Use [![License](https://img.shields.io/github/license/cylc/cylc-flow.svg?color=lightgrey)](https://github.com/cylc/cylc-flow/blob/master/COPYING) Copyright (C) 2008-2026 Earth Sciences New Zealand & British Crown (Met Office) & Contributors. Cylc is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. Cylc is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with Cylc. If not, see [GNU licenses](http://www.gnu.org/licenses/). ### Contributing [![Contributors](https://img.shields.io/github/contributors/cylc/cylc-flow.svg?color=9cf)](https://github.com/cylc/cylc-flow/graphs/contributors) [![Commit activity](https://img.shields.io/github/commit-activity/m/cylc/cylc-flow.svg?color=yellowgreen)](https://github.com/cylc/cylc-flow/commits/master) [![Last commit](https://img.shields.io/github/last-commit/cylc/cylc-flow.svg?color=ff69b4)](https://github.com/cylc/cylc-flow/commits/master) Contributions welcome: * Read the [contributing](https://github.com/cylc/cylc-flow/blob/master/CONTRIBUTING.md) page. * Development setup instructions are in the [developer docs](https://cylc.github.io/cylc-admin/#cylc-8-developer-docs). * Involved change proposals can be found in the [admin pages](https://cylc.github.io/cylc-admin/#change-proposals). * Touch base in the [developers chat](https://matrix.to/#/#cylc-general:matrix.org). This repository contains some code that was generated by GitHub Copilot. cylc-flow-8.7.0/cylc/0000775000175000017500000000000015264420307014546 5ustar alastairalastaircylc-flow-8.7.0/cylc/flow/0000775000175000017500000000000015264420307015515 5ustar alastairalastaircylc-flow-8.7.0/cylc/flow/exceptions.py0000664000175000017500000004055715264420307020263 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Exceptions for "expected" errors.""" from textwrap import ( indent, wrap, ) from typing import ( TYPE_CHECKING, Dict, Optional, Sequence, Union, ) from cylc.flow import __version__ as CYLC_VERSION from cylc.flow.util import format_cmd if TYPE_CHECKING: from cylc.flow.subprocctx import SubFuncContext class CylcError(Exception): """Generic exception for Cylc errors. This exception is raised in-place of "expected" errors where a short message to the user is more appropriate than traceback. CLI commands will catch this exception and exit with str(exception). """ class PluginError(CylcError): """Represents an error arising from a Cylc plugin. Args: entry_point: The plugin entry point as defined in pyproject.toml (e.g. 'cylc.main_loop') plugin_name: Name of the plugin exc: Original exception caught when trying to run the plugin """ def __init__(self, entry_point: str, plugin_name: str, exc: Exception): self.entry_point = entry_point self.plugin_name = plugin_name self.exc = exc def __str__(self) -> str: return ( f"Error in plugin {self.entry_point}.{self.plugin_name}\n" f"{type(self.exc).__name__}: {self.exc}" ) class InputError(CylcError): """Exception covering erroneous user input to a Cylc interface. Ideally this would be handled in the interface (e.g. argument parser). If this isn't possible raise InputError. """ class CylcConfigError(CylcError): """Generic exception to handle an error in a Cylc configuration file.""" # TODO: reference the configuration element causing the problem class WorkflowConfigError(CylcConfigError): """Exception for configuration errors in a Cylc workflow configuration.""" class GlobalConfigError(CylcConfigError): """Exception for configuration errors in a Cylc global configuration.""" class GraphParseError(WorkflowConfigError): """Exception for errors in Cylc workflow graphing.""" class TriggerExpressionError(GraphParseError): """Trigger expression syntax issue.""" class TaskProxySequenceBoundsError(CylcError): """Error on TaskProxy.__init__ with out of sequence bounds start point.""" def __init__(self, msg): CylcError.__init__( self, 'Not loading %s (out of sequence bounds)' % msg) class ParamExpandError(WorkflowConfigError): """Exception for errors in Cylc parameter expansion.""" class WorkflowEventError(CylcError): """Exception for errors in Cylc event handlers.""" class CommandFailedError(CylcError): """Exception for when scheduler commands fail.""" def __init__(self, value: Union[str, Exception]): self.value = value def __str__(self) -> str: if isinstance(self.value, Exception): return f"{type(self.value).__name__}: {self.value}" return self.value class ServiceFileError(CylcError): """Exception for errors related to workflow service files.""" class WorkflowFilesError(CylcError): """Exception for errors related to workflow files/directories.""" bullet = "\n -" class SchedulerAlive(CylcError): """Workflow contact file exists and scheduler is alive.""" class FileRemovalError(CylcError): """Exception for errors during deletion of files/directories, which are probably the filesystem's fault, not Cylc's.""" def __init__(self, exc: Exception) -> None: CylcError.__init__( self, f"{exc}. This is probably a temporary issue with the filesystem, " "not a problem with Cylc." ) class PlatformError(CylcError): """Error in the management of a remote platform. If the exception represents a command failure, provide either the ctx OR manually populate the remaining kwargs. Otherwise leave the kwargs blank. Args: message: Short description. platform_name: Name of the platform. ctx: SubFuncContext object if available. The other kwargs are derived from this. cmd: The remote command. ret_code: The command's return code. out: The command's stdout. err: The command's stderr. """ MSG_INIT = "initialisation did not complete" MSG_SELECT = "host selection failed" MSG_TIDY = "clean up did not complete" def __init__( self, message: str, platform_name: str | None, *, ctx: 'SubFuncContext | None' = None, cmd: str | Sequence[str] | None = None, ret_code: int | None = None, out: str | None = None, err: str | None = None ) -> None: self.msg = message self.platform_name = platform_name if ctx: self.cmd = ctx.cmd self.ret_code = ctx.ret_code self.out = ctx.out self.err = ctx.err else: self.cmd = cmd self.ret_code = ret_code self.out = out self.err = err # convert the cmd object to a str if needed if self.cmd and not isinstance(self.cmd, str): self.cmd = format_cmd(self.cmd) def __str__(self): # matches cylc.flow.platforms.log_platform_event format if self.platform_name: ret = f'platform: {self.platform_name} - {self.msg}' else: ret = f'{self.msg}' for label, item in [ ('COMMAND', self.cmd), ('RETURN CODE', self.ret_code), ('STDOUT', self.out), ('STDERR', self.err) ]: if item is not None: ret += f'\n{label}:' for line in str(item).splitlines(True): # keep newline chars ret += f"\n {line}" return ret class TaskDefError(WorkflowConfigError): """Exception raise for errors in TaskDef initialization.""" class XtriggerConfigError(WorkflowConfigError): """An error in an xtrigger. For example: * If the function module was not found. * If the function was not found in the xtrigger module. * If the function is not callable. * If any string template in the function context arguments are not present in the expected template values. """ def __init__(self, label: str, func: str, message: Union[str, Exception]): self.label = label self.func = func self.message = message def __str__(self) -> str: return f'[@{self.label}] {self.func}\n{self.message}' class ClientError(CylcError): """Base class for errors raised by Cylc client instances. For example, the workflow you are trying to connect to is stopped. Attributes: message: The exception message. traceback: The underlying exception instance if available. workflow: The workflow ID if available. """ def __init__( self, message: str, traceback: Optional[str] = None, workflow: Optional[str] = None ): self.message = message self.traceback = traceback # Workflow not included in string representation but useful bit of # info to attach to the exception object self.workflow = workflow def __str__(self) -> str: ret = self.message if self.traceback: # append server-side traceback ret += '\n' + self.traceback return ret class RequestError(ClientError): """Represents an error handling a request, returned by the server.""" def __init__( self, message: str, workflow_cylc_version: Optional[str] = None ): ClientError.__init__( self, message, traceback=( f"(Workflow is running in Cylc {workflow_cylc_version})" if workflow_cylc_version and workflow_cylc_version != CYLC_VERSION else None ), ) class WorkflowStopped(ClientError): """The Cylc scheduler you attempted to connect to is stopped.""" def __init__(self, workflow): self.workflow = workflow def __str__(self): return f'{self.workflow} is not running' class ClientTimeout(CylcError): """The scheduler did not respond within the timeout. This could be due to: * Network issues. * Scheduler issues. * Insufficient timeout. To increase the timeout, use the ``--comms-timeout`` option. """ def __init__(self, message: str, workflow: Optional[str] = None): self.message = message self.workflow = workflow def __str__(self) -> str: return self.message class CyclingError(CylcError): """Base class for errors in cycling configuration.""" class CyclerTypeError(CyclingError): """An error raised when incompatible cycling types are wrongly mixed.""" def __init__(self, *args): CyclingError.__init__( self, 'Incompatible cycling types: {0} ({1}), {2} ({3})'.format(*args)) class PointParsingError(CyclingError): """An error raised when a point has an incorrect value.""" def __init__(self, *args): CyclingError.__init__( self, 'Incompatible value for {0}: {1}: {2}'.format(*args)) class IntervalParsingError(CyclingError): """An error raised when an interval has an incorrect value.""" def __init__(self, *args): CyclingError.__init__( self, 'Incompatible value for {0}: {1}'.format(*args)) class SequenceParsingError(CyclingError): """Error on parsing an invalid sequence.""" class SequenceDegenerateError(CyclingError): """An error raised when adjacent points on a sequence are equal.""" def __init__(self, *args): CyclingError.__init__( self, ( '{0}, point format {1}: equal adjacent points:' ' {2} => {3}.' ).format(*args)) class CylcTimeSyntaxError(CyclingError): """An error denoting invalid ISO/Cylc input syntax.""" class CylcMissingContextPointError(CyclingError): """An error denoting a missing (but required) context cycle point.""" class CylcMissingFinalCyclePointError(CyclingError): """An error denoting a missing (but required) final cycle point.""" class PlatformLookupError(CylcConfigError): """Unable to determine the correct job platform from the information given""" class HostSelectException(CylcError): """No hosts could be selected from the provided configuration. Args: data: Mapping of hostnames to error info. ranking: The ranking expression used to select the host. """ def __init__(self, data: Dict[str, dict], ranking: Optional[str] = None): self.data = data self.ranking = ranking def __str__(self) -> str: ret = 'Could not select host from:' for host, data in sorted(self.data.items()): ret += f'\n {host}:' for key, value in data.items(): ret += f'\n {key}: {value}' hint = self.get_hint() if hint: ret += f'\n\n{hint}' return ret def get_hint(self) -> Optional[str]: """Return a hint to explain this error for certain cases.""" if all( # all procs came back with special SSH error code 255 datum.get('returncode') == 255 for datum in self.data.values() ): # likely SSH issues return ( 'Cylc could not establish SSH connection to the run hosts.' '\nEnsure you can SSH to these hosts without having to' ' answer any prompts.' ) if ( # a ranking expression was used self.ranking # and all procs came back with special 'cylc psutil' error code 2 # (which is used for errors relating to the extraction of metrics) and all( datum.get('returncode') == 2 for datum in self.data.values() ) ): # likely an issue with the ranking expression lines = wrap( self.ranking, initial_indent=' ', subsequent_indent=' ', ) ranking = "\n".join(lines) return ( 'This is likely an error in the ranking expression:' f'\n{ranking}' '\n\nConfigured by:' '\n global.cylc[scheduler][run hosts]ranking' ) return None class NoHostsError(CylcError): """None of the hosts of a given platform were reachable.""" def __init__(self, platform): self.platform_name = platform['name'] def __str__(self) -> str: return ( "Unable to find contactable host for platform: " f"{self.platform_name}" ) class NoPlatformsError(PlatformLookupError): """None of the platforms of a given set were reachable. Args: bad_hosts: The hosts that were consumed in the attempt to get a platform. If a dict, the keys are the hostnames and the values are the reasons they failed, and this info will be appended to the exception message. group: The name of the platform group, if we tried to select a platform from one. install_target: The name of the install target, if we tried to select a platform for one. place: Where the attempt to get the platform failed. """ def __init__( self, bad_hosts: set[str] | dict[str, str], *, group: str | None = None, install_target: str | None = None, place: str = '', ): self.msg = "Unable to find a contactable platform" if group: self.msg += f" from group {group}" elif install_target: self.msg += f" for install target {install_target}" if place: self.msg += f" during {place}" self.bad_hosts = set(bad_hosts) if bad_hosts and isinstance(bad_hosts, dict): self.msg += ". The following hosts were tried:\n" + "\n".join( indent(f"{host}: {reason}", 4 * ' ') for host, reason in bad_hosts.items() ) def __str__(self) -> str: return self.msg class CylcVersionError(CylcError): """Contact file is for a Cylc Version not supported by this script.""" def __init__(self, version=None): self.version = version def __str__(self): if self.version is not None: return ( f'Installed Cylc {self.version} workflow is not ' 'compatible with Cylc 8.' ) else: return "Installed workflow is not compatible with Cylc 8." class InvalidCompletionExpression(CylcError): """For the [runtime][]completion configuration. Raised when non-whitelisted syntax is present. """ def __init__(self, message, expr=None): self.message = message self.expr = expr def __str__(self): return self.message class CylcProfilerError(CylcError): """Exception for errors raised from the cylc profiler. These errors do not affect workflows functionally, just stats gathering. We don't want to panic users.""" def __init__(self, exc: Exception, error_msg: str) -> None: CylcError.__init__( self, f"{exc}. {error_msg}. This error came from the Cylc profiler" f" and is not a problem with your workflow. Statistics gathering " f"for the analysis view may be incomplete." ) cylc-flow-8.7.0/cylc/flow/id_match.py0000664000175000017500000002002415264420307017635 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Utilities for matching IDs which may reference families or include globs.""" from copy import deepcopy from fnmatch import fnmatchcase from typing import ( TYPE_CHECKING, Any, Dict, Iterable, Set, Tuple, ) from cylc.flow import LOG from cylc.flow.cycling.loader import get_point from cylc.flow.id import TaskTokens from cylc.flow.id_cli import contains_fnmatch if TYPE_CHECKING: from cylc.flow.config import WorkflowConfig def id_match( config: 'WorkflowConfig', pool: Set[TaskTokens], ids: Set[TaskTokens], only_match_pool: bool = False, ) -> Tuple[Set[TaskTokens], Set[TaskTokens]]: """New Cylc 8.6.0 task matching interface. Args: config: The workflow config. pool: The IDs in the pool of active tasks to match as a set of tokens. The Tokens should include the task state using `task_sel` to support task state matching. ids: The provided IDs to match. only_match_pool: If True, tasks outside of the provided pool will not be matched. Returns: (matched, unmatched) """ unmatched: Set[TaskTokens] = set() # mapping of family name to all contained tasks family_lookup: Dict[str, Set[str]] = _get_family_lookup(config) # set of all active cycles all_cycles: Set[str] = {tokens['cycle'] for tokens in pool} # set of all possible namespaces (tasks + families) all_namespaces: Dict[str, Any] = config.get_namespace_list( 'all namespaces' ) # separate IDs targeting active tasks ONLY from the remainder active_only_ids = { id_ for id_ in ids if id_.get('task_sel') or id_.get('cycle_sel') } plain_ids = ids - active_only_ids # match active-only IDs active_only_ids, _unmatched = _match( config, pool, active_only_ids, family_lookup, all_cycles, all_namespaces, only_match_pool=True, match_selectors=True, ) unmatched.update(_unmatched) # match IDs plain_ids, _unmatched = _match( config, pool, plain_ids, family_lookup, all_cycles, all_namespaces, only_match_pool=only_match_pool, ) unmatched.update(_unmatched) return {*active_only_ids, *plain_ids}, unmatched def _match( config: 'WorkflowConfig', pool: Set[TaskTokens], ids: Set[TaskTokens], family_lookup: Dict[str, Set[str]], all_cycles: Set[str], all_namespaces: Dict[str, Any], only_match_pool: bool = False, match_selectors: bool = False, ) -> Tuple[Set[TaskTokens], Set[TaskTokens]]: # results unmatched: Set[TaskTokens] = set() matched: Set[TaskTokens] = set() for id_ in ids: # replace the "^" token with the initial cycle point if id_['cycle'] == '^': id_ = id_.duplicate(cycle=str(config.initial_point)) # replace the "$" token with the final cycle point elif id_['cycle'] == '$': if config.final_point: id_ = id_.duplicate(cycle=str(config.final_point)) else: LOG.warning( 'ID references final cycle point, but none is set:' f' {id_.relative_id}' ) unmatched.add(id_) continue # match cycles if contains_fnmatch(id_['cycle']): _cycles = _fnmatchcase_glob(id_['cycle'], all_cycles) else: _cycles = {id_['cycle']} # match tasks _namespace = id_.get('task', '*') or 'root' _tasks = { task for namespace in _fnmatchcase_glob(_namespace, all_namespaces) for task in family_lookup.get(namespace, {namespace}) } # expand matched IDs _matched = { TaskTokens( cycle=_cycle, task=_task, task_sel=id_.get('task_sel') or id_.get('cycle_sel'), ) for _cycle in _cycles for _task in _tasks } if only_match_pool and match_selectors: # filter against active tasks _matched = _matched.intersection(pool) elif only_match_pool: _matched = _matched.intersection({ pool_task_id.duplicate(task_sel=None) for pool_task_id in pool }) else: # filter for on-sequence task instances # exclude pool IDs from filtering (they're already validated) _invalid = set() for id__ in _matched.difference({ pool_task_id.duplicate(task_sel=None) for pool_task_id in pool }): try: taskdef = config.taskdefs[id__['task']] if not taskdef.is_valid_point(get_point(id__['cycle'])): _matched.remove(id__) if ( # don't warn if the task was provided as a pattern not contains_fnmatch(id_['task']) # don't warn if the cycle was provided as a pattern and not contains_fnmatch(id_['cycle']) ): # the cycle point is not valid for this task # NOTE: only log this if the user asked for a # specific cycle/task _invalid.add(id__) except (ValueError, KeyError): _matched.remove(id__) if not _matched and _invalid: # warn if there were no matches for the provided id for id__ in sorted(_invalid): LOG.warning( 'Invalid cycle point for task:' f' {id__["task"]}, {id__["cycle"]}' ) if _matched: matched = matched.union(_matched) else: unmatched.add(id_) return matched, unmatched def _fnmatchcase_glob(pattern: str, values: Iterable[str]) -> Set[str]: """Convenience function for globbing over a list of values. This uses the "fnmatchcase" function which is shell glob like. Args: Pattern: The glob. Values: The things to evaluate the glob over. Examples: >>> sorted(_fnmatchcase_glob('*', {'a', 'b', 'c'})) ['a', 'b', 'c'] >>> sorted(_fnmatchcase_glob('a*', {'a1', 'a2', 'b1'})) ['a1', 'a2'] """ return { value for value in values if fnmatchcase(value, pattern) } def _get_family_lookup(config: 'WorkflowConfig') -> Dict[str, Set[str]]: """Return a dict mapping families to all contained tasks. This recursively expands families avoiding the need to do so later. """ lookup = deepcopy(config.runtime['descendants']) def _iter(): ret = False for namespaces in lookup.values(): for namespace in list(namespaces): if namespace in config.runtime['descendants']: ret = True namespaces.remove(namespace) namespaces.update(config.runtime['descendants'][namespace]) return ret while _iter(): pass return lookup cylc-flow-8.7.0/cylc/flow/context_node.py0000664000175000017500000001706515264420307020571 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from typing import ( Optional, Self, ) class ContextNode(): """A class for defining nested objects. Attributes: name (str): The key identifier for this node. Nodes belonging to the same parent must have not have the same name. Examples: Define the tree: >>> with ContextNode('a') as a: ... # create parent nodes using a context ... # manager ... with ContextNode('b') as b: ... # create "leaf" nodes without ... # a context manager ... c = ContextNode('c') ... d = ContextNode('d') The "key" for this node: >>> str(c) 'c' >>> repr(c) 'a/b/c' List ancestry: >>> list(c.parents()) [a/b, a] Iterate through children: >>> list(a) [a/b, a/d] Summarise the tree: >>> print(a.tree()) a b c d Node Terminology: Root/ Parent/ Leaf Leaf Class Attributes: ROOT_NAME_FMT (str): String representation for *root* nodes to be evaluated as a format string. Can use any of the instance attributes. ROOT_NAME_FMT (str): String representation for parent nodes. ROOT_NAME_FMT (str): String representation for leaf nodes. SEP (str): String to separate components in a nodes repr(). """ __slots__ = ('name', '_parent', '_children') ROOT_NAME_FMT = '{name}' NODE_NAME_FMT = '{name}' LEAF_NAME_FMT = '{name}' SEP = '/' DATA: dict = {} def __init__(self, name: str): self.name = name self._parent = None self._children = None self.DATA[self] = set(self.DATA) def __enter__(self) -> 'Self': return self def __exit__(self, *args): # list the nodes already present on this node older_siblings = self._children or {} # list the nodes which were created since we # __entered__ this node new_borns = { value.name: value for value in self.DATA if value is not self if value not in self.DATA.get(self, {}) } self._children = { **older_siblings, **new_borns } for child in new_borns.values(): # remove our children from self.DATA del self.DATA[child] # set our children to see us as their parent child._parent = self if len(self.DATA) == 1: # this is a root node, clean up after ourselves # self.DATA *should* be clean after each tree del self.DATA[self] def __iter__(self): if self._children: return iter(self._children.values()) return iter([]) def __contains__(self, name: str) -> bool: return name in self._children # type: ignore[operator] # TODO def __getitem__(self, name: str) -> 'Self': if self._children: return self._children.__getitem__(name) raise TypeError('This is not a leaf node') def get(self, *names: str) -> 'Self': """Retrieve the node given by the list of names. Example: >>> with ContextNode('a') as a: ... with ContextNode('b'): ... c = ContextNode('c') >>> a.get('b', 'c') a/b/c >>> with ContextNode('a') as a: ... with ContextNode('b'): ... with ContextNode('__MANY__'): ... c = ContextNode('c') >>> a.get('b', 'foo', 'c') a/b/__MANY__/c """ node = self for name in names: try: node = node[name] except KeyError as exc: if '__MANY__' not in node: raise exc node = node['__MANY__'] return node def __str__(self) -> str: if self.is_root(): fmt = self.ROOT_NAME_FMT elif not self.is_leaf(): fmt = self.NODE_NAME_FMT else: fmt = self.LEAF_NAME_FMT return fmt.format_map({ key: getattr(self, key) for key in self.__slots__ }) def __repr__(self) -> str: return self.SEP.join( [ str(node) for node in reversed(list(self.parents())) ] + [ str(self) ] ) def is_root(self) -> bool: """Return True if this is a root node.""" return self._parent is None def is_leaf(self) -> bool: """Return True if this is a leaf node.""" return self._children is None def parents(self): """Yield the linearised parents of this node. Yields: ContextNode Examples: >>> with ContextNode('a') as a: ... with ContextNode('b') as b: ... c = ContextNode('c') >>> c.parents().__next__() a/b >>> list(c.parents()) [a/b, a] """ pointer = self._parent while pointer: yield pointer pointer = pointer._parent def walk(self, depth: Optional[int] = None, _level: int = 0): """Walk the context tree starting at this node. Args: depth: The max depth below the current node to yield. _level: (For recursive use, do not specify.) Yields: tuple - (level, node) level (int): The tree level measured from this node. node (ContextNode): The context node. Examples: >>> with ContextNode('a') as a: ... with ContextNode('b') as b: ... c = ContextNode('c') >>> list(a.walk()) [(0, a), (1, a/b), (2, a/b/c)] >>> list(c.walk()) [(0, a/b/c)] """ if depth is not None and _level >= depth: return if _level == 0: yield (0, self) for child in self: yield (_level + 1, child) yield from child.walk(depth=depth, _level=_level + 1) def tree(self): """Return a string representation of the tree starting at this node. Returns: str Examples: >>> with ContextNode('a') as a: ... with ContextNode('b') as b: ... c = ContextNode('c') >>> print(a.tree()) a b c """ return '\n'.join(( f'{" " * level}{node}' for level, node in self.walk() )) cylc-flow-8.7.0/cylc/flow/rundb.py0000664000175000017500000012121715264420307017205 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Provide data access object for the workflow runtime database.""" from collections import defaultdict from contextlib import suppress from dataclasses import dataclass from os.path import expandvars from pprint import pformat import sqlite3 import traceback from typing import ( TYPE_CHECKING, Any, DefaultDict, Dict, Iterable, List, Optional, Set, Tuple, Union, cast, ) from cylc.flow import LOG import cylc.flow.flags from cylc.flow.flow_mgr import stringify_flow_nums from cylc.flow.util import ( deserialise_set, serialise_set, ) if TYPE_CHECKING: from collections.abc import Iterator from pathlib import Path from cylc.flow.flow_mgr import FlowNums DbArgDict = Dict[str, Any] DbUpdateTuple = Union[ Tuple[DbArgDict, DbArgDict], Tuple[str, list] ] @dataclass class CylcWorkflowDAOTableColumn: """Represent a column in a table.""" name: str datatype: str is_primary_key: bool class CylcWorkflowDAOTable: """Represent a table in the workflow runtime database.""" FMT_CREATE = "CREATE TABLE %(name)s(%(columns_str)s%(primary_keys_str)s)" FMT_DELETE = "DELETE FROM %(name)s%(where_str)s" FMT_INSERT = "INSERT OR REPLACE INTO %(name)s VALUES(%(values_str)s)" FMT_UPDATE = "UPDATE %(name)s SET %(set_str)s%(where_str)s" __slots__ = ('name', 'columns', 'delete_queues', 'insert_queue', 'update_queues') def __init__(self, name, column_items): self.name = name self.columns: List[CylcWorkflowDAOTableColumn] = [] for column_item in column_items: name = column_item[0] attrs = {} if len(column_item) > 1: attrs = column_item[1] self.columns.append(CylcWorkflowDAOTableColumn( name, attrs.get("datatype", "TEXT"), attrs.get("is_primary_key", False))) self.delete_queues = {} self.insert_queue = [] self.update_queues: DefaultDict[str, list] = defaultdict(list) def get_create_stmt(self): """Return an SQL statement to create this table.""" column_str_list = [] primary_keys = [] for column in self.columns: column_str_list.append(column.name + " " + column.datatype) if column.is_primary_key: primary_keys.append(column.name) primary_keys_str = "" if primary_keys: primary_keys_str = ", PRIMARY KEY(" + ", ".join(primary_keys) + ")" return self.FMT_CREATE % { "name": self.name, "columns_str": ", ".join(column_str_list), "primary_keys_str": primary_keys_str} def get_insert_stmt(self): """Return an SQL statement to insert a row to this table.""" return self.FMT_INSERT % { "name": self.name, "values_str": ", ".join("?" * len(self.columns))} def add_delete_item(self, where_args): """Queue a DELETE item. where_args should be a dict, delete will only apply to rows matching all these items. """ stmt_args = [] where_str = "" if where_args: where_strs = [] for column in self.columns: if column.name in where_args: where_strs.append(column.name + "==?") stmt_args.append(where_args[column.name]) if where_strs: where_str = " WHERE " + " AND ".join(where_strs) stmt = self.FMT_DELETE % {"name": self.name, "where_str": where_str} if stmt not in self.delete_queues: self.delete_queues[stmt] = [] self.delete_queues[stmt].append(stmt_args) def add_insert_item(self, args): """Queue an INSERT args. If args is a list, its length will be adjusted to be the same as the number of columns. If args is a dict, will return a list with the same length as the number of columns, the elements of which are determined by matching the column names with the keys in the dict. Empty elements are padded with None. """ if isinstance(args, list): if len(args) == len(self.columns): stmt_args = list(args) elif len(args) < len(self.columns): stmt_args = args + [None] * (len(self.columns) - len(args)) else: # len(args) > len(self.columns) stmt_args = args[0:len(self.columns)] else: stmt_args = [ args.get(column.name, None) for column in self.columns] self.insert_queue.append(stmt_args) def add_update_item(self, item: DbUpdateTuple) -> None: """Queue an UPDATE item. If stmt is not a string, it should be a tuple (set_args, where_args) - set_args should be a dict, with column keys and values to be set. where_args should be a dict, update will only apply to rows matching all these items. """ if isinstance(item[0], str): stmt = item[0] params = cast('list', item[1]) self.update_queues[stmt].extend(params) return set_args = item[0] where_args = cast('DbArgDict', item[1]) set_strs = [] stmt_args = [] for column in self.columns: if column.name in set_args: set_strs.append(column.name + "=?") stmt_args.append(set_args[column.name]) set_str = ", ".join(set_strs) where_str = "" if where_args: where_strs = [] for column in self.columns: if column.name in where_args: where_strs.append(column.name + "==?") stmt_args.append(where_args[column.name]) if where_strs: where_str = " WHERE " + " AND ".join(where_strs) stmt = self.FMT_UPDATE % { "name": self.name, "set_str": set_str, "where_str": where_str } self.update_queues[stmt].append(stmt_args) class CylcWorkflowDAO: """Data access object for the workflow runtime database.""" CONN_TIMEOUT = 0.2 DB_FILE_BASE_NAME = "db" MAX_TRIES = 100 RESTART_INCOMPAT_VERSION = "8.0rc2" # Can't restart if <= this version TABLE_BROADCAST_EVENTS = "broadcast_events" TABLE_BROADCAST_STATES = "broadcast_states" TABLE_INHERITANCE = "inheritance" TABLE_WORKFLOW_PARAMS = "workflow_params" TABLE_WORKFLOW_FLOWS = "workflow_flows" TABLE_WORKFLOW_TEMPLATE_VARS = "workflow_template_vars" TABLE_TASK_JOBS = "task_jobs" TABLE_TASK_EVENTS = "task_events" TABLE_TASK_ACTION_TIMERS = "task_action_timers" TABLE_TASK_LATE_FLAGS = "task_late_flags" TABLE_TASK_OUTPUTS = "task_outputs" TABLE_TASK_POOL = "task_pool" TABLE_TASK_PREREQUISITES = "task_prerequisites" TABLE_TASK_STATES = "task_states" TABLE_TASK_TIMEOUT_TIMERS = "task_timeout_timers" TABLE_TASKS_TO_HOLD = "tasks_to_hold" TABLE_XTRIGGERS = "xtriggers" TABLE_ABS_OUTPUTS = "absolute_outputs" TABLES_ATTRS = { TABLE_BROADCAST_EVENTS: [ ["time"], ["change"], ["point"], ["namespace"], ["key"], ["value"], ], TABLE_BROADCAST_STATES: [ ["point", {"is_primary_key": True}], ["namespace", {"is_primary_key": True}], ["key", {"is_primary_key": True}], ["value"], ], TABLE_INHERITANCE: [ ["namespace", {"is_primary_key": True}], ["inheritance"], ], TABLE_WORKFLOW_PARAMS: [ ["key", {"is_primary_key": True}], ["value"], ], TABLE_WORKFLOW_FLOWS: [ ["flow_num", {"datatype": "INTEGER", "is_primary_key": True}], ["start_time"], ["description"], ], TABLE_WORKFLOW_TEMPLATE_VARS: [ ["key", {"is_primary_key": True}], ["value"], ], TABLE_TASK_ACTION_TIMERS: [ ["cycle", {"is_primary_key": True}], ["name", {"is_primary_key": True}], ["ctx_key", {"is_primary_key": True}], ["ctx"], ["delays"], ["num", {"datatype": "INTEGER"}], ["delay"], ["timeout"], ], # NOTE: this table is used by `cylc clean`, don't rename me! TABLE_TASK_JOBS: [ ["cycle", {"is_primary_key": True}], ["name", {"is_primary_key": True}], ["submit_num", {"datatype": "INTEGER", "is_primary_key": True}], ["flow_nums"], ["is_manual_submit", {"datatype": "INTEGER"}], ["try_num", {"datatype": "INTEGER"}], # This is used to store simulation task start time across restarts. ["time_submit"], ["time_submit_exit"], ["submit_status", {"datatype": "INTEGER"}], ["time_run"], ["time_run_exit"], ["run_signal"], ["run_status", {"datatype": "INTEGER"}], # NOTE: this field is used by `cylc clean` don't rename me! ["platform_name"], ["job_runner_name"], ["job_id"], ], TABLE_TASK_EVENTS: [ ["name"], ["cycle"], ["time"], ["submit_num", {"datatype": "INTEGER"}], ["event"], ["message"], ], TABLE_TASK_LATE_FLAGS: [ ["cycle", {"is_primary_key": True}], ["name", {"is_primary_key": True}], ["value", {"datatype": "INTEGER"}], ], TABLE_TASK_OUTPUTS: [ ["cycle", {"is_primary_key": True}], ["name", {"is_primary_key": True}], ["flow_nums", {"is_primary_key": True}], ["outputs"], ], TABLE_TASK_POOL: [ ["cycle", {"is_primary_key": True}], ["name", {"is_primary_key": True}], ["flow_nums", {"is_primary_key": True}], ["status"], ["is_held", {"datatype": "INTEGER"}], ], TABLE_TASK_PREREQUISITES: [ ["cycle", {"is_primary_key": True}], ["name", {"is_primary_key": True}], ["flow_nums", {"is_primary_key": True}], ["prereq_name", {"is_primary_key": True}], ["prereq_cycle", {"is_primary_key": True}], ["prereq_output", {"is_primary_key": True}], ["satisfied"], ], # The xtriggers table holds the function signature and result of # already-satisfied (the scheduler no longer needs to call them). TABLE_XTRIGGERS: [ ["signature", {"is_primary_key": True}], ["results"], ], TABLE_TASK_STATES: [ ["name", {"is_primary_key": True}], ["cycle", {"is_primary_key": True}], ["flow_nums", {"is_primary_key": True}], ["time_created"], ["time_updated"], ["submit_num", {"datatype": "INTEGER"}], ["status"], ["flow_wait", {"datatype": "INTEGER"}], ["is_manual_submit", {"datatype": "INTEGER"}], ], TABLE_TASK_TIMEOUT_TIMERS: [ ["cycle", {"is_primary_key": True}], ["name", {"is_primary_key": True}], ["timeout", {"datatype": "REAL"}], ], TABLE_ABS_OUTPUTS: [ ["cycle"], ["name"], ["output"], ], TABLE_TASKS_TO_HOLD: [ ["name"], ["cycle"], ], } def __init__( self, db_file_name: Union['Path', str], is_public: bool = False, create_tables: bool = False ): """Initialise database access object. An instance of this class can also be opened as a context manager which will automatically close the DB connection. Args: db_file_name: Path to the database file. is_public: If True, allow retries. create_tables: If True, create the tables if they don't already exist. """ self.db_file_name = expandvars(db_file_name) self.is_public = is_public self.conn: Optional[sqlite3.Connection] = None self.n_tries = 0 self.tables = { name: CylcWorkflowDAOTable(name, attrs) for name, attrs in sorted(self.TABLES_ATTRS.items()) } if create_tables: self.create_tables() def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): """Close DB connection when leaving context manager.""" self.close() def add_delete_item(self, table_name, where_args=None): """Queue a DELETE item for a given table. where_args should be a dict, update will only apply to rows matching all these items. """ self.tables[table_name].add_delete_item(where_args) def add_insert_item(self, table_name, args): """Queue an INSERT args for a given table. If args is a list, its length will be adjusted to be the same as the number of columns. If args is a dict, will return a list with the same length as the number of columns, the elements of which are determined by matching the column names with the keys in the dict. Empty elements are padded with None. """ self.tables[table_name].add_insert_item(args) def add_update_item( self, table_name: str, item: DbUpdateTuple ) -> None: """Queue an UPDATE item for a given table. If stmt is not a string, it should be a tuple (set_args, where_args) - set_args should be a dict, with column keys and values to be set. where_args should be a dict, update will only apply to rows matching all these items. """ self.tables[table_name].add_update_item(item) def close(self) -> None: """Explicitly close the connection.""" if self.conn is not None: try: self.conn.close() except sqlite3.Error as exc: LOG.debug(f"Error closing connection to DB: {exc}") self.conn = None def connect(self) -> sqlite3.Connection: """Connect to the database.""" if self.conn is None: self.conn = sqlite3.connect( self.db_file_name, timeout=self.CONN_TIMEOUT ) return self.conn def create_tables(self): """Create tables.""" names = [] for row in self.connect().execute( "SELECT name FROM sqlite_master WHERE type==? ORDER BY name", ["table"]): names.append(row[0]) cur = None for name, table in self.tables.items(): if name not in names: cur = self.conn.execute(table.get_create_stmt()) if cur is not None: self.conn.commit() def execute_queued_items(self): """Execute queued items for each table.""" # determine the sql statements to execute sql_queue = [] # (sql_statement, values) for table in self.tables.values(): # DELETE statements may have varying number of WHERE args so we # can only executemany for each identical template statement. for stmt, stmt_args_list in table.delete_queues.items(): sql_queue.append((stmt, stmt_args_list)) # INSERT statements are uniform for each table, so all INSERT # statements can be executed using a single "executemany" call. if table.insert_queue: sql_queue.append(( table.get_insert_stmt(), table.insert_queue, )) # UPDATE statements can have varying number of SET and WHERE # args so we can only executemany for each identical template # statement. for stmt, stmt_args_list in table.update_queues.items(): sql_queue.append((stmt, stmt_args_list)) # execute the statements and commit the transaction try: for stmt, stmt_args in sql_queue: self._execute_stmt(stmt, stmt_args) # Connection should only be opened if we have executed something. if self.conn is None: return self.conn.commit() # something went wrong # (includes DB file not found, transaction processing issue, db locked) except sqlite3.Error as e: # Detailed error codes are only available for python >= 3.11 if hasattr(e, "sqlite_errorcode"): error_code = str(e.sqlite_errorcode) else: error_code = "Not available" if hasattr(e, "sqlite_errorname"): error_name = e.sqlite_errorname else: error_name = "Not available" if not self.is_public: # in case this isn't a filesystem issue, log the statements # which make up the transaction to assist debug LOG.error( 'An error occurred when writing to the database %(file)s,' ' this is probably a filesystem issue.' ' The error was: %(error)s' ' SQLite error code: %(error_code)s' ' SQLite error name: %(error_name)s' ' The attempted transaction was:\n %(transaction)s' % { "file": self.db_file_name, "error": str(e), "error_code": error_code, "error_name": error_name, "transaction": pformat(sql_queue) } ) raise self.n_tries += 1 LOG.info( "%(file)s: write attempt (%(attempt)d)" " did not complete: %(error)s\n" " SQLite error code: %(error_code)s\n" " SQLite error name: %(error_name)s" % { "file": self.db_file_name, "attempt": self.n_tries, "error": str(e), "error_code": error_code, "error_name": error_name } ) if self.conn is not None: with suppress(sqlite3.Error): self.conn.rollback() return else: # Clear the queues for table in self.tables.values(): table.delete_queues.clear() table.insert_queue.clear() table.update_queues.clear() # Report public database retry recovery if necessary if self.n_tries: LOG.info( "%(file)s: recovered after (%(attempt)d) attempt(s)\n" % { "file": self.db_file_name, "attempt": self.n_tries}) self.n_tries = 0 finally: # Note: This is not strictly necessary. But if the workflow run # directory is removed, a forced reconnection to the private # database will ensure that the workflow dies. self.close() def _execute_stmt(self, stmt, stmt_args_list): """Helper for "self.execute_queued_items". Execute a statement. If this is the public database, return True on success and False on failure. If this is the private database, return True on success, and raise on failure. """ # Filter out CYLC_TEMPLATE_VARS which breaks executemany because it's: # - a dict # - recursive (contains itself!) if stmt_args_list and stmt_args_list[0]: stmt_args_list = [ i for i in stmt_args_list if i[0] != 'CYLC_TEMPLATE_VARS' ] try: self.connect() self.conn.executemany(stmt, stmt_args_list) except sqlite3.Error as e: if not self.is_public: raise if cylc.flow.flags.verbosity > 1: traceback.print_exc() err_log = ( "cannot execute database statement:\n" "file=%(file)s:\nstmt=%(stmt)s\nerror=%(error)s" ) % {"file": self.db_file_name, "stmt": stmt, "error": str(e)} for i, stmt_args in enumerate(stmt_args_list): err_log += ("\nstmt_args[%(i)d]=%(stmt_args)s" % { "i": i, "stmt_args": stmt_args}) if self.is_public: LOG.info(err_log) else: LOG.warning(err_log) raise def pre_select_broadcast_states(self, order=None): """Query statement and args formation for select_broadcast_states.""" form_stmt = r"SELECT point,namespace,key,value FROM %s" if order == "ASC": ordering = " ORDER BY point ASC, namespace ASC, key ASC" form_stmt = form_stmt + ordering return form_stmt % self.TABLE_BROADCAST_STATES def select_broadcast_states(self, callback, sort=None): """Select from broadcast_states. Invoke callback(row_idx, row) on each row, where each row contains: [point, namespace, key, value] """ stmt = self.pre_select_broadcast_states(order=sort) for row_idx, row in enumerate(self.connect().execute(stmt)): callback(row_idx, list(row)) def select_workflow_params(self) -> Iterable[Tuple[str, Optional[str]]]: """Select all from workflow_params. E.g. a row might be ('UTC mode', '1') """ stmt = rf''' SELECT key, value FROM {self.TABLE_WORKFLOW_PARAMS} ''' # nosec B608 (table name is code constant) return self.connect().execute(stmt) def select_workflow_flows(self, flow_nums: Iterable[int]): """Return flow data for selected flows.""" stmt = rf''' SELECT flow_num, start_time, description FROM {self.TABLE_WORKFLOW_FLOWS} WHERE flow_num in ({stringify_flow_nums(flow_nums)}) ''' # nosec B608 (table name is code constant, flow_nums just ints) flows = {} for flow_num, start_time, descr in self.connect().execute(stmt): flows[flow_num] = { "start_time": start_time, "description": descr } return flows def select_workflow_flows_max_flow_num(self): """Return max flow number in the workflow_flows table.""" stmt = rf''' SELECT MAX(flow_num) FROM {self.TABLE_WORKFLOW_FLOWS} ''' # nosec B608 (table name is code constant) return self.connect().execute(stmt).fetchone()[0] def select_workflow_params_restart_count(self): """Return number of restarts in workflow_params table.""" stmt = rf""" SELECT value FROM {self.TABLE_WORKFLOW_PARAMS} WHERE key == 'n_restart' """ # nosec B608 (table name is code constant) result = self.connect().execute(stmt).fetchone() return int(result[0]) if result else 0 def select_workflow_template_vars(self, callback): """Select from workflow_template_vars. Invoke callback(row_idx, row) on each row, where each row contains: [key,value] """ for row_idx, row in enumerate(self.connect().execute( rf''' SELECT key, value FROM {self.TABLE_WORKFLOW_TEMPLATE_VARS} ''' # nosec B608 (table name is code constant) )): callback(row_idx, list(row)) def select_task_action_timers(self, callback): """Select from task_action_timers for restart. Invoke callback(row_idx, row) on each row. """ attrs = [] for item in self.TABLES_ATTRS[self.TABLE_TASK_ACTION_TIMERS]: attrs.append(item[0]) stmt = rf''' SELECT {",".join(attrs)} FROM {self.TABLE_TASK_ACTION_TIMERS} ''' # nosec B608 # * table name is code constant # * attrs are code constants for row_idx, row in enumerate(self.connect().execute(stmt)): callback(row_idx, list(row)) def select_task_job( self, cycle: str, name: str, submit_num: str | int | None = None ) -> dict[str, Any] | None: """Select items from task_jobs by (cycle, name, submit_num). :return: a dict for mapping keys to the column values """ keys = [] for column in self.tables[self.TABLE_TASK_JOBS].columns[3:]: keys.append(column.name) if submit_num in [None, "NN"]: stmt = rf''' SELECT {",".join(keys)} FROM {self.TABLE_TASK_JOBS} WHERE cycle==? AND name==? ORDER BY submit_num DESC LIMIT 1 ''' # nosec B608 # * table name is code constant # * keys are code constants stmt_args: list[Any] = [cycle, name] else: stmt = rf''' SELECT {",".join(keys)} FROM {self.TABLE_TASK_JOBS} WHERE cycle==? AND name==? AND submit_num==? ''' # nosec B608 # * table name is code constant # * keys are code constants stmt_args = [cycle, name, submit_num] with suppress(sqlite3.DatabaseError): for row in self.connect().execute(stmt, stmt_args): ret = {} for key, value in zip(keys, row): ret[key] = value return ret return None def select_jobs_for_restart(self, callback): """Select from task_pool+task_states+task_jobs for restart. Invoke callback(row_idx, row) on each row of the result. """ form_stmt = r""" SELECT %(task_pool)s.cycle, %(task_pool)s.name, %(task_jobs)s.submit_num, %(task_jobs)s.time_submit, %(task_jobs)s.submit_status, %(task_jobs)s.time_run, %(task_jobs)s.time_run_exit, %(task_jobs)s.run_status, %(task_jobs)s.job_runner_name, %(task_jobs)s.job_id, %(task_jobs)s.platform_name FROM %(task_jobs)s JOIN %(task_pool)s ON %(task_jobs)s.cycle == %(task_pool)s.cycle AND %(task_jobs)s.name == %(task_pool)s.name """ form_data = { "task_pool": self.TABLE_TASK_POOL, "task_jobs": self.TABLE_TASK_JOBS, } stmt = form_stmt % form_data for row_idx, row in enumerate(self.connect().execute(stmt)): callback(row_idx, list(row)) def select_task_job_run_times(self, callback): """Select run times of succeeded task jobs grouped by task names. Invoke callback(row_idx, row) on each row, where each row contains: [name, run_times_str] where run_times_str is a string containing comma separated list of integer run times. This method is used to re-populate elapsed run times of each task on restart. """ stmt = ( r"SELECT" r" name," r" GROUP_CONCAT(" r" CAST(strftime('%s', time_run_exit) AS NUMERIC) -" r" CAST(strftime('%s', time_run) AS NUMERIC))" r" FROM task_jobs" r" WHERE run_status==0 GROUP BY name ORDER BY time_run_exit") for row_idx, row in enumerate(self.connect().execute(stmt)): callback(row_idx, list(row)) def select_task_job_platforms(self): """Return the set of platform names from task_jobs table. Warning: This interface is used by `cylc clean` which does not upgrade the DB first (it could, this would only extend backwards compatibility but would not help with forwards compatibility). Keep this query to the minimum of tables/fields to avoid breaking compatibility with older/newer versions of Cylc. """ stmt = rf''' SELECT DISTINCT platform_name FROM {self.TABLE_TASK_JOBS} ''' # nosec B608 (table name is code constant) return {i[0] for i in self.connect().execute(stmt)} def select_prev_instances( self, name: str, point: str ) -> List[Tuple[int, bool, Set[int], str]]: """Select task_states table info about previous instances of a task. Flow merge results in multiple entries for the same submit number. """ # Ignore bandit false positive: B608: hardcoded_sql_expressions # Not an injection, simply putting the table name in the SQL query # expression as a string constant local to this module. stmt = ( # nosec B608 r"SELECT flow_nums,submit_num,flow_wait,status FROM %(name)s" r" WHERE name==? AND cycle==?" ) % {"name": self.TABLE_TASK_STATES} return [ ( submit_num, flow_wait == 1, deserialise_set(flow_nums_str), status ) for flow_nums_str, submit_num, flow_wait, status in ( self.connect().execute(stmt, (name, point,)) ) ] def select_latest_flow_nums(self) -> Optional['FlowNums']: """Return a list of the most recent previous flow numbers.""" stmt = rf''' SELECT flow_nums, MAX(time_created) FROM {self.TABLE_TASK_STATES} WHERE flow_nums != ? ''' # nosec B608 (table name is code constant) # Exclude flow=none: params = [serialise_set()] flow_nums_str = self.connect().execute(stmt, params).fetchone()[0] if flow_nums_str: return deserialise_set(flow_nums_str) return None def select_task_outputs( self, name: str, point: str ) -> 'Iterator[tuple[str, FlowNums]]': """Select task outputs for each flow. Yields: (outputs_dict_str, flow_nums_set) """ stmt = rf''' SELECT flow_nums,outputs FROM {self.TABLE_TASK_OUTPUTS} WHERE name==? AND cycle==? ''' # nosec B608 (table name is code constant) for flow_nums, outputs in self.connect().execute( stmt, (name, point,) ): yield (outputs, deserialise_set(flow_nums)) def select_xtriggers_for_restart(self, callback): stmt = rf''' SELECT signature, results FROM {self.TABLE_XTRIGGERS} ''' # nosec B608 (table name is code constant) for row_idx, row in enumerate(self.connect().execute(stmt, [])): callback(row_idx, list(row)) def select_abs_outputs_for_restart(self, callback): stmt = rf''' SELECT cycle, name, output FROM {self.TABLE_ABS_OUTPUTS} ''' # nosec B608 (table name is code constant) for row_idx, row in enumerate(self.connect().execute(stmt, [])): callback(row_idx, list(row)) def select_task_pool(self, callback): """Select from task_pool. Invoke callback(row_idx, row) on each row, where each row contains: [cycle, name, status] """ form_stmt = r"SELECT cycle,name,status,is_held FROM %s" stmt = form_stmt % self.TABLE_TASK_POOL for row_idx, row in enumerate(self.connect().execute(stmt)): callback(row_idx, list(row)) def select_task_pool_for_restart( self, ) -> Iterable[tuple[ str, str, str, int, int, Any | None, str, int, int, int | None, str | None, str | None, str | None, float | None, str, ]]: """Select from task_pool+task_states+task_jobs for restart.""" form_stmt = r""" SELECT %(task_pool)s.cycle, %(task_pool)s.name, %(task_pool)s.flow_nums, %(task_states)s.flow_wait, %(task_states)s.is_manual_submit, %(task_late_flags)s.value, %(task_pool)s.status, %(task_pool)s.is_held, %(task_states)s.submit_num, %(task_jobs)s.try_num, %(task_jobs)s.platform_name, %(task_jobs)s.time_submit, %(task_jobs)s.time_run, %(task_timeout_timers)s.timeout, %(task_outputs)s.outputs FROM %(task_pool)s JOIN %(task_states)s ON %(task_pool)s.cycle == %(task_states)s.cycle AND %(task_pool)s.name == %(task_states)s.name AND %(task_pool)s.flow_nums == %(task_states)s.flow_nums LEFT OUTER JOIN %(task_late_flags)s ON %(task_pool)s.cycle == %(task_late_flags)s.cycle AND %(task_pool)s.name == %(task_late_flags)s.name LEFT OUTER JOIN %(task_jobs)s ON %(task_pool)s.cycle == %(task_jobs)s.cycle AND %(task_pool)s.name == %(task_jobs)s.name AND %(task_states)s.submit_num == %(task_jobs)s.submit_num LEFT OUTER JOIN %(task_timeout_timers)s ON %(task_pool)s.cycle == %(task_timeout_timers)s.cycle AND %(task_pool)s.name == %(task_timeout_timers)s.name LEFT OUTER JOIN %(task_outputs)s ON %(task_pool)s.cycle == %(task_outputs)s.cycle AND %(task_pool)s.name == %(task_outputs)s.name AND %(task_pool)s.flow_nums == %(task_outputs)s.flow_nums """ form_data = { "task_pool": self.TABLE_TASK_POOL, "task_states": self.TABLE_TASK_STATES, "task_late_flags": self.TABLE_TASK_LATE_FLAGS, "task_timeout_timers": self.TABLE_TASK_TIMEOUT_TIMERS, "task_jobs": self.TABLE_TASK_JOBS, "task_outputs": self.TABLE_TASK_OUTPUTS, } stmt = form_stmt % form_data return self.connect().execute(stmt) def select_task_prerequisites( self, cycle: str, name: str, flow_nums: str ) -> List[Tuple[str, str, str, str]]: """Return prerequisites of a task of the given name & cycle point.""" stmt = rf""" SELECT prereq_name, prereq_cycle, prereq_output, satisfied FROM {self.TABLE_TASK_PREREQUISITES} WHERE cycle == ? AND name == ? AND flow_nums == ? """ # nosec B608 (table name is code constant) stmt_args = [cycle, name, flow_nums] return list(self.connect().execute(stmt, stmt_args)) def select_tasks_to_hold(self) -> List[Tuple[str, str]]: """Return all tasks to hold stored in the DB.""" stmt = rf''' SELECT name, cycle FROM {self.TABLE_TASKS_TO_HOLD} ''' # nosec B608 (table name is code constant) return list(self.connect().execute(stmt)) def select_task_times(self): """Select submit/start/stop times to compute job timings. To make data interpretation easier, choose the most recent succeeded task to sample timings from. """ stmt = rf""" SELECT name, cycle, platform_name, job_runner_name, time_submit, time_run, time_run_exit FROM {self.TABLE_TASK_JOBS} WHERE run_status = 0 """ # nosec B608 (table name is code constant) columns = ( 'name', 'cycle', 'host', 'job_runner', 'submit_time', 'start_time', 'succeed_time' ) return columns, list(self.connect().execute(stmt)) def select_tasks_for_datastore( self, task_ids ): """Select state and outputs of specified tasks.""" if not task_ids: return [] form_stmt = r""" SELECT %(task_states)s.cycle, %(task_states)s.name, %(task_states)s.flow_nums, %(task_states)s.status, MAX(%(task_states)s.submit_num), %(task_outputs)s.outputs FROM %(task_states)s LEFT OUTER JOIN %(task_outputs)s ON %(task_states)s.cycle == %(task_outputs)s.cycle AND %(task_states)s.name == %(task_outputs)s.name WHERE %(task_states)s.cycle || '/' || %(task_states)s.name IN ( %(task_ids)s ) GROUP BY %(task_states)s.cycle, %(task_states)s.name """ form_data = { "task_states": self.TABLE_TASK_STATES, "task_outputs": self.TABLE_TASK_OUTPUTS, "task_ids": ', '.join(f"'{val}'" for val in task_ids), } stmt = form_stmt % form_data return list(self.connect().execute(stmt)) def select_prereqs_for_datastore( self, prereq_ids ): """Select prerequisites of specified tasks.""" if not prereq_ids: return [] form_stmt = r""" SELECT cycle, name, prereq_name, prereq_cycle, prereq_output, satisfied FROM %(prerequisites)s WHERE cycle || '/' || name || '/' || flow_nums IN ( %(prereq_tasks_args)s ) """ form_data = { "prerequisites": self.TABLE_TASK_PREREQUISITES, "prereq_tasks_args": ', '.join(f"'{val}'" for val in prereq_ids), } stmt = form_stmt % form_data return list(self.connect().execute(stmt)) def select_jobs_for_datastore( self, task_ids ): """Select jobs of of specified tasks.""" if not task_ids: return [] form_stmt = r""" SELECT %(task_states)s.cycle, %(task_states)s.name, %(task_jobs)s.submit_num, %(task_jobs)s.time_submit, %(task_jobs)s.submit_status, %(task_jobs)s.time_run, %(task_jobs)s.time_run_exit, %(task_jobs)s.run_status, %(task_jobs)s.job_runner_name, %(task_jobs)s.job_id, %(task_jobs)s.platform_name FROM %(task_jobs)s JOIN %(task_states)s ON %(task_jobs)s.cycle == %(task_states)s.cycle AND %(task_jobs)s.name == %(task_states)s.name WHERE %(task_states)s.cycle || '/' || %(task_states)s.name IN ( %(task_ids)s ) ORDER BY %(task_states)s.submit_num DESC """ form_data = { "task_states": self.TABLE_TASK_STATES, "task_jobs": self.TABLE_TASK_JOBS, "task_ids": ', '.join(f"'{val}'" for val in task_ids), } stmt = form_stmt % form_data return list(self.connect().execute(stmt)) def vacuum(self): """Vacuum to the database.""" return self.connect().execute("VACUUM") cylc-flow-8.7.0/cylc/flow/install.py0000664000175000017500000005217415264420307017546 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Functionality for (local) workflow installation.""" import logging import os import re import shlex from contextlib import suppress from pathlib import Path from subprocess import ( PIPE, Popen, ) from typing import ( Any, Dict, List, Optional, Tuple, Union, ) from cylc.flow import LOG from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.exceptions import ( InputError, WorkflowFilesError, ) from cylc.flow.loggingutil import ( CylcLogFormatter, close_log, get_next_log_number, get_sorted_logs_by_time, ) from cylc.flow.pathutil import ( expand_path, get_cylc_run_dir, get_next_rundir_number, get_workflow_run_dir, make_localhost_symlinks, ) from cylc.flow.remote import ( DEFAULT_RSYNC_OPTS, ) from cylc.flow.workflow_files import ( WorkflowFiles, abort_if_flow_file_in_path, check_flow_file, get_cylc_run_abs_path, is_valid_run_dir, validate_workflow_name, ) NESTED_DIRS_MSG = ( "Nested {dir_type} directories not allowed - cannot install workflow" " in '{dest}' as '{existing}' is already a valid {dir_type} directory." ) def _get_logger(rund, log_name, open_file=True): """Get log and create and open if necessary. Args: rund: The workflow run directory of the associated workflow. log_name: The name of the log to open. open_file: Open the appropriate log file and add it as a file handler to the logger. I.E. Start writing the log to a file if not already doing so. """ logger = logging.getLogger(log_name) logger.setLevel(logging.INFO) if open_file and not logger.hasHandlers(): _open_install_log(rund, logger) return logger def _open_install_log(rund, logger): """Open Cylc log handlers for install/reinstall.""" rund = Path(rund).expanduser() log_type = logger.name[logger.name.startswith('cylc-') and len('cylc-'):] log_dir = Path( rund, WorkflowFiles.LogDir.DIRNAME, WorkflowFiles.LogDir.INSTALL) log_files = get_sorted_logs_by_time(log_dir, '*.log') log_num = get_next_log_number(log_files[-1]) if log_files else 1 log_path = Path(log_dir, f"{log_num:02d}-{log_type}.log") log_parent_dir = log_path.parent log_parent_dir.mkdir(exist_ok=True, parents=True) handler = logging.FileHandler(log_path) handler.setFormatter(CylcLogFormatter()) logger.addHandler(handler) def get_rsync_rund_cmd(src, dst, reinstall=False, dry_run=False): """Create and return the rsync command used for cylc install/re-install. Args: src (str): file path location of source directory dst (str): file path location of destination directory reinstall (bool): indicate reinstall (--delete option added) dry-run (bool): indicate dry-run, rsync will not take place but report output if a real run were to be executed Return: list: command to use for rsync. """ rsync_cmd = shlex.split( glbl_cfg().get(['platforms', 'localhost', 'rsync command']) ) rsync_cmd += DEFAULT_RSYNC_OPTS if dry_run: rsync_cmd.append("--dry-run") if reinstall: rsync_cmd.append('--delete') exclusions = [ '.git', '.svn', '.cylcignore', 'opt/rose-suite-cylc-install.conf', WorkflowFiles.LogDir.DIRNAME, WorkflowFiles.WORK_DIR, WorkflowFiles.SHARE_DIR, WorkflowFiles.Install.DIRNAME, WorkflowFiles.Service.DIRNAME ] # This is a hack to make sure that changes to rose-suite.conf # are considered when re-installing. # It should be removed after https://github.com/cylc/cylc-rose/issues/149 if not dry_run: exclusions.append('rose-suite.conf') for exclude in exclusions: if ( Path(src).joinpath(exclude).exists() or Path(dst).joinpath(exclude).exists() ): # Note '/' is the rsync "anchor" to the top level: rsync_cmd.append(f"--exclude=/{exclude}") cylcignore_file = Path(src).joinpath('.cylcignore') if cylcignore_file.exists(): rsync_cmd.append(f"--exclude-from={cylcignore_file}") rsync_cmd.append(f"{src}/") rsync_cmd.append(f"{dst}/") return rsync_cmd def reinstall_workflow( source: Path, named_run: str, rundir: Path, dry_run: bool = False ) -> str: """Reinstall workflow. Args: source: source directory named_run: name of the run e.g. my-flow/run1 rundir: run directory dry_run: if True, will not execute the file transfer but report what would be changed. Raises: WorkflowFilesError: If rsync returns non-zero. Returns: Stdout from the rsync command. """ validate_source_dir(source, named_run) check_nested_dirs(rundir) reinstall_log = _get_logger( rundir, 'cylc-reinstall', open_file=not dry_run, # don't open the log file for --dry-run ) reinstall_log.info( f'Reinstalling "{named_run}", from "{source}" to "{rundir}"' ) rsync_cmd = get_rsync_rund_cmd( source, rundir, reinstall=True, dry_run=dry_run, ) rsync_cmd.append('--out-format=%i %o %n%L') # %i: itemized changes - needed for rsync to report files with # changed permissions # Run rsync command: reinstall_log.info(shlex.join(rsync_cmd)) LOG.debug(shlex.join(rsync_cmd)) proc = Popen(rsync_cmd, stdout=PIPE, stderr=PIPE, text=True) # nosec # * command is constructed via internal interface stdout, stderr = (i.strip() for i in proc.communicate()) reinstall_log.info( f"Copying files from {source} to {rundir}" f"\n{stdout}" ) if proc.returncode != 0: raise WorkflowFilesError( f'An error occurred reinstalling from {source} to {rundir}' f'\n{stderr}' ) check_flow_file(rundir) reinstall_log.info(f'REINSTALLED {named_run} from {source}') print( f'REINSTALL{"ED" if not dry_run else ""} {named_run} from {source}' ) close_log(reinstall_log) return stdout def install_workflow( source: Path, workflow_name: Optional[str] = None, run_name: Optional[str] = None, no_run_name: bool = False, cli_symlink_dirs: Optional[Dict[str, Dict[str, Any]]] = None ) -> Tuple[Path, Path, str, str]: """Install a workflow, or renew its installation. Install workflow into new run directory. Create symlink to workflow source location, creating any symlinks for run, work, log, share, share/cycle directories. Args: source: absolute path to workflow source directory. workflow_name: workflow name, default basename($PWD). run_name: name of the run, overrides run1, run2, run 3 etc... If specified, cylc install will not create runN symlink. rundir: for overriding the default cylc-run directory. no_run_name: Flag as True to install workflow into ~/cylc-run/ cli_symlink_dirs: Symlink dirs, if entered on the cli. Return: source: absolute path to source directory. rundir: absolute path to run directory, where the workflow has been installed into. workflow_name: installed workflow name (which may be computed here). named_run: Name of the run. Raise: WorkflowFilesError: No flow.cylc file found in source location. Illegal name (can look like a relative path, but not absolute). Another workflow already has this name. Trying to install a workflow that is nested inside of another. """ abort_if_flow_file_in_path(source) source = Path(expand_path(source)).resolve() if not workflow_name: workflow_name = get_source_workflow_name(source) if run_name is not None: if len(Path(run_name).parts) != 1: raise WorkflowFilesError( f'Run name cannot be a path. (You used {run_name})' ) validate_workflow_name( os.path.join(workflow_name, run_name), check_reserved_names=True ) else: validate_workflow_name(workflow_name, check_reserved_names=True) validate_source_dir(source, workflow_name) run_path_base = Path(get_workflow_run_dir(workflow_name)) relink, run_num, rundir = get_run_dir_info( run_path_base, run_name, no_run_name ) max_scan_depth = glbl_cfg().get(['install', 'max depth']) workflow_id = rundir.relative_to(get_cylc_run_dir()) if len(workflow_id.parts) > max_scan_depth: raise WorkflowFilesError( f"Cannot install: workflow ID '{workflow_id}' would exceed " f"global.cylc[install]max depth = {max_scan_depth}" ) check_nested_dirs(rundir, run_path_base) if rundir.exists(): raise WorkflowFilesError( f"'{rundir}' already exists\n" "To reinstall, use `cylc reinstall`" ) symlinks_created = {} named_run = workflow_name if run_name: named_run = os.path.join(named_run, run_name) elif run_num: named_run = os.path.join(named_run, f'run{run_num}') symlinks_created = make_localhost_symlinks( rundir, named_run, symlink_conf=cli_symlink_dirs) install_log = _get_logger(rundir, 'cylc-install') if symlinks_created: for target, symlink in symlinks_created.items(): install_log.info(f"Symlink created: {symlink} -> {target}") try: rundir.mkdir(exist_ok=True, parents=True) except FileExistsError: # This occurs when the file exists but is _not_ a directory. raise WorkflowFilesError( f"Cannot install as there is an existing file at {rundir}." ) from None if relink: link_runN(rundir) rsync_cmd = get_rsync_rund_cmd(source, rundir) proc = Popen(rsync_cmd, stdout=PIPE, stderr=PIPE, text=True) # nosec # * command is constructed via internal interface stdout, stderr = proc.communicate() install_log.info( f"Copying files from {source} to {rundir}" f"\n{stdout}" ) if proc.returncode != 0: install_log.warning( f"An error occurred when copying files from {source} to {rundir}") install_log.warning(f" Warning: {stderr}") cylc_install = Path(rundir.parent, WorkflowFiles.Install.DIRNAME) if no_run_name: cylc_install = Path(rundir, WorkflowFiles.Install.DIRNAME) source_link = cylc_install.joinpath(WorkflowFiles.Install.SOURCE) # check source link matches the source symlink from workflow dir. cylc_install.mkdir(parents=True, exist_ok=True) if not source_link.exists(): if source_link.is_symlink(): # Condition represents a broken symlink. raise WorkflowFilesError( f'Symlink broken: {source_link} -> {source_link.resolve()}.' ) install_log.info(f"Creating symlink from {source_link}") source_link.symlink_to(source.resolve()) else: if source_link.resolve() != source.resolve(): raise WorkflowFilesError( f"Failed to install from {source.resolve()}: " f"previous installations were from {source_link.resolve()}" ) install_log.info( f'Symlink from "{source_link}" to "{source}" in place.') install_log.info(f'INSTALLED {named_run} from {source}') close_log(install_log) return source, rundir, workflow_name, named_run def get_run_dir_info( run_path_base: Path, run_name: Optional[str], no_run_name: bool ) -> Tuple[bool, Optional[int], Path]: """Get (numbered, named or unnamed) run directory info for current install. Args: run_path_base: The workflow directory absolute path. run_name: Name of the run. no_run_name: Flag as True to indicate no run name - workflow installed into ~/cylc-run/. Returns: relink: True if runN symlink needs updating. run_num: Run number of the current install, if using numbered runs. rundir: Run directory absolute path. """ relink = False run_num = None if no_run_name: rundir = run_path_base elif run_name: rundir = run_path_base.joinpath(run_name) if run_path_base.exists() and detect_flow_exists(run_path_base, True): raise WorkflowFilesError( f"--run-name option not allowed as '{run_path_base}' contains " "installed numbered runs." ) else: run_num = get_next_rundir_number(run_path_base) rundir = Path(run_path_base, f'run{run_num}') if run_path_base.exists() and detect_flow_exists(run_path_base, False): raise WorkflowFilesError( f"Path: \"{run_path_base}\" contains an installed" " workflow. Use --run-name to create a new run." ) unlink_runN(run_path_base) relink = True return relink, run_num, rundir def get_source_dirs() -> List[str]: return glbl_cfg().get(['install', 'source dirs']) def search_install_source_dirs(workflow_name: Union[Path, str]) -> Path: """Return the path of a workflow source dir if it is present in the 'global.cylc[install]source dirs' search path.""" abort_if_flow_file_in_path(Path(workflow_name)) search_path: List[str] = get_source_dirs() if not search_path: raise WorkflowFilesError( "Cannot find workflow as 'global.cylc[install]source dirs' " "does not contain any paths") for path in search_path: try: return check_flow_file(Path(path, workflow_name)).parent except WorkflowFilesError: continue raise WorkflowFilesError( f"Could not find workflow '{workflow_name}' in: " f"{', '.join(search_path)}") def get_source_workflow_name(source: Path) -> str: """Return workflow name relative to configured source dirs if possible, else the basename of the given path. Note the source path provided should be fully expanded (user and env vars) and normalised. """ for dir_ in get_source_dirs(): try: return str(source.relative_to(Path(expand_path(dir_)).resolve())) except ValueError: continue return source.name def unlink_runN(path: Union[Path, str]) -> bool: """Remove symlink runN if it exists. Args: path: Absolute path to workflow dir containing runN. """ try: Path(expand_path(path, WorkflowFiles.RUN_N)).unlink() except OSError: return False return True def link_runN(latest_run: Union[Path, str]): """Create symlink runN, pointing at the latest run""" latest_run = Path(latest_run) run_n = Path(latest_run.parent, WorkflowFiles.RUN_N) with suppress(OSError): run_n.symlink_to(latest_run.name) def validate_source_dir( source: Union[Path, str], workflow_name: str, ) -> None: """Ensure the source directory is valid: - has flow file - does not contain reserved dir names Args: source: Path to source directory workflow_name: Workflow ID without the run identifier. Raises: WorkflowFilesError: If log, share, work or _cylc-install directories exist in the source directory. """ # Source dir must not contain reserved run dir names (as file or dir). for dir_ in WorkflowFiles.RESERVED_DIRNAMES: if Path(source, dir_).exists(): raise WorkflowFilesError( f"{workflow_name} installation failed " f"- {dir_} exists in source directory." ) check_flow_file(source, allow_suite_rc=False) def parse_cli_sym_dirs(symlink_dirs: str) -> Dict[str, Dict[str, Any]]: """Converts command line entered symlink dirs to a dictionary. Args: symlink_dirs: As entered by user on cli, e.g. "log=$DIR, share=$DIR2". Raises: WorkflowFilesError: If directory to be symlinked is not in permitted dirs: run, log, share, work, share/cycle Returns: dict: In the same form as would be returned by global config. e.g. {'localhost': {'log': '$DIR', 'share': '$DIR2' } } """ # Ensures the same nested dict format which is returned by the glb cfg symdict: Dict[str, Dict[str, Any]] = {'localhost': {'run': None}} if symlink_dirs == "": return symdict symlist = symlink_dirs.strip(',').split(',') possible_symlink_dirs = set(WorkflowFiles.SYMLINK_DIRS.union( {WorkflowFiles.RUN_DIR}) ) possible_symlink_dirs.remove('') for pair in symlist: try: key, val = pair.split("=") key = key.strip() except ValueError: raise InputError( 'There is an error in --symlink-dirs option:' f' {pair}. Try entering option in the form ' '--symlink-dirs=\'log=$DIR, share=$DIR2, ...\'' ) from None if key not in possible_symlink_dirs: dirs = ', '.join(possible_symlink_dirs) raise InputError( f"{key} not a valid entry for --symlink-dirs. " f"Configurable symlink dirs are: {dirs}" ) symdict['localhost'][key] = val.strip() or None return symdict def detect_flow_exists( run_path_base: Union[Path, str], numbered: bool ) -> bool: """Returns True if installed flow already exists. Args: run_path_base: Absolute path of the parent of the workflow's run dir, i.e ~/cylc-run/ numbered: If True, will detect if numbered runs exist. If False, will detect if non-numbered runs exist, i.e. runs installed by --run-name. """ for entry in Path(run_path_base).iterdir(): is_numbered = bool(re.search(r'^run\d+$', entry.name)) if ( entry.is_dir() and entry.name not in { WorkflowFiles.Install.DIRNAME, WorkflowFiles.RUN_N } and Path(entry, WorkflowFiles.FLOW_FILE).exists() and is_numbered == numbered ): return True return False def check_nested_dirs( run_dir: Path, install_dir: Optional[Path] = None ) -> None: """Disallow nested dirs: - Nested installed run dirs - Nested installed workflow dirs Args: run_dir: Absolute workflow run directory path. install_dir: Absolute workflow install directory path (contains _cylc-install). If None, will not check for nested install dirs. Raises: WorkflowFilesError if run_dir is nested inside an existing run dir, or install dirs are nested. """ if install_dir is not None: install_dir = Path(os.path.normpath(install_dir)) # Check parents: for parent_dir in run_dir.parents: # Stop searching at ~/cylc-run if parent_dir == Path(get_cylc_run_dir()): break # check for run directories: if is_valid_run_dir(parent_dir): raise WorkflowFilesError( NESTED_DIRS_MSG.format( dir_type='run', dest=run_dir, existing=get_cylc_run_abs_path(parent_dir) ) ) # Check for install directories: if ( install_dir and parent_dir in install_dir.parents and (parent_dir / WorkflowFiles.Install.DIRNAME).is_dir() ): raise WorkflowFilesError( NESTED_DIRS_MSG.format( dir_type='install', dest=run_dir, existing=get_cylc_run_abs_path(parent_dir) ) ) if install_dir: # Search child tree for install directories: for depth in range(glbl_cfg().get(['install', 'max depth'])): search_pattern = f'*/{"*/" * depth}{WorkflowFiles.Install.DIRNAME}' for result in install_dir.glob(search_pattern): raise WorkflowFilesError( NESTED_DIRS_MSG.format( dir_type='install', dest=run_dir, existing=get_cylc_run_abs_path(result.parent) ) ) cylc-flow-8.7.0/cylc/flow/templatevars.py0000664000175000017500000001451215264420307020601 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Load custom variables for template processor.""" from ast import literal_eval from typing import Any, Dict, List, Optional, Set, Union, TYPE_CHECKING from cylc.flow import LOG from cylc.flow.exceptions import InputError from cylc.flow.rundb import CylcWorkflowDAO from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager from cylc.flow.workflow_files import WorkflowFiles if TYPE_CHECKING: from optparse import Values from pathlib import Path OVERWRITE_WARNING = ( 'Template variable {} redefined:' ' the previous value will be overwritten.' ) def get_template_vars_from_db(run_dir: 'Path') -> dict: """Get template vars stored in a workflow run database. """ pub_db_file = ( run_dir / WorkflowFiles.LogDir.DIRNAME / WorkflowFiles.LogDir.DB ) template_vars: dict = {} if not pub_db_file.exists(): return template_vars WorkflowDatabaseManager.check_db_compatibility(pub_db_file) with CylcWorkflowDAO(pub_db_file, is_public=True) as dao: dao.select_workflow_template_vars( lambda _, row: template_vars.__setitem__(row[0], eval_var(row[1])) ) return template_vars def eval_var(var): """Wrap ast.literal_eval to provide more helpful error. Examples: >>> eval_var('42') 42 >>> eval_var('"string"') 'string' >>> eval_var('string') Traceback (most recent call last): cylc.flow.exceptions.InputError: Invalid template variable: string (note string values must be quoted) >>> eval_var('[') Traceback (most recent call last): cylc.flow.exceptions.InputError: Invalid template variable: [ (values must be valid Python literals) """ try: return literal_eval(var) except ValueError: raise InputError( f'Invalid template variable: {var}' '\n(note string values must be quoted)' ) from None except SyntaxError: raise InputError( f'Invalid template variable: {var}' '\n(values must be valid Python literals)' ) from None def parse_string_list(stringlist: str) -> list[str]: """Parse a comma separated string list into a Python string list. Examples: >>> parse_string_list('a,b,c') ['a', 'b', 'c'] >>> parse_string_list('a,"b,b",c') ['a', 'b,b', 'c'] >>> parse_string_list("a,'b,b','c'") ['a', 'b,b', 'c'] """ list_ = [] in_quotes = False buffer = '' for char in stringlist: if char in {'"', "'"}: in_quotes = not in_quotes elif not in_quotes and char == ',': list_.append(buffer) buffer = '' else: buffer += char list_.append(buffer) return list_ def load_template_vars( template_vars: Optional[List[str]] = None, template_vars_file: 'Union[Path, str, None]' = None, templatevars_lists: Optional[List[str]] = None ) -> Dict[str, Any]: """Load template variables from key=value strings.""" keys: Set[str] = set() invalid_vars: str = '' # Parse Template vars set by file (-S) file_tvars: Dict[str, Any] = {} if template_vars_file: with open(template_vars_file, 'r') as handle: for line in handle: line = line.strip().split("#", 1)[0] if not line: continue try: key, val = line.split("=", 1) except ValueError: invalid_vars += f'\n * {line}' continue file_tvars[key.strip()] = eval_var(val.strip()) keys.add(key.strip()) cli_tvars: Dict[str, Any] = {} tvars_lists: dict[str, list[str]] = {} for input_, result, func in ( (template_vars, cli_tvars, eval_var), (templatevars_lists, tvars_lists, parse_string_list) ): for pair in input_ or []: # Handle ValueError try: key, val = pair.split("=", 1) except ValueError: invalid_vars += f'\n * {pair}' continue key, val = key.strip(), val.strip() if key in result: LOG.warning(OVERWRITE_WARNING.format(key)) result[key] = func(val) keys.add(key) # Raise an error if there are any args without the form key=value.: if invalid_vars: raise InputError( 'Template variables must be set with key=value(s):' + invalid_vars ) # Explicitly record which source of tvars over-rides which. res = {} badkeys = '' for key in keys: if key in cli_tvars and key in tvars_lists: badkeys += ( f"\n * {key}={cli_tvars[key]} and {key}={tvars_lists[key]}") else: res[key] = cli_tvars.get( key, tvars_lists.get( key, file_tvars.get(key))) # Raise an error if there are any key-clashes between tvars and tvars_list if badkeys: raise InputError( "You can't set the same template variable with both -s and -z:" + badkeys ) return res def get_template_vars(options: 'Values') -> Dict[str, Any]: """Convenience wrapper for ``load_template_vars``. Args: options: Options passed to the Cylc script which is using this function. Returns: template_vars: Template variables to give to a Cylc config. """ return load_template_vars( options.templatevars, options.templatevars_file, options.templatevars_lists ) cylc-flow-8.7.0/cylc/flow/job_runner_mgr.py0000664000175000017500000010341415264420307021102 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Manage submission, poll and kill of a job to the job runners. The job runner interface is documented in cylc.flow.job_runner_handlers.documentation. Please update this file as the interface changes. """ from contextlib import suppress import json import os from pathlib import Path import shlex import stat import sys import traceback from shutil import rmtree from signal import SIGKILL from subprocess import DEVNULL # nosec from cylc.flow.task_message import ( CYLC_JOB_PID, CYLC_JOB_INIT_TIME, CYLC_JOB_EXIT_TIME, CYLC_JOB_EXIT, CYLC_MESSAGE) from cylc.flow.cylc_subproc import procopen from cylc.flow.task_job_logs import ( JOB_LOG_ERR, JOB_LOG_JOB, JOB_LOG_OUT, JOB_LOG_STATUS) from cylc.flow.task_outputs import TASK_OUTPUT_SUCCEEDED from cylc.flow.wallclock import get_current_time_string from cylc.flow.parsec.OrderedDict import OrderedDict JOB_FILES_REMOVED_MESSAGE = 'ERR_JOB_FILES_REMOVED' class JobPollContext(): """Context object for a job poll.""" CONTEXT_ATTRIBUTES = ( 'job_log_dir', # cycle/task/submit_num 'job_runner_name', 'job_id', # job id in job runner 'job_runner_exit_polled', # 0 for false, 1 for true 'run_status', # 0 for success, 1 for failure 'run_signal', # signal received on run failure 'time_submit_exit', # submit (exit) time 'time_run', # run start time 'time_run_exit', # run exit time 'job_runner_call_no_lines', # line count in job runner call stdout ) __slots__ = CONTEXT_ATTRIBUTES + ( 'pid', 'messages' ) def __init__(self, job_log_dir, **attrs): self.job_log_dir = job_log_dir self.job_runner_name = None self.job_id = None self.job_runner_exit_polled = None self.pid = None self.run_status = None self.run_signal = None self.time_submit_exit = None self.time_run = None self.time_run_exit = None self.job_runner_call_no_lines = None self.messages = [] if attrs: for key, value in attrs.items(): if key not in self.CONTEXT_ATTRIBUTES: raise ValueError('Invalid kwarg "%s"' % key) setattr(self, key, value) def update(self, other): """Update my data from given file context.""" for i in self.__slots__: setattr(self, i, getattr(other, i)) def get_summary_str(self): """Return the poll context as a summary string delimited by "|".""" ret = OrderedDict() for key in self.CONTEXT_ATTRIBUTES: value = getattr(self, key) if key == 'job_log_dir' or value is None: continue ret[key] = value return '%s|%s' % (self.job_log_dir, json.dumps(ret)) class JobRunnerManager(): """Job submission, poll and kill. Manage the importing of job submission method modules. """ CYLC_JOB_RUNNER_NAME = "CYLC_JOB_RUNNER_NAME" CYLC_JOB_ID = "CYLC_JOB_ID" CYLC_JOB_RUNNER_SUBMIT_TIME = "CYLC_JOB_RUNNER_SUBMIT_TIME" CYLC_JOB_RUNNER_EXIT_POLLED = "CYLC_JOB_RUNNER_EXIT_POLLED" FAIL_SIGNALS = ("EXIT", "ERR", "TERM", "XCPU") LINE_PREFIX_JOB_RUNNER_NAME = "# Job runner: " LINE_PREFIX_JOB_RUNNER_CMD_TMPL = "# Job runner command template: " LINE_PREFIX_EXECUTION_TIME_LIMIT = "# Execution time limit: " LINE_PREFIX_EOF = "#EOF: " LINE_PREFIX_JOB_LOG_DIR = "# Job log directory: " OUT_PREFIX_COMMAND = "[TASK JOB COMMAND]" OUT_PREFIX_MESSAGE = "[TASK JOB MESSAGE]" OUT_PREFIX_SUMMARY = "[TASK JOB SUMMARY]" OUT_PREFIX_CMD_ERR = "[TASK JOB ERROR]" _INSTANCES: dict = {} @classmethod def configure_workflow_run_dir(cls, workflow_run_dir): """Add local python module paths if not already done.""" for sub_dir in ["python", os.path.join("lib", "python")]: # TODO - eventually drop the deprecated "python" sub-dir. workflow_py = os.path.join(workflow_run_dir, sub_dir) if os.path.isdir(workflow_py) and workflow_py not in sys.path: sys.path.append(workflow_py) def __init__(self, clean_env=False, env=None, path=None): """Initialise JobRunnerManager.""" # Job submission environment. self.clean_env = clean_env self.path = path self.env = env def _get_sys(self, job_runner_name): """Return an instance of the class for "job_runner_name".""" if job_runner_name in self._INSTANCES: return self._INSTANCES[job_runner_name] for key in [f"cylc.flow.job_runner_handlers.{job_runner_name}", job_runner_name]: try: mod_of_name = __import__(key, fromlist=[key]) self._INSTANCES[job_runner_name] = getattr( mod_of_name, "JOB_RUNNER_HANDLER", None) return self._INSTANCES[job_runner_name] except ImportError: if key == job_runner_name: raise def format_directives(self, job_conf): """Format the job directives for a job file, if relevant.""" job_runner = self._get_sys(job_conf['platform']['job runner']) if hasattr(job_runner, "format_directives"): job_conf = { # strip $HOME from the job file path # paths in directives should be interpreted relative to $HOME # https://github.com/cylc/cylc-flow/issues/4247 **job_conf, 'job_file_path': ( job_conf["job_file_path"].replace(r"$HOME/", "") ) } return job_runner.format_directives(job_conf) def get_fail_signals(self, job_conf): """Return a list of failure signal names to trap in the job file.""" job_runner = self._get_sys(job_conf['platform']['job runner']) return getattr(job_runner, "FAIL_SIGNALS", self.FAIL_SIGNALS) def get_vacation_signal(self, job_conf): """Return the vacation signal name for a job file.""" job_runner = self._get_sys(job_conf['platform']['job runner']) if hasattr(job_runner, "get_vacation_signal"): return job_runner.get_vacation_signal(job_conf) def is_job_local_to_host(self, job_runner_name): """Return True if job runner runs jobs local to the submit host.""" return getattr( self._get_sys(job_runner_name), "SHOULD_KILL_PROC_GROUP", False) def jobs_kill(self, job_log_root, job_log_dirs): """Kill multiple jobs. job_log_root -- The log/job/ sub-directory of the workflow. job_log_dirs -- A list containing point/name/submit_num for jobs. """ # Note: The more efficient way to do this is to group the jobs by their # job runners, and call the kill command for each job runner once. # However, this will make it more difficult to determine if the kill # command for a particular job is successful or not. if "$" in job_log_root: job_log_root = os.path.expandvars(job_log_root) self.configure_workflow_run_dir(job_log_root.rsplit(os.sep, 2)[0]) now = get_current_time_string() for job_log_dir in job_log_dirs: ret_code, err = self.job_kill( os.path.join(job_log_root, job_log_dir, JOB_LOG_STATUS)) sys.stdout.write("%s%s|%s|%d\n" % ( self.OUT_PREFIX_SUMMARY, now, job_log_dir, ret_code)) # Note: Print STDERR to STDOUT may look a bit strange, but it # requires less logic for the workflow to parse the output. for line in err.strip().splitlines(): sys.stdout.write( f"{self.OUT_PREFIX_CMD_ERR}{now}|{job_log_dir}|{line}\n" ) def jobs_poll(self, job_log_root, job_log_dirs): """Poll multiple jobs. job_log_root -- The log/job/ sub-directory of the workflow. job_log_dirs -- A list containing point/name/submit_num for jobs. """ if "$" in job_log_root: job_log_root = os.path.expandvars(job_log_root) self.configure_workflow_run_dir(job_log_root.rsplit(os.sep, 2)[0]) ctx_list = [] # Contexts for all relevant jobs ctx_list_by_job_runner = {} # {job_runner_name1: [ctx1, ...], ...} for job_log_dir in job_log_dirs: ctx = self._jobs_poll_status_files(job_log_root, job_log_dir) if ctx is None: continue ctx_list.append(ctx) if not ctx.job_runner_name or not ctx.job_id: # Lost job runner information for some reason. # Mark the job as if it is no longer in the job runner. ctx.job_runner_exit_polled = 1 sys.stderr.write( "%s/%s: incomplete job runner info\n" % ( ctx.job_log_dir, JOB_LOG_STATUS)) # We can trust: # * Jobs previously polled to have exited the job runner. # * Jobs succeeded or failed with ERR/EXIT. if (ctx.job_runner_exit_polled or ctx.run_status == 0 or ctx.run_signal in ["ERR", "EXIT"]): continue if ctx.job_runner_name not in ctx_list_by_job_runner: ctx_list_by_job_runner[ctx.job_runner_name] = [] ctx_list_by_job_runner[ctx.job_runner_name].append(ctx) for job_runner_name, my_ctx_list in ctx_list_by_job_runner.items(): self._jobs_poll_runner( job_log_root, job_runner_name, my_ctx_list) cur_time_str = get_current_time_string() for ctx in ctx_list: for message in ctx.messages: sys.stdout.write("%s%s|%s|%s\n" % ( self.OUT_PREFIX_MESSAGE, cur_time_str, ctx.job_log_dir, message)) sys.stdout.write("%s%s|%s\n" % ( self.OUT_PREFIX_SUMMARY, cur_time_str, ctx.get_summary_str())) def jobs_submit(self, job_log_root, job_log_dirs, remote_mode=False, utc_mode=False): """Submit multiple jobs. job_log_root -- The log/job/ sub-directory of the workflow. job_log_dirs -- A list containing point/name/submit_num for jobs. remote_mode -- am I running on the remote job host? utc_mode -- is the workflow running in UTC mode? """ if "$" in job_log_root: job_log_root = os.path.expandvars(job_log_root) self.configure_workflow_run_dir(job_log_root.rsplit(os.sep, 2)[0]) if remote_mode: items = self._jobs_submit_prep_by_stdin(job_log_root, job_log_dirs) else: items = self._jobs_submit_prep_by_args(job_log_root, job_log_dirs) now = get_current_time_string(override_use_utc=utc_mode) for job_log_dir, job_runner_name, submit_opts in items: job_file_path = os.path.join( job_log_root, job_log_dir, JOB_LOG_JOB) if not job_runner_name: sys.stdout.write("%s%s|%s|1|\n" % ( self.OUT_PREFIX_SUMMARY, now, job_log_dir)) continue ret_code, out, err, job_id = self._job_submit_impl( job_file_path, job_runner_name, submit_opts) sys.stdout.write("%s%s|%s|%d|%s\n" % ( self.OUT_PREFIX_SUMMARY, now, job_log_dir, ret_code, job_id)) for key, value in [("STDERR", err), ("STDOUT", out)]: if value is None: continue for line in value.strip().splitlines(): sys.stdout.write( f"{self.OUT_PREFIX_COMMAND}{now}" f"|{job_log_dir}|[{key}] {line}\n" ) def job_kill(self, st_file_path): """Ask job runner to terminate the job specified in "st_file_path". Return 0 on success, non-zero integer on failure. """ # WORKFLOW_RUN_DIR/log/job/CYCLE/TASK/SUBMIT/job.status self.configure_workflow_run_dir(st_file_path.rsplit(os.sep, 6)[0]) try: with open(st_file_path) as st_file: for line in st_file: if line.startswith(f"{self.CYLC_JOB_RUNNER_NAME}="): job_runner = self._get_sys( line.strip().split("=", 1)[1] ) break else: return ( 1, "Cannot determine job runner from " f"{JOB_LOG_STATUS} file" ) st_file.seek(0, 0) # rewind if getattr(job_runner, "SHOULD_KILL_PROC_GROUP", False): for line in st_file: if line.startswith(CYLC_JOB_PID + "="): pid = line.strip().split("=", 1)[1] try: os.killpg(os.getpgid(int(pid)), SIGKILL) except (OSError, ValueError) as exc: traceback.print_exc() return (1, str(exc)) else: return (0, "") st_file.seek(0, 0) # rewind if hasattr(job_runner, "KILL_CMD_TMPL"): for line in st_file: if not line.startswith(f"{self.CYLC_JOB_ID}="): continue job_id = line.strip().split("=", 1)[1] command = shlex.split( job_runner.KILL_CMD_TMPL % {"job_id": job_id}) try: proc = procopen(command, stdindevnull=True, stderrpipe=True) except OSError as exc: # subprocess.Popen has a bad habit of not setting # the filename of the executable when it raises an # OSError. if not exc.filename: exc.filename = command[0] traceback.print_exc() return (1, str(exc)) else: return ( proc.wait(), proc.communicate()[1].decode() ) return (1, f"Cannot determine job ID from {JOB_LOG_STATUS} file") except IOError as exc: return (1, str(exc)) @classmethod def _create_nn(cls, job_file_path): """Create NN symbolic link if necessary, and remove any old job logs. If NN => 01, remove numbered dirs with submit numbers greater than 01. Helper for "self._job_submit_impl". """ job_file_dir = os.path.dirname(job_file_path) source = os.path.basename(job_file_dir) task_log_dir = os.path.dirname(job_file_dir) nn_path = os.path.join(task_log_dir, "NN") try: old_source = os.readlink(nn_path) except OSError: old_source = None if old_source is not None and old_source != source: os.unlink(nn_path) old_source = None if old_source is None: os.symlink(source, nn_path) # On submit 1, remove any left over digit directories from prev runs if source == "01": for name in os.listdir(task_log_dir): if name != source and name.isdigit(): # Ignore errors, not disastrous if rmtree fails rmtree( os.path.join(task_log_dir, name), ignore_errors=True) # Delete old job logs if necessary for name in JOB_LOG_ERR, JOB_LOG_OUT: with suppress(FileNotFoundError): os.unlink(os.path.join(job_file_dir, name)) @classmethod def _filter_submit_output(cls, st_file_path, job_runner, out, err): """Filter submit command output, if relevant.""" job_id = None if hasattr(job_runner, "REC_ID_FROM_SUBMIT_ERR"): text = err rec_id = job_runner.REC_ID_FROM_SUBMIT_ERR elif hasattr(job_runner, "REC_ID_FROM_SUBMIT_OUT"): text = out rec_id = job_runner.REC_ID_FROM_SUBMIT_OUT if rec_id: for line in str(text).splitlines(): match = rec_id.match(line) if match: job_id = match.group("id") if hasattr(job_runner, "manip_job_id"): job_id = job_runner.manip_job_id(job_id) with open(st_file_path, "a") as job_status_file: job_status_file.write("{0}={1}\n".format( cls.CYLC_JOB_ID, job_id)) job_status_file.write("{0}={1}\n".format( cls.CYLC_JOB_RUNNER_SUBMIT_TIME, get_current_time_string())) break if hasattr(job_runner, "filter_submit_output"): out, err = job_runner.filter_submit_output(out, err) return out, err, job_id def _jobs_poll_status_files(self, job_log_root, job_log_dir): """Helper 1 for self.jobs_poll(job_log_root, job_log_dirs).""" ctx = JobPollContext(job_log_dir) # If the log directory has been deleted prematurely, return a task # failure and an explanation: if not os.path.exists(os.path.join(job_log_root, ctx.job_log_dir)): # The job may still be in the job runner and may yet succeed, # but we assume it failed & exited because it's the best we # can do as it is no longer possible to poll it. ctx.run_status = 1 ctx.job_runner_exit_polled = 1 ctx.run_signal = JOB_FILES_REMOVED_MESSAGE return ctx try: with open( os.path.join(job_log_root, ctx.job_log_dir, JOB_LOG_STATUS) ) as handle: for line in handle: if "=" not in line: continue key, value = line.strip().split("=", 1) if key == self.CYLC_JOB_RUNNER_NAME: ctx.job_runner_name = value elif key == self.CYLC_JOB_ID: ctx.job_id = value elif key == self.CYLC_JOB_RUNNER_EXIT_POLLED: ctx.job_runner_exit_polled = 1 elif key == CYLC_JOB_PID: ctx.pid = value elif key == self.CYLC_JOB_RUNNER_SUBMIT_TIME: ctx.time_submit_exit = value elif key == CYLC_JOB_INIT_TIME: ctx.time_run = value elif key == CYLC_JOB_EXIT_TIME: ctx.time_run_exit = value elif key == CYLC_JOB_EXIT: if value == TASK_OUTPUT_SUCCEEDED.upper(): ctx.run_status = 0 else: ctx.run_status = 1 ctx.run_signal = value elif key == CYLC_MESSAGE: ctx.messages.append(value) except IOError as exc: sys.stderr.write(f"{exc}\n") return return ctx def _jobs_poll_runner(self, job_log_root, job_runner_name, my_ctx_list): """Helper 2 for self.jobs_poll(job_log_root, job_log_dirs).""" exp_job_ids = [ctx.job_id for ctx in my_ctx_list] bad_job_ids = list(exp_job_ids) exp_pids = [] bad_pids = [] items = [[self._get_sys(job_runner_name), exp_job_ids, bad_job_ids]] if getattr(items[0][0], "SHOULD_POLL_PROC_GROUP", False): exp_pids = [ctx.pid for ctx in my_ctx_list if ctx.pid is not None] bad_pids.extend(exp_pids) items.append([self._get_sys("background"), exp_pids, bad_pids]) debug_messages = [] for job_runner, exp_ids, bad_ids in items: if hasattr(job_runner, "get_poll_many_cmd"): # Some poll commands may not be as simple cmd = job_runner.get_poll_many_cmd(exp_ids) else: # if hasattr(job_runner, "POLL_CMD"): # Simple poll command that takes a list of job IDs cmd = [job_runner.POLL_CMD, *exp_ids] try: proc = procopen(cmd, stdindevnull=True, stderrpipe=True, stdoutpipe=True) except OSError as exc: # subprocess.Popen has a bad habit of not setting the # filename of the executable when it raises an OSError. if not exc.filename: exc.filename = cmd[0] sys.stderr.write(f"{exc}\n") return ret_code = proc.wait() out, err = (f.decode() for f in proc.communicate()) debug_messages.append('{0} - {1}'.format( job_runner, len(out.split('\n'))) ) sys.stderr.write(err) if (ret_code and hasattr(job_runner, "POLL_CANT_CONNECT_ERR") and job_runner.POLL_CANT_CONNECT_ERR in err): # Poll command failed because it cannot connect to job runner # Assume jobs are still healthy until the job runner is back. bad_ids[:] = [] elif hasattr(job_runner, "filter_poll_many_output"): # Allow custom filter for id_ in job_runner.filter_poll_many_output(out): with suppress(ValueError): bad_ids.remove(id_) else: # Just about all poll commands return a table, with column 1 # being the job ID. The logic here should be sufficient to # ensure that any table header is ignored. for line in out.splitlines(): try: head = line.split(None, 1)[0] except IndexError: continue if head in exp_ids: with suppress(ValueError): bad_ids.remove(head) debug_flag = False for ctx in my_ctx_list: ctx.job_runner_exit_polled = int( ctx.job_id in bad_job_ids) # Exited job runner, but process still running # This can happen to jobs in some "at" implementation if ctx.job_runner_exit_polled and ctx.pid in exp_pids: if ctx.pid not in bad_pids: ctx.job_runner_exit_polled = 0 else: debug_flag = True # Add information to "job.status" if ctx.job_runner_exit_polled: try: with open(os.path.join( job_log_root, ctx.job_log_dir, JOB_LOG_STATUS), "a" ) as handle: handle.write("{0}={1}\n".format( self.CYLC_JOB_RUNNER_EXIT_POLLED, get_current_time_string()) ) except IOError as exc: sys.stderr.write(f"{exc}\n") # Re-read the status file in case the job started and exited # between the file and batch system checks, which would be # interpreted as submit-failed (job exited without starting). # Possible if polling many jobs and/or system heavily loaded. file_ctx = self._jobs_poll_status_files( job_log_root, ctx.job_log_dir) ctx.update(file_ctx) if debug_flag: ctx.job_runner_call_no_lines = ', '.join(debug_messages) def _job_submit_impl( self, job_file_path, job_runner_name, submit_opts): """Helper for self.jobs_submit() and self.job_submit().""" # Create NN symbolic link, if necessary self._create_nn(job_file_path) # Start new status file with open(f"{job_file_path}.status", "w") as job_status_file: job_status_file.write( "{0}={1}\n".format( self.CYLC_JOB_RUNNER_NAME, job_runner_name ) ) # Submit job job_runner = self._get_sys(job_runner_name) if not self.clean_env: # Pass the whole environment to the job submit subprocess. # (Note this runs on the job host). # NOTE:: We modify this below so use a copy! # See: https://github.com/cylc/cylc-flow/issues/7333 env = dict(os.environ) else: # $HOME is required by job.sh on the job host. env = {'HOME': os.environ.get('HOME', '')} # Pass selected extra variables to the job submit subprocess. for var in self.env: env[var] = os.environ.get(var, '') if self.path is not None: # Append to avoid overriding an inherited PATH (e.g. in a venv) env['PATH'] = env.get('PATH', '') + ':' + ':'.join(self.path) if hasattr(job_runner, "submit"): submit_opts['env'] = env # job_runner.submit should handle OSError, if relevant. ret_code, out, err = job_runner.submit(job_file_path, submit_opts) else: proc_stdin_arg = None # Set command STDIN to DEVNULL by default to prevent leakage of # STDIN from current environment. proc_stdin_value = DEVNULL # nosec if hasattr(job_runner, "get_submit_stdin"): proc_stdin_arg, proc_stdin_value = job_runner.get_submit_stdin( job_file_path, submit_opts) if isinstance(proc_stdin_value, str): proc_stdin_value = proc_stdin_value.encode() if hasattr(job_runner, "SUBMIT_CMD_ENV"): env.update(job_runner.SUBMIT_CMD_ENV) job_runner_cmd_tmpl = submit_opts.get("job_runner_cmd_tmpl") if job_runner_cmd_tmpl: # No need to catch OSError when using shell. It is unlikely # that we do not have a shell, and still manage to get as far # as here. job_runner_cmd = job_runner_cmd_tmpl % {"job": job_file_path} proc = procopen(job_runner_cmd, stdin=proc_stdin_arg, stdoutpipe=True, stderrpipe=True, usesh=True, env=env) # calls to open a shell are aggregated in # cylc_subproc.procopen() else: command = shlex.split( job_runner.SUBMIT_CMD_TMPL % {"job": job_file_path}) try: proc = procopen( command, stdin=proc_stdin_arg, stdoutpipe=True, stderrpipe=True, env=env, # paths in directives should be interpreted relative to # $HOME # https://github.com/cylc/cylc-flow/issues/4247 cwd=Path('~').expanduser() ) except OSError as exc: # subprocess.Popen has a bad habit of not setting the # filename of the executable when it raises an OSError. if not exc.filename: exc.filename = command[0] return 1, "", str(exc), "" out, err = (f.decode() for f in proc.communicate(proc_stdin_value)) ret_code = proc.wait() with suppress(AttributeError, IOError): proc_stdin_arg.close() # Filter submit command output, if relevant # Get job ID, if possible job_id = None if out or err: try: out, err, job_id = self._filter_submit_output( f"{job_file_path}.status", job_runner, out, err) except OSError: ret_code = 1 self.job_kill(f"{job_file_path}.status") return ret_code, out, err, job_id def _jobs_submit_prep_by_args(self, job_log_root, job_log_dirs): """Prepare job files for submit by reading files in arguments. Job files are specified in the arguments in local mode. Extract job submission methods and job submission command templates from each job file. Return a list, where each element contains something like: (job_log_dir, job_runner_name, submit_opts) """ items = [] for job_log_dir in job_log_dirs: job_file_path = os.path.join(job_log_root, job_log_dir, "job") job_runner_name = None submit_opts = {} with open(job_file_path, 'r') as job_file: for line in job_file: if line.startswith(self.LINE_PREFIX_JOB_RUNNER_NAME): job_runner_name = line.replace( self.LINE_PREFIX_JOB_RUNNER_NAME, "").strip() elif line.startswith(self.LINE_PREFIX_JOB_RUNNER_CMD_TMPL): submit_opts["job_runner_cmd_tmpl"] = line.replace( self.LINE_PREFIX_JOB_RUNNER_CMD_TMPL, "").strip() elif line.startswith( self.LINE_PREFIX_EXECUTION_TIME_LIMIT ): submit_opts["execution_time_limit"] = float( line.replace( self.LINE_PREFIX_EXECUTION_TIME_LIMIT, "" ).strip() ) items.append((job_log_dir, job_runner_name, submit_opts)) return items def _jobs_submit_prep_by_stdin(self, job_log_root, job_log_dirs): """Prepare job files for submit by reading from STDIN. Job files are uploaded via STDIN in remote mode. Extract job submission methods and job submission command templates from each job file. Return a list, where each element contains something like: (job_log_dir, job_runner_name, submit_opts) """ items = [[job_log_dir, None, {}] for job_log_dir in job_log_dirs] items_map = {} for item in items: items_map[item[0]] = item handle = None job_runner_name = None submit_opts = {} job_log_dir = None lines = [] # Get job files from STDIN. # Get job runner name and job runner command template from each job # file. # Write job file in correct location. while True: # Note: "for cur_line in sys.stdin:" may hang cur_line = sys.stdin.readline() if not cur_line: if handle is not None: handle.close() break if cur_line.startswith(self.LINE_PREFIX_JOB_RUNNER_NAME): job_runner_name = cur_line.replace( self.LINE_PREFIX_JOB_RUNNER_NAME, "").strip() elif cur_line.startswith(self.LINE_PREFIX_JOB_RUNNER_CMD_TMPL): submit_opts["job_runner_cmd_tmpl"] = cur_line.replace( self.LINE_PREFIX_JOB_RUNNER_CMD_TMPL, "").strip() elif cur_line.startswith(self.LINE_PREFIX_EXECUTION_TIME_LIMIT): submit_opts["execution_time_limit"] = float(cur_line.replace( self.LINE_PREFIX_EXECUTION_TIME_LIMIT, "").strip()) elif cur_line.startswith(self.LINE_PREFIX_JOB_LOG_DIR): job_log_dir = cur_line.replace( self.LINE_PREFIX_JOB_LOG_DIR, "").strip() os.makedirs( os.path.join(job_log_root, job_log_dir), exist_ok=True) if handle is not None: handle.close() handle = open( # noqa: SIM115 (can't convert to with open) os.path.join(job_log_root, job_log_dir, "job.tmp"), "wb") if handle is None: lines.append(cur_line) else: for line in lines + [cur_line]: handle.write(line.encode()) lines = [] if cur_line.startswith(self.LINE_PREFIX_EOF + job_log_dir): handle.close() # Make it executable os.chmod(handle.name, ( os.stat(handle.name).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)) # Rename from "*/job.tmp" to "*/job" os.rename(handle.name, handle.name[:-4]) try: items_map[job_log_dir][1] = job_runner_name items_map[job_log_dir][2] = submit_opts except KeyError: pass handle = None job_log_dir = None job_runner_name = None submit_opts = {} return items cylc-flow-8.7.0/cylc/flow/config.py0000664000175000017500000033420715264420307017345 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Parse and validate the workflow definition file Set local values of variables to give workflow context before parsing config, i.e for template filters (Jinja2, python ...etc) and possibly needed locally by event handlers. This is needed for both running and non-running workflow parsing (obtaining config/graph info). Potentially task-specific due to different directory paths on different task hosts, however, they are overridden by tasks prior to job submission. Do some consistency checking, then construct task proxy objects and graph structures. """ from contextlib import suppress from copy import copy from fnmatch import fnmatchcase import os from pathlib import Path import re from textwrap import wrap import traceback from types import SimpleNamespace from typing import ( TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Literal, Mapping, Optional, Set, Tuple, Union, ) from metomi.isodatetime.data import Calendar from metomi.isodatetime.dumpers import TimePointDumper from metomi.isodatetime.exceptions import IsodatetimeError from metomi.isodatetime.parsers import DurationParser from metomi.isodatetime.timezone import get_local_time_zone_format from cylc.flow import LOG from cylc.flow.c3mro import C3 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.cfgspec.workflow import RawWorkflowConfig from cylc.flow.cycling.integer import IntegerInterval from cylc.flow.cycling.iso8601 import ( ISO8601Interval, ingest_time, ) from cylc.flow.cycling.loader import ( INTEGER_CYCLING_TYPE, ISO8601_CYCLING_TYPE, get_dump_format, get_interval, get_interval_cls, get_point, get_point_relative, get_sequence, get_sequence_cls, init_cyclers, ) from cylc.flow.exceptions import ( CylcError, InputError, IntervalParsingError, ParamExpandError, TaskDefError, WorkflowConfigError, ) import cylc.flow.flags from cylc.flow.graph_parser import GraphParser from cylc.flow.graphnode import GraphNodeParser from cylc.flow.id import Tokens from cylc.flow.listify import listify from cylc.flow.log_level import verbosity_to_env from cylc.flow.param_expand import NameExpander from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults from cylc.flow.parsec.exceptions import ItemNotFoundError from cylc.flow.parsec.upgrade import upgrader from cylc.flow.parsec.util import ( dequote, replicate, ) from cylc.flow.pathutil import ( get_cylc_run_dir, get_workflow_name_from_id, is_relative_to, ) from cylc.flow.run_modes import ( WORKFLOW_ONLY_MODES, RunMode, ) from cylc.flow.run_modes.simulation import configure_sim_mode from cylc.flow.run_modes.skip import skip_mode_validate from cylc.flow.subprocctx import SubFuncContext from cylc.flow.task_events_mgr import ( EventData, TaskEventsManager, get_event_handler_data, ) from cylc.flow.task_id import TaskID from cylc.flow.task_outputs import ( TASK_OUTPUT_FAILED, TASK_OUTPUT_FINISHED, TASK_OUTPUT_SUCCEEDED, TaskOutputs, get_completion_expression, get_optional_outputs, get_trigger_completion_variable_maps, trigger_to_completion_variable, ) from cylc.flow.task_qualifiers import ( ALT_QUALIFIERS, TASK_QUALIFIERS, ) from cylc.flow.task_trigger import ( Dependency, TaskTrigger, ) from cylc.flow.taskdef import TaskDef from cylc.flow.unicode_rules import ( TaskMessageValidator, TaskNameValidator, TaskOutputValidator, XtriggerNameValidator, ) from cylc.flow.wallclock import ( get_current_time_string, get_utc_mode, set_utc_mode, ) from cylc.flow.workflow_events import WorkflowEventHandler from cylc.flow.workflow_files import ( NO_TITLE, WorkflowFiles, ) from cylc.flow.xtrigger_mgr import XtriggerCollator if TYPE_CHECKING: from optparse import Values from cylc.flow.cycling import ( IntervalBase, PointBase, SequenceBase, ) RE_CLOCK_OFFSET = re.compile( rf''' ^ \s* ({TaskID.NAME_RE}) # task name (?:\(\s*(.+)\s*\))? # optional (arguments, ...) \s* $ ''', re.X, ) RE_EXT_TRIGGER = re.compile( r''' ^ \s* (.*) # task name \s* \(\s*(.+)\s*\) # required (arguments, ...) \s* $ ''', re.X, ) RE_SEC_MULTI_SEQ = re.compile(r'(?![^(]+\)),') RE_WORKFLOW_ID_VAR = re.compile(r'\${?CYLC_WORKFLOW_(REG_)?ID}?') RE_TASK_NAME_VAR = re.compile(r'\${?CYLC_TASK_NAME}?') RE_VARNAME = re.compile(r'^[a-zA-Z_][\w]*$') def check_varnames(env: Iterable[str]) -> List[str]: """Check a list of env var names for legality. Return a list of bad names (empty implies success). Examples: >>> check_varnames(['foo', 'BAR', '+baz']) ['+baz'] """ bad = [] for varname in env: if not RE_VARNAME.match(varname): bad.append(varname) return bad def interpolate_template(tmpl, params_dict): """Try the string interpolation/formatting operator `%` on a template string with a dictionary of parameters. E.g. 'a_%(foo)d' % {'foo': 12} If it fails, raises ParamExpandError, but if the string does not contain `%(`, it just returns the string. Examples: >>> interpolate_template('_%(a)s_', {'a': 'A'}) '_A_' >>> interpolate_template('%(a)s', {'b': 'B'}) Traceback (most recent call last): cylc.flow.exceptions.ParamExpandError: bad parameter >>> interpolate_template('%(a)d', {'a': 'A'}) Traceback (most recent call last): cylc.flow.exceptions.ParamExpandError: wrong data type for parameter >>> interpolate_template('%(as', {}) Traceback (most recent call last): cylc.flow.exceptions.ParamExpandError: bad template syntax """ if '%(' not in tmpl: return tmpl # User probably not trying to use param template try: return tmpl % params_dict except KeyError: raise ParamExpandError('bad parameter') from None except TypeError: raise ParamExpandError('wrong data type for parameter') from None except ValueError: raise ParamExpandError('bad template syntax') from None def _parse_iso_cycle_point(value: str) -> str: """Helper for parsing initial/start cycle point option in datetime cycling mode.""" if value == 'now': return get_current_time_string() try: return ingest_time(value, get_current_time_string()) except IsodatetimeError as exc: raise WorkflowConfigError(str(exc)) from None class WorkflowConfig: """Class for workflow configuration items and derived quantities.""" CHECK_CIRCULAR_LIMIT = 100 # If no. tasks > this, don't check circular VIS_N_POINTS = 3 MAX_WARNING_LINES = 5 def __init__( self, workflow: str, fpath: Union[Path, str], options: 'Values', template_vars: Optional[Mapping[str, Any]] = None, output_fname: Optional[str] = None, mem_log_func: Optional[Callable[[str], None]] = None, run_dir: Optional[str] = None, log_dir: Optional[str] = None, work_dir: Optional[str] = None, share_dir: Optional[str] = None, ) -> None: """ Initialize the workflow config object. Args: workflow: workflow ID fpath: workflow config file path options: CLI options """ self.mem_log = mem_log_func if self.mem_log is None: self.mem_log = lambda x: None self.mem_log("config.py:config.py: start init config") self.workflow = workflow self.workflow_name = get_workflow_name_from_id(self.workflow) self.fpath: Path = Path(fpath) self.fdir = str(self.fpath.parent) self.run_dir = run_dir self.log_dir = log_dir self.share_dir = share_dir self.work_dir = work_dir self.options = options self.implicit_tasks: Set[str] = set() self.edges: Dict[ 'SequenceBase', Set[Tuple[str, str, bool, bool]] ] = {} self.taskdefs: Dict[str, TaskDef] = {} self.expiration_offsets = {} self.ext_triggers = {} # Old external triggers (client/server) self.xtrigger_collator = XtriggerCollator() self.workflow_polling_tasks = {} # type: ignore # TODO figure out type self.initial_point: 'PointBase' self.start_point: 'PointBase' self.stop_point: Optional['PointBase'] = None self.final_point: Optional['PointBase'] = None self.sequences: List['SequenceBase'] = [] self.actual_first_point: Optional['PointBase'] = None self._start_point_for_actual_first_point: Optional['PointBase'] = None self.task_param_vars = {} # type: ignore # TODO figure out type self.runahead_limit: Optional['IntervalBase'] = None # runtime hierarchy dicts keyed by namespace name: self.runtime: Dict[str, dict] = { # TODO figure out type # lists of parent namespaces 'parents': {}, # lists of C3-linearized ancestor namespaces 'linearized ancestors': {}, # lists of first-parent ancestor namespaces 'first-parent ancestors': {}, # sets of all descendant namespaces # (not including the final tasks) 'descendants': {}, # sets of all descendant namespaces from the first-parent # hierarchy (first parents are collapsible in visualization) 'first-parent descendants': {}, } # tasks self.leaves = [] # TODO figure out type # one up from root self.feet = [] # type: ignore # TODO figure out type # Export local environmental workflow context before config parsing. self.process_workflow_env() # parse, upgrade, validate the workflow, but don't expand with default # items self.mem_log("config.py: before RawWorkflowConfig init") if output_fname: output_fname = os.path.expandvars(output_fname) self.pcfg = RawWorkflowConfig( fpath, output_fname, template_vars, self.options ) self.mem_log("config.py: after RawWorkflowConfig init") self.mem_log("config.py: before get(sparse=True") self.cfg = self.pcfg.get(sparse=True) self.mem_log("config.py: after get(sparse=True)") if 'scheduler' in self.cfg and 'install' in self.cfg['scheduler']: self.get_validated_rsync_includes() # First check for the essential scheduling section. if 'scheduling' not in self.cfg: raise WorkflowConfigError("missing [scheduling] section.") if 'graph' not in self.cfg['scheduling']: raise WorkflowConfigError("missing [scheduling][[graph]] section.") # (The check that 'graph' is defined is below). # Override the workflow defn with an initial point from the CLI # or from reload/restart: icp_str = getattr(self.options, 'icp', None) if icp_str is not None: self.cfg['scheduling']['initial cycle point'] = icp_str self.prelim_process_graph() # allow test workflows with no [runtime]: if 'runtime' not in self.cfg: self.cfg['runtime'] = OrderedDictWithDefaults() if 'root' not in self.cfg['runtime']: self.cfg['runtime']['root'] = OrderedDictWithDefaults() try: # Ugly hack to avoid templates getting included in parameters parameter_values = { key: value for key, value in self.cfg['task parameters'].items() if key != 'templates' } except KeyError: # (Workflow config defaults not put in yet.) parameter_values = {} try: parameter_templates = self.cfg['task parameters']['templates'] except KeyError: parameter_templates = {} # parameter values and templates are normally needed together. self.parameters = (parameter_values, parameter_templates) LOG.debug("Expanding [runtime] namespace lists and parameters") # Set default parameter expansion templates if necessary. for pname, pvalues in parameter_values.items(): if pvalues and pname not in parameter_templates: if any(not isinstance(pvalue, int) for pvalue in pvalues): # Strings, bare parameter values parameter_templates[pname] = r'_%%(%s)s' % pname elif any(pvalue < 0 for pvalue in pvalues): # Integers, with negative value(s) # Prefix values with signs and parameter names parameter_templates[pname] = r'_%s%%(%s)+0%dd' % ( pname, pname, max(len(str(pvalue)) for pvalue in pvalues)) else: # Integers, positive only # Prefix values with parameter names parameter_templates[pname] = r'_%s%%(%s)0%dd' % ( pname, pname, len(str(max(pvalues)))) # Expand parameters in 'special task' lists. if 'special tasks' in self.cfg['scheduling']: for spec, names in self.cfg['scheduling']['special tasks'].items(): self.cfg['scheduling']['special tasks'][spec] = ( self._expand_name_list(names)) # Expand parameters in internal queue member lists. if 'queues' in self.cfg['scheduling']: for queue, cfg in self.cfg['scheduling']['queues'].items(): if 'members' not in cfg: continue self.cfg['scheduling']['queues'][queue]['members'] = ( self._expand_name_list(cfg['members'])) # Check environment variable names and parameter environment templates # Done before inheritance to avoid repetition self.check_env_names() self.check_param_env_tmpls() self.check_for_owner(self.cfg['runtime']) self.mem_log("config.py: before _expand_runtime") self._expand_runtime() self.mem_log("config.py: after _expand_runtime") self.ns_defn_order = list(self.cfg['runtime']) self.mem_log("config.py: before compute_family_tree") # do sparse inheritance self.compute_family_tree() self.mem_log("config.py: after compute_family_tree") self.mem_log("config.py: before inheritance") self.compute_inheritance() self.mem_log("config.py: after inheritance") # filter task environment variables after inheritance self.filter_env() # Now add config defaults. Items added prior to this end up in the # sparse dict (e.g. parameter-expanded namespaces). self.mem_log("config.py: before get(sparse=False)") self.cfg = self.pcfg.get(sparse=False) self.mem_log("config.py: after get(sparse=False)") # These 2 must be called before call to init_cyclers(self.cfg): self.set_experimental_features() self.process_utc_mode() self.process_cycle_point_tz() # after the call to init_cyclers, we can start getting proper points. init_cyclers(self.cfg) self.cycling_type: Literal['integer', 'iso8601'] = ( get_interval_cls().get_null().TYPE ) self.cycle_point_dump_format = get_dump_format(self.cycling_type) # Initial point from workflow definition (or CLI override above). self.process_initial_cycle_point() self.process_start_cycle_point() self.process_final_cycle_point() self.process_stop_cycle_point() # Parse special task cycle point offsets, and replace family names. LOG.debug("Parsing [special tasks]") for s_type in self.cfg['scheduling']['special tasks']: result = copy(self.cfg['scheduling']['special tasks'][s_type]) extn = '' for item in self.cfg['scheduling']['special tasks'][s_type]: name = item if s_type == 'external-trigger': match = RE_EXT_TRIGGER.match(item) if match is None: raise WorkflowConfigError( "Illegal %s spec: %s" % (s_type, item) ) name, ext_trigger_msg = match.groups() extn = "(" + ext_trigger_msg + ")" elif s_type in ['clock-trigger', 'clock-expire']: match = RE_CLOCK_OFFSET.match(item) if match is None: raise WorkflowConfigError( "Illegal %s spec: %s" % (s_type, item) ) if ( self.cfg['scheduling']['cycling mode'] != Calendar.MODE_GREGORIAN ): raise WorkflowConfigError( "%s tasks require " "[scheduling]cycling mode=%s" % ( s_type, Calendar.MODE_GREGORIAN) ) name, offset_string = match.groups() if not offset_string: offset_string = "PT0M" if ( cylc.flow.flags.verbosity > 0 and offset_string.startswith("-") ): LOG.warning( "%s offsets are normally positive: %s" % ( s_type, item)) try: offset_interval = ( get_interval(offset_string).standardise()) except IntervalParsingError: raise WorkflowConfigError( "Illegal %s spec: %s" % (s_type, offset_string) ) from None extn = "(" + offset_string + ")" # Replace family names with members. if name in self.runtime['descendants']: result.remove(item) for member in self.runtime['descendants'][name]: if member in self.runtime['descendants']: # (sub-family) continue result.append(member + extn) if s_type == 'clock-expire': self.expiration_offsets[member] = offset_interval if s_type == 'external-trigger': self.ext_triggers[member] = ext_trigger_msg elif s_type == 'clock-expire': self.expiration_offsets[name] = offset_interval elif s_type == 'external-trigger': self.ext_triggers[name] = dequote(ext_trigger_msg) self.cfg['scheduling']['special tasks'][s_type] = result self.process_config_env() self._upg_wflow_event_names() self.mem_log("config.py: before load_graph()") self._load_graph() self.mem_log("config.py: after load_graph()") self._set_completion_expressions() self.process_runahead_limit() run_mode = RunMode.get(self.options) if run_mode in {RunMode.SIMULATION, RunMode.DUMMY}: for taskdef in self.taskdefs.values(): configure_sim_mode(taskdef.rtconfig, None, False) self.configure_workflow_state_polling_tasks() for taskdef in self.taskdefs.values(): self._check_task_event_names(taskdef) self._check_task_event_handlers(taskdef) self._check_special_tasks() # adds to self.implicit_tasks self._check_explicit_cycling() self._warn_if_queues_have_implicit_tasks( self.cfg, self.taskdefs, self.MAX_WARNING_LINES) self._check_implicit_tasks() self._check_sequence_bounds() self.validate_namespace_names() # Check that external trigger messages are only used once (they have to # be discarded immediately to avoid triggering the next instance of the # just-triggered task). seen = {} for name, tdef in self.taskdefs.items(): for msg in tdef.external_triggers: if msg not in seen: seen[msg] = name else: LOG.error( "External trigger '%s'\n used in tasks %s and %s." % ( msg, name, seen[msg])) raise WorkflowConfigError( "external triggers must be used only once.") self.upgrade_clock_triggers() self.leaves = self.get_task_name_list() for ancestors in self.runtime['first-parent ancestors'].values(): try: foot = ancestors[-2] # one back from 'root' except IndexError: pass else: if foot not in self.feet: self.feet.append(foot) self.feet.sort() # sort effects get_graph_raw output self.process_metadata_urls() if getattr(self.options, 'is_validate', False): self.mem_log("config.py: before _check_circular()") self._check_circular() self.mem_log("config.py: after _check_circular()") self.mem_log("config.py: end init config") skip_mode_validate(self.taskdefs) def set_experimental_features(self): all_ = self.cfg['scheduler']['experimental']['all'] self.experimental = SimpleNamespace(**{ key.replace(' ', '_'): value or all_ for key, value in self.cfg['scheduler']['experimental'].items() if key != 'all' }) @staticmethod def _warn_if_queues_have_implicit_tasks( config, taskdefs, max_warning_lines ): """Warn if queues contain implicit tasks. """ implicit_q_msg = '' # Get the names of the first N implicit queue tasks: for queue in config["scheduling"]["queues"]: for name in config["scheduling"]["queues"][queue][ "members" ]: if ( name not in taskdefs and name not in config['runtime'] and len(implicit_q_msg.split('\n')) <= max_warning_lines ): implicit_q_msg += f'\n * task {name!r} in queue {queue!r}' # Warn users if tasks are implied by queues. if implicit_q_msg: truncation_msg = ( f"\n...showing first {max_warning_lines} tasks..." if len(implicit_q_msg.split('\n')) > max_warning_lines else "" ) LOG.warning( 'Queues contain tasks not defined in' f' runtime: {implicit_q_msg}{truncation_msg}' ) def prelim_process_graph(self) -> None: """Ensure graph is not empty; set integer cycling mode and icp/fcp = 1 for simplest "R1 = foo" type graphs. """ graphdict = self.cfg['scheduling']['graph'] if not any(graphdict.values()): raise WorkflowConfigError('No workflow dependency graph defined.') if ( 'cycling mode' not in self.cfg['scheduling'] and self.cfg['scheduling'].get('initial cycle point', '1') == '1' and all(item in ['graph', '1', 'R1'] for item in graphdict) ): # Pure acyclic graph, assume integer cycling mode with '1' cycle for key in ('initial cycle point', 'final cycle point'): if key not in self.cfg['scheduling']: self.cfg['scheduling'][key] = '1' self.cfg['scheduling']['cycling mode'] = INTEGER_CYCLING_TYPE def process_utc_mode(self): """Set UTC mode from config or from stored value on restart. Sets: self.cfg['scheduler']['UTC mode'] The UTC mode flag """ cfg_utc_mode = self.cfg['scheduler']['UTC mode'] # Get the original UTC mode if restart: orig_utc_mode = getattr(self.options, 'utc_mode', None) if orig_utc_mode is None: # Not a restart - will save config value if cfg_utc_mode is not None: orig_utc_mode = cfg_utc_mode else: orig_utc_mode = glbl_cfg().get(['scheduler', 'UTC mode']) elif cfg_utc_mode is not None and cfg_utc_mode != orig_utc_mode: LOG.warning( "UTC mode = {0} specified in configuration, but is stored as " "{1} from the initial run. The workflow will continue to use " "UTC mode = {1}" .format(cfg_utc_mode, orig_utc_mode) ) self.cfg['scheduler']['UTC mode'] = orig_utc_mode set_utc_mode(orig_utc_mode) def process_cycle_point_tz(self): """Set the cycle point time zone from config or from stored value on restart. Ensure workflows restart with the same cycle point time zone even after system time zone changes e.g. DST (the value is put in db by Scheduler). Sets: self.cfg['scheduler']['cycle point time zone'] """ cfg_cp_tz = self.cfg['scheduler'].get('cycle point time zone') if not cfg_cp_tz: cfg_cp_tz = 'Z' # Get the original workflow run time zone if restart: orig_cp_tz = getattr(self.options, 'cycle_point_tz', None) if orig_cp_tz is None: # Not a restart if cfg_cp_tz is None: if get_utc_mode() is True: orig_cp_tz = 'Z' else: orig_cp_tz = get_local_time_zone_format() else: orig_cp_tz = cfg_cp_tz elif cfg_cp_tz is not None: dmp = TimePointDumper() if dmp.get_time_zone(cfg_cp_tz) != dmp.get_time_zone(orig_cp_tz): LOG.warning( "cycle point time zone = {0} specified in configuration, " "but there is a stored value of {1} from the initial run. " "The workflow will continue to run in {1}" .format(cfg_cp_tz, orig_cp_tz) ) self.cfg['scheduler']['cycle point time zone'] = orig_cp_tz def process_initial_cycle_point(self) -> None: """Validate and set initial cycle point from flow.cylc or options. Sets: self.initial_point self.cfg['scheduling']['initial cycle point'] Raises: WorkflowConfigError - if it fails to validate """ orig_icp = self.cfg['scheduling']['initial cycle point'] if self.cycling_type == INTEGER_CYCLING_TYPE: if orig_icp is None: orig_icp = '1' icp = orig_icp else: if orig_icp is None: raise WorkflowConfigError( "This workflow requires an initial cycle point.") icp = _parse_iso_cycle_point(orig_icp) self.initial_point = get_point(icp).standardise() self.cfg['scheduling']['initial cycle point'] = str(self.initial_point) # Validate initial cycle point against any constraints constraints = self.cfg['scheduling']['initial cycle point constraints'] if constraints: valid_icp = False for entry in constraints: possible_pt = get_point_relative( entry, self.initial_point ).standardise() if self.initial_point == possible_pt: valid_icp = True break if not valid_icp: raise WorkflowConfigError( f"Initial cycle point {self.initial_point} does not meet " f"the constraints {constraints}") def process_start_cycle_point(self) -> None: """Set the start cycle point from options. Sets: self.options.startcp self.start_point """ startcp = getattr(self.options, 'startcp', None) starttask = getattr(self.options, 'starttask', None) if startcp is not None and starttask is not None: raise InputError( "--start-cycle-point and --start-task are mutually exclusive" ) if startcp: # Start from a point later than initial point. if self.cycling_type == ISO8601_CYCLING_TYPE: self.options.startcp = _parse_iso_cycle_point(startcp) self.start_point = get_point(self.options.startcp).standardise() elif starttask: # Start from designated task(s). # Select the earliest start point for use in pre-initial ignore. try: cycle_points = [ Tokens(taskid, relative=True)['cycle'] for taskid in self.options.starttask ] except ValueError as exc: raise InputError(str(exc)) from None self.start_point = min( get_point(cycle).standardise() for cycle in cycle_points if cycle ) else: # Start from the initial point. self.start_point = self.initial_point def process_final_cycle_point(self) -> None: """Validate and set the final cycle point from flow.cylc or options. Sets: self.final_point self.cfg['scheduling']['final cycle point'] Raises: WorkflowConfigError - if it fails to validate """ if self.cfg['scheduling']['final cycle point'] == '': # (Unlike other cycle point settings in config, fcp is treated as # a string by parsec to allow for expressions like '+P1Y') self.cfg['scheduling']['final cycle point'] = None fcp_str = getattr(self.options, 'fcp', None) if fcp_str == 'reload': fcp_str = self.options.fcp = None if fcp_str is None: fcp_str = self.cfg['scheduling']['final cycle point'] if fcp_str is not None: # Is the final "point"(/interval) relative to initial? if self.cycling_type == INTEGER_CYCLING_TYPE: if "P" in fcp_str: # Relative, integer cycling. self.final_point = get_point_relative( self.cfg['scheduling']['final cycle point'], self.initial_point ).standardise() else: with suppress(IsodatetimeError): # Relative, ISO8601 cycling. self.final_point = get_point_relative( fcp_str, self.initial_point).standardise() if self.final_point is None: # Must be absolute. self.final_point = get_point(fcp_str).standardise() self.cfg['scheduling']['final cycle point'] = str(self.final_point) if (self.final_point is not None and self.initial_point > self.final_point): raise WorkflowConfigError( f"The initial cycle point:{self.initial_point} is after the " f"final cycle point:{self.final_point}.") # Validate final cycle point against any constraints constraints = self.cfg['scheduling']['final cycle point constraints'] if constraints and self.final_point is not None: valid_fcp = False for entry in constraints: possible_pt = get_point_relative( entry, self.final_point).standardise() if self.final_point == possible_pt: valid_fcp = True break if not valid_fcp: raise WorkflowConfigError( f"Final cycle point {self.final_point} does not " f"meet the constraints {constraints}") def process_stop_cycle_point(self) -> None: """Set the stop after cycle point. In decreasing priority, it is set: * From the command line option (``--stopcp=XYZ``) or database. * From the flow.cylc file (``[scheduling]stop after cycle point``). However, if ``--stopcp=reload`` on the command line during restart, the ``[scheduling]stop after cycle point`` value is used. """ stopcp_str: Optional[str] = getattr(self.options, 'stopcp', None) if stopcp_str == 'reload': stopcp_str = self.options.stopcp = None if stopcp_str is None: stopcp_str = self.cfg['scheduling']['stop after cycle point'] if stopcp_str: self.stop_point = get_point_relative( stopcp_str, self.initial_point, ).standardise() if ( self.final_point is not None and self.stop_point is not None and self.stop_point > self.final_point ): LOG.warning( f"Stop cycle point '{self.stop_point}' will have no " "effect as it is after the final cycle " f"point '{self.final_point}'." ) self.stop_point = None stopcp_str = str(self.stop_point) if self.stop_point else None self.cfg['scheduling']['stop after cycle point'] = stopcp_str def _check_implicit_tasks(self) -> None: """Raise WorkflowConfigError if implicit tasks are found in graph or queue config, unless allowed by config.""" if not self.implicit_tasks: return print_limit = 10 tasks_str = '\n * '.join(list(self.implicit_tasks)[:print_limit]) num = len(self.implicit_tasks) if num > print_limit: tasks_str += f"\n and {num} more" msg = ( "implicit tasks detected (no entry under [runtime]):\n" f" * {tasks_str}" ) if self.cfg['scheduler']['allow implicit tasks']: LOG.debug(msg) return # Check if implicit tasks explicitly disallowed try: is_disallowed = self.pcfg.get( ['scheduler', 'allow implicit tasks'], sparse=True ) is False except ItemNotFoundError: is_disallowed = False if is_disallowed: raise WorkflowConfigError(msg) # Otherwise "[scheduler]allow implicit tasks" is not set msg += ( "\nTo allow implicit tasks, use " f"'{WorkflowFiles.FLOW_FILE}[scheduler]allow implicit tasks'" ) raise WorkflowConfigError(msg) def _check_circular(self): """Check for circular dependence in graph.""" if (len(self.taskdefs) > self.CHECK_CIRCULAR_LIMIT and not getattr(self.options, 'check_circular', False)): LOG.info( f"Number of tasks is > {self.CHECK_CIRCULAR_LIMIT}; will not " "check graph for circular dependencies. To run this check" " anyway use the option --check-circular.") return start_point_str = self.cfg['scheduling']['initial cycle point'] raw_graph = self.get_graph_raw( start_point_str, stop_point_str=None, sort=False, ) lhs2rhss = {} # left hand side to right hand sides rhs2lhss = {} # right hand side to left hand sides for lhs, rhs in raw_graph: lhs2rhss.setdefault(lhs, set()) lhs2rhss[lhs].add(rhs) rhs2lhss.setdefault(rhs, set()) rhs2lhss[rhs].add(lhs) self._check_circular_helper(lhs2rhss, rhs2lhss) if rhs2lhss: # Before reporting circular dependence, pick out all the edges with # no outgoings. self._check_circular_helper(rhs2lhss, lhs2rhss) err_msg = '' for rhs, lhss in sorted(rhs2lhss.items()): for lhs in sorted(lhss): err_msg += ' %s => %s' % ( Tokens( cycle=str(lhs[1]), task=lhs[0] ).relative_id, Tokens( cycle=str(rhs[1]), task=rhs[0] ).relative_id, ) if err_msg: raise WorkflowConfigError( 'circular edges detected:' + err_msg) @staticmethod def _check_circular_helper(x2ys, y2xs): """Topological elimination. An implementation of Kahn's algorithm for topological sorting, but only use the part for pulling out starter nodes with no incoming edges. See https://en.wikipedia.org/wiki/Topological_sorting x2ys is a map of {x1: [y1, y2, ...], ...} to map edges using x's as keys, such as x1 => y1, x1 => y2, etc y2xs is a map of {y3: [x4, x5, ...], ...} to map edges using y's as keys, such as x4 => y3, x5 => y3, etc """ # Starter x nodes are those with no incoming, i.e. # x nodes that never appear as a y. sxs = set(x2ys).difference(y2xs) while sxs: sx01 = sxs.pop() for y01 in x2ys[sx01]: y2xs[y01].remove(sx01) if not y2xs[y01]: if y01 in x2ys: # No need to look at this again if it does not have any # outgoing. sxs.add(y01) del y2xs[y01] del x2ys[sx01] def _check_sequence_bounds(self) -> None: """Check bounds of sequences against the initial cycle point.""" out_of_bounds = [ str(seq) for seq in self.sequences if seq.get_first_point(self.initial_point) is None ] if out_of_bounds: if len(out_of_bounds) > 1: # avoid spamming users with multiple warnings out_of_bounds_str = '\n'.join( wrap(', '.join(out_of_bounds), 70) ) msg = ( "multiple sequences out of bounds for initial cycle point " f"{self.initial_point}:\n{out_of_bounds_str}" ) else: msg = ( f"{out_of_bounds[0]}: sequence out of bounds for " f"initial cycle point {self.initial_point}" ) LOG.warning(msg) def _set_completion_expressions(self): """Sets and checks completion expressions for each task. If a task does not have a user-defined completion expression, then set one according to the default rules. If a task does have a used-defined completion expression, then ensure it is consistent with the use of outputs in the graph. """ for name, taskdef in self.taskdefs.items(): expr = taskdef.rtconfig['completion'] if expr: any_suicide = any( dep.suicide for d in taskdef.dependencies.values() for dep in d ) # check the user-defined expression self._check_completion_expression(name, expr, any_suicide) else: # derive a completion expression for this taskdef expr = get_completion_expression(taskdef) # update both the sparse and dense configs to make these values # visible to "cylc config" to make the completion expression more # transparent to users. # NOTE: we have to update both because we are setting this value # late on in the process after the dense copy has been made self.pcfg.sparse.setdefault( 'runtime', {} ).setdefault( name, {} )['completion'] = expr self.pcfg.dense['runtime'][name]['completion'] = expr # update the task's runtime config to make this value visible to # the data store # NOTE: we have to do this because we are setting this value late # on after the TaskDef has been created taskdef.rtconfig['completion'] = expr def _check_completion_expression( self, task_name: str, expr: str, any_suicide: bool ) -> None: """Checks a user-defined completion expression. Args: task_name: The name of the task we are checking. expr: The completion expression as defined in the config. any_suicide: Does this task have any suicide triggers """ # check for invalid triggers in the expression if 'submit-failed' in expr: raise WorkflowConfigError( f'Error in [runtime][{task_name}]completion:' f'\nUse "submit_failed" rather than "submit-failed"' ' in completion expressions.' ) elif '-' in expr: raise WorkflowConfigError( f'Error in [runtime][{task_name}]completion:' f'\n {expr}' '\nReplace hyphens with underscores in task outputs when' ' used in completion expressions.' ) # get the outputs and completion expression for this task try: outputs = self.taskdefs[task_name].outputs except KeyError: # this is a family -> we'll check integrity for each task that # inherits from it return ( _trigger_to_completion_variable, _completion_variable_to_trigger, ) = get_trigger_completion_variable_maps(outputs.keys()) # get the optional/required outputs defined in the graph graph_optionals = { # completion_variable: is_optional _trigger_to_completion_variable[trigger]: ( None if is_required is None else not is_required ) for trigger, (_, is_required) in outputs.items() } if ( graph_optionals[TASK_OUTPUT_SUCCEEDED] is True and graph_optionals[TASK_OUTPUT_FAILED] is None ): # failed is implicitly optional if succeeded is optional # https://github.com/cylc/cylc-flow/pull/6046#issuecomment-2059266086 graph_optionals[TASK_OUTPUT_FAILED] = True # get the optional/required outputs defined in the expression try: # this involves running the expression which also validates it expression_optionals = get_optional_outputs(expr, outputs) except NameError as exc: # expression references an output which has not been registered error = exc.args[0][5:] if f"'{TASK_OUTPUT_FINISHED}'" in error: # the finished output cannot be used in completion expressions # see proposal point 5:: # https://cylc.github.io/cylc-admin/proposal-optional-output-extension.html#proposal raise WorkflowConfigError( f'Error in [runtime][{task_name}]completion:' f'\n {expr}' '\nThe "finished" output cannot be used in completion' ' expressions, use "succeeded or failed".' ) from None for alt_qualifier, qualifier in ALT_QUALIFIERS.items(): _alt_compvar = trigger_to_completion_variable(alt_qualifier) _compvar = trigger_to_completion_variable(qualifier) if re.search(rf'\b{_alt_compvar}\b', error): raise WorkflowConfigError( f'Error in [runtime][{task_name}]completion:' f'\n {expr}' f'\nUse "{_compvar}" not "{_alt_compvar}" ' 'in completion expressions.' ) from None raise WorkflowConfigError( # NOTE: str(exc) == "name 'x' is not defined" tested in # tests/integration/test_optional_outputs.py f'Error in [runtime][{task_name}]completion:' f'\n{error}' ) from None except Exception as exc: # includes InvalidCompletionExpression # expression contains non-whitelisted syntax or any other error in # the expression e.g. SyntaxError raise WorkflowConfigError( f'Error in [runtime][{task_name}]completion:' f'\n{str(exc)}' ) from None # ensure consistency between the graph and the completion expression for compvar in ( { *graph_optionals, *expression_optionals } ): # is the output optional in the graph? graph_opt = graph_optionals.get(compvar) # is the output optional in the completion expression? expr_opt = expression_optionals.get(compvar) # True = is optional # False = is required # None = is not referenced # graph_opt expr_opt # True True ok # True False not ok # True None not ok [1] # False True not ok [1] # False False ok # False None not ok # None True ok # None False ok # None None ok # [1] applies only to "submit-failed" and "expired" trigger = _completion_variable_to_trigger[compvar] if graph_opt is True and expr_opt is False: raise WorkflowConfigError( f'{task_name}:{trigger} is optional in the graph' ' (? symbol), but required in the completion' f' expression:\n{expr}' ) if graph_opt is False and expr_opt is None: raise WorkflowConfigError( f'{task_name}:{trigger} is required in the graph,' ' but not referenced in the completion' f' expression\n{expr}' ) if ( graph_opt is True and expr_opt is None and compvar in {'submit_failed', 'expired'} ): msg = ( f'{task_name}:{trigger} is permitted in the graph' ' but is not referenced in the completion.' ) if ( any_suicide and trigger == "expired" and self.experimental.expire_triggers ): msg += ( "\nThis may be due to use of an expire " "(formerly suicide) trigger." ) msg += f'\nTry: completion = "{expr} or {compvar}"' raise WorkflowConfigError(msg) if ( graph_opt is False and expr_opt is True and compvar not in {'submit_failed', 'expired'} ): raise WorkflowConfigError( f'{task_name}:{trigger} is required in the graph,' ' but optional in the completion expression' f'\n{expr}' ) def _expand_name_list(self, orig_names): """Expand any parameters in lists of names.""" name_expander = NameExpander(self.parameters) exp_names = [] for orig_name in orig_names: exp_names += [name for name, _ in name_expander.expand(orig_name)] return exp_names def _update_task_params(self, task_name, params): """Update the dict of parameters used in a task definition. # Used to expand parameter values in task environments. """ self.task_param_vars.setdefault( task_name, {} ).update( params ) def _expand_runtime(self): """Expand [runtime] name lists or parameterized names. This makes individual runtime namespaces out of any headings that represent multiple namespaces, like [[foo, bar]] or [[foo]]. It requires replicating the sparse runtime OrderedDict into a new OrderedDict - we can't just stick expanded names on the end because the order matters (for add-or-override by repeated namespaces). TODO - this will have an impact on memory footprint for large workflows with a lot of runtime config. We should consider ditching OrderedDict and instead using an ordinary dict """ if (not self.parameters[0] and not any(',' in ns for ns in self.cfg['runtime'])): # No parameters, no namespace lists: no expansion needed. return newruntime = OrderedDictWithDefaults() name_expander = NameExpander(self.parameters) for namespace_heading, namespace_dict in self.cfg['runtime'].items(): for name, indices in name_expander.expand(namespace_heading): if name not in newruntime: newruntime[name] = OrderedDictWithDefaults() replicate(newruntime[name], namespace_dict) if indices: self._update_task_params(name, indices) new_environ = OrderedDictWithDefaults() if 'environment' in newruntime[name]: new_environ = newruntime[name]['environment'].copy() newruntime[name]['environment'] = new_environ if 'inherit' in newruntime[name]: # Allow inheritance from parameterized namespaces. parents = newruntime[name]['inherit'] origin = 'inherit = %s' % ', '.join(parents) repl_parents = [] for parent in parents: used_indices, expanded = ( name_expander.expand_parent_params( parent, indices, origin) ) repl_parents.append(expanded) if used_indices: self._update_task_params(name, used_indices) newruntime[name]['inherit'] = repl_parents self.cfg['runtime'] = newruntime def validate_namespace_names(self): """Validate task and family names.""" for name in self.implicit_tasks: success, message = TaskNameValidator.validate(name) if not success: raise WorkflowConfigError( f'invalid task name "{name}"\n{message}' ) for name in self.cfg['runtime']: if name == 'root': # root is allowed to be defined in the runtime section continue success, message = TaskNameValidator.validate(name) if not success: raise WorkflowConfigError( f'task/family name {message}\n[runtime][[{name}]]' ) def check_env_names(self): """Check for illegal environment variable names""" bad = {} for label, item in self.cfg['runtime'].items(): for key in ('environment', 'parameter environment templates'): if key in item: res = check_varnames(item[key]) if res: bad[(label, key)] = res if bad: raise WorkflowConfigError( "Illegal environment variable name(s) detected:\n* " # f"\n{err_msg}" + '\n* '.join( f'[runtime][{label}][{key}]{name}' for (label, key), names in bad.items() for name in names ) ) def check_param_env_tmpls(self): """Check for illegal parameter environment templates""" parameter_values = { key: values[0] for key, values in self.parameters[0].items() if values } bads = set() for task_name, task_items in self.cfg['runtime'].items(): if 'environment' not in task_items: continue for name, tmpl in task_items['environment'].items(): try: interpolate_template(tmpl, parameter_values) except ParamExpandError as descr: bads.add((task_name, name, tmpl, descr)) if bads: LOG.warning( 'bad parameter environment template:\n ' + '\n '.join( '[runtime][%s][environment]%s = %s # %s' % bad for bad in sorted(bads) ) ) def filter_env(self): """Filter environment variables after sparse inheritance""" for ns in self.cfg['runtime'].values(): try: oenv = ns['environment'] except KeyError: # no environment to filter continue try: fincl = ns['environment filter']['include'] except KeyError: # empty include-filter means include all fincl = [] try: fexcl = ns['environment filter']['exclude'] except KeyError: # empty exclude-filter means exclude none fexcl = [] if not fincl and not fexcl: # no filtering to do continue nenv = OrderedDictWithDefaults() for key, val in oenv.items(): if (not fincl or key in fincl) and key not in fexcl: nenv[key] = val ns['environment'] = nenv def compute_family_tree(self): first_parents = {} demoted = {} for name in self.cfg['runtime']: if name == 'root': self.runtime['parents'][name] = [] first_parents[name] = [] continue # get declared parents, with implicit inheritance from root. pts = self.cfg['runtime'][name].get('inherit', ['root']) if not pts: pts = ['root'] for p in pts: if p == "None": # see just below continue if p not in self.cfg['runtime']: raise WorkflowConfigError( "undefined parent for " + name + ": " + p) if pts[0] == "None": if len(pts) < 2: raise WorkflowConfigError( "null parentage for " + name) demoted[name] = pts[1] pts = pts[1:] first_parents[name] = ['root'] else: first_parents[name] = [pts[0]] self.runtime['parents'][name] = pts if cylc.flow.flags.verbosity > 0 and demoted: log_msg = "First parent(s) demoted to secondary:" for n, p in demoted.items(): log_msg += "\n + %s as parent of '%s'" % (p, n) LOG.debug(log_msg) c3 = C3(self.runtime['parents']) c3_single = C3(first_parents) for name in self.cfg['runtime']: try: self.runtime['linearized ancestors'][name] = c3.mro(name) self.runtime['first-parent ancestors'][name] = ( c3_single.mro(name)) except RecursionError: raise WorkflowConfigError( "circular [runtime] inheritance?" ) from None except Exception as exc: # catch inheritance errors # TODO - specialise MRO exceptions raise WorkflowConfigError(str(exc)) from None for name in self.cfg['runtime']: ancestors = self.runtime['linearized ancestors'][name] for p in ancestors[1:]: self.runtime['descendants'].setdefault(p, set()).add(name) first_ancestors = self.runtime['first-parent ancestors'][name] for p in first_ancestors[1:]: self.runtime['first-parent descendants'].setdefault( p, set() ).add(name) def compute_inheritance(self): LOG.debug("Parsing the runtime namespace hierarchy") # TODO: Note an unused alternative mechanism was removed here # (March 2020). It stored the result of each completed MRO and # re-used these wherever possible. This could be more efficient # for full namespaces in deep hierarchies. We should go back and # look if inheritance computation becomes a problem. results = OrderedDictWithDefaults() # Loop through runtime members, 'root' first. nses = list(self.cfg['runtime']) nses.sort(key=lambda ns: ns != 'root') for ns in nses: # for each namespace ... hierarchy = copy(self.runtime['linearized ancestors'][ns]) hierarchy.reverse() result = OrderedDictWithDefaults() # Go up the linearized MRO from root, replicating or # overriding each namespace element as we go. for name in hierarchy: replicate(result, self.cfg['runtime'][name]) # n_reps += 1 results[ns] = result # replace pre-inheritance namespaces with the post-inheritance result self.cfg['runtime'] = results # uncomment this to compare the simple and efficient methods # print ' Number of namespace replications:', n_reps # def print_inheritance(self): # # (use for debugging) # for foo in self.runtime: # log_msg = '\t' + foo # for item, val in self.runtime[foo].items(): # log_msg += '\t\t' + item + '\t' + val # LOG.info(log_msg) def process_runahead_limit(self): """Extract runahead limit information.""" limit = self.cfg['scheduling']['runahead limit'] if limit.isdigit(): limit = f'PT{limit}H' LOG.warning( 'Use of a raw number of hours for the runahead limit is ' f'deprecated. Use "{limit}" instead') number_limit_regex = re.compile(r'^P\d+$') time_limit_regexes = DurationParser.DURATION_REGEXES if number_limit_regex.fullmatch(limit): self.runahead_limit = IntegerInterval(limit) elif ( # noqa: SIM106 self.cycling_type == ISO8601_CYCLING_TYPE and any(tlr.fullmatch(limit) for tlr in time_limit_regexes) ): self.runahead_limit = ISO8601Interval(limit) else: raise WorkflowConfigError( f'bad runahead limit "{limit}" for {self.cycling_type} ' 'cycling type') def get_config(self, args, sparse=False): return self.pcfg.get(args, sparse) def adopt_orphans(self, orphans): # Called by the scheduler after reloading the workflow definition # at run time and finding any live task proxies whose # definitions have been removed from the workflow. Keep them # in the default queue and under the root family, until they # run their course and disappear. for orphan in orphans: self.runtime['linearized ancestors'][orphan] = [orphan, 'root'] def configure_workflow_state_polling_tasks(self): # Deprecation warning - automatic workflow state polling tasks don't # necessarily have deprecated config items outside the graph string. if ( self.workflow_polling_tasks and getattr(self.options, 'is_validate', False) ): LOG.warning( "Workflow state polling tasks are deprecated." " Please convert to workflow_state xtriggers:\n * " + "\n * ".join(self.workflow_polling_tasks) ) # Check custom script not defined for automatic workflow polling tasks. for l_task in self.workflow_polling_tasks: try: cs = self.pcfg.get(sparse=True)['runtime'][l_task]['script'] except KeyError: pass else: if cs: # (allow explicit blanking of inherited script) raise WorkflowConfigError( "script cannot be defined for automatic" + " workflow polling task '%s':\n%s" % (l_task, cs)) # Generate the automatic scripting. for name, tdef in self.taskdefs.items(): if name not in self.workflow_polling_tasks: continue rtc = tdef.rtconfig comstr = ( "cylc workflow-state " f"{tdef.workflow_polling_cfg['workflow']}//" "$CYLC_TASK_CYCLE_POINT/" f"{tdef.workflow_polling_cfg['task']}" ) graph_selector = tdef.workflow_polling_cfg['status'] config_message = rtc['workflow state polling']['message'] if ( graph_selector is not None and ( config_message is not None ) and ( graph_selector != config_message ) ): raise WorkflowConfigError( f'Polling task "{name}" must configure a target status or' f' output message in the graph (:{graph_selector}) or task' f' definition (message = "{config_message}") but not both.' ) if graph_selector is not None: comstr += f":{graph_selector}" elif config_message is not None: # quote: may contain spaces comstr += f':"{config_message}" --messages' else: # default to :succeeded comstr += f":{TASK_OUTPUT_SUCCEEDED}" for key, fmt in [ ('interval', ' --%s=%d'), ('max-polls', ' --%s=%s'), ('alt-cylc-run-dir', ' --%s=%s')]: if rtc['workflow state polling'][key]: comstr += fmt % (key, rtc['workflow state polling'][key]) script = f"echo {comstr}\n{comstr}" rtc['script'] = script def get_parent_lists(self): return self.runtime['parents'] def get_first_parent_ancestors(self, pruned=False): if pruned: # prune non-task namespaces from ancestors dict pruned_ancestors = {} for key, val in self.runtime['first-parent ancestors'].items(): if key not in self.taskdefs: continue pruned_ancestors[key] = val return pruned_ancestors else: return self.runtime['first-parent ancestors'] def get_linearized_ancestors(self): return self.runtime['linearized ancestors'] def get_first_parent_descendants(self): return self.runtime['first-parent descendants'] def add_tree_titles(self, tree): for key, val in tree.items(): if val == {}: tree[key] = self.cfg['runtime'][key]['meta'].get( 'title', NO_TITLE) elif isinstance(val, dict): self.add_tree_titles(val) def get_namespace_list(self, which): names = [] if which == 'graphed tasks': # tasks used only in the graph names = list(self.taskdefs) elif which == 'all namespaces': # all namespaces names = list(self.cfg['runtime']) elif which == 'all tasks': for ns in self.cfg['runtime']: if ns not in self.runtime['descendants']: # tasks have no descendants names.append(ns) result = {} for ns in names: result[ns] = self.cfg['runtime'][ns]['meta'].get( 'title', NO_TITLE) return result def get_mro(self, ns): try: mro = self.runtime['linearized ancestors'][ns] except KeyError: mro = ["no such namespace: " + ns] return mro def process_workflow_env(self): """Export Workflow context to the local environment. A source workflow has only a name. Once installed it also has an ID and a run directory. And at scheduler start-up it has work, share, and log sub-dirs too. """ for key, value in { **verbosity_to_env(cylc.flow.flags.verbosity), 'CYLC_WORKFLOW_NAME': self.workflow_name, 'CYLC_WORKFLOW_NAME_BASE': str(Path(self.workflow_name).name), }.items(): os.environ[key] = value if is_relative_to(self.fdir, get_cylc_run_dir()): # This is an installed workflow. # - self.run_dir is only defined by the scheduler # - but the run dir exists, created at installation # - run sub-dirs may exist, if this installation was run already # but if the scheduler is not running they shouldn't be used. for key, value in { 'CYLC_WORKFLOW_ID': self.workflow, 'CYLC_WORKFLOW_RUN_DIR': str(self.fdir), }.items(): os.environ[key] = value if self.run_dir is not None: # Run directory is only defined if the scheduler is running; in # which case the following run sub-directories must exist. for key, value in { 'CYLC_WORKFLOW_LOG_DIR': str(self.log_dir), 'CYLC_WORKFLOW_WORK_DIR': str(self.work_dir), 'CYLC_WORKFLOW_SHARE_DIR': str(self.share_dir), }.items(): os.environ[key] = value def process_config_env(self): """Set local config derived environment.""" os.environ['CYLC_UTC'] = str(get_utc_mode()) os.environ['CYLC_WORKFLOW_INITIAL_CYCLE_POINT'] = str( self.initial_point ) os.environ['CYLC_WORKFLOW_FINAL_CYCLE_POINT'] = str(self.final_point) os.environ['CYLC_CYCLING_MODE'] = self.cfg['scheduling'][ 'cycling mode'] # Add workflow bin directory to PATH for workflow and event handlers if self.share_dir is not None: os.environ['PATH'] = os.pathsep.join( [ os.path.join(self.share_dir, 'bin'), os.path.join(self.fdir, 'bin'), os.environ['PATH'] ] ) else: os.environ['PATH'] = os.pathsep.join( [ os.path.join(self.fdir, 'bin'), os.environ['PATH'] ] ) def _check_task_event_names(self, taskdef: 'TaskDef') -> None: """Validate task handler/mail event names.""" for setting in ('handler events', 'mail events'): event_names: Optional[List[str]] = taskdef.rtconfig['events'][ setting ] if not event_names: continue invalid = set(event_names).difference( TaskEventsManager.STD_EVENTS, taskdef.rtconfig['outputs'], ) if invalid: LOG.warning( "Invalid event name(s) for " f"[runtime][{taskdef.name}][events]{setting}: " + ', '.join(sorted(invalid)) ) def _check_task_event_handlers(self, taskdef: 'TaskDef') -> None: """Check custom event handler templates can be expanded. Ensures that any %(template_variables)s in task event handlers are present in the data that will be passed to them when called (otherwise they will fail). """ if not taskdef.rtconfig['events']: return handler_data = {item.value: '' for item in EventData} handler_data.update( get_event_handler_data(taskdef.rtconfig, self.cfg) ) for key, values in taskdef.rtconfig['events'].items(): if values and (key == 'handlers' or key.endswith(' handlers')): for handler_template in values: try: handler_template % handler_data except (KeyError, ValueError) as exc: raise WorkflowConfigError( f'bad task event handler template' f' {taskdef.name}: {handler_template}: {repr(exc)}' ) from None def _check_special_tasks(self): """Check declared special tasks are valid, and detect special implicit tasks""" for task_type in self.cfg['scheduling']['special tasks']: for name in self.cfg['scheduling']['special tasks'][task_type]: if task_type in ['clock-trigger', 'clock-expire', 'external-trigger']: name = name.split('(', 1)[0] if not TaskID.is_valid_name(name): raise WorkflowConfigError( f'Illegal {task_type} task name: {name}') if (name not in self.taskdefs and name not in self.cfg['runtime']): self.implicit_tasks.add(name) def _check_explicit_cycling(self): """Check that inter-cycle offsets refer to cycling tasks. E.G. foo[-P1] => bar requires foo to be defined in the graph somewhere. """ for taskdef in self.taskdefs.values(): taskdef.check_for_explicit_cycling() def get_task_name_list(self): """Return a sorted list of all tasks used in the dependency graph. Note: the sort order may effect get_graph_raw output. """ return sorted(self.taskdefs) def generate_edges(self, lexpr, orig_lexpr, left_nodes, right, seq, suicide=False): """Generate edges. Add nodes from this graph section to the abstract graph edges structure. """ conditional = False if '|' in lexpr: # plot conditional triggers differently conditional = True if seq not in self.edges: self.edges[seq] = set() if not left_nodes: # Right is a lone node. self.edges[seq].add((right, None, suicide, conditional)) conditional_xtriggers = [] for left in left_nodes: # if left is None: # continue # TODO - RIGHT CANNOT BE NONE NOW? # if right is not None: # Check for self-edges. if left == right or left.startswith(right + ':'): # (This passes inter-cycle offsets: left[-P1D] => left) # (TODO - but not explicit null offsets like [-P0D]!) if suicide: continue if orig_lexpr != lexpr: LOG.error(f"{orig_lexpr} => {right}") raise WorkflowConfigError( f"self-edge detected: {left} => {right}") if left.startswith('@') and conditional: conditional_xtriggers.append(f'{lexpr} => {right}') else: self.edges[seq].add((left, right, suicide, conditional)) if conditional_xtriggers: raise WorkflowConfigError( 'Xtriggers cannot be used in conditional graph' ' expressions:\n' + '\n * '.join(conditional_xtriggers) ) def generate_taskdef(self, orig_expr, node): """Generate task definition for node.""" name = GraphNodeParser.get_inst().parse(node)[0] taskdef = self.get_taskdef(name, orig_expr) if name in self.workflow_polling_tasks: taskdef.workflow_polling_cfg = { 'workflow': self.workflow_polling_tasks[name][0], 'task': self.workflow_polling_tasks[name][1], 'status': self.workflow_polling_tasks[name][2] } def add_sequence(self, nodes, seq, suicide): """Add valid sequences to taskdefs.""" graph_node_parser = GraphNodeParser.get_inst() for node in nodes: name, offset = graph_node_parser.parse(node)[:2] taskdef = self.get_taskdef(name) # Only add sequence to taskdef if explicit (not an offset). if offset: taskdef.used_in_offset_trigger = True elif not suicide: # "foo => !bar" does not define a sequence for bar taskdef.add_sequence(seq) def generate_triggers(self, lexpression, left_nodes, right, seq, suicide, task_triggers): """Create Dependency and TaskTrigger objects. Register dependency with the relevant TaskDef object. """ if not right or not left_nodes: # Lone nodes have no triggers. return # Convert expression to a (nested) list. try: expr_list = listify(lexpression) except SyntaxError: raise WorkflowConfigError( 'Error in expression "%s"' % lexpression ) from None triggers = {} xtrig_labels = set() graph_node_parser = GraphNodeParser.get_inst() for left in left_nodes: if left.startswith('@'): xtrig_labels.add(left[1:]) continue # (GraphParseError checked above) (name, offset, output, offset_is_from_icp, offset_is_irregular, offset_is_absolute) = ( graph_node_parser.parse(left) ) # Qualifier. outputs = self.cfg['runtime'][name]['outputs'] if outputs and (output in outputs): # Qualifier is a custom task message. qualifier = outputs[output] elif output: if not TaskOutputs.is_valid_std_name(output): raise WorkflowConfigError( f"Undefined custom output: {name}:{output}" ) qualifier = output else: # No qualifier specified => use "succeeded". qualifier = TASK_OUTPUT_SUCCEEDED # Generate TaskTrigger if not already done. key = (name, offset, qualifier, offset_is_irregular, offset_is_absolute, offset_is_from_icp, self.initial_point) try: task_trigger = task_triggers[key] except KeyError: task_trigger = TaskTrigger(*key) task_triggers[key] = task_trigger triggers[left] = task_trigger # (name is left name) self.taskdefs[name].add_graph_child(task_trigger, right, seq) # graph_parents not currently used but might be needed soon: self.taskdefs[right].add_graph_parent(task_trigger, name, seq) # Walk down "expr_list" depth first, and replace any items matching a # key in "triggers" ("left" values) with the trigger. stack = [expr_list] while stack: item_list = stack.pop() for i, item in enumerate(item_list): if isinstance(item, list): stack.append(item) elif item in triggers: item_list[i] = triggers[item] if triggers: dependency = Dependency(expr_list, set(triggers.values()), suicide) self.taskdefs[right].add_dependency(dependency, seq) validator = XtriggerNameValidator.validate xtrigs = self.cfg['scheduling']['xtriggers'] for label in xtrigs: valid, msg = validator(label) if not valid: raise WorkflowConfigError( f'Invalid xtrigger name "{label}" - {msg}' ) self.xtrigger_collator.sequential_xtriggers_default = ( self.cfg['scheduling']['sequential xtriggers'] ) for label in xtrig_labels: try: xtrig = xtrigs[label] except KeyError: if label != 'wall_clock': raise WorkflowConfigError( f"xtrigger not defined: {label}" ) from None else: # Allow "@wall_clock" in graph as implicit zero-offset. xtrig = SubFuncContext('wall_clock', 'wall_clock', [], {}) if ( xtrig.func_name == 'wall_clock' and self.cycling_type == INTEGER_CYCLING_TYPE ): raise WorkflowConfigError( "Clock xtriggers require datetime cycling:" f" {label} = {xtrig.get_signature()}" ) self.xtrigger_collator.add_trig(label, xtrig, self.fdir) self.taskdefs[right].add_xtrig_label(label, seq) def get_actual_first_point(self, start_point): """Get actual first cycle point for the workflow Get all sequences to adjust the putative start time upward. """ if (self._start_point_for_actual_first_point is not None and self._start_point_for_actual_first_point == start_point and self.actual_first_point is not None): return self.actual_first_point self._start_point_for_actual_first_point = start_point adjusted = [] for seq in self.sequences: point = seq.get_first_point(start_point) if point: adjusted.append(point) if len(adjusted) > 0: adjusted.sort() self.actual_first_point = adjusted[0] else: self.actual_first_point = start_point return self.actual_first_point def _get_stop_point(self, start_point, stop_point_str=None): """Get stop point from string value or interval, or return None.""" if stop_point_str is None: stop_point = None elif "P" in stop_point_str: # Is the final point(/interval) relative to initial? if self.cfg['scheduling']['cycling mode'] == 'integer': # Relative, integer cycling. stop_point = get_point_relative( stop_point_str, start_point ).standardise() else: # Relative, ISO8601 cycling. stop_point = get_point_relative( stop_point_str, start_point ).standardise() else: stop_point = get_point(stop_point_str).standardise() return stop_point def get_graph_raw( self, start_point_str=None, stop_point_str=None, grouping=None, flatten_icp_dependence=False, sort=True, ): """Return concrete graph edges between specified cycle points. Return a family-collapsed graph if the grouping arg is not None: * ['FAM1', 'FAM2']: group (collapse) specified families * ['']: group (collapse) all families above root The "flatten_icp_dependence" option detects ICP dependencies (e.g. "start[^] => foo") and prefixes the cycle point on the left side of the edge with "R1.". This allows graph visualisation tools to handle these nodes differently. For validation, return non-suicide edges with left and right nodes. """ start_point = get_point( start_point_str or self.cfg['scheduling']['initial cycle point'] ) stop_point = self._get_stop_point(start_point, stop_point_str) actual_first_point = self.get_actual_first_point(start_point) if grouping is None: grouping = [] elif grouping == ['']: grouping = [ fam for fam in self.runtime["first-parent descendants"].keys() if fam != "root" ] else: for bad in ( set(grouping).difference( self.runtime["first-parent descendants"].keys() ) ): LOG.warning(f"Ignoring undefined family {bad}") grouping.remove(bad) is_validate = getattr( self.options, 'is_validate', False) # this is for _check_circular if is_validate: grouping = [] # Now define the concrete graph edges (pairs of nodes) for plotting. workflow_final_point = get_point( self.cfg['scheduling']['final cycle point']) # For the computed stop point, store VIS_N_POINTS of each sequence, # and then cull later to the first VIS_N_POINTS over all sequences. # For nested closed families, only consider the outermost one fpd = self.runtime['first-parent descendants'] clf_map = {} for name in grouping: if all( name not in fpd[i] for i in grouping ): clf_map[name] = fpd[name] gr_edges = {} start_point_offset_cache = {} point_offset_cache = None graph_node_parser = GraphNodeParser.get_inst() for sequence, edges in self.edges.items(): # Get initial cycle point for this sequence point = sequence.get_first_point(start_point) new_points = set() while point is not None: new_points.add(point) if stop_point is None: if len(new_points) > self.VIS_N_POINTS: # Take VIS_N_POINTS cycles from each sequence. break elif point > stop_point: # Beyond requested final cycle point. break if (workflow_final_point is not None and point > workflow_final_point): # Beyond workflow final cycle point. break point_offset_cache = {} for left, right, suicide, cond in edges: if is_validate and (not right or suicide): continue if right: r_id = (right, point) else: r_id = None if left[0] == '@': # @xtrigger node. name = left offset_is_from_icp = False offset = None else: name, offset, _, offset_is_from_icp, _, _ = ( graph_node_parser.parse(left) ) if offset: if offset_is_from_icp: cache = start_point_offset_cache rel_point = start_point else: cache = point_offset_cache rel_point = point try: l_point = cache[offset] except KeyError: l_point = get_point_relative(offset, rel_point) cache[offset] = l_point else: l_point = point if ( flatten_icp_dependence and point != self.initial_point and offset and offset_is_from_icp and get_interval_cls()(offset) == get_interval_cls().get_null() ): # this is an R1 dependency (e.g, start[^] => foo). When # "collapse_icp = True", we prefix the point with "R1." # so graph visualization tools can apply special logic. l_id = (name, f'R1.{point}') # print('%', str(l_id)) else: l_id = (name, l_point) if actual_first_point > l_point: # Check that l_id is not earlier than start time. if ( is_validate or r_id is None or r_id[1] < actual_first_point ): continue # Pre-initial dependency; # keep right hand node. l_id = r_id r_id = None gr_edges.setdefault(point, []) if is_validate: gr_edges[point].append((l_id, r_id)) else: lstr, rstr = self._close_families(l_id, r_id, clf_map) gr_edges[point].append( (lstr, rstr, None, suicide, cond)) # Increment the cycle point. point = sequence.get_next_point_on_sequence(point) del clf_map del start_point_offset_cache del point_offset_cache GraphNodeParser.get_inst().clear() if stop_point is None: # Prune to VIS_N_POINTS points in total. graph_raw_edges = [] for point in sorted(gr_edges)[:self.VIS_N_POINTS]: graph_raw_edges.extend(gr_edges[point]) else: # Flatten nested list. graph_raw_edges = ( [i for sublist in gr_edges.values() for i in sublist]) if sort: graph_raw_edges.sort( key=lambda x: [y if y else '' for y in x[:2]] ) return graph_raw_edges def get_node_labels(self, start_point_str=None, stop_point_str=None): """Return dependency graph node labels.""" ret = set() for edge in self.get_graph_raw( start_point_str, stop_point_str, ): left, right = edge[0:2] if left: ret.add(left) if right: ret.add(right) return ret @staticmethod def _close_families(l_id, r_id, clf_map): """Turn (name, point) to 'name.point' for edge. Replace close family members with family nodes if relevant. """ lret = None lname, lpoint = None, None if l_id: lname, lpoint = l_id lret = Tokens( cycle=str(lpoint), task=lname, ).relative_id rret = None rname, rpoint = None, None if r_id: rname, rpoint = r_id rret = Tokens( cycle=str(rpoint), task=rname, ).relative_id for fam_name, fam_members in clf_map.items(): if lname in fam_members and rname in fam_members: # l and r are both members lret = Tokens( cycle=str(lpoint), task=fam_name, ).relative_id rret = Tokens( cycle=str(rpoint), task=fam_name, ).relative_id break elif lname in fam_members: # l is a member lret = Tokens( cycle=str(lpoint), task=fam_name, ).relative_id elif rname in fam_members: # r is a member rret = Tokens( cycle=str(rpoint), task=fam_name, ).relative_id return lret, rret def _load_graph(self): """Parse and load dependency graph.""" LOG.debug("Parsing the dependency graph") # Generate a map of *task* members of each family. # Note we could exclude 'root' from this and disallow use of 'root' in # the graph (which would probably be quite reasonable). family_map = { family: [ task for task in sorted(tasks) if ( task in self.runtime['parents'] and task not in self.runtime['descendants'] ) ] for family, tasks in self.runtime['descendants'].items() if family != 'root' } graphdict = self.cfg['scheduling']['graph'] if 'graph' in graphdict: section = get_sequence_cls().get_async_expr() graphdict[section] = graphdict.pop('graph') icp = str(self.initial_point) fcp = self.cfg['scheduling']['final cycle point'] # Make a stack of sections and graphs [(sec1, graph1), ...] sections = [] for section, value in self.cfg['scheduling']['graph'].items(): # Substitute initial and final cycle points. section = section.replace("^", icp) if fcp: section = section.replace("$", fcp) elif "$" in section: raise WorkflowConfigError("Final cycle point referenced" " ($) but not defined.") # If the section consists of more than one sequence, split it up. new_sections = RE_SEC_MULTI_SEQ.split(section) if len(new_sections) > 1: for new_section in new_sections: sections.append((new_section.strip(), value)) else: sections.append((section, value)) # Parse and process each graph section. task_triggers = {} task_output_opt = {} for section, graph in sections: try: seq = get_sequence(section, icp, fcp) except (AttributeError, TypeError, ValueError, CylcError) as exc: if cylc.flow.flags.verbosity > 1: traceback.print_exc() msg = 'Cannot process recurrence %s' % section msg += ' (initial cycle point=%s)' % icp msg += ' (final cycle point=%s)' % fcp if isinstance(exc, CylcError): msg += ' %s' % exc.args[0] raise WorkflowConfigError(msg) from None self.sequences.append(seq) parser = GraphParser( family_map, self.parameters, task_output_opt=task_output_opt, expire_triggers=self.experimental.expire_triggers, ) parser.parse_graph(graph) task_output_opt.update(parser.task_output_opt) self.workflow_polling_tasks.update( parser.workflow_state_polling_tasks) self._proc_triggers(parser, seq, task_triggers) # Checking for undefined outputs for terminal tasks. Tasks with # dependencies are checked in generate_triggers: self.check_terminal_outputs(parser.terminals) # set of all cycling intervals contained within the workflow cycling_intervals = { sequence.get_interval() for sequence in self.sequences } | { # add a null interval to handle async workflows get_interval_cls().get_null() } # determine the longest cycling interval in the workflow self.interval_of_longest_sequence: 'IntervalBase' = max({ interval for interval in cycling_intervals # NOTE: None type sorts above a null interval for strange # historical reasons so must be filtered out if interval is not None }) self.set_required_outputs(task_output_opt) if cylc.flow.flags.verbosity > 1: # Print inferred output optionality for debugging the graph parser. # (This does not includes tasks that default to success-required.) optionals = [ f" \u2022 {name}:{output}" for (name, output), (optional, _, _) in task_output_opt.items() if optional ] requireds = [ f" \u2022 {name}:{output}" for (name, output), (optional, _, _) in task_output_opt.items() if not optional ] if optionals: LOG.debug( "Optional outputs inferred from the graph:\n" f"{'\n'.join(optionals)}" ) if requireds: LOG.debug( "Required outputs inferred from the graph:\n" f"{'\n'.join(requireds)}" ) # Detect use of xtrigger names with '@' prefix (creates a task). overlap = set(self.taskdefs.keys()).intersection( list(self.cfg['scheduling']['xtriggers'])) if overlap: LOG.error(', '.join(overlap)) raise WorkflowConfigError('task and @xtrigger names clash') for tdef in self.taskdefs.values(): tdef.tweak_outputs() def check_terminal_outputs(self, terminals: Iterable[str]) -> None: """Check that task outputs have been registered with tasks. Where a "terminal output" is an output for a task at the end of a graph string, such as "end" in `start => middle => end`. Raises: WorkflowConfigError if a custom output is not defined. """ for t in terminals: if ':' not in t: continue task, output = t.split(':') if (output := output.strip('?')) in TASK_QUALIFIERS: continue task = task.strip('!') if output not in self.cfg['runtime'][task]['outputs']: raise WorkflowConfigError( f"Undefined custom output: {task}:{output}" ) def _proc_triggers(self, parser, seq, task_triggers): """Define graph edges, taskdefs, and triggers, from graph sections.""" suicides = 0 for right, val in parser.triggers.items(): for expr, trigs in val.items(): orig = parser.original[right][expr] lefts, suicide = trigs # (lefts, right) e.g.: # for """ # foo|bar => baz # @x => baz # """ # - ([], foo) # - ([], bar) # - (['foo:succeeded', 'bar:succeeded'], baz) # - (['@x'], baz) self.generate_edges(expr, orig, lefts, right, seq, suicide) # Lefts can be null; all appear on RHS once so can generate # taskdefs with right only. Right is never None or @xtrigger. self.generate_taskdef(orig, right) self.add_sequence( [ node for node in lefts + [right] if node and not node.startswith('@') ], seq, suicide ) # RHS quals not needed now (used already for taskdef outputs) right = parser.REC_QUAL.sub('', right) self.generate_triggers( expr, lefts, right, seq, suicide, task_triggers) if suicide: suicides += 1 if suicides: LOG.info( f"{suicides} suicide trigger(s) detected. These are rarely " "needed in Cylc 8 - see https://cylc.github.io/cylc-doc/" "stable/html/7-to-8/major-changes/suicide-triggers.html" ) def set_required_outputs( self, task_output_opt: Dict[Tuple[str, str], Tuple[bool, bool, bool]] ) -> None: """set optional/required status of parsed task outputs. Args: task_output_opt: {(task, output): (is-optional, default, is_set)} """ for name, taskdef in self.taskdefs.items(): for output in taskdef.outputs: try: optional, _, _ = task_output_opt[(name, output)] except KeyError: # Output not used in graph. continue taskdef.set_required_output(output, not optional) def find_taskdefs(self, name: str) -> Set[TaskDef]: """Find TaskDef objects in family "name" or matching "name". Return a list of TaskDef objects which: * have names that glob matches "name". * are in a family that glob matches "name". """ if name in self.taskdefs: # Match a task name return {self.taskdefs[name]} fams = self.runtime['first-parent descendants'] if name in fams: # Match a family name return { self.taskdefs[member] for member in fams[name] if member in self.taskdefs } # Glob match from_task_names = { taskdef for key, taskdef in self.taskdefs.items() if fnmatchcase(key, name) } from_family_names = { self.taskdefs[member] for key, members in fams.items() if fnmatchcase(key, name) for member in members if member in self.taskdefs } return from_task_names.union(from_family_names) def get_taskdef( self, name: str, orig_expr: Optional[str] = None ) -> TaskDef: """Return an instance of TaskDef for task name.""" if name not in self.taskdefs: if name == 'root': self.implicit_tasks.add(name) elif name not in self.cfg['runtime']: # implicit inheritance from root self.implicit_tasks.add(name) # These can't just be a reference to root runtime as we have to # make some items task-specific: e.g. subst task name in URLs. self.cfg['runtime'][name] = OrderedDictWithDefaults() replicate(self.cfg['runtime'][name], self.cfg['runtime']['root']) if 'root' not in self.runtime['descendants']: # (happens when no runtimes are defined in flow.cylc) self.runtime['descendants']['root'] = set() if 'root' not in self.runtime['first-parent descendants']: # (happens when no runtimes are defined in flow.cylc) self.runtime['first-parent descendants']['root'] = set() self.runtime['parents'][name] = ['root'] self.runtime['linearized ancestors'][name] = [name, 'root'] self.runtime['first-parent ancestors'][name] = [name, 'root'] self.runtime['descendants']['root'].add(name) self.runtime['first-parent descendants']['root'].add(name) self.ns_defn_order.append(name) try: self.taskdefs[name] = self._get_taskdef(name) except TaskDefError as exc: if orig_expr: LOG.error(orig_expr) raise WorkflowConfigError(str(exc)) from None else: # Record custom message outputs from [runtime]. messages = set(self.cfg['runtime'][name]['outputs'].values()) for output, message in ( self.cfg['runtime'][name]['outputs'].items() ): try: messages.remove(message) except KeyError: raise WorkflowConfigError( 'Duplicate task message in' f' "[runtime][{name}][outputs]' f'{output} = {message}" - messages must be unique' ) from None valid, msg = TaskOutputValidator.validate(output) if not valid: raise WorkflowConfigError( f'Invalid task output "' f'[runtime][{name}][outputs]' f'{output} = {message}" - {msg}' ) valid, msg = TaskMessageValidator.validate(message) if not valid: raise WorkflowConfigError( f'Invalid task message "' f'[runtime][{name}][outputs]' f'{output} = {message}" - {msg}' ) self.taskdefs[name].add_output(output, message) return self.taskdefs[name] def _get_taskdef(self, name: str) -> TaskDef: """Get the dense task runtime.""" # (TaskDefError caught above) try: rtcfg = self.cfg['runtime'][name] # If the workflow mode is simulation or dummy always # override the task config: workflow_run_mode = RunMode.get(self.options) if workflow_run_mode.value in WORKFLOW_ONLY_MODES: rtcfg['run mode'] = workflow_run_mode.value except KeyError: raise WorkflowConfigError("Task not defined: %s" % name) from None # We may want to put in some handling for cases of changing the # initial cycle via restart (accidentally or otherwise). # Get the taskdef object for generating the task proxy class taskd = TaskDef( name, rtcfg, self.start_point, self.initial_point) # TODO - put all taskd.foo items in a single config dict if name in self.expiration_offsets: taskd.expiration_offset = self.expiration_offsets[name] if name in self.ext_triggers: taskd.external_triggers.append(self.ext_triggers[name]) taskd.sequential = ( name in self.cfg['scheduling']['special tasks']['sequential']) taskd.namespace_hierarchy = list( reversed(self.runtime['linearized ancestors'][name])) if name in self.task_param_vars: taskd.param_var.update(self.task_param_vars[name]) return taskd def describe(self, name): """Return title and description of the named task.""" return self.taskdefs[name].describe() def get_ref_log_name(self): """Return path to reference log (for reference test).""" return os.path.join(self.fdir, 'reference.log') def get_expected_failed_tasks(self): """Return list of expected failed tasks. Return: - An empty list if NO task is expected to fail. - A list of NAME.CYCLE for the tasks that are expected to fail in reference test mode. - None if there is no expectation either way. """ if self.options.reftest: return self.cfg['scheduler']['events']['expected task failures'] elif self.options.abort_if_any_task_fails: return [] else: return None def get_validated_rsync_includes(self): """Validate and return items to be included in the file installation""" includes = self.cfg['scheduler']['install'] illegal_includes = {} for include in includes: include_path = Path(include) if include_path.is_absolute(): illegal_includes[include] = 'Paths cannot be absolute' elif len(include_path.parts) > 1: illegal_includes[include] = ( 'Only top-level files/directories can be configured' ) if len(illegal_includes) > 0: raise WorkflowConfigError( 'Error in [scheduler]install:\n\t' + '\n\t'.join( f'"{path}" - {message}.' for path, message in illegal_includes.items() ) ) return includes def process_metadata_urls(self): """Process [meta]URL items.""" # workflow metadata url = self.cfg['meta']['URL'] try: self.cfg['meta']['URL'] = url % { 'workflow': self.workflow, } except (KeyError, ValueError): try: # Replace workflow and task name in workflow and task URLs. # BACK COMPAT: suite_name # url: https://github.com/cylc/cylc-flow/pull/4724 # from: 7 # to: 8.0 # remove at: 8.9 self.cfg['meta']['URL'] = url % { # cylc 7 'suite_name': self.workflow, # cylc 8 'workflow': self.workflow, } except (KeyError, ValueError): raise InputError( f'Invalid template [meta]URL: {url}' ) from None else: LOG.warning( 'Detected deprecated template variables in [meta]URL. ' 'Support will be removed in Cylc 8.9.' '\nSee the configuration documentation for details.' ) # task metadata self.cfg['meta']['URL'] = RE_WORKFLOW_ID_VAR.sub( self.workflow, self.cfg['meta']['URL']) for name, cfg in self.cfg['runtime'].items(): try: cfg['meta']['URL'] = cfg['meta']['URL'] % { 'workflow': self.workflow, 'task': name, } except (KeyError, ValueError): # BACK COMPAT: suite_name, task_name # url: https://github.com/cylc/cylc-flow/pull/4724 # from: 7 # to: 8.0 # remove at: 8.9 try: cfg['meta']['URL'] = cfg['meta']['URL'] % { # cylc 7 'suite_name': self.workflow, 'task_name': name, # cylc 8 'workflow': self.workflow, 'task': name, } except (KeyError, ValueError): raise InputError( f'Invalid template [meta]URL: {url}' ) from None else: LOG.warning( 'Detected deprecated template variables in' f' [runtime][{name}][meta]URL. Support will be ' 'removed in Cylc 8.9.' '\nSee the configuration documentation for details.' ) cfg['meta']['URL'] = RE_WORKFLOW_ID_VAR.sub( self.workflow, cfg['meta']['URL']) cfg['meta']['URL'] = RE_TASK_NAME_VAR.sub( name, cfg['meta']['URL']) @staticmethod def check_for_owner(tasks: Dict) -> None: """Raise exception if [runtime][task][remote]owner """ owners = {} for task, tdef in tasks.items(): owner = tdef.get('remote', {}).get('owner', None) if owner: owners[task] = owner if owners: # TODO: Convert URL to a stable or latest release doc after 8.0 # https://github.com/cylc/cylc-flow/issues/4663 msg = ( '"[runtime][task][remote]owner" is obsolete at Cylc 8.' '\nsee https://cylc.github.io/cylc-doc/stable/' 'html/7-to-8/major-changes/remote-owner.html' f'\nFirst {min(len(owners), 5)} tasks:' ) for task, _ in list(owners.items())[:5]: msg += f'\n * {task}"' raise WorkflowConfigError(msg) def upgrade_clock_triggers(self): """Convert old-style clock triggers to clock xtriggers. [[special tasks]] clock-trigger = foo(PT1D) becomes: [[xtriggers]] _cylc_wall_clock_foo = wall_clock(PT1D) Not done by parsec upgrade because the graph has to be parsed first. """ # BACK COMPAT: clock-trigger has been deprecated # from: 8.0.0 # remove at: x if len(self.cfg['scheduling']['special tasks']['clock-trigger']) > 0: LOG.warning(r"Clock-trigger is deprecated; " r"please use @wall_clock") for item in self.cfg['scheduling']['special tasks']['clock-trigger']: match = RE_CLOCK_OFFSET.match(item) # (Already validated during "special tasks" parsing above.) task_name, offset = match.groups() # Derive an xtrigger label. label = '_'.join(('_cylc', 'wall_clock', task_name)) # Define the xtrigger function. args = [] if offset is None else [offset] xtrig = SubFuncContext(label, 'wall_clock', args, {}) self.xtrigger_collator.add_trig(label, xtrig, self.fdir) # Add it to the task, for each sequence that the task appears in. taskdef = self.get_taskdef(task_name) for seq in taskdef.sequences: taskdef.add_xtrig_label(label, seq) # BACK COMPAT # from: 7 # to: 8.0 # remove at: 8.9 def _upg_wflow_event_names(self) -> None: """Upgrade any Cylc 7 workflow handler/mail events names.""" for setting in ('handler events', 'mail events'): event_names: list[str] | None = self.cfg['scheduler']['events'][ setting ] if not event_names: continue upgraded: dict[str, str] = {} for i, event in enumerate(event_names): if event in WorkflowEventHandler.EVENTS_DEPRECATED: event_names[i] = upgraded[event] = ( WorkflowEventHandler.EVENTS_DEPRECATED[event] ) if upgraded: LOG.warning( f"{upgrader.depr_msg}\n" f" * (8.0.0) [scheduler][events][{setting}] " + ', '.join(f'{k} -> {v}' for k, v in upgraded.items()) + " - support for the old item will be removed at Cylc 8.9" ) cylc-flow-8.7.0/cylc/flow/scripts/0000775000175000017500000000000015264420307017204 5ustar alastairalastaircylc-flow-8.7.0/cylc/flow/scripts/tui.py0000664000175000017500000000551115264420307020361 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc tui [WORKFLOW] View and control running workflows in the terminal. (Tui = Terminal User Interface) Tui allows you to monitor and interact with workflows in a manner similar to the GUI. Press "h" whilst running Tui to bring up the help screen, use the arrow keys to navigate. """ from getpass import getuser from textwrap import indent from typing import TYPE_CHECKING, Optional from cylc.flow.id import Tokens from cylc.flow.id_cli import parse_id from cylc.flow.option_parsers import ( OPT_WORKFLOW_ID_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.terminal import cli_function from cylc.flow.tui import TUI from cylc.flow.tui.util import suppress_logging from cylc.flow.tui.app import ( TuiApp, ) if TYPE_CHECKING: from optparse import Values __doc__ += indent(TUI, ' ') def get_option_parser() -> COP: parser = COP( __doc__, argdoc=[OPT_WORKFLOW_ID_ARG_DOC], # auto_add=False, NOTE: at present auto_add can not be turned off color=False ) parser.add_option( '--comms-timeout', metavar='SEC', help=( # NOTE: Tui overrides the default client timeout "Set the timeout for communication with the running workflow." " The default is 5 seconds, you may need to increase this in" " order for Tui to keep up with especially busy workflows." ), action='store', default=5, dest='comms_timeout', type=int, ) return parser @cli_function(get_option_parser) def main(_, options: 'Values', workflow_id: Optional[str] = None) -> None: # get workflow ID if specified if workflow_id: workflow_id, *_ = parse_id( workflow_id, constraint='workflows', ) tokens = Tokens(workflow_id) workflow_id = tokens.duplicate(user=getuser()).id # start Tui with suppress_logging(), TuiApp().main( workflow_id, client_timeout=options.comms_timeout, ): # tui stops according to user input pass cylc-flow-8.7.0/cylc/flow/scripts/get_workflow_contact.py0000775000175000017500000000356715264420307024020 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc get-workflow-contact [OPTIONS] ARGS Print contact information of a running workflow. """ from typing import TYPE_CHECKING from cylc.flow.exceptions import CylcError, ServiceFileError from cylc.flow.id_cli import parse_id from cylc.flow.option_parsers import ( WORKFLOW_ID_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.workflow_files import load_contact_file from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values def get_option_parser(): return COP(__doc__, argdoc=[WORKFLOW_ID_ARG_DOC]) @cli_function(get_option_parser) def main(parser: COP, options: 'Values', workflow_id: str) -> None: """CLI for "cylc get-workflow-contact".""" workflow_id, *_ = parse_id( workflow_id, constraint='workflows', ) try: data = load_contact_file(workflow_id) except ServiceFileError as exc: raise CylcError( f"{workflow_id}: cannot get contact info, workflow not running?" ) from exc else: for key, value in sorted(data.items()): print("%s=%s" % (key, value)) cylc-flow-8.7.0/cylc/flow/scripts/reload.py0000775000175000017500000001102615264420307021027 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc reload [OPTIONS] ARGS Reload the configuration of a running workflow. Example: # install and run the workflow $ cylc install $ cylc play my_workflow # make changes to the workflow source directory # reinstall the workflow $ cylc reinstall my_workflow # reload the workflow to pick up changes $ cylc reload my_workflow # the workflow is now running with the new config All settings including task definitions, with the exception of workflow log config, can be changed on reload. Changes to task definitions take effect immediately, unless a task is already running at reload time. Upon reload, remote file installation will be triggered for all relevant platforms on the next job submit. Any changed files that are configured to be included in the file installation will be transferred to the appropriate remote platform(s). If the workflow was started with Jinja2 template variables set on the command line (cylc play --set 'FOO="bar"' WORKFLOW_ID) the same template settings apply to the reload (only changes to the flow.cylc file itself are reloaded). If the modified workflow definition does not parse, failure to reload will be reported but no harm will be done to the running workflow. """ from functools import partial import sys from typing import TYPE_CHECKING from packaging.version import parse as parse_version from cylc.flow.exceptions import InputError from cylc.flow.network.client_factory import get_client from cylc.flow.network.multi import call_multi from cylc.flow.option_parsers import ( WORKFLOW_ID_MULTI_ARG_DOC, CylcOptionParser as COP, OptionSettings, ) from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values RELOAD_OPTIONS = [ OptionSettings( ['-g', '--global'], help='also reload global configuration.', action="store_true", default=False, dest="reload_global", sources={'reload'} ), ] MUTATION = ''' mutation ( $wFlows: [WorkflowID]!, ) { reload ( workflows: $wFlows ) { result } } ''' # Separate mutation, not backwards compatible with Cylc < 8.5.0: MUTATION_GLOBAL = ''' mutation ( $wFlows: [WorkflowID]!, $reloadGlobal: Boolean, ) { reload ( workflows: $wFlows reloadGlobal: $reloadGlobal ) { result } } ''' def get_option_parser(): parser = COP( __doc__, comms=True, multiworkflow=True, argdoc=[WORKFLOW_ID_MULTI_ARG_DOC], ) for option in RELOAD_OPTIONS: parser.add_option(*option.args, **option.kwargs) return parser async def run(options: 'Values', workflow_id: str): pclient = get_client(workflow_id, timeout=options.comms_timeout) if options.reload_global: if parse_version(pclient.scheduler_version) < parse_version( '8.5.0.dev' ): raise InputError( "The --global option is not supported by the version of Cylc " f"running the workflow ({pclient.scheduler_version})." ) mutation_kwargs = { 'request_string': MUTATION_GLOBAL, 'variables': { 'wFlows': [workflow_id], 'reloadGlobal': options.reload_global, } } else: mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [workflow_id], } } return await pclient.async_request('graphql', mutation_kwargs) @cli_function(get_option_parser) def main(parser: COP, options: 'Values', *ids) -> None: reload_cli(options, *ids) def reload_cli(options: 'Values', *ids) -> None: rets = call_multi( partial(run, options), *ids, constraint='workflows', ) sys.exit(all(rets.values()) is False) cylc-flow-8.7.0/cylc/flow/scripts/dump.py0000775000175000017500000002401015264420307020523 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc dump [OPTIONS] ARGS Print state of a running workflow including tasks in the active (n=0) window. For a more detailed view of the workflow see `cylc tui` or `cylc gui`. For more information about a specific task, such as the current state of its prerequisites and outputs, see 'cylc show'. For command line monitoring: * `cylc tui` * `watch cylc dump -t WORKFLOW_ID` works for small simple workflows Examples: # Display the state of all active tasks, sorted by cycle point: $ cylc dump --tasks --sort WORKFLOW_ID | grep running # Display the state of all active tasks in a particular cycle point: $ cylc dump -t WORKFLOW_ID | grep 2010082406 """ import asyncio import json from typing import TYPE_CHECKING from graphene.utils.str_converters import to_snake_case from cylc.flow.exceptions import CylcError from cylc.flow.id_cli import parse_id_async from cylc.flow.option_parsers import ( WORKFLOW_ID_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.network.client_factory import get_client from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values TASK_SUMMARY_FRAGMENT = ''' fragment tProxy on TaskProxy { id name cyclePoint state graphDepth isHeld isQueued isRunahead flowNums firstParent { id } jobSubmits jobs(sort: {keys: ["submitNum"], reverse: true}) { id state submitNum submittedTime startedTime finishedTime jobLogDir platform executionTimeLimit jobRunnerName jobId } } ''' FAMILY_SUMMARY_FRAGMENT = ''' fragment fProxy on FamilyProxy { id name cyclePoint state } ''' WORKFLOW_SUMMARY_FRAGMENT = ''' fragment wFlow on Workflow { name oldestActiveCyclePoint newestActiveCyclePoint timeZoneInfo { hours minutes stringBasic stringExtended } lastUpdated runMode states namespaceDefinitionOrder: nsDefOrder reloaded stateTotals meta { title description URL userDefined } status statusMsg families { name meta { title description URL userDefined } firstParent { name } } tasks { name meta { title description URL userDefined } meanElapsedTime firstParent { name } } } ''' def get_option_parser(): parser = COP( __doc__, comms=True, argdoc=[WORKFLOW_ID_ARG_DOC], ) parser.add_option( "-g", "--global", help="Global information only.", action="store_const", const="global", dest="disp_form") parser.add_option( "-t", "--tasks", help="Task states only.", action="store_const", const="tasks", dest="disp_form") parser.add_option( "-i", "--full-ids", help="With -t/--tasks (task states only)," " print full workflow//cycle/name IDs without other attributes" " for easy cut-and-paste to other commands.", action="store_true", default=False, dest="full_ids") parser.add_option( "-l", "--legacy", help="Tasks states only; use legacy format.", action="store_true", default=False, dest="legacy_format") parser.add_option( "-f", "--flows", help="Print flow numbers with tasks.", action="store_true", default=False, dest="show_flows") parser.add_option( "-r", "--raw", "--raw-format", help='Display raw format.', action="store_const", const="raw", dest="disp_form") parser.add_option( "-p", "--pretty", "--pretty-print", help='Display raw format with indents and newlines.', action="store_true", default=False, dest="pretty") parser.add_option( "-s", "--sort", help="Task states only; sort by cycle point instead of name.", action="store_true", default=False, dest="sort_by_cycle") return parser @cli_function(get_option_parser) def main(_, options: 'Values', workflow_id: str) -> None: asyncio.run(dump(workflow_id, options)) async def dump(workflow_id, options, write=print): workflow_id, *_ = await parse_id_async( workflow_id, constraint='workflows', ) pclient = get_client( workflow_id, # set the default timeout a bit higher for "cylc dump" as it can # take a little while for Cylc to formulate the response timeout=options.comms_timeout or 15, ) if options.sort_by_cycle: sort_args = {'keys': ['cyclePoint', 'name']} else: sort_args = {'keys': ['name', 'cyclePoint']} # restrict to the n=0 window graph_depth = 0 if options.disp_form == "raw": query = f''' {TASK_SUMMARY_FRAGMENT} {FAMILY_SUMMARY_FRAGMENT} {WORKFLOW_SUMMARY_FRAGMENT} query ($wFlows: [ID]!, $sortBy: SortArgs) {{ workflows (ids: $wFlows, stripNull: false) {{ ...wFlow taskProxies (sort: $sortBy, graphDepth: {graph_depth}) {{ ...tProxy }} familyProxies (sort: $sortBy, graphDepth: {graph_depth}) {{ ...fProxy }} }} }}''' elif options.disp_form != "tasks": query = f''' {WORKFLOW_SUMMARY_FRAGMENT} query ($wFlows: [ID]!) {{ workflows (ids: $wFlows, stripNull: false) {{ ...wFlow }} }}''' else: query = f''' {TASK_SUMMARY_FRAGMENT} query ($wFlows: [ID]!, $sortBy: SortArgs) {{ workflows (ids: $wFlows, stripNull: false) {{ taskProxies (sort: $sortBy, graphDepth: {graph_depth}) {{ ...tProxy }} }} }}''' query_kwargs = { 'request_string': query, 'variables': {'wFlows': [workflow_id], 'sortBy': sort_args} } workflows = await pclient.async_request('graphql', query_kwargs) try: for summary in workflows['workflows']: if options.disp_form == "raw": if options.pretty: write(json.dumps(summary, indent=4)) else: write(summary) else: if options.disp_form != "tasks": node_urls = { node['name']: node['meta']['URL'] for node in summary['tasks'] + summary['families'] } summary['workflowUrls'] = { node_name: node_urls[node_name] for node_name in summary['namespaceDefinitionOrder'] if node_name in node_urls } summary['workflowUrls']['workflow_id'] = ( summary['meta']['URL']) del summary['tasks'] del summary['families'] del summary['meta'] for key, value in sorted(summary.items()): write( f'{to_snake_case(key).replace("_", " ")}={value}') elif options.legacy_format: for item in summary['taskProxies']: if options.sort_by_cycle: values = [ item['cyclePoint'], item['name'], item['state']] else: values = [ item['name'], item['cyclePoint'], item['state']] values.append('held' if item['isHeld'] else 'not-held') values.append('queued' if item['isQueued'] else 'not-queued') values.append('runahead' if item['isRunahead'] else 'not-runahead') if options.show_flows: values.append(item['flowNums']) write(', '.join(values)) else: for item in summary['taskProxies']: result = "" attrs = [] if options.full_ids: result = f"{workflow_id}//" result += f"{item['cyclePoint']}/{item['name']}" if not options.full_ids: result += f":{item['state']}" if item['isHeld']: attrs.append("held") if item['isQueued']: attrs.append("queued") if item['isRunahead']: attrs.append("runahead") if attrs: result += " (" + ",".join(attrs) + ")" if options.show_flows and not options.full_ids: result += ( f" flows={item['flowNums'].replace(' ', '')}" ) write(result) except Exception as exc: raise CylcError( json.dumps(workflows, indent=4) + '\n' + str(exc) + '\n' ) from None cylc-flow-8.7.0/cylc/flow/scripts/install.py0000775000175000017500000002375315264420307021241 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc install [OPTIONS] ARGS Install a workflow into ~/cylc-run. The workflow can then be started, stopped, and targeted by name. Normal installation creates a numbered run directory "~/cylc-run//run". If a SOURCE_NAME is supplied, Cylc will search for the workflow source in the list of directories given by "global.cylc[install]source dirs", and install the first match. The installed workflow ID will start with SOURCE_NAME, unless --workflow-name is used. If a PATH is supplied, Cylc will install the workflow from the source directory given by the path. Relative paths must start with "./" to avoid ambiguity with SOURCE_NAME (i.e. "foo/bar" will be interpreted as a source name, whereas "./foo/bar" will be interpreted as a path). The installed workflow ID will start with the basename of the path, unless --workflow-name is used. If no argument is supplied, Cylc will install the workflow from the source in the current working directory. A "_cylc-install/source" symlink to the source directory will be created in "~/cylc-run/". Any files or directories (excluding .git, .svn) from the source directory are copied to the new run directory. A ".service" directory will also be created in the run directory; this is used for server authentication files at runtime. Examples: # Install workflow "dogs/fido" from the first match in # `global.cylc[install]source dirs`, e.g. ~/cylc-src/dogs/fido/flow.cylc, # with run directory ~/cylc-run/dogs/fido/run1 (if "run1" already exists, # this will increment) $ cylc install dogs/fido # Install $PWD as "rabbit", if $PWD is ~/bunny/rabbit, with # run directory ~/cylc-run/rabbit/run1 $ cylc install # Install $PWD as "rabbit", if $PWD is ~/bunny/rabbit, with # run directory ~/cylc-run/rabbit (note: no "run1" sub-directory) $ cylc install --no-run-name # Install $PWD as "fido", regardless of what $PWD is called, with # run directory ~/cylc-run/fido/run1 $ cylc install --workflow-name=fido # Install $PWD/bunny/rabbit/ as "rabbit", with run directory # ~/cylc-run/rabbit/run1 $ cylc install ./bunny/rabbit # Install /home/somewhere/badger as "badger", with run directory # ~/cylc-run/badger/run1 $ cylc install /home/somewhere/badger # Install $PWD as "cats", if $PWD is ~/cats, with run directory # ~/cylc-run/cats/paws $ cylc install --run-name=paws The same workflow can be installed with multiple names; this results in multiple workflow run directories that link to the same workflow definition. """ from ansimarkup import ansiprint as cprint import asyncio from optparse import Values from pathlib import Path from typing import Any, Dict, Optional, Tuple from cylc.flow import LOG from cylc.flow.exceptions import InputError from cylc.flow.loggingutil import CylcLogFormatter, set_timestamps from cylc.flow.option_parsers import ( CylcOptionParser as COP, OptionSettings, Options ) from cylc.flow.pathutil import ( EXPLICIT_RELATIVE_PATH_REGEX, expand_path, get_workflow_run_dir ) from cylc.flow.plugins import run_plugins_async from cylc.flow.install import ( install_workflow, parse_cli_sym_dirs, search_install_source_dirs, ) from cylc.flow.scripts.scan import ( get_pipe, _format_plain, FLOW_STATE_SYMBOLS, FLOW_STATE_CMAP ) from cylc.flow.terminal import cli_function INSTALL_OPTIONS = [ OptionSettings( ["--workflow-name", "-n"], help="Install into ~/cylc-run//runN ", action="store", metavar="WORKFLOW_NAME", default=None, dest="workflow_name", sources={'install'}, ), OptionSettings( ["--symlink-dirs"], help=( "Enter a comma-delimited list, in the form" " 'log=path/to/store, share = $HOME/some/path, ...'." " Use this option to override the global.cylc configuration" " for local symlinks for the run, log, work, share and" " share/cycle directories. Enter" " an empty list '' to skip making localhost symlink dirs."), action="store", dest="symlink_dirs", sources={'install'}, ), OptionSettings( ["--run-name", "-r"], help=( "Give the run a custom name instead of automatically" " numbering it."), action="store", metavar="RUN_NAME", default=None, dest="run_name", sources={'install'}, ), OptionSettings( ["--no-run-name"], help=( "Install the workflow directly into" " ~/cylc-run/," " without an automatic run number or custom run name."), action="store_true", default=False, dest="no_run_name", sources={'install'}, ), OptionSettings( ['--no-ping'], help=( "When scanning for active instances of the workflow, " "do not attempt to contact the schedulers to get status."), action='store_true', default=False, dest='no_ping', sources={'install'}, ) ] def get_option_parser() -> COP: parser = COP( __doc__, comms=True, argdoc=[ COP.optional( ('SOURCE_NAME | PATH', 'Workflow source name or path to source directory') ) ] ) options = parser.get_cylc_rose_options() + INSTALL_OPTIONS for option in options: parser.add_option(*option.args, **option.kwargs) return parser def get_source_location(path: Optional[str]) -> Path: """Return the workflow source location as an absolute path. Note: does not check that the source actually exists. """ if path is None: return Path.cwd() path = str(path).strip() expanded_path = Path(expand_path(path)) if expanded_path.is_absolute(): return expanded_path if EXPLICIT_RELATIVE_PATH_REGEX.match(path): return Path.cwd() / expanded_path return search_install_source_dirs(expanded_path) async def scan(wf_name: str, ping: bool = True) -> None: """Print any instances of wf_name that are already active.""" opts = Values({ 'name': [f'{wf_name}/*'], 'states': {'running', 'paused', 'stopping'}, 'source': False, 'ping': ping, # get status of scanned workflows }) active = sorted( [ item async for item in get_pipe( opts, None, scan_dir=get_workflow_run_dir(wf_name) # restricted scan ) ], key=lambda flow: flow['name'] ) if active: n = len(active) grammar = ( ["s", "are", "them all"] if n > 1 else ["", "is", "it"] ) cprint( CylcLogFormatter.COLORS['WARNING'].format( f'NOTE: {n} run%s of "{wf_name}"' ' %s already active:' % tuple(grammar[:2]) ) ) for item in active: if opts.ping: status = item['status'] tag = FLOW_STATE_CMAP[status] symbol = f" <{tag}>{FLOW_STATE_SYMBOLS[status]}" else: symbol = " " cprint(symbol, _format_plain(item, opts)) pattern = ( f"'{wf_name}/*'" if n > 1 else f"{item['name']}" ) print( f'You can stop %s with:\n cylc stop {pattern}' '\nSee "cylc stop --help" for options.' % grammar[-1] ) InstallOptions = Options(get_option_parser()) @cli_function(get_option_parser) def main( _parser: COP, opts: 'Values', id_: Optional[str] = None ) -> None: """CLI wrapper.""" asyncio.run(install_cli(opts, id_)) async def install_cli( opts: 'Values', id_: Optional[str] = None ) -> Tuple[str, str]: """Install workflow and scan for already-running instances.""" wf_name, wf_id = await install(opts, id_) await scan(wf_name, not opts.no_ping) return wf_name, wf_id async def install( opts: 'Values', id_: Optional[str] = None ) -> Tuple[str, str]: set_timestamps(LOG, opts.log_timestamp and opts.verbosity > 1) if opts.no_run_name and opts.run_name: raise InputError( "options --no-run-name and --run-name are mutually exclusive." ) source = get_source_location(id_) async for _entry_point, _plugin_result in run_plugins_async( 'cylc.pre_configure', srcdir=source, opts=opts, ): pass cli_symdirs: Optional[Dict[str, Dict[str, Any]]] = None if opts.symlink_dirs == '': cli_symdirs = {} elif opts.symlink_dirs: cli_symdirs = parse_cli_sym_dirs(opts.symlink_dirs) source_dir, rundir, workflow_name, workflow_id = install_workflow( source=source, workflow_name=opts.workflow_name, run_name=opts.run_name, no_run_name=opts.no_run_name, cli_symlink_dirs=cli_symdirs ) async for _entry_point, _plugin_result in run_plugins_async( 'cylc.post_install', srcdir=source_dir, opts=opts, rundir=str(rundir), async_block=True, ): pass print(f'INSTALLED {workflow_id} from {source_dir}') return workflow_name, workflow_id cylc-flow-8.7.0/cylc/flow/scripts/check_versions.py0000775000175000017500000001162215264420307022570 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc check-versions [OPTIONS] ARGS Check that Cylc versions match on different platforms. Check that the remote versions of Cylc invoked on each of the platforms used by a workflow matches the version this script is run with. Note: Cylc supports multiple parallel installations at different versions via the Cylc wrapper script. For more information see the installation section of the documentation. Use -v/--verbose to see the command invoked to determine the remote version (all remote Cylc command invocations will be of the same form, which may be site dependent -- see cylc global config documentation. """ import sys from typing import TYPE_CHECKING import cylc.flow.flags from cylc.flow.option_parsers import ( WORKFLOW_ID_OR_PATH_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.cylc_subproc import procopen, PIPE, DEVNULL from cylc.flow import __version__ as CYLC_VERSION from cylc.flow.config import WorkflowConfig from cylc.flow.exceptions import NoHostsError from cylc.flow.id_cli import parse_id from cylc.flow.platforms import get_platform, get_host_from_platform from cylc.flow.remote import construct_ssh_cmd from cylc.flow.templatevars import get_template_vars from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values def get_option_parser(): parser = COP( __doc__, jset=True, argdoc=[WORKFLOW_ID_OR_PATH_ARG_DOC], ) parser.add_option( "-e", "--error", help=( f"Exit with error status if {CYLC_VERSION} is not available " "on all remote platforms." ), action="store_true", default=False, dest="error" ) return parser @cli_function(get_option_parser) def main(_, options: 'Values', *ids) -> None: workflow_id, _, flow_file = parse_id( *ids, src=True, constraint='workflows', ) # extract task host platforms from the workflow_id config = WorkflowConfig( workflow_id, flow_file, options, get_template_vars(options)) platforms = { config.get_config(['runtime', name, 'platform']) for name in config.get_namespace_list('all tasks') } - {None, 'localhost'} # When "workflow run hosts" are formalised as "flow platforms" # we can substitute `localhost` for this, in the mean time # we will have to assume that flow hosts are configured correctly. if not platforms: sys.exit(0) verbose = cylc.flow.flags.verbosity > 0 versions = check_versions(platforms, verbose) report_results(platforms, versions, options.error) def check_versions(platforms, verbose): # get the cylc version on each platform versions = {} for platform_name in sorted(platforms): platform = get_platform(platform_name) try: host = get_host_from_platform( platform, bad_hosts=None ) except NoHostsError: print( f'Could not connect to {platform["name"]}', file=sys.stderr ) continue cmd = construct_ssh_cmd( ['version'], platform, host ) if verbose: print(cmd) proc = procopen(cmd, stdin=DEVNULL, stdout=PIPE, stderr=PIPE) out, err = proc.communicate() out = out.decode() err = err.decode() if proc.wait() == 0: if verbose: print(" %s" % out) versions[platform_name] = out.strip() else: versions[platform_name] = f'ERROR: {err.strip()}' return versions def report_results(platforms, versions, exit_error): # report results max_len = max((len(platform_name) for platform_name in platforms)) print(f'{"platform".rjust(max_len)}: cylc version') print('-' * (max_len + 14)) for platform_name, result in versions.items(): print(f'{platform_name.rjust(max_len)}: {result}') if all((version == CYLC_VERSION for version in versions.values())): ret_code = 0 elif exit_error: ret_code = 1 else: ret_code = 0 sys.exit(ret_code) cylc-flow-8.7.0/cylc/flow/scripts/config.py0000775000175000017500000001477315264420307021042 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc config [OPTIONS] ARGS Parse and print Cylc configuration files. Print parsed configuration, after runtime inheritance. If WORKFLOW is specified, print the workflow configuration, otherwise print the global configuration. Note: This is different to `cylc view` which doesn't parse the configuration, so is useful for debugging Jinja2. By default, unset values are printed as an empty string, or (for historical reasons) as "None" with -o/--one-line. These defaults can be changed with the -n/--null-value option. Examples: # print global configuration $ cylc config # print workflow configuration $ cylc config myflow # print specific setting from the global config $ cylc config -i '[platforms][myplatform]hosts # print specific section from workflow config $ cylc config -i '[scheduling][graph]' myflow # print workflow config, first setting the initial cycle point $ cylc config --initial-cycle-point=now myflow """ import asyncio import os.path from pathlib import Path from typing import List, Optional, TYPE_CHECKING from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.config import WorkflowConfig from cylc.flow.id_cli import parse_id_async from cylc.flow.exceptions import InputError from cylc.flow.option_parsers import ( AGAINST_SOURCE_OPTION, WORKFLOW_ID_OR_PATH_ARG_DOC, CylcOptionParser as COP, icp_option, ) from cylc.flow.pathutil import get_workflow_run_dir from cylc.flow.templatevars import get_template_vars from cylc.flow.terminal import cli_function from cylc.flow.workflow_files import WorkflowFiles if TYPE_CHECKING: from optparse import Values def get_option_parser() -> COP: parser = COP( __doc__, argdoc=[COP.optional(WORKFLOW_ID_OR_PATH_ARG_DOC)], jset=True, ) parser.add_option( "-i", "--item", metavar="[SEC...]ITEM", help="Item or section to print (multiple use allowed).", action="append", dest="item", default=[]) parser.add_option( '-d', '--defaults', help='Include the hard-coded Cylc default values in the output.', action='store_true', default=False ) parser.add_option( "-n", "--null-value", help="The string to print for unset values (default nothing).", metavar="STRING", action="store", default='', dest="none_str") parser.add_option( "-o", "--one-line", help="Print multiple single-value items at once.", action="store_true", default=False, dest="oneline") parser.add_option( "--print-hierarchy", "--print-filenames", "--hierarchy", help=( "Print the list of locations in which configuration files are " "looked for. An existing configuration file lower down the list " "overrides any settings it shares with those higher up."), action="store_true", default=False, dest="print_hierarchy") parser.add_option(icp_option) platform_listing_options_group = parser.add_option_group( 'Platform printing options') platform_listing_options_group.add_option( '--platform-names', help=( 'Print a list of platforms and platform group names from the ' 'configuration.' ), action='store_true', default=False, dest='print_platform_names' ) platform_listing_options_group.add_option( '--platforms', help=( 'Print platform and platform group configurations, ' 'including metadata.' ), action='store_true', default=False, dest='print_platforms' ) parser.add_option( *AGAINST_SOURCE_OPTION.args, **AGAINST_SOURCE_OPTION.kwargs) parser.add_cylc_rose_options() return parser def get_config_file_hierarchy(workflow_id: Optional[str] = None) -> List[str]: filepaths = [os.path.join(path, glbl_cfg().CONF_BASENAME) for _, path in glbl_cfg().conf_dir_hierarchy] if workflow_id is not None: filepaths.append( get_workflow_run_dir(workflow_id, WorkflowFiles.FLOW_FILE) ) return filepaths @cli_function(get_option_parser) def main( parser: COP, options: 'Values', *ids, ) -> None: asyncio.run(_main(options, *ids)) async def _main( options: 'Values', *ids, ) -> None: if options.print_platform_names and options.print_platforms: options.print_platform_names = False if options.print_platform_names or options.print_platforms: # Get platform information: if ids: raise InputError( "Workflow IDs are incompatible with --platform options." ) glbl_cfg().platform_dump( options.print_platform_names, options.print_platforms ) return if not ids: if options.print_hierarchy: print("\n".join(get_config_file_hierarchy())) return glbl_cfg().idump( options.item, not options.defaults, oneline=options.oneline, none_str=options.none_str ) return workflow_id, _, flow_file = await parse_id_async( *ids, src=True, constraint='workflows', ) if options.print_hierarchy: print("\n".join(get_config_file_hierarchy(workflow_id))) return # Save the location of the existing workflow run dir in the # against source option: if options.against_source: options.against_source = Path(get_workflow_run_dir(workflow_id)) config = WorkflowConfig( workflow_id, flow_file, options, get_template_vars(options) ) config.pcfg.idump( options.item, not options.defaults, oneline=options.oneline, none_str=options.none_str ) cylc-flow-8.7.0/cylc/flow/scripts/verbosity.py0000775000175000017500000000512315264420307021610 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc verbosity [OPTIONS] ARGS Set the logging severity level of a running scheduler. Messages at or above the chosen level are logged. If you choose WARNING (say) only WARNING and CRITICAL messages will be logged. """ from functools import partial import sys from typing import TYPE_CHECKING from cylc.flow import LOG_LEVELS from cylc.flow.exceptions import InputError from cylc.flow.network.client_factory import get_client from cylc.flow.network.multi import call_multi from cylc.flow.option_parsers import ( WORKFLOW_ID_MULTI_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values MUTATION = ''' mutation ( $wFlows: [WorkflowID]!, $level: LogLevels!, ) { setVerbosity ( workflows: $wFlows, level: $level, ) { result } } ''' def get_option_parser() -> COP: parser = COP( __doc__, comms=True, multiworkflow=True, argdoc=[ ('LEVEL', ', '.join(LOG_LEVELS.keys())), WORKFLOW_ID_MULTI_ARG_DOC, ] ) return parser async def run(options: 'Values', severity: str, workflow_id: str): pclient = get_client(workflow_id, timeout=options.comms_timeout) mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [workflow_id], 'level': severity, } } return await pclient.async_request('graphql', mutation_kwargs) @cli_function(get_option_parser) def main(parser: COP, options: 'Values', severity: str, *ids: str) -> None: if severity not in LOG_LEVELS: raise InputError(f"Illegal logging level, {severity}") rets = call_multi( partial(run, options, severity), *ids, constraint='workflows', ) sys.exit(all(rets.values()) is False) cylc-flow-8.7.0/cylc/flow/scripts/ping.py0000775000175000017500000000510715264420307020521 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc ping [OPTIONS] ARGS Test communication with running workflows. Print the HOST:PORT of running workflows. If any are not running, exit with error status. """ from functools import partial import sys from typing import Any, Dict, TYPE_CHECKING from cylc.flow.network.client_factory import get_client from cylc.flow.network.multi import call_multi from cylc.flow.option_parsers import ( ID_MULTI_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values FLOW_QUERY = ''' query ($wFlows: [ID]) { workflows(ids: $wFlows) { id name port } } ''' def get_option_parser() -> COP: parser = COP( __doc__, comms=True, multiworkflow=True, argdoc=[ID_MULTI_ARG_DOC], ) return parser async def run( options: 'Values', workflow_id: str, client=None ) -> Dict: pclient = client or get_client(workflow_id, timeout=options.comms_timeout) ret: Dict[str, Any] = { 'stdout': [], 'stderr': [], 'exit': 0 } flow_kwargs: Dict[str, Any] = { 'request_string': FLOW_QUERY, 'variables': {'wFlows': [workflow_id]} } result = await pclient.async_request('graphql', flow_kwargs) for flow in result['workflows']: ret['stdout'].append(f"{pclient.host}:{flow['port']}") return ret def report(response): return ( '\n'.join(response['stdout']), '\n'.join(response['stderr']), response['exit'] == 0, ) @cli_function(get_option_parser) def main( parser: COP, options: 'Values', *ids, ) -> None: rets = call_multi( partial(run, options), *ids, report=report, constraint='workflows', ) sys.exit(all(rets.values()) is False) cylc-flow-8.7.0/cylc/flow/scripts/report_timings.py0000775000175000017500000002761415264420307022640 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc report-timings [OPTIONS] ARGS Display workflow timing information. Retrieve workflow timing information for wait and run time performance analysis. Raw output and summary output (in text or HTML format) are available. Output is sent to standard output, unless an output filename is supplied. Summary Output (the default): Data stratified by host and job runner that provides a statistical summary of: 1. Queue wait time (duration between task submission and start times) 2. Task run time (duration between start and succeed times) 3. Total run time (duration between task submission and succeed times) Summary tables can be output in plain text format, or HTML with embedded SVG boxplots. Both summary options require the Pandas library, and the HTML summary option requires the Matplotlib library. Raw Output: A flat list of tabular data that provides (for each task and cycle) the 1. Time of successful submission 2. Time of task start 3. Time of task successful completion as well as information about the job runner and remote host to permit stratification/grouping if desired by downstream processors. Timings are shown only for succeeded tasks. For long-running or large workflows (i.e. workflows with many task events), the database query to obtain the timing information may take some time. """ import contextlib import io as StringIO import sys from collections import Counter from typing import TYPE_CHECKING from cylc.flow.exceptions import CylcError from cylc.flow.id_cli import parse_id from cylc.flow.option_parsers import ( WORKFLOW_ID_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.pathutil import get_workflow_run_pub_db_path from cylc.flow.rundb import CylcWorkflowDAO from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values @contextlib.contextmanager def smart_open(filename=None): """ Allow context management for output to either a file or to standard output transparently. See https://stackoverflow.com/a/17603000 """ if filename and filename != '-': fh = open(filename, 'w') # noqa SIM115 (close done in finally block) else: fh = sys.stdout try: yield fh finally: if fh is not sys.stdout: fh.close() def format_raw(row_buf, output): """Implement --format=raw""" output.write(row_buf.getvalue()) def format_summary(row_buf, output): """Implement --format=summary""" summary = TextTimingSummary(row_buf) summary.write_summary(output) def format_html(row_buf, output): """Implement --format=html""" summary = HTMLTimingSummary(row_buf) summary.write_summary(output) # supported output formats FORMATS = { 'raw': format_raw, 'summary': format_summary, 'html': format_html, } def get_option_parser() -> COP: parser = COP( __doc__, argdoc=[WORKFLOW_ID_ARG_DOC] ) parser.add_option( '--format', '-t', help=( f"Select output format. Choose from: {', '.join(FORMATS)}. " r"Default: %default." ), action='store', default='summary', choices=list(FORMATS) ) parser.add_option( "-O", "--output-file", help="Output to a specific file", action="store", default=None, dest="output_filename") return parser @cli_function(get_option_parser) def main(parser: COP, options: 'Values', workflow_id: str) -> None: _main(options, workflow_id) def _main(options: 'Values', workflow_id: str) -> None: workflow_id, *_ = parse_id( workflow_id, constraint='workflows', ) db_file = get_workflow_run_pub_db_path(workflow_id) with CylcWorkflowDAO(db_file, is_public=True) as dao: row_buf = format_rows(*dao.select_task_times()) with smart_open(options.output_filename) as output: FORMATS[options.format](row_buf, output) def format_rows(header, rows): """Write the rows in tabular format to a string buffer. Ensure that each column is wide enough to contain the widest data value and the widest header value. """ sio = StringIO.StringIO() max_lengths = [ max(data_len, head_len) for data_len, head_len in zip( (max(len(r[i]) for r in rows) for i in range(len(header))), (len(h) for h in header) ) ] formatter = ' '.join('%%-%ds' % line for line in max_lengths) + '\n' sio.write(formatter % header) for r in sorted(rows): sio.write(formatter % r) sio.seek(0) return sio class TimingSummary: """Base class for summarizing timing output from cylc.flow run database.""" def __init__(self, filepath_or_buffer): """Set up internal dataframe storage for time durations.""" self._check_imports() self.read_timings(filepath_or_buffer) def read_timings(self, filepath_or_buffer): """ Set up time duration dataframe storage based on start/stop/succeed times from flat database output. """ import pandas as pd # Avoid truncation of content in columns. pd.set_option('display.max_colwidth', 10000) df = pd.read_csv( filepath_or_buffer, sep=r'\s+', index_col=[0, 1, 2, 3], parse_dates=[4, 5, 6] ) self.df = pd.DataFrame({ 'queue_time': ( df['start_time'] - df['submit_time']).apply(self._dt_to_s), 'run_time': ( df['succeed_time'] - df['start_time']).apply(self._dt_to_s), 'total_time': ( df['succeed_time'] - df['submit_time']).apply(self._dt_to_s), }) self.df = self.df.rename_axis('timing_category', axis='columns') self.by_host_and_job_runner = self.df.groupby( level=['host', 'job_runner'] ) def write_summary(self, buf=sys.stdout): """Using the stored timings dataframe, output the data summary.""" self.write_summary_header(buf) for group, df in self.by_host_and_job_runner: self.write_group_header(buf, group) df_reshape = self._reshape_timings(df) df_describe = df.groupby(level='name').describe() df_describe.index.rename(None, inplace=True) for timing_category in self.df.columns: self.write_category( buf, timing_category, df_reshape, df_describe ) self.write_summary_footer(buf) def write_summary_header(self, buf): pass def write_summary_footer(self, buf): pass def write_group_header(self, buf, group): pass def write_category(self, buf, category, df_reshape, df_describe): pass def _check_imports(self): try: import pandas except ImportError: raise CylcError( 'Cannot import pandas - summary unavailable.' ' try: pip install cylc-flow[report-timings]' ) from None else: del pandas @staticmethod def _reshape_timings(timings): """ Given a dataframe of timings returned from the Cylc DAO methods indexed by (task, cycle point, ...) with columns for the various timing categories, return a dataframe reshaped with an index of (timing category, cycle point, ...) with columns for each task. Need a special method rather than standard Pandas pivot-table to handle duplicated index entries properly. """ if timings.index.duplicated().any(): # The 'unstack' used to pivot the dataframe gives an error if # there are duplicate entries in the index (see #2509). The # best way around this seems to be to add an intermediate index # level (in this case a retry counter) to de-duplicate indices. counts = Counter() retry = [] for t in timings.index: counts[t] += 1 retry.append(counts[t]) timings = timings.assign(retry=retry) timings = timings.set_index('retry', append=True) return timings.unstack('name').stack(level=0, future_stack=True) @staticmethod def _dt_to_s(dt): return dt.total_seconds() class TextTimingSummary(TimingSummary): """Timing summary in text form.""" line_width = 80 def write_group_header(self, buf, group): title = 'Host: %s\tJob Runner: %s' % group buf.write('=' * self.line_width + '\n') buf.write(title.center(self.line_width - 1) + '\n') buf.write('=' * self.line_width + '\n') def write_category(self, buf, category, df_reshape, df_describe): buf.write(category.center(self.line_width) + '\n') buf.write(('-' * len(category)).center(self.line_width) + '\n') buf.write(df_describe[category].to_string()) buf.write('\n\n') class HTMLTimingSummary(TimingSummary): """Timing summary in HTML form.""" def write_summary_header(self, buf): css = """ body { font-family: Sans-Serif; text-align: center; } h1 { background-color: grey; } h2 { width: 85%; background-color: #f0f0f0; margin: auto; } table { width: 75%; margin: auto; border-collapse: collapse; } tr:nth-child(even) { background-color: #f2f2f2; } td { text-align: right; } th, td { border-bottom: 1px solid grey; } svg { width: 65%; height: auto; margin: auto; display: block; } .timing { padding-bottom: 40px; } """ buf.write('' % css) def write_summary_footer(self, buf): buf.write('') def write_group_header(self, buf, group): buf.write('

Timings for host %s using job runner %s

' % group) def write_category(self, buf, category, df_reshape, df_describe): import matplotlib.pyplot as plt buf.write('
' % category) buf.write('

%s

\n' % (category.replace('_', ' ').title())) ax = ( df_reshape .xs(category, level='timing_category') .plot(kind='box', orientation='vertical') ) ax.invert_yaxis() ax.set_xlabel('Seconds') plt.tight_layout() plt.gcf().savefig(buf, format='svg') table = df_describe[category].to_html( classes="summary", index_names=False, border=0 ) buf.write(table) buf.write('
') pass def _check_imports(self): try: import matplotlib matplotlib.use('Agg') except ImportError: raise CylcError( 'Cannot import matplotlib - HTML summary unavailable.' ) from None super(HTMLTimingSummary, self)._check_imports() cylc-flow-8.7.0/cylc/flow/scripts/cycle_point.py0000775000175000017500000002137415264420307022100 0ustar alastairalastair#!/usr/bin/env python3 # -*- coding: utf-8 -*- # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc cycle-point [OPTIONS] ARGS Utility for simple date-time cycle point arithmetic. For more generic date-time manipulations see the "isodatetime" command. Filename templating replaces elements of a template string with corresponding elements of the current or given cycle point. Use ISO 8601 or posix date-time format elements: $ cylc cycle-point 2010080T00 --template foo-CCYY-MM-DD-Thh.nc foo-2010-08-08-T00.nc $ cylc cycle-point 2010080T00 --template foo-%Y-%m-%d-T%H.nc foo-2010-08-08-T00.nc Other examples: # print offset from an explicit cycle point: $ cylc cycle-point --offset-hours=6 20100823T1800Z 20100824T0000Z # print offset from $CYLC_TASK_CYCLE_POINT (as in workflow tasks): $ export CYLC_TASK_CYCLE_POINT=20100823T1800Z $ cylc cycle-point --offset-hours=-6 20100823T1200Z # cycle point filename templating, explicit template: $ export CYLC_TASK_CYCLE_POINT=2010-08 $ cylc cycle-point --offset-years=2 --template=foo-CCYY-MM.nc foo-2012-08.nc # cycle point filename templating, template in a variable: $ export CYLC_TASK_CYCLE_POINT=2010-08 $ export MYTEMPLATE=foo-CCYY-MM.nc $ cylc cycle-point --offset-years=2 --template=MYTEMPLATE foo-2012-08.nc """ import os import sys import cylc.flow.cycling.iso8601 from cylc.flow.option_parsers import CylcOptionParser as COP from cylc.flow.terminal import cli_function import metomi.isodatetime.data import metomi.isodatetime.dumpers import metomi.isodatetime.parsers from metomi.isodatetime.exceptions import IsodatetimeError def get_option_parser() -> COP: parser = COP( __doc__, color=False, argdoc=[ COP.optional( ('POINT', 'ISO8601 date-time, default=$CYLC_TASK_CYCLE_POINT') ) ] ) parser.add_option( "--offset-hours", metavar="HOURS", help="Add N hours to CYCLE (may be negative)", action="store", dest="offsethours") parser.add_option( "--offset-days", metavar="DAYS", help="Add N days to CYCLE (N may be negative)", action="store", dest="offsetdays") parser.add_option( "--offset-months", metavar="MONTHS", help="Add N months to CYCLE (N may be negative)", action="store", dest="offsetmonths") parser.add_option( "--offset-years", metavar="YEARS", help="Add N years to CYCLE (N may be negative)", action="store", dest="offsetyears") parser.add_option( "--offset", metavar="ISO_OFFSET", help="Add an ISO 8601-based interval representation to CYCLE", action="store", dest="offset") parser.add_option( "--equal", metavar="POINT2", help="Succeed if POINT2 is equal to POINT (format agnostic).", action="store", dest="point2") parser.add_option( "--template", metavar="TEMPLATE", help="Filename template string or variable", action="store", dest="template") parser.add_option( "--time-zone", metavar="TEMPLATE", help="Control the formatting of the result's timezone (e.g. " "Z, +13:00, -hh)", action="store", default=None, dest="time_zone") parser.add_option( "--num-expanded-year-digits", metavar="NUMBER", help="Specify a number of expanded year digits to print in the result", action="store", default=0, dest="num_expanded_year_digits") parser.add_option( "--print-year", help="Print only CCYY of result", action="store_true", default=False, dest="print_year") parser.add_option( "--print-month", help="Print only MM of result", action="store_true", default=False, dest="print_month") parser.add_option( "--print-day", help="Print only DD of result", action="store_true", default=False, dest="print_day") parser.add_option( "--print-hour", help="Print only hh of result", action="store_true", default=False, dest="print_hour") return parser @cli_function(get_option_parser) def main(parser, options, *args): if len(args) == 0: # input cycle point must be defined in the environment. if 'CYLC_TASK_CYCLE_POINT' not in os.environ: parser.error("Provide CYCLE arg, or define $CYLC_TASK_CYCLE_POINT") cycle_point_string = os.environ['CYLC_TASK_CYCLE_POINT'] else: # must be cycle point cycle_point_string = args[0] # template string template = None if options.template: if (options.print_month or options.print_year or options.print_day or options.print_hour): parser.error( '"print only" options are incompatible with templating') tmp = options.template template = os.environ.get(tmp, tmp) else: n_chosen = 0 if options.print_year: n_chosen += 1 if options.num_expanded_year_digits: template = "±XCCYY" else: template = "CCYY" if options.print_month: n_chosen += 1 template = "MM" if options.print_day: n_chosen += 1 template = "DD" if options.print_hour: n_chosen += 1 template = "%H" if n_chosen != 0 and n_chosen != 1: parser.error("Choose NONE or ONE of print_(year|month|day|hour)") cylc.flow.cycling.iso8601.init( num_expanded_year_digits=options.num_expanded_year_digits, time_zone=options.time_zone ) iso_point_parser = metomi.isodatetime.parsers.TimePointParser( num_expanded_year_digits=options.num_expanded_year_digits ) iso_point_dumper = metomi.isodatetime.dumpers.TimePointDumper( num_expanded_year_digits=options.num_expanded_year_digits ) try: cycle_point = iso_point_parser.parse( cycle_point_string, dump_as_parsed=(template is None)) except IsodatetimeError as exc: parser.error('ERROR: invalid cycle: %s' % exc) if options.point2: try: cycle_point2 = iso_point_parser.parse( options.point2, dump_as_parsed=(template is None)) except IsodatetimeError as exc: parser.error('ERROR: invalid cycle: %s' % exc) if cycle_point2 == cycle_point: sys.exit(0) else: sys.exit(1) offset_props = {} if options.offsethours: try: offset_props["hours"] = int(options.offsethours) except ValueError: parser.error('ERROR: offset must be integer') if options.offsetdays: try: offset_props["days"] = int(options.offsetdays) except ValueError: parser.error('ERROR: offset must be integer') if options.offsetmonths: try: offset_props["months"] = int(options.offsetmonths) except ValueError: parser.error('ERROR: offset must be integer') if options.offsetyears: try: offset_props["years"] = int(options.offsetyears) except ValueError: parser.error('ERROR: offset must be integer') offset = metomi.isodatetime.data.Duration(**offset_props) if options.offset: opt_offset = options.offset sign_factor = 1 if options.offset.startswith("-"): opt_offset = options.offset[1:] sign_factor = -1 try: offset += metomi.isodatetime.parsers.DurationParser().parse( opt_offset) * sign_factor except IsodatetimeError as exc: parser.error('ERROR: offset not valid: %s' % exc) cycle_point += offset if template is None: print(cycle_point) else: dump_string = iso_point_dumper.dump(cycle_point, template) if (dump_string == template and not any(_ in template for _ in ["CCYY", "MM", "DD", "%"])): # A pure time string, no date - not well handled in isodatetime. print(iso_point_dumper.dump(cycle_point, "T" + template)[1:]) else: print(dump_string) cylc-flow-8.7.0/cylc/flow/scripts/validate_reinstall.py0000664000175000017500000002031615264420307023426 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc validate-reinstall [OPTIONS] ARGS Validate, reinstall, and reload or restart a workflow. This command updates a workflow to reflect any new changes made in the workflow source directory since it was installed. The workflow will be reinstalled, then either: * Reloaded (if the workflow is running), * or restarted (if it is stopped). Cylc will show you the list of files changed and prompt for confirmation before reinstalling. Use the '--yes' option to bypass this. This command is equivalent to: $ cylc validate myworkflow --against-source $ cylc reinstall myworkflow # if myworkflow is running: $ cylc reload myworkflow # else: $ cylc play myworkflow Note: "cylc validate --against-source" checks the code in the workflow source directory against any options (e.g. template variables) which have been set in the installed workflow to ensure the change can be safely applied. """ from ansimarkup import parse as cparse import asyncio from pathlib import Path import sys from typing import TYPE_CHECKING, Union if TYPE_CHECKING: from optparse import Values from cylc.flow import LOG from cylc.flow.exceptions import ( WorkflowStopped, ) from cylc.flow.id_cli import parse_id_async from cylc.flow.loggingutil import set_timestamps from cylc.flow.option_parsers import ( WORKFLOW_ID_ARG_DOC, CylcOptionParser as COP, combine_options, log_subcommand, cleanup_sysargv ) from cylc.flow.scheduler_cli import PLAY_OPTIONS, cylc_play from cylc.flow.scripts.ping import run as cylc_ping from cylc.flow.scripts.validate import ( VALIDATE_OPTIONS, VALIDATE_AGAINST_SOURCE_OPTION, run as cylc_validate, ) from cylc.flow.scripts.reinstall import ( REINSTALL_CYLC_ROSE_OPTIONS, REINSTALL_OPTIONS, reinstall_cli as cylc_reinstall, ) from cylc.flow.scripts.reload import ( RELOAD_OPTIONS, run as cylc_reload ) from cylc.flow.terminal import cli_function, is_terminal from cylc.flow.workflow_files import get_workflow_run_dir CYLC_ROSE_OPTIONS = COP.get_cylc_rose_options() VR_OPTIONS = combine_options( VALIDATE_OPTIONS, REINSTALL_OPTIONS, REINSTALL_CYLC_ROSE_OPTIONS, RELOAD_OPTIONS, PLAY_OPTIONS, CYLC_ROSE_OPTIONS, modify={'cylc-rose': 'validate, install'} ) _input = input # to enable testing def get_option_parser() -> COP: parser = COP( __doc__, comms=True, jset=True, argdoc=[WORKFLOW_ID_ARG_DOC], ) for option in VR_OPTIONS: parser.add_option(*option.args, **option.kwargs) parser.set_defaults(is_validate=True) return parser def check_tvars_and_workflow_stopped( is_running: bool, tvars: list, tvars_file: list ) -> bool: """are template variables set and workflow stopped? Template vars set by --set (options.templatevars) or --set-file (options.templatevars_file) are only valid if the workflow is stopped and vr will play it. args: is_running: Is workflow running? tvars: options.tvars, from `--set` tvars_file: options.tvars_file, from `--set-file` """ if is_running and (tvars or tvars_file): LOG.warning( 'Template variables (from --set/--set-file) can ' 'only be changed if the workflow is stopped.' ) return False return True @cli_function(get_option_parser) def main(parser: COP, options: 'Values', workflow_id: str): ret = asyncio.run(vr_cli(parser, options, workflow_id)) if isinstance(ret, str): # NOTE: cylc_play must be called from sync code (not async code) cylc_play(options, ret, parse_workflow_id=False) elif ret is False: sys.exit(1) async def vr_cli( parser: COP, options: 'Values', workflow_id: str ) -> Union[bool, str]: """Validate and reinstall and optionally reload workflow. Runs: * Validate * Reinstall * Reload (if the workflow is already running) Returns: The workflow_id or a True/False outcome. workflow_id: If the workflow is stopped and requires restarting. True: If workflow is running and does not require restarting. False: If this command should "exit 1". """ unparsed_wid = workflow_id workflow_id, *_ = await parse_id_async( workflow_id, constraint='workflows', ) # First attempt to work out whether the workflow is running. # We are trying to avoid reinstalling then subsequently being # unable to play or reload because we cannot identify workflow state. log_subcommand('ping', workflow_id) try: result = await cylc_ping(options, workflow_id) # (don't catch CylcError: unable to determine if running or not) except WorkflowStopped: print(cparse(f"{workflow_id} is not running")) workflow_running = False else: print(cparse(f"{workflow_id}: {result['stdout'][0]}")) workflow_running = True # options.tvars and tvars_file are _only_ valid when playing a stopped # workflow: Fail if they are set and workflow running: if not check_tvars_and_workflow_stopped( workflow_running, options.templatevars, options.templatevars_file ): return False # Save the location of the existing workflow run dir in the # against source option: options.against_source = Path(get_workflow_run_dir(workflow_id)) # Run "cylc validate" log_subcommand('validate --against-source', workflow_id) await cylc_validate(options, workflow_id) # Unset options that do not apply after validation: del options.against_source del options.is_validate # Run "cylc reinstall" log_subcommand('reinstall', workflow_id) reinstall_ok = await cylc_reinstall( options, workflow_id, [], print_reload_tip=False ) if not reinstall_ok: # No changes OR user said No to the reinstall prompt. # If not a terminal and not --yes do nothing. if not is_terminal() and not options.skip_interactive: return False # Else if not --yes, prompt user to continue or not. if not options.skip_interactive: usr = None if not workflow_running: # No changes to install, and the workflow is not running. # Can still restart with no changes. action = "Restart" else: # No changes to install, and the workflow is running. # Reload is probably pointless, but not necessarily (the user # could modify the run dir and do 'vr' to restart or reload). action = "Reload" while usr not in ['y', 'n']: usr = _input( cparse(f'{action} anyway? [y/n]: ') ).lower() if usr == 'n': return False # Run "cylc reload" (if workflow is running or paused) if workflow_running: log_subcommand('reload', workflow_id) await cylc_reload(options, workflow_id) return True # Run "cylc play" (if workflow is stopped) else: set_timestamps(LOG, options.log_timestamp) cleanup_sysargv( 'play', unparsed_wid, options, compound_script_opts=[*VR_OPTIONS, VALIDATE_AGAINST_SOURCE_OPTION], script_opts=(*PLAY_OPTIONS, *parser.get_std_options()), source='', # Intentionally blank ) log_subcommand(*sys.argv[1:]) return workflow_id cylc-flow-8.7.0/cylc/flow/scripts/remote_tidy.py0000775000175000017500000000302515264420307022105 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc remote-tidy [OPTIONS] ARGS (This command is for internal use.) Remove ".service/contact" from a task remote (i.e. a [owner@]host). Remove ".service" directory on the remote if emptied. Remove authentication keys. """ from cylc.flow.option_parsers import CylcOptionParser as COP from cylc.flow.task_remote_cmd import remote_tidy from cylc.flow.terminal import cli_function INTERNAL = True def get_option_parser() -> COP: parser = COP( __doc__, argdoc=[ ("INSTALL_TARGET", "target platform to be tidied"), ("RUND", "The run directory of the workflow") ] ) return parser @cli_function(get_option_parser) def main(parser, _, install_target, rund): remote_tidy(install_target, rund) cylc-flow-8.7.0/cylc/flow/scripts/workflow_state.py0000775000175000017500000003746515264420307022652 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . r"""cylc workflow-state [OPTIONS] ARGS Check or poll a workflow database for task statuses or completed outputs. The ID argument can target a workflow, or a cycle point, or a specific task, with an optional selector on cycle or task to match final task statuses, output trigger (with --triggers), or output message (with --messages). You cannot poll for transient states such as "submitted" and "running"; poll for the corresponding output triggers instead ("submitted", "started"). All matching results will be printed. If no results match, the command will repeatedly check (poll) until a match is found or polling is exhausted (see --max-polls and --interval). For a one-off check set --max-polls=1. If the database does not exist at first, polls are consumed waiting for it so you can start checking before the target workflow is started. Legacy (pre-8.3.0) options are supported, but deprecated, for existing scripts: cylc workflow-state --task=NAME --point=CYCLE --status=STATUS --output=MESSAGE --message=MESSAGE --task-point WORKFLOW (Note from 8.0 until 8.3.0 --output and --message both matched task messages). In "cycle/task:selector" the selector will match task statuses, unless: - with --triggers, it will only match task output triggers. - with --messages, it will only match task output messages. It is recommended to use triggers instead - they match both naturally & manually set outputs. Selector does not default to "succeeded". If omitted, any status will match. The "finished" pseudo-output is an alias for "succeeded or failed". In the ID, both cycle and task can include "*" to match any sequence of zero or more characters. Quote the pattern to protect it from shell expansion. Note tasks get recorded in the DB once they enter the active window (n=0). Flow numbers are only printed for flow numbers > 1. Use in task scripting: - To poll a task at the same cycle point in another workflow, just use $CYLC_TASK_CYCLE_POINT in the ID. - To poll a task at an offset cycle point, use the --offset option to have Cylc do the datetime arithmetic for you. - However, see also the workflow_state xtrigger for this use case. Warnings: - Typos in the workflow or task ID will result in fruitless polling. - Cycle points are auto-converted to the DB point format (and UTC mode). - Task outputs manually completed by "cylc set" have "(force-completed)" recorded as the task message in the DB, so it is best to query trigger names, not messages, unless specifically interested in forced outputs. Examples: # Print the status of all tasks in WORKFLOW: $ cylc workflow-state WORKFLOW # Print the status of all tasks in cycle point 2033: $ cylc workflow-state WORKFLOW//2033 # Print the status of all tasks named foo: $ cylc workflow-state "WORKFLOW//*/foo" # Print all succeeded tasks: $ cylc workflow-state "WORKFLOW//*/*:succeeded" # Print all tasks foo that completed output (trigger name) file1: $ cylc workflow-state "WORKFLOW//*/foo:file1" # Print if task 2033/foo completed output (trigger name) file1: $ cylc workflow-state WORKFLOW//2033/foo:file1 See also: - the workflow_state xtrigger, for state polling within workflows - "cylc dump -t", to query a scheduler for current statuses - "cylc show", to query a scheduler for task prerequisites and outputs """ import asyncio import os import sqlite3 import sys from typing import TYPE_CHECKING, List, Optional from cylc.flow.pathutil import get_cylc_run_dir from cylc.flow.id import Tokens from cylc.flow.exceptions import InputError from cylc.flow.option_parsers import ( ID_SEL_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow import LOG from cylc.flow.command_polling import Poller from cylc.flow.dbstatecheck import CylcWorkflowDBChecker from cylc.flow.terminal import cli_function from cylc.flow.workflow_files import infer_latest_run_from_id from cylc.flow.task_state import ( TASK_STATUSES_FINAL, TASK_STATUSES_ALL, ) if TYPE_CHECKING: from optparse import Values WILDCARD = "*" # polling defaults MAX_POLLS = 12 INTERVAL = 5 OPT_DEPR_MSG = "DEPRECATED, use ID" OPT_DEPR_MSG1 = 'DEPRECATED, use "ID:STATUS"' OPT_DEPR_MSG2 = 'DEPRECATED, use "ID:MSG"' def unquote(s: str) -> str: """Remove leading & trailing quotes from a string. Examples: >>> unquote('"foo"') 'foo' >>> unquote("'foo'") 'foo' >>> unquote('foo') 'foo' >>> unquote("'tis a fine morning") "'tis a fine morning" """ if ( s.startswith('"') and s.endswith('"') or s.startswith("'") and s.endswith("'") ): return s[1:-1] return s class WorkflowPoller(Poller): """An object that polls for task states or outputs in a workflow DB.""" def __init__( self, id_: str, offset: Optional[str], flow_num: Optional[int], alt_cylc_run_dir: Optional[str], default_status: Optional[str], is_trigger: bool, is_message: bool, old_format: bool = False, pretty_print: bool = False, accept_pre_start_tasks: bool = False, **kwargs ): self.id_ = id_ self.offset = offset self.flow_num = flow_num self.alt_cylc_run_dir = alt_cylc_run_dir self.old_format = old_format self.pretty_print = pretty_print self.is_message = is_message self.is_trigger = is_trigger self.accept_pre_start_tasks = accept_pre_start_tasks try: tokens = Tokens(self.id_) except ValueError as exc: raise InputError(exc) from None self.workflow_id_raw = tokens.workflow_id self.selector = ( tokens["cycle_sel"] or tokens["task_sel"] or default_status ) if self.selector: self.selector = unquote(self.selector) self.cycle_raw = tokens["cycle"] self.task = tokens["task"] self.workflow_id: Optional[str] = None self.cycle: Optional[str] = None self.result: Optional[List[List[str]]] = None self._db_checker: Optional[CylcWorkflowDBChecker] = None super().__init__(**kwargs) def _find_workflow(self) -> bool: """Find workflow and infer run directory, return True if found.""" try: self.workflow_id = infer_latest_run_from_id( self.workflow_id_raw, self.alt_cylc_run_dir ) except InputError: LOG.debug("Workflow not found") return False if self.workflow_id != self.workflow_id_raw: # Print inferred ID. sys.stderr.write(f"Inferred workflow ID: {self.workflow_id}\n") return True @property def db_checker(self) -> Optional[CylcWorkflowDBChecker]: """Connect to workflow DB if not already connected. Returns DB checker if connected. """ if not self._db_checker: try: self._db_checker = CylcWorkflowDBChecker( get_cylc_run_dir(self.alt_cylc_run_dir), self.workflow_id, accept_pre_start_tasks=self.accept_pre_start_tasks, ) except (OSError, sqlite3.Error): LOG.debug("DB not connected") return None return self._db_checker async def check(self) -> bool: """Return True if requested state achieved, else False. Called once per poll by super() so only find and connect once. Store self.result for external access. """ if self.workflow_id is None and not self._find_workflow(): return False if self.db_checker is None: return False if self.cycle is None: # Adjust target cycle point to the DB format. self.cycle = self.db_checker.adjust_point_to_db( self.cycle_raw, self.offset) self.result = self.db_checker.workflow_state_query( self.task, self.cycle, self.selector, self.is_trigger, self.is_message, self.flow_num ) if self.result: # End the polling dot stream and print inferred runN workflow ID. self.db_checker.display_maps( self.result, self.old_format, self.pretty_print) return bool(self.result) def get_option_parser() -> COP: parser = COP( __doc__, argdoc=[ID_SEL_ARG_DOC] ) # --run-dir for pre-8.3.0 back-compat parser.add_option( "-d", "--alt-cylc-run-dir", "--run-dir", help="Alternate cylc-run directory, e.g. for other users' workflows.", metavar="DIR", action="store", dest="alt_cylc_run_dir", default=None) parser.add_option( "-s", "--offset", help="Offset from ID cycle point as an ISO8601 duration for datetime" " cycling (e.g. 'PT30M' for 30 minutes) or an integer interval for" " integer cycling (e.g. 'P2'). This can be used in task job scripts" " to poll offset cycle points without doing the cycle arithmetic" " yourself - but see also the workflow_state xtrigger.", action="store", dest="offset", metavar="DURATION", default=None) parser.add_option( "--flow", help="Flow number, for target tasks. By default, any flow.", action="store", type="int", dest="flow_num", default=None) parser.add_option( "--triggers", help="Task selector should match output trigger names rather than " "status.", action="store_true", dest="is_trigger", default=False) parser.add_option( "--messages", help="Task selector should match output messages rather than status.", action="store_true", dest="is_message", default=False) parser.add_option( "--accept-pre-start-tasks", help=( "Assume queried task statuses or outputs before the " "start cycle point are succeeded or complete." ), action="store_true", dest="accept_pre_start_tasks", default=False ) parser.add_option( "--pretty", help="Pretty-print outputs (the default is single-line output).", action="store_true", dest="pretty_print", default=False) parser.add_option( "--old-format", help="Print results in legacy comma-separated format.", action="store_true", dest="old_format", default=False) # BACK COMPAT: workflow_state_backcompat, support for pre-8.3.0 CLI options # from: 8.0.0 # to: 8.3.0 # remove at: 8.9 parser.add_option( "-t", "--task", help=f"Task name. {OPT_DEPR_MSG}.", metavar="NAME", action="store", dest="depr_task", default=None) parser.add_option( "-p", "--point", metavar="CYCLE", help=f"Cycle point. {OPT_DEPR_MSG}.", action="store", dest="depr_point", default=None) parser.add_option( "-T", "--task-point", help="Get cycle point from the environment variable" " $CYLC_TASK_CYCLE_POINT (e.g. in task job scripts)", action="store_true", dest="depr_env_point", default=False) parser.add_option( "-S", "--status", metavar="STATUS", help=f"Task status. {OPT_DEPR_MSG1}.", action="store", dest="depr_status", default=None) # Prior to 8.3.0 --output was just an alias for --message parser.add_option( "-O", "--output", "-m", "--message", metavar="MSG", help=f"Task output message. {OPT_DEPR_MSG2}.", action="store", dest="depr_msg", default=None) WorkflowPoller.add_to_cmd_options( parser, d_interval=INTERVAL, d_max_polls=MAX_POLLS ) return parser @cli_function(get_option_parser, remove_opts=["--db"]) def main(parser: COP, options: 'Values', *ids: str) -> None: # Note it would be cleaner to use 'id_cli.parse_ids()' here to get the # workflow ID and tokens, but that function infers run number and fails # if the workflow is not installed yet. We want to be able to start polling # before the workflow is installed, which makes it easier to get a set of # interdependent workflows up and running, so runN inference is done inside # the poller. TODO: consider using id_cli.parse_ids inside the poller. # (Note this applies to polling tasks, which use the CLI, not xtriggers). id_ = ids[0].rstrip('/') # might get 'id/' due to autocomplete if any( [ options.depr_task, options.depr_status, options.depr_msg, options.depr_point, options.depr_env_point ] ): depr_opts = ( "--task, --status, --message, --output, --point, --task-point" ) if id_ != Tokens(id_)["workflow"]: raise InputError( f"with deprecated {depr_opts}, the argument must be a" " plain workflow ID (i.e. no cycle, task, or :selector)." ) if options.depr_status and options.depr_msg: raise InputError("set --status or --message, not both.") if options.depr_env_point: if options.depr_point: raise InputError( "set --task-point or --point=CYCLE, not both.") try: options.depr_point = os.environ["CYLC_TASK_CYCLE_POINT"] except KeyError: raise InputError( "--task-point: $CYLC_TASK_CYCLE_POINT is not defined" ) from None if options.depr_point is not None: id_ += f"//{options.depr_point}" elif ( options.depr_task is not None or options.depr_status is not None or options.depr_msg is not None ): id_ += "//*" if options.depr_task is not None: id_ += f"/{options.depr_task}" if options.depr_status is not None: id_ += f":{options.depr_status}" elif options.depr_msg is not None: id_ += f":{options.depr_msg}" options.is_message = True msg = ( f"{depr_opts} are deprecated and will be removed in Cylc 8.9. " "Please use an ID: " ) if not options.depr_env_point: msg += id_ else: msg += id_.replace(options.depr_point, "$CYLC_TASK_CYCLE_POINT") if ( options.depr_status and options.depr_status in TASK_STATUSES_ALL and options.depr_status not in TASK_STATUSES_FINAL ): # polling for non-final task statuses is flaky msg += ' and the --triggers option' LOG.warning(msg) poller = WorkflowPoller( id_, options.offset, options.flow_num, options.alt_cylc_run_dir, default_status=None, is_trigger=options.is_trigger, is_message=options.is_message, old_format=options.old_format, pretty_print=options.pretty_print, condition=id_, interval=options.interval, max_polls=options.max_polls, args=None, accept_pre_start_tasks=options.accept_pre_start_tasks, ) if not asyncio.run( poller.poll() ): sys.exit(1) cylc-flow-8.7.0/cylc/flow/scripts/stop.py0000775000175000017500000002114215264420307020546 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc stop [OPTIONS] ARGS Tell a workflow to shut down. Examples: # wait for active tasks to finish, then shut down $ cylc stop my_workflow # kill active tasks, then shut down $ cylc stop my_workflow --kill # shut down immediately, leave active tasks alone $ cylc stop my_workflow --now # shut down all workflows $ cylc stop '*' # shut down after the cycle 1234 has been passed $ cylc stop my_workflow//1234 # shut down after the task foo in cycle 1234 has succeeded $ cylc stop my_workflow//1234/foo By default stopping workflows wait for submitted and running tasks to complete before shutting down. You can change this behaviour with the --now or --kill options. There are several shutdown methods: 1. (default) stop after current active tasks finish 2. (--now) stop immediately, orphaning current active tasks 3. (--kill) stop after killing current active tasks 4. (if ID specifies a cycle point) stop after the cycle point 5. (if ID specifies a task ID) stop after the task has succeeded 6. (--wall-clock=T) stop after time T (an ISO 8601 date-time format e.g. CCYYMMDDThh:mm, CCYY-MM-DDThh, etc). Tasks that become ready after the shutdown is ordered will be submitted immediately if the workflow is restarted. Remaining task event handlers and job poll and kill commands will be executed prior to shutdown, unless --now is used twice. This command exits immediately unless --max-polls is greater than zero, in which case it polls to wait for workflow shutdown. """ from functools import partial import sys from typing import TYPE_CHECKING, Optional from cylc.flow.command_polling import Poller from cylc.flow.exceptions import ( ClientError, ClientTimeout, CylcError, InputError, WorkflowStopped, ) from cylc.flow.network.client_factory import get_client from cylc.flow.network.multi import call_multi from cylc.flow.network.schema import WorkflowStopMode from cylc.flow.option_parsers import ( CylcOptionParser as COP, ID_MULTI_ARG_DOC, ) from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values MUTATION = ''' mutation ( $wFlows: [WorkflowID]!, $stopMode: WorkflowStopMode, $cyclePoint: CyclePoint, $clockTime: TimePoint, $task: TaskID, $flowNum: Int, ) { stop ( workflows: $wFlows, mode: $stopMode, cyclePoint: $cyclePoint, clockTime: $clockTime, task: $task, flowNum: $flowNum ) { result } } ''' POLLER_QUERY = ''' query ($wFlows: [ID]) { workflows(ids: $wFlows) { id } } ''' class StopPoller(Poller): """A polling object that checks if a workflow has stopped yet.""" def __init__(self, pclient, condition, interval, max_polls): Poller.__init__(self, condition, interval, max_polls, None) self.pclient = pclient self.query = { 'request_string': POLLER_QUERY, 'variables': {'wFlows': [self.pclient.workflow]} } async def check(self): """Return True if workflow has stopped (success) else False""" try: await self.pclient.async_request('graphql', self.query) except (ClientError, ClientTimeout, CylcError): # failed to ping - workflow stopped or (CylcError) restarted on # another host:port (in which case it must have stopped first). return True else: # pinged - workflow must be alive return False def get_option_parser() -> COP: parser = COP( __doc__, comms=True, multiworkflow=True, argdoc=[ ID_MULTI_ARG_DOC, ], ) parser.add_option( "-k", "--kill", help="Shut down after killing currently active tasks.", action="store_true", default=False, dest="kill") parser.add_option( "--flow", metavar="INT", help="Stop flow number INT from spawning more tasks. " "The scheduler will shut down if it is the only flow.", action="store", dest="flow_num") parser.add_option( "-n", "--now", help=( "Shut down without waiting for active tasks to complete." + " If this option is specified once," + " wait for task event handler, job poll/kill to complete." + " If this option is specified more than once," + " tell the workflow to terminate immediately."), action="count", default=0, dest="now") parser.add_option( "-w", "--wall-clock", metavar="STOP", help="Shut down after time STOP (ISO 8601 formatted)", action="store", dest="wall_clock") StopPoller.add_to_cmd_options(parser, d_max_polls=0) return parser def _validate( options: 'Values', stop_task: Optional[str], stop_cycle: Optional[str], *globs: str ) -> None: """Check option choices are valid.""" if stop_task is not None and options.kill: raise InputError("--kill is not compatible with stop-task") if stop_cycle is not None and options.kill: raise InputError("--kill is not compatible with stop-cycle") if stop_task and stop_cycle: raise InputError('stop-task is not compatible with stop-cycle') if options.kill and options.now: raise InputError("--kill is not compatible with --now") if options.flow_num and int(options.max_polls) > 0: raise InputError("--flow is not compatible with --max-polls") if options.flow_num and globs: raise InputError("--flow is not compatible with task IDs") async def run( options: 'Values', workflow_id, *tokens_list, ) -> object: return await _run(options, workflow_id, *tokens_list) async def _run( options: 'Values', workflow_id, *tokens_list, ) -> object: # parse the stop-task or stop-cycle if provided stop_task = stop_cycle = None if tokens_list: tokens = tokens_list[0] if tokens['task']: stop_task = tokens.relative_id elif tokens['cycle']: stop_cycle = tokens['cycle'] _validate(options, stop_task, stop_cycle, *tokens_list) pclient = get_client(workflow_id, timeout=options.comms_timeout) if int(options.max_polls) > 0: # (test to avoid the "nothing to do" warning for # --max-polls=0) spoller = StopPoller( pclient, "workflow stopped", options.interval, options.max_polls, ) if stop_task or stop_cycle: mode: Optional[str] = None elif options.kill: mode = WorkflowStopMode.Kill.name elif options.now > 1: mode = WorkflowStopMode.NowNow.name elif options.now: mode = WorkflowStopMode.Now.name else: mode = WorkflowStopMode.Clean.name if ( options.flow_num is not None and hasattr(options.flow_num, 'isdigit') and options.flow_num.isdigit() ): # flow num gql type is int options.flow_num = int(options.flow_num) mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [workflow_id], 'stopMode': mode, 'cyclePoint': stop_cycle, 'clockTime': options.wall_clock, 'task': stop_task, 'flowNum': options.flow_num }, } ret = await pclient.async_request('graphql', mutation_kwargs) if int(options.max_polls) > 0 and not await spoller.poll(): # (test to avoid the "nothing to do" warning for # --max-polls=0) raise CylcError( f'Workflow did not shut down after {options.max_polls} polls' ) return ret @cli_function(get_option_parser) def main( parser: COP, options: 'Values', *ids, ) -> None: rets = call_multi( partial(run, options), *ids, constraint='mixed', max_tasks=1, success_exceptions=(WorkflowStopped,), ) sys.exit(all(rets.values()) is False) cylc-flow-8.7.0/cylc/flow/scripts/hold.py0000775000175000017500000001117115264420307020510 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc hold [OPTIONS] ARGS Hold task(s) in a workflow. Held tasks do not submit jobs even if ready. Any task can be held. Note: a held running task will not submit more jobs itself (e.g. by retries) until released, but the running job may still complete outputs that cause non-held downstream tasks to trigger. To pause an entire workflow use "cylc pause". Examples: # Hold mytask at cycle point 1234 in my_workflow (if it has not yet spawned, # it will hold as soon as it spawns): $ cylc hold my_workflow//1234/mytask # Hold all active tasks at cycle 1234 in my_workflow (note: tasks # before/after this cycle point will not be held): $ cylc hold 'my_workflow//1234/*' # Hold all active instances of mytask in my_workflow (note: this will not # hold any unspawned tasks that might spawn in the future): $ cylc hold 'my_workflow//*/mytask' # Hold all active failed tasks: $ cylc hold 'my_workflow//*:failed' # Hold all tasks after cycle point 1234 in my_workflow: $ cylc hold my_workflow// --after=1234 # Hold cycles 1, 2 & 3 in my_workflow: $ cylc hold my_workflow// //1 //2 //3 # Hold cycle "1" in "my_workflow_1" and "my_workflow_2": $ cylc hold my_workflow_1//1 my_workflow_2//1 Note: To pause a workflow (immediately preventing all job submission), use 'cylc pause' instead. See also 'cylc release'. """ from functools import partial import sys from typing import TYPE_CHECKING from cylc.flow.exceptions import InputError from cylc.flow.network.client_factory import get_client from cylc.flow.option_parsers import ( FULL_ID_MULTI_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.terminal import cli_function from cylc.flow.network.multi import call_multi if TYPE_CHECKING: from optparse import Values HOLD_MUTATION = ''' mutation ( $wFlows: [WorkflowID]!, $tasks: [NamespaceIDGlob]! ) { hold ( workflows: $wFlows, tasks: $tasks ) { result } } ''' SET_HOLD_POINT_MUTATION = ''' mutation ( $wFlows: [WorkflowID]!, $point: CyclePoint! ) { setHoldPoint ( workflows: $wFlows, point: $point ) { result } } ''' def get_option_parser() -> COP: parser = COP( __doc__, comms=True, multitask=True, multiworkflow=True, argdoc=[FULL_ID_MULTI_ARG_DOC], ) parser.add_option( "--after", help="Hold all tasks after this cycle point.", metavar="CYCLE_POINT", action="store", dest="hold_point_string") return parser def _validate(options: 'Values', *task_globs: str) -> None: """Check combination of options and task globs is valid.""" if options.hold_point_string: if task_globs: raise InputError( "Cannot combine --after with Cylc/Task IDs.\n" "`cylc hold --after` holds all tasks after the given " "cycle point.") elif not task_globs: raise InputError( "Must define Cycles/Tasks. See `cylc hold --help`.") async def run(options, workflow_id, *tokens_list): _validate(options, *tokens_list) pclient = get_client(workflow_id, timeout=options.comms_timeout) if options.hold_point_string: mutation = SET_HOLD_POINT_MUTATION args = {'point': options.hold_point_string} else: mutation = HOLD_MUTATION args = { 'tasks': [ id_.relative_id_with_selectors for id_ in tokens_list ] } mutation_kwargs = { 'request_string': mutation, 'variables': { 'wFlows': [workflow_id], **args } } return await pclient.async_request('graphql', mutation_kwargs) @cli_function(get_option_parser) def main(parser: COP, options: 'Values', *ids): rets = call_multi( partial(run, options), *ids, constraint='mixed', ) sys.exit(all(rets.values()) is False) cylc-flow-8.7.0/cylc/flow/scripts/show.py0000775000175000017500000003562015264420307020547 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc show [OPTIONS] ARGS Display workflow and task information, for tasks in the current n-window. Query a running workflow for: # view workflow metadata $ cylc show my_workflow # view task metadata $ cylc show my_workflow --task-def my_task # view prerequisites & outputs for a live task $ cylc show my_workflow//1/my_task Output completion status is shown for all tasks in the current n-window. Prerequisite satisfaction is not shown for past tasks reloaded from the workflow database. """ import asyncio import re import json import sys from textwrap import indent from typing import Any, Dict, TYPE_CHECKING from ansimarkup import ansiprint from metomi.isodatetime.data import ( get_timepoint_from_seconds_since_unix_epoch as seconds2point) from cylc.flow.exceptions import InputError from cylc.flow.id import Tokens from cylc.flow.id_cli import parse_ids from cylc.flow.network.client_factory import get_client from cylc.flow.run_modes import RunMode from cylc.flow.task_outputs import TaskOutputs from cylc.flow.task_state import ( TASK_STATUSES_ORDERED, TASK_STATUS_RUNNING ) from cylc.flow.option_parsers import ( CylcOptionParser as COP, ID_MULTI_ARG_DOC, Options, ) from cylc.flow.terminal import cli_function from cylc.flow.util import BOOL_SYMBOLS if TYPE_CHECKING: from optparse import Values WORKFLOW_META_QUERY = ''' query ($wFlows: [ID]!) { workflows (ids: $wFlows, stripNull: false) { meta { title description URL userDefined } } } ''' TASK_META_QUERY = ''' query ($wFlows: [ID]!, $taskIds: [ID]) { tasks (workflows: $wFlows, ids: $taskIds, stripNull: false) { name meta { title description URL userDefined } } } ''' TASK_PREREQS_QUERY = ''' query ($wFlows: [ID]!, $taskIds: [ID]) { taskProxies (workflows: $wFlows, ids: $taskIds, stripNull: false) { id name cyclePoint state isHeld isQueued isRunahead flowNums task { meta { title description URL userDefined } } prerequisites { expression conditions { exprAlias taskId reqState message satisfied } satisfied } outputs { label message satisfied } externalTriggers { id label message satisfied } xtriggers { id label satisfied } runtime { completion runMode } } } ''' SATISFIED = BOOL_SYMBOLS[True] UNSATISFIED = BOOL_SYMBOLS[False] def print_msg_state(msg, state): if state: ansiprint(f' {SATISFIED} {msg}') else: ansiprint(f' {UNSATISFIED} {msg}') def print_completion_state(t_proxy): # create task outputs object outputs = TaskOutputs(t_proxy["runtime"]["completion"]) for output in t_proxy['outputs']: outputs.add(output['label'], output['message']) if output['satisfied']: outputs.set_message_complete(output['message']) ansiprint( f'output completion:' f' {"complete" if outputs.is_complete() else "incomplete"}' f'\n{indent(outputs.format_completion_status(ansimarkup=2), " ")}' ) def flatten_data(data, flat_data=None): """Reduce a nested data structure to a flat one. Examples: It flattens out nested dictionaries: >>> flatten_data({}) {} >>> flatten_data({'a': 1}) {'a': 1} >>> flatten_data({'a': {'b': 2, 'c': {'d': 4}}}) {'b': 2, 'd': 4} It iterates through any lists that it finds: >>> flatten_data({'a': [{'b': 2}, {'c': 3}]}) {'b': 2, 'c': 3} Overriding is determined by iteration order (don't rely on it): >>> flatten_data({'a': 1, 'b': {'a': 2}}) {'a': 2} It can't flatten things which aren't dicts: >>> flatten_data({'a': ['x', 'y']}) Traceback (most recent call last): AttributeError: 'str' object has no attribute 'items' """ if flat_data is None: flat_data = {} for key, value in data.items(): if isinstance(value, dict): flatten_data(value, flat_data) elif isinstance(value, list): for member in value: flatten_data(member, flat_data) else: flat_data[key] = value return flat_data def get_option_parser(): parser = COP( __doc__, comms=True, multitask=True, argdoc=[ID_MULTI_ARG_DOC], ) parser.add_option( '--list-prereqs', action="store_true", default=False, help="Print a task's pre-requisites as a list.", ) parser.add_option( '--json', action="store_true", default=False, help="Print output in JSON format.", ) parser.add_option( '--task-def', action="append", default=None, dest='task_defs', metavar='TASK_NAME', help="View metadata for a task definition (can be used multiple times)" ) return parser ShowOptions = Options(get_option_parser()) async def workflow_meta_query(workflow_id, pclient, options, json_filter): query = WORKFLOW_META_QUERY query_kwargs = { 'request_string': query, 'variables': {'wFlows': [workflow_id]} } # Print workflow info. results = await pclient.async_request('graphql', query_kwargs) for workflow_id in results['workflows']: flat_data = flatten_data(workflow_id) if options.json: json_filter.update(flat_data) else: for key, value in sorted(flat_data.items(), reverse=True): ansiprint( f'{key}: {value or "(not given)"}' ) return 0 async def prereqs_and_outputs_query( workflow_id, tokens_list, pclient, options, json_filter, ): ids_list = [ # convert the tokens into standardised IDs tokens.relative_id_with_selectors for tokens in tokens_list ] tp_query = TASK_PREREQS_QUERY tp_kwargs = { 'request_string': tp_query, 'variables': { 'wFlows': [workflow_id], 'taskIds': ids_list, } } results = await pclient.async_request('graphql', tp_kwargs) task_proxies = sorted(results['taskProxies'], key=lambda proxy: proxy['id']) multi = len(task_proxies) > 1 for t_proxy in task_proxies: task_id = Tokens(t_proxy['id']).relative_id state = t_proxy['state'] if options.json: json_filter.update({task_id: t_proxy}) else: if multi: ansiprint(f'\nTask ID: {task_id}') prereqs = [] for item in t_proxy['prerequisites']: prefix = '' multi_cond = len(item['conditions']) > 1 if multi_cond: prereqs.append([ True, '', item['expression'].replace('c', ''), item['satisfied'] ]) for cond in item['conditions']: if multi_cond and not options.list_prereqs: prefix = f'\t{cond["exprAlias"].strip("c")} = ' prereqs.append([ False, prefix, f'{cond["taskId"]} {cond["reqState"]}', cond['satisfied'] ]) if options.list_prereqs: for composite, _, msg, _ in prereqs: if not composite: print(msg) else: flat_meta = flatten_data(t_proxy['task']['meta']) for key, value in sorted(flat_meta.items(), reverse=True): ansiprint( f'{key}:' f' {value or "(not given)"}') # state and state attributes attrs = [] if t_proxy['isHeld']: attrs.append("held") if t_proxy['isQueued']: attrs.append("queued") if t_proxy['isRunahead']: attrs.append("runahead") run_mode = t_proxy['runtime']['runMode'] if run_mode and RunMode(run_mode) != RunMode.LIVE: attrs.append(f"run mode={run_mode}") state_msg = state if attrs: state_msg += f" ({','.join(attrs)})" ansiprint(f'state: {state_msg}') # flow numbers, if not just 1 if t_proxy["flowNums"] != "[1]": ansiprint( f"flows: " f"{t_proxy['flowNums'].replace(' ', '')}" ) # prerequisites pre_txt = "prerequisites:" if not prereqs: ansiprint(f"{pre_txt} (None)") elif ( TASK_STATUSES_ORDERED.index(state) > TASK_STATUSES_ORDERED.index(TASK_STATUS_RUNNING) ): # We only store prerequisites in the DB for n>0. ansiprint(f"{pre_txt} (n/a for past tasks)") else: ansiprint( f"{pre_txt}" f" ('{UNSATISFIED}': not satisfied)" ) for _, prefix, msg, state in prereqs: print_msg_state(f'{prefix}{msg}', state) # outputs ansiprint( 'outputs:' f" ('{UNSATISFIED}': not completed)") if not t_proxy['outputs']: # (Not possible - standard outputs) print(' (None)') for output in t_proxy['outputs']: info = f'{task_id} {output["label"]}' print_msg_state(info, output['satisfied']) if ( t_proxy['externalTriggers'] or t_proxy['xtriggers'] ): ansiprint( "other:" f" ('{UNSATISFIED}': not satisfied)" ) for ext_trig in t_proxy['externalTriggers']: state = ext_trig['satisfied'] print_msg_state( f'{ext_trig["label"]} ... {state}', state) for xtrig in t_proxy['xtriggers']: label = get_wallclock_label(xtrig) or xtrig['id'] state = xtrig['satisfied'] print_msg_state( f'xtrigger "{xtrig["label"]} = {label}"', state) print_completion_state(t_proxy) if not task_proxies: ansiprint( "No matching active tasks found: " f"{', '.join(ids_list)}", file=sys.stderr, ) return 1 return 0 def get_wallclock_label(xtrig: Dict[str, Any]) -> str: """Return a label for an xtrigger if it is a wall_clock trigger. Returns: A label or False. Examples: >>> this = get_wallclock_label >>> this({'id': 'wall_clock(trigger_time=0)'}) 'wall_clock(trigger_time=1970-01-01T00:00:00Z)' >>> this({'id': 'wall_clock(trigger_time=440143843)'}) 'wall_clock(trigger_time=1983-12-13T06:10:43Z)' """ wallclock_trigger = re.findall( r'wall_clock\(trigger_time=(.*)\)', xtrig['id']) if wallclock_trigger: return ( 'wall_clock(trigger_time=' f'{str(seconds2point(wallclock_trigger[0], True))})' ) return '' async def task_meta_query( workflow_id, task_names, pclient, options, json_filter, ): tasks_query = TASK_META_QUERY tasks_kwargs = { 'request_string': tasks_query, 'variables': { 'wFlows': [workflow_id], 'taskIds': task_names, }, } # Print workflow info. results = await pclient.async_request('graphql', tasks_kwargs) multi = len(results['tasks']) > 1 for task in results['tasks']: flat_data = flatten_data(task['meta']) if options.json: json_filter.update({task['name']: flat_data}) else: if multi: print(f'\nTASK NAME: {task["name"]}') for key, value in sorted(flat_data.items(), reverse=True): ansiprint( f'{key}: {value or "(not given)"}') return 0 async def show(workflow_id, tokens_list, opts): pclient = get_client( workflow_id, timeout=opts.comms_timeout, ) json_filter: 'Dict' = {} ret = 0 if opts.task_defs: ret = await task_meta_query( workflow_id, opts.task_defs, pclient, opts, json_filter, ) elif not tokens_list: ret = await workflow_meta_query( workflow_id, pclient, opts, json_filter, ) else: ret = await prereqs_and_outputs_query( workflow_id, tokens_list, pclient, opts, json_filter, ) if opts.json: print(json.dumps(json_filter, indent=4)) return ret @cli_function(get_option_parser) def main(_, options: 'Values', *ids) -> None: """Implement "cylc show" CLI.""" workflow_args, _ = parse_ids( *ids, constraint='mixed', max_workflows=1, ) workflow_id = next(iter(workflow_args)) tokens_list = workflow_args[workflow_id] if tokens_list and options.task_defs: raise InputError( 'Cannot query both live tasks and task definitions.' ) ret = asyncio.run(show(workflow_id, tokens_list, options)) sys.exit(ret) cylc-flow-8.7.0/cylc/flow/scripts/poll.py0000775000175000017500000000513715264420307020535 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc poll [OPTIONS] ARGS Poll (query) jobs to verify and update their statuses. This checks the job status file and queries the job runner on the job platform. Pollable tasks are those in the n=0 window with an associated job ID, including incomplete finished tasks. Examples: # poll all pollable tasks in my_workflow $ cylc poll 'my_workflow//*' # poll specific tasks in my_workflow $ cylc poll my_workflow// //1/a //1/b """ from functools import partial import sys from typing import TYPE_CHECKING from cylc.flow.network.client_factory import get_client from cylc.flow.network.multi import call_multi from cylc.flow.option_parsers import ( FULL_ID_MULTI_ARG_DOC, CylcOptionParser as COP ) from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values MUTATION = ''' mutation ( $wFlows: [WorkflowID]!, $tasks: [NamespaceIDGlob]!, ) { poll ( workflows: $wFlows, tasks: $tasks, ) { result } } ''' def get_option_parser() -> COP: parser = COP( __doc__, comms=True, multitask=True, multiworkflow=True, argdoc=[FULL_ID_MULTI_ARG_DOC], ) return parser async def run(options: 'Values', workflow_id: str, *tokens_list): pclient = get_client(workflow_id, timeout=options.comms_timeout) mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [workflow_id], 'tasks': [ tokens.relative_id_with_selectors for tokens in tokens_list ], } } return await pclient.async_request('graphql', mutation_kwargs) @cli_function(get_option_parser) def main(parser: COP, options: 'Values', *ids): rets = call_multi( partial(run, options), *ids, ) sys.exit(all(rets.values()) is False) cylc-flow-8.7.0/cylc/flow/scripts/jobs_submit.py0000775000175000017500000000546615264420307022114 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc jobs-submit [OPTIONS] ARGS (This command is for internal use.) Submit jobs to relevant job runners. On a remote job host, this command reads the job files from STDIN. """ from cylc.flow.option_parsers import CylcOptionParser as COP from cylc.flow.terminal import cli_function from cylc.flow.job_runner_mgr import JobRunnerManager INTERNAL = True def get_option_parser() -> COP: parser = COP( __doc__, argdoc=[ ("JOB-LOG-ROOT", "The log/job sub-directory for the workflow"), COP.optional( ("JOB-LOG-DIR ...", "A point/name/submit_num sub-directory") ) ], ) parser.add_option( "--remote-mode", help="Is this being run on a remote job host?", action="store_true", dest="remote_mode", default=False, ) parser.add_option( "--utc-mode", help="(for remote mode) is the workflow running in UTC mode?", action="store_true", dest="utc_mode", default=False, ) parser.add_option( "--clean-env", help="Clean job submission environment.", action="store_true", dest="clean_env", default=False, ) parser.add_option( "--env", help="Variable to pass from parent environment to job submit " "environment. This option can be used multiple times.", action="append", metavar="VAR=VALUE", dest="env", default=[] ) parser.add_option( "--path", help="Executable location to pass to job submit environment. " "This option can be used multiple times.", action="append", metavar="PATH", dest="path", default=[] ) return parser @cli_function(get_option_parser) def main(parser, opts, job_log_root, *job_log_dirs): """CLI main.""" JobRunnerManager(opts.clean_env, opts.env, opts.path).jobs_submit( job_log_root, job_log_dirs, remote_mode=opts.remote_mode, utc_mode=opts.utc_mode, ) cylc-flow-8.7.0/cylc/flow/scripts/cylc.py0000664000175000017500000005327315264420307020522 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc main entry point""" import os import sys def pythonpath_manip(): """Stop PYTHONPATH contaminating the Cylc Environment * Remove PYTHONPATH items from sys.path to prevent PYTHONPATH contaminating the Cylc Environment. * Add items from CYLC_PYTHONPATH to sys.path. See Also: https://github.com/cylc/cylc-flow/issues/5124 """ if 'CYLC_PYTHONPATH' in os.environ: paths = [ os.path.abspath(item) for item in os.environ['CYLC_PYTHONPATH'].split(os.pathsep) ] paths.extend(sys.path) sys.path = paths if 'PYTHONPATH' in os.environ: for item in os.environ['PYTHONPATH'].split(os.pathsep): abspath = os.path.abspath(item) if abspath in sys.path: sys.path.remove(abspath) pythonpath_manip() import argparse from contextlib import contextmanager from importlib.metadata import ( entry_points, files, ) from typing import ( Iterator, Optional, Tuple, ) from ansimarkup import parse as cparse from cylc.flow import ( __version__, iter_entry_points, ) from cylc.flow.option_parsers import ( format_help_headings, format_shell_examples, ) from cylc.flow.scripts.common import cylc_header def get_version(long=False): """Return version string, and (if long is True) install location. The install location returned is the top directory of the virtual environment, obtained from the Python executable path. (cylc-flow file locations are buried deep in the library and don't always give the right result, e.g. if installed with `pip install -e .`). """ from pathlib import Path version = f"{__version__}" if long: version += f" ({Path(sys.argv[0])})" return version USAGE = f"""{cylc_header()} Cylc ("silk") efficiently manages distributed cycling workflows. Cylc is Open Source software (GPL-3.0): see "cylc help license". Version: $ cylc version --long {get_version(True)} Quick Start: $ cylc install # install a workflow $ cylc play # run or resume a workflow $ cylc stop # stop a workflow $ cylc clean # delete an installed workflow $ cylc gui # start the in-browser web UI $ cylc tui # start the in-terminal UI $ cylc help all # see all cylc commands $ cylc --help # specific command help Cylc IDs: Workflows and tasks are identified by IDs of the form: workflow//cycle/task You can split an ID at the // so following two IDs are equivalent: workflow//cycle1 workflow//cycle2 workflow// //cycle1 //cycle2 IDs can be written as globs: *// # All workflows workflow//* # All cycle points in "workflow" workflow//cycle/* # All tasks in cycle point "cycle" of "workflow" $ cylc help id # More information on IDs Cylc commands can be abbreviated: $ cylc trigger workflow//cycle/task # trigger task in workflow $ cylc trig workflow//cycle/task # trigger task in workflow $ cylc t # error: trigger or tui? """ ID_HELP = ''' Workflow IDs: Every Installed Cylc workflow has an ID. For example if we install a workflow like so: $ cylc install --workflow-name=foo We will end up with a workflow with the ID "foo/run1". This ID can be used to interact with the workflow: $ cylc play foo/run1 $ cylc pause foo/run1 $ cylc stop foo/run1 If you omit run number ("run1", "run2", ...) Cylc will infer latest run. $ cylc play foo $ cylc pause foo $ cylc stop foo Workflows can be installed hierarchically: $ cylc install --workflow-name=foo/bar/baz # play the workflow with the ID "foo/bar/baz" $ cylc play foo/bar/baz The full format of a workflow ID is: ~user/workflow-id You can omit the user name when working on your own workflows. Cycle / Family / Task / Job IDs: Just as workflows have IDs, so do objects within workflows: cycle/task_or_family/job Examples: 1 # The cycle point "1" 1/a # The task "a" in cycle point "1" 1/a/1 # The first job of the task "a" in the cycle point "1". Full ID Join workflow and cycle/task/job IDs with //: workflow//cycle/task/job Examples: w// # The workflow "w" w//1 # The cycle "1" in the workflow "w" w//1/a # The task "a" in the cycle "1" in the workflow "w" w//1/a/1 # The first job of w//1/a/1 ~alice/test # The workflow "test" installed under the user # account "alice" Patterns Patterns can be used in Cylc IDs: * # Matches everything. ? # Matches any single character. [seq] # Matches any character in "seq". [!seq] # Matches any character not in "seq". Examples: * # All workflows test* # All workflows starting "test". test/* # All workflows starting "test/". workflow//* # All active cycles in workflow workflow//cycle/* # All tasks in workflow//cycle workflow//cycle/task/* # All jobs in workflow//cycle/job Warning: Quote IDs on the command line to protect them from shell expansion. Selectors Selectors allow you to filter for specific states. Selectors are prefixed by a colon (:). Examples: *:running # All running workflows workflow//*:running # All running tasks in workflow workflow//cycle/*:running # All running tasks in workflow//cycle workflow//cycle/task/*:running # All running jobs in # workflow//cycle/task When selectors are applied to tasks, they will only match active tasks. I.e, "workflow//*:failed" will only select active failed tasks, not all historical failed tasks going back to the start of the workflow. ''' # because this command is not served from behind cli_function like the # other cylc commands we have to manually patch in colour support USAGE = cparse(format_help_headings(format_shell_examples(USAGE))) # all sub-commands # {name: entry_point} COMMANDS: dict = { entry_point.name: entry_point for entry_point in iter_entry_points('cylc.command') } # aliases for sub-commands # {alias_name: command_name} ALIASES = { 'bcast': 'broadcast', 'compare': 'diff', 'cyclepoint': 'cycle-point', 'cycletime': 'cycle-point', 'datetime': 'cycle-point', 'external-trigger': 'ext-trigger', 'get-contact': 'get-workflow-contact', 'get-cylc-version': 'get-workflow-version', 'log': 'cat-log', 'ls': 'list', 'shutdown': 'stop', 'task-message': 'message', 'unhold': 'release', 'validate-install-play': 'vip', 'validate-reinstall': 'vr', } # aliases for sub-commands which no longer exist # {alias_name: message_to_user} # fmt: off DEAD_ENDS = { 'check-software': 'use standard tools to inspect the environment' ' e.g. https://pypi.org/project/pipdeptree/', 'checkpoint': 'DB checkpoints have been removed. You can now "rewind" a' ' workflow by triggering the flow anywhere in the graph.', 'conditions': 'cylc conditions has been replaced by cylc help license', 'documentation': 'Cylc documentation is now at https://cylc.org', 'edit': 'Command removed, please edit the workflow in source directory', 'get-directory': 'cylc get-directory has been removed.', 'get-config': 'cylc get-config has been replaced by cylc config', 'get-site-config': 'cylc get-site-config has been replaced by cylc config', 'get-suite-config': 'cylc get-suite-config has been replaced by cylc config', 'get-global-config': 'cylc get-global-config has been replaced by cylc config', 'graph-diff': 'cylc graph-diff has been removed,' ' use cylc graph --diff ', 'gscan': 'cylc gscan has been removed, use the web UI', 'insert': 'Insertion is no longer required, `cylc set` and `cylc trigger`' ' will insert tasks automatically.', 'jobscript': 'cylc jobscript has been removed', 'nudge': 'cylc nudge has been removed', 'print': 'cylc print has been removed; use `cylc scan --states=all`', 'register': 'cylc register has been removed; use cylc install or cylc play', 'reset': 'cylc reset has been replaced by cylc set', 'set-outputs': 'cylc set-outputs (cylc 8.0-8.2) has been replaced by cylc set', 'restart': 'cylc run & cylc restart have been replaced by cylc play', 'suite-state': 'cylc suite-state has been replaced by cylc workflow-state', 'run': 'cylc run & cylc restart have been replaced by cylc play', 'search': 'cylc search has been removed; please use `grep` or a text editor', 'spawn': 'cylc spawn has been removed; spawning is now performed automatically', 'submit': 'cylc submit has been removed', 'start': 'cylc start & cylc restart have been replaced by cylc play', 'set-verbosity': 'cylc set-verbosity has been replaced by cylc verbosity', 'warranty': 'cylc warranty has been replaced by cylc help license', } # fmt: on def execute_cmd(cmd: str, *args: str) -> int: """Execute a sub-command. Args: cmd: The name of the command. args: Command line arguments to pass to that command. """ entry_point = COMMANDS[cmd] try: entry_point.load()(*args) except ModuleNotFoundError as exc: msg = handle_missing_dependency(entry_point, exc) print(msg, file=sys.stderr) return 1 return 0 def match_command(command): """Permit abbreviated commands (e.g. tri -> trigger). Args: command (string): The input string to match. Returns: string - The matched command. Exits: 1: If the number of command matches != 1 """ possible_cmds = { *{ # search commands cmd for cmd in COMMANDS if cmd.startswith(command) }, } if len(possible_cmds) == 0: print( f"cylc {command}: unknown utility. Abort.\n" 'Type "cylc help all" for a list of utilities.', file=sys.stderr ) sys.exit(1) elif len(possible_cmds) > 1: print( "cylc {}: is ambiguous for:\n{}".format( command, "\n".join( [ f" cylc {cmd}" for cmd in sorted(possible_cmds) ] ), ), file=sys.stderr, ) sys.exit(1) else: command = possible_cmds.pop() return command def parse_docstring(docstring): """Extract the description and usage lines from a sub-command docstring. Args: docstring (str): Multiline string i.e. __doc__ Returns: tuple - (usage, description) """ lines = [ line for line in docstring.splitlines() if line ] usage = None desc = None if len(lines) > 0: usage = lines[0] if len(lines) > 1: desc = lines[1] return (usage, desc) def iter_commands() -> Iterator[Tuple[str, Optional[str], Optional[str]]]: """Yield all Cylc sub-commands that are available. Skips sub-commands that require missing optional dependencies. Yields: (command, description, usage) """ for cmd, entry_point in sorted(COMMANDS.items()): try: module = __import__(entry_point.module, fromlist=['']) except ModuleNotFoundError as exc: handle_missing_dependency(entry_point, exc) continue if getattr(module, 'INTERNAL', False): # do not list internal commands continue usage, desc = parse_docstring(module.__doc__) yield (cmd, desc, usage) def print_id_help(): print(ID_HELP) def print_license() -> None: for file in files('cylc-flow') or []: if file.name == 'COPYING': print(file.read_text()) return def print_command_list(commands=None, indent=0): """Print list of Cylc commands. Args: commands (list): List of commands to display. indent (int): Number of spaces to put at the start of each line. """ from cylc.flow.terminal import print_contents contents = [ (cmd, desc) for cmd, desc, _, in iter_commands() if not commands or cmd in commands ] print_contents(contents, indent=indent, char=cparse('.')) def cli_help(): """Display the main Cylc help page.""" # add a splash of colour # we need to do this explicitly as this command is not behind cli_function # (assume the cylc help is only ever requested interactively in a # modern terminal) from colorama import init as color_init color_init(autoreset=True, strip=False) print(USAGE) def cli_version(long_fmt=False): """Wrapper for get_version.""" print(get_version(long_fmt)) if long_fmt: print(cparse(list_plugins())) def list_plugins(): from cylc.flow.terminal import DIM, format_grid # go through all Cylc entry points _dists = set() __entry_points = {} for entry_point in entry_points(): if ( # all Cylc entry points are under the "cylc" namespace entry_point.group.startswith('cylc.') # don't list cylc-flow entry-points (i.e. built-ins) and not entry_point.value.startswith('cylc.flow') ): _dists.add(entry_point.dist) __entry_points.setdefault( entry_point.group, [], ).append(entry_point) # list all the distributions which provide Cylc entry points _plugins = [] for dist in _dists: _plugins.append(( '', f'{dist.name}', dist.version, f'<{DIM}>{dist.locate_file("__init__.py").parent}', )) # list all of the entry points by "group" (e.g. "cylc.command") _entry_points = [] for group, points in sorted(__entry_points.items()): _entry_points.append((f' {group}:', '', '')) for entry_point in points: _entry_points.append(( f' {entry_point.name}', f'{entry_point.dist.name}', f'<{DIM}>{entry_point.value}', )) return '\n'.join(( '\nPlugins:', *format_grid(_plugins), '\nEntry Points:', *format_grid( _entry_points ), )) @contextmanager def pycoverage(cmd_args): # pragma: no cover """Capture code coverage if configured to do so. This requires Cylc to be installed in editable mode (i.e. `pip install -e`) in order to access the coverage configuration file, etc. $ pip install -e /cylc/working/directory Set the CYLC_COVERAGE env var as appropriate before running tests $ export CYLC_COVERAGE=1 Coverage files will be written out to the working copy irrespective of where in the filesystem the `cylc` command was run. $ cd /cylc/working/directory $ coverage combine $ coverage report For remote tasks the coverage files will be written to the cylc working directory on the remote so you will have to scp them back to your local working directory before running coverage combine: $ cd /cylc/working/directory $ ssh remote-host cd /cylc/remote/working/directory && coverage combine $ scp \ > remote-host/cylc/remote/working/directory/.coverage \ > .coverage.remote-host.12345.12345 $ coverage combine $ coverage report Environment Variables: CYLC_COVERAGE: '0' Do nothing / run as normal. '1' Collect coverage data. '2' Collect coverage data and log every command for which coverage data was successfully recorded to a .coverage_commands_captured file in the Cylc working directory. """ cylc_coverage = os.environ.get('CYLC_COVERAGE') if cylc_coverage not in ('1', '2'): yield return # import here to avoid unnecessary imports when not running coverage import cylc.flow import coverage from pathlib import Path import signal # the cylc working directory if 'CYLC_COVERAGE_BASE' in os.environ: cylc_wc = Path(os.environ['CYLC_COVERAGE_BASE']) else: cylc_wc = Path(cylc.flow.__file__).parents[2] # initiate coverage try: cov = coverage.Coverage( # NOTE: coverage paths are all relative so we must hack them here # to absolute values, otherwise when `cylc` scripts are run # elsewhere on the filesystem they will fail to capture coverage # data and will dump empty coverage files where they run. config_file=str(cylc_wc / '.coveragerc'), data_file=str(cylc_wc / '.coverage'), source=[str(cylc_wc / 'cylc')] ) except coverage.misc.CoverageException as exc: raise Exception( # make sure this exception is visible in the traceback '\n\n*****************************\n\n' 'Could not initiate coverage, likely because Cylc was not ' 'installed in editable mode.' '\n\n*****************************\n\n' ) from exc def cov_stop(*a, **k): cov.stop() cov.save() if cylc_coverage == '2': with open(cylc_wc / '.coverage_commands_captured', 'a+') as ccc: ccc.write( '$ cylc ' + (' '.join(cmd_args) + '\n'), ) # start the coverage running cov.start() signal.signal(signal.SIGINT, cov_stop) try: # yield control back to cylc, return once the command exits yield finally: # stop the coverage and save the data cov_stop() def get_arg_parser(): parser = argparse.ArgumentParser(add_help=False) parser.add_argument( '--help', '-h', action='store_true', default=False, dest='help_' ) parser.add_argument( '--version', '-V', action='store_true', default=False, dest='version' ) return parser def main() -> None: # pragma: no cover opts, cmd_args = get_arg_parser().parse_known_args() with pycoverage(cmd_args): ret = _main(opts, cmd_args) sys.exit(ret) def _main(opts, cmd_args: list[str]) -> int: """Implement the Cylc CLI. Returns the exit code as an integer. """ if not cmd_args: if opts.version: cli_version() return 0 else: cli_help() return 0 else: cmd_args = list(cmd_args) command: str = cmd_args.pop(0) if command == "version": cli_version("--long" in cmd_args) return 0 if command == "help": opts.help_ = True if not len(cmd_args): cli_help() return 0 elif cmd_args == ['all']: print_command_list() return 0 elif cmd_args == ['id']: print_id_help() return 0 if cmd_args in (['license'], ['licence']): print_license() return 0 else: command = cmd_args.pop(0) # this is an alias to a command command = ALIASES.get(command, command) if command in DEAD_ENDS: # this command has been removed but not aliased # display a helpful message and move on# print( cparse( f'{DEAD_ENDS[command]}' ) ) return 42 if command not in COMMANDS: # check if this is a command abbreviation or exit command = match_command(command) if opts.help_: cmd_args.append('--help') return execute_cmd(command, *cmd_args) def handle_missing_dependency( entry_point, err: ModuleNotFoundError ) -> str: """Return a suitable error message for a missing optional dependency. Args: entry_point: The entry point that was attempted to load but caused a ModuleNotFoundError. err: The ModuleNotFoundError that was caught. Re-raises the given ModuleNotFoundError if it is unexpected. """ msg = f'"cylc {entry_point.name}" requires "{entry_point.dist.name}' if entry_point.extras: msg += f'[{",".join(entry_point.extras)}]' msg += f'"\n\n{err.__class__.__name__}: {err}' return msg cylc-flow-8.7.0/cylc/flow/scripts/message.py0000775000175000017500000001536515264420307021217 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . r"""cylc message [OPTIONS] -- ARGS Command to record and send messages from task jobs back to the scheduler. Messages are sent to: - job stdout/stderr - the job status file, if there is one - the scheduler, if communication is possible Jobs automatically use this command to record and report job status: started (running) and success or failure. Applications run by jobs can use this command to report custom messages and and registered task output messages. For custom outputs, use the task message not the associated output name: [runtime] [[my-task]] # ... [[[outputs]]] # = x = "file x completed and archived" Messages can be specified as arguments. A '-' indicates that the command should read messages from STDIN. When reading from STDIN, multiple messages are separated by empty lines. Examples: # Single message as an argument: $ cylc message -- "${CYLC_WORKFLOW_ID}" "${CYLC_TASK_JOB}" 'Hello world!' # Multiple messages as arguments: $ cylc message -- "${CYLC_WORKFLOW_ID}" "${CYLC_TASK_JOB}" \ > 'Hello world!' 'Hi' 'WARNING:Hey!' # Multiple messages on STDIN: $ cylc message -- "${CYLC_WORKFLOW_ID}" "${CYLC_TASK_JOB}" - <<'__STDIN__' > Hello > world! > > Hi > > WARNING:Hey! >__STDIN__ Note "${CYLC_WORKFLOW_ID}" and "${CYLC_TASK_JOB}" are available in job environments - you do not need to write their actual values in task scripting. Each message can be prefixed with a severity level using the syntax 'SEVERITY:MESSAGE' (colons cannot be used unless such a prefix is provided). The default message severity is INFO. The --severity=SEVERITY option can be used to set the default severity level for all unprefixed messages. Increased severity will make messages more visible in workflow logs, using colour and format changes. DEBUG messages will not be shown in logs by default. The severity levels are those of the Python Logging Library https://docs.python.org/3/library/logging.html#logging-levels: - CRITICAL - ERROR - WARNING - INFO - DEBUG Note: To abort a job script with a custom error message, use cylc__job_abort: cylc__job_abort 'message...' (For technical reasons this is a shell function, not a cylc sub-command). For backward compatibility, if number of arguments is less than or equal to 2, the command assumes the classic interface, where all arguments are messages. Otherwise, the first 2 arguments are assumed to be workflow ID and job identifier. """ import asyncio from logging import getLevelName, INFO import os import sys from typing import TYPE_CHECKING, Sequence from cylc.flow.id_cli import parse_id_async from cylc.flow.option_parsers import ( WORKFLOW_ID_ARG_DOC, CylcOptionParser as COP ) from cylc.flow.task_message import record_messages from cylc.flow.terminal import cli_function from cylc.flow.exceptions import InputError from cylc.flow.unicode_rules import TaskMessageValidator if TYPE_CHECKING: from optparse import Values def get_option_parser() -> COP: parser = COP( __doc__, comms=True, argdoc=[ COP.optional(WORKFLOW_ID_ARG_DOC), COP.optional( ('JOB', 'Job ID - CYCLE/TASK_NAME/SUBMIT_NUM') ), COP.optional( ('[SEVERITY:]MESSAGE ...', 'Messages') ) ] ) parser.add_option( '-s', '--severity', '-p', '--priority', metavar='SEVERITY', help='Set severity levels for messages that do not have one', action='store', dest='severity') return parser @cli_function(get_option_parser) def main(parser: COP, options: 'Values', *args: str) -> None: """CLI.""" if not args: parser.error('No message supplied') return asyncio.run(_main(options, args)) async def _main(options: 'Values', args: Sequence[str]) -> None: if len(args) <= 2: # BACK COMPAT: args <= 2 # from: # 7.6? # remove at: # 9.0? # (As of Dec 2020 some functional tests still use the classic # two arg interface) workflow_id = os.getenv('CYLC_WORKFLOW_ID') job_id = os.getenv('CYLC_TASK_JOB') if not workflow_id or not job_id: raise InputError( "Must set $CYLC_WORKFLOW_ID and $CYLC_TASK_JOB if not " "specified as arguments" ) message_strs = list(args) else: workflow_id, job_id, *message_strs = args workflow_id, *_ = await parse_id_async( workflow_id, constraint='workflows', ) # Read messages from STDIN if '-' in message_strs: current_message_str = '' while True: # Note: `for line in sys.stdin:` can hang message_str = sys.stdin.readline() if message_str.strip(): # non-empty line current_message_str += message_str elif message_str: # empty line, start next message if current_message_str: message_strs.append(current_message_str) current_message_str = '' # reset else: # end of file if current_message_str: message_strs.append(current_message_str) break # Separate "severity: message" messages = [] # [(severity, message_str), ...] for message_str in message_strs: if message_str == '-': pass elif ':' in message_str: valid, err_msg = TaskMessageValidator.validate(message_str) if not valid: raise InputError( f'Invalid task message "{message_str}" - {err_msg}') messages.append( [item.strip() for item in message_str.split(':', 1)]) elif options.severity: messages.append([options.severity, message_str.strip()]) else: messages.append([getLevelName(INFO), message_str.strip()]) await record_messages(workflow_id, job_id, messages) cylc-flow-8.7.0/cylc/flow/scripts/psutil.py0000664000175000017500000000574115264420307021105 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Report system utilisation information. (This command is for internal use.) This is for use in situations where Cylc needs to extract information from the `psutil` on remote platforms. Exits: 0 - If successful. 2 - For errors in extracting results from psutil 1 - For all other errors. """ import json import sys import psutil # from cylc.flow.host_select import _simple_eval from cylc.flow.option_parsers import OptionParser from cylc.flow.terminal import ( cli_function, parse_dirty_json ) INTERNAL = True def get_option_parser(): return OptionParser(__doc__) def _call_psutil_interface(interface, *args): """Call a psutil interface with the provided arguments. Args: interface: The psutil method we want to call, e.g. "cpu_percent". In the case of objects, this may include attributes, e.g. "Process.cmdline". args: The arguments to provide to the psutil method. Returns: The result of the psutil method call. """ result = psutil is_first = True for fcn in interface.split('.'): try: method = getattr(result, fcn) except AttributeError as exc: # error obtaining interfaces from psutil e.g: # * requesting a method which does not exist print(exc, file=sys.stderr) sys.exit(2) if is_first: is_first = False else: args = () result = method(*args) return result def _psutil(metrics_json): metrics = parse_dirty_json(metrics_json) try: ret = [_call_psutil_interface(*key) for key in metrics] except Exception as exc: # error extracting metrics from psutil e.g: # * requesting information on a resource which does not exist print(exc, file=sys.stderr) sys.exit(2) # serialise for ind, item in enumerate(ret): if hasattr(item, '_asdict'): ret[ind] = item._asdict() elif hasattr(item, 'as_dict'): ret[ind] = item.as_dict() return ret @cli_function(get_option_parser) def main(*_): print( json.dumps( _psutil(sys.stdin.read()) ) ) cylc-flow-8.7.0/cylc/flow/scripts/__init__.py0000664000175000017500000000152615264420307021321 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . _copyright_year = 2026 # This is set by GH Actions update_copyright workflow cylc-flow-8.7.0/cylc/flow/scripts/scan.py0000664000175000017500000003741615264420307020515 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc scan [OPTIONS] List your source, and/or installed, and/or running workflows. By default, running workflows are listed as indicated by the presence of scheduler contact files, ~/cylc-run//.service/contact. With "--ping" or "-t rich", attempt to contact the schedulers. If any are not found to be running, their contact files will be removed (these files may left behind if the scheduler did not shut down cleanly). Examples: # list all active workflows (i.e. running or paused) $ cylc scan # show more information about active workflows $ cylc scan -t rich # don't rely on colour for job state totals $ cylc scan -t rich --colour-blind # list workflows that are installed but stopped or not run yet $ cylc scan --state stopped # list all workflows (active or inactive) $ cylc scan --state=running,paused,stopped $ cylc scan --state=all # or using the shorthand # filter workflows by name $ cylc scan --name '^f.*' # show only flows starting with "f" # list source workflows in tree format # (looks in the dirs configured by "global.cylc[install]source dirs") $ cylc scan --source -t tree # print contact file data in JSON format $ cylc scan -t json """ import asyncio import json from pathlib import Path from typing import Callable, Optional, TYPE_CHECKING from ansimarkup import ansiprint as cprint from cylc.flow import LOG from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.exceptions import InputError from cylc.flow.network.scan import ( contact_info, filter_name, graphql_query, is_active, scan, scan_multi, ) from cylc.flow.option_parsers import ( CylcOptionParser as COP, Options ) from cylc.flow.print_tree import get_tree from cylc.flow.terminal import cli_function, DIM from cylc.flow.util import natural_sort_key from cylc.flow.workflow_files import ContactFileFields as Cont if TYPE_CHECKING: from optparse import Values # all supported workflow states FLOW_STATES = { 'running', 'paused', 'stopping', 'stopped' } # workflow status colours FLOW_STATE_CMAP = { # workflow state: term colour 'running': 'green', 'paused': 'fg 172', 'stopping': 'fg 201', 'stopped': 'red' } # workflow status symbols FLOW_STATE_SYMBOLS = { # NOTE: the standard media control characters ▶️,, ⏸️,, ⏹️ # can appear wildly different font-depending and may not # even be monospace 'running': '▶', 'paused': '‖', 'stopping': '◧', 'stopped': '■' } # document the flow states __doc__ += '\n '.join( [ '\nWorkflow States:' ] + [ ( f'<{FLOW_STATE_CMAP[state]}>' f'{FLOW_STATE_SYMBOLS[state]}' f' {state}' f'' ) for state in FLOW_STATES ] ) # job icon colours JOB_COLOURS = { # job status: term colour 'submitted': 'fg 44', 'submit-failed': 'fg 13', 'running': 'fg 32', 'succeeded': 'fg 35', 'failed': 'fg 124' } # document the task states __doc__ += '\n '.join( [ '\n\nTask States:' ] + [ f'<{tag}>■ {state}' for state, tag in JOB_COLOURS.items() ] ) # graphql fields to request from the workflow for the "rich" format RICH_FIELDS = { 'status': None, 'stateTotals': None, 'cylcVersion': None, 'meta': { 'title', 'description', }, } BAD_CONTACT_FILE_MSG = "Bad contact file: {flow_name}" def get_option_parser() -> COP: """CLI opts for "cylc scan".""" parser = COP( __doc__, comms=True ) parser.add_option( '--name', '-n', help=( 'Filter workflows by registered name using a regex.' ' Can be used multiple times, workflows will be displayed if' ' their name matches ANY of the provided regexes.' ), action='append' ) parser.add_option( '--states', help=( 'Choose which workflows to display by providing a list of states' ' or "all" to show everything. See the full `cylc scan` help' ' for a list of supported states.' ), default='running,paused,stopping', action='store' ) parser.add_option( '--source', '-s', help=( 'List source workflows from configured source' ' directories (overrides the --states option).' ), default=False, action='store_true' ) parser.add_option( '--format', '-t', help=( r'Output data and format (default "%default").' ' ("name": list the workflow IDs only)' ' ("plain": name,host:port,PID on one line)' ' ("tree": name,host:port,PID in tree format)' ' ("json": full contact data in JSON format)' ' ("rich": include task state summary data)' ), choices=('rich', 'plain', 'json', 'tree', 'name'), default='plain' ) parser.add_option( '--sort', help='Sort flows by name before writing them (takes longer).', action='store_true' ) parser.add_option( '--colour-blind', '--color-blind', help=( "Don't depend on colour to convey information. " 'Use this rather than --color=never so you still get bold text.' ), action='store_true' ) parser.add_option( '--ping', help=( "Connect to schedulers and remove contact files if they are not " "found to be running. Contact files can be left behind when " "schedulers get killed." ), action='store_true' ) return parser def state_totals(totals, colour_blind=False): """Return a string with a visual representation of workflow state totals. Args: totals (dict): State totals dictionary. colour_blind (bool): If True then we will not depend on colour to convey information. Returns: str """ ret = [] for state, tag in JOB_COLOURS.items(): number = totals[state] if number == 0: continue if colour_blind: ret.append(f'<{tag}>{state}:{number}') else: ret.append(f'<{tag}>{number} ■') return ' '.join(ret) def state_totals_key(): """Return a key to accompany state_totals.""" return 'Job State Key: ' + ' '.join( f'<{tag}>\u25A0 {name}' for name, tag in JOB_COLOURS.items() ) def _format_json(items, _): """A JSON formatter.""" return json.dumps( [ { key: str(value) if isinstance(value, Path) else value for key, value in flow.items() } for flow in items ], indent=4, sort_keys=True ) def _format_plain(flow, _): """A single line format of the form: [:]""" if flow.get('contact'): try: return ( f"{flow['name']} " f"{flow[Cont.HOST]}:{flow[Cont.PORT]} " f"{flow[Cont.PID]}" ) except KeyError: LOG.warning(BAD_CONTACT_FILE_MSG.format(flow_name=flow['name'])) return None else: return f'<{DIM}>{flow["name"]}' def _format_name_only(flow, _): """A single line format of the form: [:]""" return flow['name'] def _format_rich(flow, opts): """A multiline format which pulls in metadata.""" status = flow.get('status', 'stopped') if opts.colour_blind: name = f'{flow["name"]} ({status})' else: symbol = FLOW_STATE_SYMBOLS[status] tag = FLOW_STATE_CMAP[status] name = f'<{tag}>{symbol} {flow["name"]}' if not flow.get('contact') or 'status' not in flow: ret = [f'<{DIM}>{name}'] else: ret = [f'{name}'] display = { 'state totals': state_totals( flow['stateTotals'], opts.colour_blind ), **{ key: flow['meta'][key] or f'<{DIM}>*null*' for key in ( 'title', 'description' ) }, **{ name: flow[key] for name, key in ( ('version', 'cylcVersion'), ('host', Cont.HOST), ('port', Cont.PORT), ('PID', Cont.PID) ) } } maxlen = max(len(key) for key in display) for key, value in display.items(): # format multiline strings by whitespace padding the lines value = ('\n' + (' ' * (maxlen + 7))).join(value.splitlines()) # write out the key: value pairs ret.append(f' {key: <{maxlen}} {value}') return '\n'.join(ret) def sort_function(flow): if ( flow.get('status') == 'running' or flow.get('status') == 'paused' ): state = 0 elif flow.get('contact'): state = 2 else: state = 3 return (state, *natural_sort_key(flow['name'])) async def _sorted(pipe, formatter, opts, write): """List all flows, sort, then print them individually.""" ret = [] async for item in pipe: ret.append(item) for flow in sorted(ret, key=sort_function): out = formatter(flow, opts) if out is not None: write(out) async def _serial(pipe, formatter, opts, write): """List all flows, then print them as one.""" ret = [] async for item in pipe: ret.append(item) out = formatter(ret, opts) if out is not None: write(out) async def _async(pipe, formatter, opts, write): """List and print flows individually.""" async for flow in pipe: out = formatter(flow, opts) if out is not None: write(out) async def _tree(pipe, formatter, opts, write): """List all flows, sort, then print them as a tree.""" # get list of flows ret = [] async for flow in pipe: ret.append(flow) tree = {} _construct_tree(ret, tree, formatter, opts, write) # print tree ret = get_tree(tree, '', sort=False, use_unicode=True) if ret: write('\n'.join(ret)) def _construct_tree(flows, tree, formatter, opts, write): """Construct the tree, for given flows and formatter.""" for flow in sorted(flows, key=lambda f: f['name']): parts = Path(flow['name']).parts pointer = tree for part in parts[:-1]: if part not in pointer: pointer[part] = {} pointer = pointer[part] flow['name'] = parts[-1] item = formatter(flow, opts) if item is None: continue if len(parts) > 1: item = f' {item}' pointer[item] = '' def get_pipe(opts, formatter, scan_dir=None): """Construct a pipe for listing flows.""" if scan_dir: pipe = scan(scan_dir=scan_dir) elif opts.source: pipe = scan_multi( Path(path).expanduser() for path in glbl_cfg().get(['install', 'source dirs']) ) opts.states = {'stopped'} else: pipe = scan show_running = 'running' in opts.states show_paused = 'paused' in opts.states show_active = show_running or show_paused or 'stopping' in opts.states # show_active = bool({'running', 'paused'} & opts.states) show_inactive = bool({'stopped'} & opts.states) # filter by flow name if opts.name: pipe |= filter_name(*opts.name) # filter by flow state if show_active: pipe |= is_active(True, filter_stop=(not show_inactive)) elif show_inactive: pipe |= is_active(False) # get contact file information if show_active: pipe |= contact_info graphql_fields = {} graphql_filters = set() # filter paused/running flows if show_active and not (show_running and show_paused): graphql_fields['status'] = None graphql_filters.add((('status',), tuple(opts.states))) # get fancy data if requested if formatter == _format_rich: # graphql_fields['status'] = None graphql_fields.update(RICH_FIELDS) # add graphql queries / filters to the pipe if show_active and graphql_fields: pipe |= graphql_query(graphql_fields, filters=graphql_filters) elif opts.ping: # check the flow is running even if not required # by display format or filters pipe |= graphql_query({'status': None}) # yield results as they are processed pipe.preserve_order = False return pipe def get_formatter(opts): """Return the appropriate formatter and method for the provided opts.""" formatter = None method = None if opts.format == 'plain': formatter = _format_plain method = _async elif opts.format == 'json': formatter = _format_json method = _serial elif opts.format == 'rich': formatter = _format_rich method = _async elif opts.format == 'tree': formatter = _format_plain method = _tree elif opts.format == 'name': # noqa: SIM106 formatter = _format_name_only method = _async else: raise NotImplementedError if opts.sort and method == _serial: raise ValueError(f'--sort is incompatible with --format {opts.format}') elif opts.sort: method = _sorted return formatter, method async def scanner(opts, write, scan_dir=None): """Print workflows to stdout.""" formatter, method = get_formatter(opts) pipe = get_pipe(opts, formatter, scan_dir) LOG.debug(f'pipe: {repr(pipe)}') await method(pipe, formatter, opts, write) async def main( opts: 'Values', color: bool = False, scan_dir: Optional[Path] = None, write: Callable = cprint ) -> None: """Open up a Python API for testing purposes. Note: Don't use this API for anything other than testing, there is a proper Python API for these purposes. """ # validate / standardise the list of workflow states opts.states = set(opts.states.split(',')) if 'all' in opts.states: opts.states = FLOW_STATES elif ( not opts.states or not all( state in FLOW_STATES for state in opts.states ) ): raise InputError( '--states must be a comma separated list of workflow states.' '\nSee `cylc scan --help` for supported states.' ) if not color: # we cannot support colour or have been requested not to use it opts.colour_blind = True # print state totals key as needed if opts.format == 'rich' and not opts.colour_blind: cprint(state_totals_key() + '\n') await scanner(opts, write, scan_dir) @cli_function(get_option_parser) def cli(_, opts: 'Values', color: bool) -> None: """Implement `cylc scan`.""" asyncio.run( main(opts, color) ) ScanOptions = Options(get_option_parser()) cylc-flow-8.7.0/cylc/flow/scripts/profiler.py0000775000175000017500000002300615264420307021404 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) NIWA & British Crown (Met Office) & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc profiler [OPTIONS] Profiler which periodically polls cgroups to track the resource usage of jobs running on the node. """ import asyncio from contextlib import suppress from dataclasses import dataclass import json import os from pathlib import Path import re import signal import psutil from cylc.flow import LOG from cylc.flow.exceptions import CylcProfilerError import cylc.flow.flags from cylc.flow.option_parsers import CylcOptionParser as COP from cylc.flow.remote import watch_and_kill from cylc.flow.task_message import record_messages from cylc.flow.terminal import cli_function from metomi.isodatetime.parsers import DurationParser dp = DurationParser() INTERNAL = True PID_REGEX = re.compile(r"([^:]*\d{6,}.*)") RE_CPU_USAGE = re.compile(r'usage_usec\s*(\d+)') @dataclass class Process: """Class for representing CPU and Memory usage of a process""" cgroup_memory_path: Path max_rss: int cgroup_cpu_path: Path memory_allocated_path: Path cgroup_version: int async def report_to_scheduler(process: Process, comms_timeout: int): """Return the profiler's data to the scheduler.""" # extract the stats profiler_data = get_profiler_data(process) # send a task message to the scheduler / write message to job.status file await record_messages( os.environ['CYLC_WORKFLOW_ID'], os.environ['CYLC_TASK_JOB'], [['DEBUG', f'_cylc_profiler: {json.dumps(profiler_data)}']], comms_timeout=comms_timeout, ) def get_profiler_data(process: Process): if ( process.cgroup_memory_path is None or process.cgroup_cpu_path is None or process.memory_allocated_path is None ): # If a task fails instantly, or finishes very quickly (< 1 second), # the get config function doesn't have time to run max_rss = cpu_time = memory_allocated = 0 else: max_rss = process.max_rss cpu_time = parse_cpu_file(process) memory_allocated = parse_memory_allocated(process) return { 'max_rss': max_rss, 'cpu_time': cpu_time, 'memory_allocated': memory_allocated, } def parse_memory_file(process: Process): """Open the memory stat file and copy the appropriate data""" try: if process.cgroup_version == 2: with open(process.cgroup_memory_path, 'r') as f: for line in f: if "anon" in line: return int(''.join(filter(str.isdigit, line))) else: with open(process.cgroup_memory_path, 'r') as f: for line in f: if "total_rss" in line: return int(''.join(filter(str.isdigit, line))) except Exception as err: raise CylcProfilerError( err, "Unable to find memory usage data") from err def parse_memory_allocated(process: Process) -> int: """Open the memory stat file and copy the appropriate data""" if process.cgroup_version == 2: cgroup_memory_path = process.memory_allocated_path for _ in range(5): memory_max_file = cgroup_memory_path / "memory.max" with open(memory_max_file, 'r') as f: line = f.readline() if "max" not in line: return int(line) cgroup_memory_path = cgroup_memory_path.parent return 0 else: # Memory limit not tracked for cgroups v1 return 0 def parse_cpu_file(process: Process) -> int: """Open the CPU stat file and return the appropriate data""" try: if process.cgroup_version == 2: with open(process.cgroup_cpu_path, 'r') as f: for line in f: if match := RE_CPU_USAGE.search(line): return round(int(match.group(1)) / 1000) raise FileNotFoundError(process.cgroup_cpu_path) elif process.cgroup_version == 1: with open(process.cgroup_cpu_path, 'r') as f: for line in f: # Cgroups v1 uses nanoseconds return round(int(line) / 1000000) raise FileNotFoundError(process.cgroup_cpu_path) except Exception as err: raise CylcProfilerError( err, "Unable to find cpu usage data") from err return 0 def get_cgroup_version(cgroup_location: Path, cgroup_name: str) -> int: # Strip leading '/' so the name is treated as relative when joined cgroup_name = cgroup_name.lstrip('/') try: if (cgroup_location / cgroup_name).exists(): return 2 elif (cgroup_location / "memory" / cgroup_name).exists(): return 1 raise FileNotFoundError(cgroup_location / cgroup_name) except Exception as err: raise CylcProfilerError( err, f"Cgroup not found at {cgroup_location / cgroup_name}" ) from err def get_cgroup_name(): """Get the cgroup directory for the current process""" # fugly hack to allow functional tests to use test data if 'profiler_test_env_var' in os.environ: return os.environ['profiler_test_env_var'] # Get the PID of the current process pid = os.getpid() cgroup_path = Path('/proc') / str(pid) / 'cgroup' try: # Get the cgroup information for the current process result = cgroup_path.read_text() return PID_REGEX.search(result).group().lstrip('/') except Exception as err: raise CylcProfilerError( err, f'{cgroup_path} not found') from err def get_cgroup_paths(location: Path) -> Process: cgroup_name = get_cgroup_name().lstrip('/') cgroup_version = get_cgroup_version(location, cgroup_name) if cgroup_version == 2: return Process( cgroup_memory_path=location / cgroup_name / "memory.stat", cgroup_cpu_path=location / cgroup_name / "cpu.stat", memory_allocated_path=location / cgroup_name, cgroup_version=cgroup_version, max_rss=0, ) elif cgroup_version == 1: return Process( cgroup_memory_path=( location / "memory" / cgroup_name / "memory.stat" ), cgroup_cpu_path=( location / "cpu" / cgroup_name / "cpuacct.usage" ), memory_allocated_path=Path(), cgroup_version=cgroup_version, max_rss=0, ) raise CylcProfilerError(FileNotFoundError(), "Unable to determine cgroup version") async def profile(process: Process, delay, keep_looping=lambda: True): # The infinite loop that will constantly poll the cgroup # The lambda function is used to allow the loop to be stopped in unit tests while keep_looping(): # Polling the cgroup for memory and keeping track of the max rss value max_rss = parse_memory_file(process) if max_rss is not None and max_rss > process.max_rss: process.max_rss = max_rss await asyncio.sleep(delay) def get_option_parser() -> COP: parser = COP( __doc__, comms=True, argdoc=[ ], ) parser.add_option( "-i", type=str, help="interval between query cycles in seconds", dest="delay") parser.add_option( "-m", type=str, help="Location of cgroups directory", dest="cgroup_location") return parser @cli_function(get_option_parser) def main(_parser: COP, options) -> None: """CLI main.""" try: asyncio.run(_main(options)) except asyncio.CancelledError: pass except Exception as exc: # Only log at info level as not very important (show traceback if -vv) LOG.info(exc, exc_info=(cylc.flow.flags.verbosity > 1)) async def _main(options) -> None: # convert from ISO8601 duration to integer seconds delay = int(dp.parse(options.delay).get_seconds()) # get cgroup information process = get_cgroup_paths(Path(options.cgroup_location)) # the profiler will run until one of these coroutines calls `sys.exit`: tasks = [ # run the profiler itself asyncio.create_task( profile(process, delay), name="profiler", ), # kill the profiler if its PPID changes # (i.e, if the job exits before the profiler does) asyncio.create_task( watch_and_kill(psutil.Process(os.getpid())), name="profiler_watchdog", ), ] # The signal library doesn't work with asyncio, so we have to use the # loop's add_signal_handler function instead loop = asyncio.get_running_loop() for sig in (signal.SIGINT, signal.SIGHUP, signal.SIGTERM): loop.add_signal_handler(sig, lambda: {t.cancel() for t in tasks}) with suppress(asyncio.CancelledError): await asyncio.gather(*tasks) await report_to_scheduler(process, options.comms_timeout) cylc-flow-8.7.0/cylc/flow/scripts/function_run.py0000775000175000017500000000276015264420307022277 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """USAGE: cylc function-run (This command is for internal use.) Run a Python xtrigger function "(*args, **kwargs)" in the process pool. Positional and keyword arguments must be passed in as JSON strings. Python entry points are the preferred way to make xtriggers available to the scheduler, but local xtriggers can be stored in . """ import sys from cylc.flow.subprocpool import run_function INTERNAL = True def main(*api_args): if api_args: args = [None] + list(api_args) else: args = sys.argv if args[1] in ["help", "--help"] or len(args) != 6: print(__doc__) sys.exit(0) run_function(*args[1:]) cylc-flow-8.7.0/cylc/flow/scripts/view.py0000664000175000017500000001114015264420307020525 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc view [OPTIONS] ARGS Print a processed workflow configuration. Print workflow configurations as processed before full parsing by Cylc. This includes Jinja2 template processing, and inlining of include-files. Some explanatory markup may also be requested. Warning: This command will fail if `CYLC_` template variables are referenced without default values, because they are only defined for full parsing. E.g. (Jinja2): `{{CYLC_WORKFLOW_ID | default("not defined")}}`. See also `cylc config`, which displays the fully parsed configuration. """ import asyncio from pathlib import Path from typing import TYPE_CHECKING from cylc.flow.id_cli import parse_id_async from cylc.flow.option_parsers import ( AGAINST_SOURCE_OPTION, WORKFLOW_ID_OR_PATH_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.pathutil import get_workflow_run_dir from cylc.flow.parsec.fileparse import read_and_proc from cylc.flow.templatevars import get_template_vars from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values def get_option_parser(): parser = COP( __doc__, jset=True, argdoc=[WORKFLOW_ID_OR_PATH_ARG_DOC], ) parser.add_option( "--inline", "-i", help="Inline include-files.", action="store_true", default=False, dest="inline") parser.add_option( "--jinja2", "-j", help="View after Jinja2 template processing " "(implies '-i/--inline' as well).", action="store_true", default=False, dest="jinja2") parser.add_option( "-p", "--process", help="View after all processing (Jinja2, inlining, " "line-continuation joining).", action="store_true", default=False, dest="process") parser.add_option( "--mark", "-m", help="(With '-i') Mark inclusions in the left margin.", action="store_true", default=False, dest="mark") parser.add_option( "--label", "-l", help="(With '-i') Label file inclusions with the file name. Line " "numbers will not correspond to those reported by the parser.", action="store_true", default=False, dest="label") parser.add_option( "--single", help="(With '-i') Inline only the first instances of any " "multiply-included files. Line numbers will not correspond to " "those reported by the parser.", action="store_true", default=False, dest="single") parser.add_option( "--cat", "-c", help="Concatenate continuation lines (line numbers will " "not correspond to those reported by the parser).", action="store_true", default=False, dest="cat") parser.add_option( *AGAINST_SOURCE_OPTION.args, **AGAINST_SOURCE_OPTION.kwargs) parser.add_cylc_rose_options() return parser @cli_function(get_option_parser) def main(parser: COP, options: 'Values', workflow_id: str) -> None: asyncio.run(_main(options, workflow_id)) async def _main(options: 'Values', workflow_id: str) -> None: workflow_id, _, flow_file = await parse_id_async( workflow_id, src=True, constraint='workflows', ) # Save the location of the existing workflow run dir in the # against source option: if options.against_source: options.against_source = Path(get_workflow_run_dir(workflow_id)) # read in the flow.cylc file for line in read_and_proc( flow_file, get_template_vars(options), viewcfg={ 'mark': options.mark, 'single': options.single, 'label': options.label, 'jinja2': options.jinja2 or options.process, 'contin': options.cat or options.process, 'inline': options.jinja2 or options.inline or options.process, }, opts=options, ): print(line) cylc-flow-8.7.0/cylc/flow/scripts/play.py0000664000175000017500000000177715264420307020537 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # (hack to import the __doc__ from another file) # type: ignore from cylc.flow.scheduler_cli import ( # noqa: F401 play as main, PLAY_DOC as __doc__ ) # CLI of "cylc play". See cylc.flow.scheduler_cli for details. cylc-flow-8.7.0/cylc/flow/scripts/jobs_kill.py0000775000175000017500000000332215264420307021531 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc jobs-kill [OPTIONS] ARGS (This command is for internal use. Users should use "cylc kill".) Read job status files to obtain the names of the job runners and the job IDs in the runners. Invoke the relevant job runner commands to ask the job runners to terminate the jobs. """ from cylc.flow.job_runner_mgr import JobRunnerManager from cylc.flow.option_parsers import CylcOptionParser as COP from cylc.flow.terminal import cli_function INTERNAL = True def get_option_parser() -> COP: parser = COP( __doc__, argdoc=[ ("JOB-LOG-ROOT", "The log/job sub-directory for the workflow"), COP.optional( ("JOB-LOG-DIR ...", "A point/name/submit_num sub-directory") ) ], ) return parser @cli_function(get_option_parser) def main(parser, options, job_log_root, *job_log_dirs): """CLI main.""" JobRunnerManager().jobs_kill(job_log_root, job_log_dirs) cylc-flow-8.7.0/cylc/flow/scripts/common.py0000664000175000017500000000374315264420307021055 0ustar alastairalastair# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Shared utilities for Cylc scripts.""" from itertools import zip_longest from ansimarkup import strip from cylc.flow import __version__ from cylc.flow.scripts import _copyright_year from cylc.flow.terminal import get_width # fmt: off CYLC_LOGO = [ " ▪ ■ ", " ██ ", "▝▘ ", ] # fmt: on CYLC = ( "" "C" "y" "l" "c" "" ) VERSION = f"{__version__}" LICENSE = [ f"{CYLC} Workflow Engine {VERSION}", f"Copyright (C) 2008-{_copyright_year} Earth Sciences New Zealand", "& British Crown (Met Office) & Contributors" ] def cylc_header(width=None): """Print copyright and license information.""" width = width or get_width() lmax = ( max(len(strip(line)) for line in LICENSE) + len(strip(CYLC_LOGO[0])) ) if width >= lmax + 1: header = '\n'.join( ('{0} {1}').format(*x) for x in zip_longest( CYLC_LOGO, LICENSE ) ) else: header = '\n'.join(CYLC_LOGO) + '\n' + '\n'.join(LICENSE) return f"\n{header}\n" cylc-flow-8.7.0/cylc/flow/scripts/reinstall.py0000664000175000017500000003012615264420307021555 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc reinstall [OPTIONS] ARGS Reinstall a previously installed workflow. Examples: # Having previously installed: $ cylc install myflow # To reinstall the latest run: $ cylc reinstall myflow # Or, to reinstall a specific run: $ cylc reinstall myflow/run2 # If the workflow is running: $ cylc reinstall myflow # reinstall as usual $ cylc reload myflow # pick up changes in the workflow config What reinstall does: Reinstall synchronises files between the workflow source and the specified run directory. Any files which have been added, updated or removed in the source directory will be added, updated or removed in the run directory. Cylc uses "rsync" to do this (run in "--debug" mode to see the exact command used). How changes are displayed: Reinstall will first perform a dry run showing the files it would change. This is displayed in "rsync" format e.g: send foo # this means the file "foo" would be added/updated del. bar # this means the file "bar" would be deleted How to prevent reinstall deleting files: Reinstall will delete any files which are not present in the source directory (i.e. if you delete a file from the source directory, a reinstall would remove the file from the run directory too). The "work/" and "share/" directory are excluded from this. These are the recommended locations for any files created at runtime. You can extend the list of "excluded" paths by creating a ".cylcignore" file. For example the following file would exclude "data/" and any ".csv" files from being overwritten by a reinstallation: $ cat .cylcignore data *.csv Note any paths listed in ".cylcignore" will not be installed by "cylc install" even if present in the source directory. """ from functools import partial from pathlib import Path import re import sys from typing import ( TYPE_CHECKING, Callable, List, Optional, ) from ansimarkup import parse as cparse from cylc.flow.exceptions import ( ServiceFileError, WorkflowFilesError, ) import cylc.flow.flags from cylc.flow.install import reinstall_workflow from cylc.flow.network.multi import call_multi from cylc.flow.option_parsers import ( ID_MULTI_ARG_DOC, CylcOptionParser as COP, OptionSettings, ) from cylc.flow.pathutil import get_workflow_run_dir from cylc.flow.plugins import run_plugins_async from cylc.flow.terminal import ( DIM, cli_function, handle_sigint, interrupt, is_terminal, ) from cylc.flow.workflow_files import ( get_workflow_source_dir, load_contact_file, ) if TYPE_CHECKING: from optparse import Values _input = input # to enable testing REINSTALL_CYLC_ROSE_OPTIONS = [ OptionSettings( ['--clear-rose-install-options'], help="Clear options previously set by cylc-rose.", action='store_true', default=False, dest="clear_rose_install_opts", sources={'reinstall'} ) ] REINSTALL_OPTIONS = [ OptionSettings( ["--yes", "-y"], help='Skip interactive prompts.', action="store_true", default=False, dest="skip_interactive", sources={'reinstall'} ), ] def get_option_parser() -> COP: parser = COP( __doc__, comms=True, multiworkflow=True, argdoc=[ ID_MULTI_ARG_DOC, ], ) try: # If cylc-rose plugin is available __import__('cylc.rose') except ImportError: options = REINSTALL_OPTIONS else: parser.add_cylc_rose_options() options = REINSTALL_CYLC_ROSE_OPTIONS + REINSTALL_OPTIONS for option in options: parser.add_option(*option.args, **option.kwargs) return parser @cli_function(get_option_parser) def main( _parser: COP, opts: 'Values', *ids: str ) -> None: """CLI wrapper.""" rets = call_multi( partial(reinstall_cli, opts), *ids, constraint='workflows', report=lambda x: ('Done', None, True), ) sys.exit(all(rets.values()) is False) async def reinstall_cli( opts: 'Values', workflow_id, *tokens_list, print_reload_tip: bool = True ) -> bool: """Implement cylc reinstall. This is the bit which contains all the CLI logic. """ run_dir: Optional[Path] run_dir = Path(get_workflow_run_dir(workflow_id)) if not run_dir.is_dir(): raise WorkflowFilesError( f'"{workflow_id}" is not an installed workflow.') source, source_symlink = get_workflow_source_dir(run_dir) if not source: raise WorkflowFilesError( f'"{workflow_id}" was not installed with cylc install.') source_: Path = Path(source) if not source_.is_dir(): raise WorkflowFilesError( f'Workflow source dir is not accessible: "{source_}".\n' f'Restore the source or modify the "{source_symlink}"' ' symlink to continue.' ) usr: str = '' try: if is_terminal() and not opts.skip_interactive: # interactive mode - perform dry-run and prompt # dry-mode reinstall if not await reinstall( opts, workflow_id, source_, run_dir, dry_run=True, ): # no rsync output == no changes => exit print(cparse( '' 'No changes made:' f' {workflow_id} is up to date with {source_}' '' )) return False display_rose_warning(source_) display_cylcignore_tip() # prompt for permission to continue with handle_sigint(interrupt): while usr not in ['y', 'n']: usr = _input( cparse('Continue [y/n]: ') ).lower() else: # non interactive-mode - no dry-run, no prompt usr = 'y' except KeyboardInterrupt: # ensure the "reinstall canceled" message shows for ctrl+c usr = 'n' # cancel the reinstall if usr == 'y': # reinstall for real await reinstall(opts, workflow_id, source_, run_dir, dry_run=False) print(cparse('Successfully reinstalled.')) if print_reload_tip: display_cylc_reload_tip(workflow_id) return True else: # no reinstall print( cparse('No changes made: reinstall cancelled.') ) return False async def reinstall( opts: 'Values', workflow_id: str, src_dir: Path, run_dir: Path, dry_run: bool = False, write: Callable = print, ) -> bool: """Perform reinstallation. This is the purely functional bit without the CLI logic. Args: opts: CLI options. workflow_id: Workflow ID as a string. src_dir: Workflow source directory path. run_dir: Installed workflow run directory path. dry_run: If True perform a "dry run" which doesn't change anything. write: Used to display dry_run output. Returns: reinstall_needed - In dry_run mode returns False if rsync *would* update anything, else returns True. """ if not dry_run: # run pre_configure plugins async for _entry_point, _plugin_result in run_plugins_async( 'cylc.pre_configure', srcdir=src_dir, opts=opts, ): pass # reinstall from src_dir (will raise WorkflowFilesError on error) stdout: str = reinstall_workflow( source=src_dir, named_run=workflow_id, rundir=run_dir, dry_run=dry_run, ) if dry_run: # display changes changes = format_reinstall_output(stdout) if not changes: return False # display rsync output write('\n'.join(changes), file=sys.stdout) if not dry_run: # run post_install plugins async for _entry_point, _plugin_result in run_plugins_async( 'cylc.post_install', srcdir=src_dir, opts=opts, rundir=str(run_dir), async_block=True, ): pass return True def format_reinstall_output(out: str) -> List[str]: r"""Format rsync stdout for presenting to users. Note: Output formats of different rsync implementations may differ so keep this code simple and robust. Example: >>> format_reinstall_output( ... '>f+++++++++ send foo\n*deleting del. bar\nbaz' ... '\ncannot delete non-empty directory: opt' ... ) == [ ... cparse('send foo'), ... cparse('del. bar'), ... 'baz', ... ] True """ lines = [] for line in out.splitlines(): if not line or line.startswith('cannot delete non-empty directory:'): # This can arise as a result of deleting a subdir when we are # excluding files within the subdir. # This is likely to occur for the opt dir when a rose-suite.conf # file is present. # Skip this line as nothing will happen to this dir. continue match = re.match(r'^(.{11}) (send|del\.|recv) (.*)$', line) # (On some systems, may get "recv" instead of "send") if match: summary, operation, file = match.groups() color = 'green' if operation == 'send' else 'red' formatted_line = f"<{color}>{operation} {file}" if cylc.flow.flags.verbosity > 0: formatted_line = f"<{DIM}>{summary} {formatted_line}" lines.append(cparse(formatted_line)) else: # tolerate unknown output from different rsync implementations lines.append(line) return lines def display_rose_warning(src_dir: Path) -> None: """Explain why rose installed files are marked as deleted.""" if (src_dir / 'rose-suite.conf').is_file(): # TODO: remove this in combination with # https://github.com/cylc/cylc-rose/issues/149 print( cparse( f'\n<{DIM}>' 'NOTE: Files created by Rose file installation will show' ' as deleted.' '\n They will be re-created during the reinstall' ' process.' f'', ), file=sys.stderr, ) def display_cylcignore_tip(): print( cparse( f'\n<{DIM}>TIP: You can "exclude" files/dirs to prevent' ' Cylc from installing or overwriting\n them by adding' ' them to the .cylcignore file. See cylc reinstall --help.' f'\n' '\nReinstall would make the above changes.' ) ) def display_cylc_reload_tip(workflow_id: str) -> None: """Recommend use of "cylc reload" if applicable. Uses a quick and simple way to tell if a workflow is running. It would be better to "ping" the workflow, however, this is sufficient for our purposes. """ try: load_contact_file(workflow_id) except ServiceFileError: return print(cparse( '\n' f'Run "cylc reload {workflow_id}" to pick up changes.' '' )) cylc-flow-8.7.0/cylc/flow/scripts/remote_init.py0000775000175000017500000000434515264420307022105 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc remote-init [OPTIONS] ARGS (This command is for internal use.) Initialise an install target. Initialisation creates a workflow run directory on the install target, "$HOME/cylc-run//". The .service directory is also created and populated with the install target authentication files and the contact file. Symlinks are created for run, work, share, share/cycle, log directories, configured in the global.flow. Return: 0: On success or if initialisation not required (REMOTE_INIT_DONE) 1: Error occurred when symlinking. 2: Unexpected authentication keys exist. 3: Any other unexpected error occurred. """ import sys from cylc.flow.option_parsers import CylcOptionParser as COP from cylc.flow.task_remote_cmd import remote_init from cylc.flow.terminal import cli_function INTERNAL = True def get_option_parser() -> COP: parser = COP( __doc__, argdoc=[ ("INSTALL_TARGET", "Target to be initialised"), ("RUND", "The run directory of the workflow"), COP.optional( ('DIRS_TO_BE_SYMLINKED ...', "Directories to be symlinked") ) ], color=False ) return parser @cli_function(get_option_parser) def main(parser, options, install_target, rund, *dirs_to_be_symlinked): sys.exit( remote_init( install_target, rund, *dirs_to_be_symlinked, ) ) cylc-flow-8.7.0/cylc/flow/scripts/clean.py0000664000175000017500000002030315264420307020636 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc clean [OPTIONS] ARGS Delete a stopped workflow. Remove workflow files from the local scheduler filesystem and any remote hosts the workflow was installed on. NOTE: this command is intended for workflows installed with `cylc install`. If this is run for a workflow that was instead written directly in ~/cylc-run and not backed up elsewhere, it will be lost. It will also remove any symlink directory targets. Workflow IDs can be hierarchical, corresponding to the path under ~/cylc-run. Examples: # Remove the workflow at ~/cylc-run/foo/bar $ cylc clean foo/bar # Remove multiple workflows $ cylc clean one two three # Remove the workflow's log directory $ cylc clean foo/bar --rm log # Remove the log and work directories $ cylc clean foo/bar --rm log:work # or $ cylc clean foo/bar --rm log --rm work # Remove all job log files from the 2020 cycle points $ cylc clean foo/bar --rm 'log/job/2020*' # Remove all .csv files $ cylc clean foo/bar --rm '**/*.csv' # Only remove the workflow on the local filesystem $ cylc clean foo/bar --local-only # Only remove the workflow on remote install targets $ cylc clean foo/bar --remote-only """ import asyncio from optparse import SUPPRESS_HELP import sys from textwrap import indent from typing import ( TYPE_CHECKING, Iterable, List, Tuple, ) from metomi.isodatetime.exceptions import ISO8601SyntaxError from metomi.isodatetime.parsers import DurationParser from cylc.flow import LOG from cylc.flow.clean import ( get_contained_workflows, init_clean, ) from cylc.flow.exceptions import ( CylcError, InputError, ) import cylc.flow.flags from cylc.flow.id_cli import parse_ids_async from cylc.flow.loggingutil import set_timestamps from cylc.flow.option_parsers import ( WORKFLOW_ID_MULTI_ARG_DOC, CylcOptionParser as COP, Options, ) from cylc.flow.terminal import ( handle_sigint, cli_function, is_terminal, ) from cylc.flow.util import natural_sort_key if TYPE_CHECKING: from optparse import Values def get_option_parser(): parser = COP( __doc__, multiworkflow=True, argdoc=[WORKFLOW_ID_MULTI_ARG_DOC], segregated_log=True, ) parser.add_option( '--rm', metavar='DIR[:DIR:...]', help=("Only clean the specified subdirectories (or files) in the " "run directory, rather than the whole run directory. " "Accepts quoted globs."), action='append', dest='rm_dirs', default=[] ) parser.add_option( '--local-only', '--local', help="Only clean on the local filesystem (not remote hosts).", action='store_true', dest='local_only' ) parser.add_option( '--remote-only', '--remote', help="Only clean on remote hosts (not the local filesystem).", action='store_true', dest='remote_only' ) parser.add_option( '--yes', '-y', help=( "Skip interactive prompt if trying to clean multiple " "run directories at once." ), action='store_true', dest='skip_interactive' ) parser.add_option( '--timeout', help=( "The length of time to wait for cleaning to take place on " r"remote hosts before cancelling. Default: %default." ), action='store', default='PT5M', dest='remote_timeout' ) parser.add_option( '--no-scan', help=SUPPRESS_HELP, action='store_true', dest='no_scan' # Used on remote re-invocation - do not scan for workflows, just # clean exactly what you were told to clean ) return parser CleanOptions = Options(get_option_parser()) def parse_timeout(opts: 'Values') -> None: """Parse timeout as ISO 8601 duration or number of seconds.""" if opts.remote_timeout: try: timeout = int( DurationParser().parse(opts.remote_timeout).get_seconds() ) except ISO8601SyntaxError: try: timeout = int(opts.remote_timeout) except ValueError: raise InputError( f"Invalid timeout: {opts.remote_timeout}. Must be " "an ISO 8601 duration or number of seconds." ) from None opts.remote_timeout = str(timeout) def prompt(workflows: Iterable[str]) -> None: """Ask user if they want to clean the given set of workflows.""" print("Would clean the following workflows:") for workflow in workflows: print(f' {workflow}') if is_terminal(): with handle_sigint(): while True: ret = input('Remove these workflows (y/n): ').lower() if ret == 'y': return if ret == 'n': sys.exit(1) else: print( "Use --yes to remove multiple workflows in non-interactive mode.", file=sys.stderr ) sys.exit(1) async def scan( workflows: Iterable[str], multi_mode: bool ) -> Tuple[List[str], bool]: """Expand truncated workflow IDs For example "one" might expand to "one/run1" & "one/run2" or "one/two/run1". Returns (workflows, multi_mode) """ ret = [] for workflow in list(workflows): contained_flows = await get_contained_workflows(workflow) if contained_flows: ret.extend(contained_flows) multi_mode = True else: ret.append(workflow) ret.sort(key=natural_sort_key) return ret, multi_mode async def run(*ids: str, opts: 'Values') -> None: if opts.no_scan: workflows: Iterable[str] = ids else: # parse ids from the CLI workflows, multi_mode = await parse_ids_async( *ids, constraint='workflows', match_workflows=True, match_active=False, infer_latest_runs=False, # don't infer latest runs like other cmds ) # expand partial workflow ids (including run names) workflows, multi_mode = await scan(workflows, multi_mode) if not workflows: LOG.warning(f"No stopped workflows matching {', '.join(ids)}") return if multi_mode and not opts.skip_interactive: prompt(workflows) # prompt for approval or exit tasks_map = { workflow: asyncio.create_task(init_clean(workflow, opts)) for workflow in workflows } await asyncio.gather(*tasks_map.values(), return_exceptions=True) if failed := { workflow: exc for workflow, task in tasks_map.items() if (exc := task.exception()) }: raise CylcError( '\n'.join( ( f"{workflow}: clean incomplete\n" f"{indent(str(exc), ' ' * 4).rstrip()}" ) for workflow, exc in failed.items() ) ) @cli_function(get_option_parser) def main(_parser, opts: 'Values', *ids: str): _main(opts, *ids) def _main(opts: 'Values', *ids: str): """Run the clean command. This is a separate function for ease of testing. """ if cylc.flow.flags.verbosity < 2: set_timestamps(LOG, False) if opts.local_only and opts.remote_only: raise InputError( "--local and --remote options are mutually exclusive" ) parse_timeout(opts) try: asyncio.run(run(*ids, opts=opts)) except KeyboardInterrupt: sys.exit(1) cylc-flow-8.7.0/cylc/flow/scripts/graph.py0000664000175000017500000004041415264420307020662 0ustar alastairalastair#!/usr/bin/env python3 # # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc graph [OPTIONS] ARGS Produces graphical and textual representations of workflow dependencies. Examples: # generate a graphical representation of workflow dependencies # (requires graphviz to be installed in the Cylc environment) $ cylc graph one # print a textual representation of the graph of the flow one $ cylc graph one --reference # display the difference between the flows one and two $ cylc graph one --diff two # render the graph to a svg file $ cylc graph one -o 'one.svg' """ import asyncio from difflib import unified_diff from pathlib import Path from shutil import which from subprocess import Popen, PIPE import sys from tempfile import NamedTemporaryFile from typing import Dict, List, Optional, TYPE_CHECKING, Tuple, Callable from ansimarkup import ansiprint as cprint from cylc.flow.config import WorkflowConfig from cylc.flow.exceptions import InputError, CylcError from cylc.flow.id import Tokens from cylc.flow.id_cli import parse_id_async from cylc.flow.option_parsers import ( AGAINST_SOURCE_OPTION, WORKFLOW_ID_OR_PATH_ARG_DOC, CylcOptionParser as COP, icp_option, ) from cylc.flow.pathutil import get_workflow_run_dir from cylc.flow.templatevars import get_template_vars from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values def sort_integer_node(id_): """Return sort tokens for nodes with cyclepoints in integer format. Example: >>> sort_integer_node('11/foo') ('foo', 11) >>> sort_integer_node('R1.1/prep') ('prep', 1) """ tokens = Tokens(id_.removeprefix("R1."), relative=True) return (tokens['task'], int(tokens['cycle'])) def sort_integer_edge(id_): """Return sort tokens for edges with cyclepoints in integer format. Example: >>> sort_integer_edge(('11/foo', '12/foo', None)) (('foo', 11), ('foo', 12)) >>> sort_integer_edge(('11/foo', None , None)) (('foo', 11), ('', 0)) """ return ( sort_integer_node(id_[0]), sort_integer_node(id_[1]) if id_[1] else ('', 0) ) def sort_datetime_edge(item): """Return sort tokens for edges with cyclepoints in ISO8601 format. Example: >>> sort_datetime_edge(('a', None, None)) ('a', '') """ return (item[0], item[1] or '') # node/edge types Node = str # node ID Edge = Tuple[str, str] # left, right def get_nodes_and_edges( opts, workflow_id, start, stop, flow_file, ) -> Tuple[List[Node], List[Edge]]: """Return graph sorted nodes and edges.""" config = get_config(workflow_id, opts, flow_file) if opts.namespaces: nodes, edges = _get_inheritance_nodes_and_edges(config) else: nodes, edges = _get_graph_nodes_edges( config, start, stop, grouping=opts.grouping, show_suicide=opts.show_suicide, flatten_icp_dependence=opts.flatten_icp_dependence, ) return nodes, edges def _get_graph_nodes_edges( config, start_point_str=None, stop_point_str=None, grouping=None, show_suicide=False, flatten_icp_dependence=False, ) -> Tuple[List[Node], List[Edge]]: """Return nodes and edges for a workflow graph.""" graph = config.get_graph_raw( start_point_str, stop_point_str, grouping, flatten_icp_dependence=flatten_icp_dependence, ) if not graph: return [], [] # set sort keys based on cycling mode # (note sorting is very important for the reference format) node_sort: Optional[Callable] edge_sort: Optional[Callable] if config.cfg['scheduling']['cycling mode'] == 'integer': # integer sorting node_sort = sort_integer_node edge_sort = sort_integer_edge else: # datetime sorting node_sort = None # lexicographically sortable edge_sort = sort_datetime_edge # get nodes nodes = sorted( { node for left, right, _, suicide, _ in graph for node in (left, right) if node if show_suicide or not suicide }, key=node_sort, ) # get edges edges = sorted( { (left, right) for left, right, _, suicide, _ in graph if right if show_suicide or not suicide }, key=edge_sort, ) return nodes, edges def _get_inheritance_nodes_and_edges( config ) -> Tuple[List[Node], List[Edge]]: """Return nodes and edges for an inheritance graph.""" nodes = set() edges = set() for namespace, tasks in config.get_parent_lists().items(): nodes.add(namespace) for task in tasks: edges.add((task, namespace)) nodes.add(task) return sorted(nodes), sorted(edges) def get_config(workflow_id: str, opts: 'Values', flow_file) -> WorkflowConfig: """Return a WorkflowConfig object for the provided id_ / path.""" template_vars = get_template_vars(opts) return WorkflowConfig( workflow_id, flow_file, opts, template_vars=template_vars ) def format_graphviz( opts, nodes: List[Node], edges: List[Edge], ) -> List[str]: """Write graph in graphviz format.""" # write graph header dot_lines = [ 'digraph {', ' graph [fontname="sans" fontsize="25"]', ' node [fontname="sans"]', ] if opts.transpose: dot_lines.append(' rankdir="LR"') if opts.namespaces: dot_lines.append(' node [shape="rect"]') dot_lines.append('') # write nodes if opts.namespaces: dot_lines.extend([ rf' "{node}"' for node in nodes ]) dot_lines.append('') else: # group by cycle cycles: Dict[str, List[str]] = {} for node in nodes: tokens = Tokens(node, relative=True) cycle: str = tokens['cycle'] task: str = tokens['task'] cycles.setdefault(cycle, []).append(task) # write nodes by cycle if opts.cycles: indent = ' ' else: indent = ' ' for cycle, tasks in cycles.items(): r1 = cycle.startswith('R1') if opts.cycles and not r1: dot_lines.extend( [ f' subgraph "cluster_{cycle}" {{ ', f' label="{cycle}"', ' style="dashed"', ] ) if r1: cycle_label = 'R1' else: cycle_label = cycle dot_lines.extend( rf'{indent}"{cycle}/{task}" [label="{task}\n{cycle_label}"]' for task in tasks ) if opts.cycles and not r1: dot_lines.append(' }') dot_lines.append('') # write edges for left, right in edges: dot_lines.append(f' "{left}" -> "{right}"') # close graph dot_lines.append('}') return dot_lines def format_cylc_reference( opts, nodes: List[Node], edges: List[Edge], ) -> List[str]: """Write graph in cylc reference format.""" lines = [] # write edges for left, right in edges: lines.append(f'edge "{left}" "{right}"') # write separator lines.append('graph') # write nodes if opts.namespaces: for node in nodes: lines.append(f'node "{node}" "{node}"') else: for node in nodes: tokens = Tokens(node, relative=True) lines.append( f'node "{node}" "{tokens["task"]}\\n{tokens["cycle"]}"' ) # write terminator lines.append('stop') return lines def render_dot(dot_lines, filename, fmt): """Render graph using `dot`.""" # check graphviz-dot is installed if not which('dot'): sys.exit('Graphviz must be installed to render graphs.') # render graph with graphviz proc = Popen( # nosec ['dot', f'-T{fmt}', '-o', filename], stdin=PIPE, text=True ) proc.communicate('\n'.join(dot_lines)) proc.wait() if proc.returncode: raise CylcError('Graphing Failed') def open_image(filename): """Open an image file.""" try: from PIL import Image except ModuleNotFoundError: # dependencies required to display images not present pass else: img = Image.open(filename) img.show() def graph_render(opts, workflow_id, start, stop, flow_file) -> int: """Render the workflow graph to the specified format. Graph is rendered to the specified format. The Graphviz "dot" format does not require Graphviz to be installed. All other formats require Graphviz. Supported formats depend on your Graphviz installation. """ # get nodes and edges nodes, edges = get_nodes_and_edges( opts, workflow_id, start, stop, flow_file ) if not nodes: msg = 'No tasks to display.' if start or stop: msg += ' Try changing the start and stop values.' cprint(f'{msg}', file=sys.stderr) return 1 # format the graph in graphviz-dot format dot_lines = format_graphviz(opts, nodes, edges) # set filename and output format if opts.output: filename = opts.output try: fmt = filename.rsplit('.', 1)[1] except IndexError: cprint( 'Output filename requires a format.', file=sys.stderr, ) return 1 else: filename = NamedTemporaryFile().name fmt = 'png' if fmt == 'dot': # output in dot format (graphviz not needed for this) with open(filename, 'w+') as dot_file: dot_file.write('\n'.join(dot_lines) + '\n') return 0 # render with graphviz render_dot(dot_lines, filename, fmt) # notify the user / open the graph cprint(f'Graph rendered to {opts.output or filename}') if not opts.output: open_image(filename) return 0 def graph_reference( opts, workflow_id, start, stop, flow_file, write=print, ) -> int: """Format the workflow graph using the cylc reference format.""" # get nodes and edges nodes, edges = get_nodes_and_edges( opts, workflow_id, start, stop, flow_file ) for line in format_cylc_reference(opts, nodes, edges): write(line) return 0 async def graph_diff( opts, workflow_a, workflow_b, start, stop, flow_file ) -> int: """Difference the workflow graphs using the cylc reference format.""" workflow_b, _, flow_file_b = await parse_id_async( workflow_b, src=True, constraint='workflows', ) # load graphs graph_a: List[str] = [] graph_b: List[str] = [] graph_reference( opts, workflow_a, start, stop, flow_file, write=graph_a.append, ) graph_reference( opts, workflow_b, start, stop, flow_file_b, write=graph_b.append, ) # compare graphs diff_lines = list( unified_diff( [f'{line}\n' for line in graph_a], [f'{line}\n' for line in graph_b], fromfile=workflow_a, tofile=workflow_b, ) ) # return results if diff_lines: sys.stdout.writelines(diff_lines) return 1 return 0 def get_option_parser() -> COP: """CLI.""" parser = COP( __doc__, jset=True, argdoc=[ WORKFLOW_ID_OR_PATH_ARG_DOC, COP.optional( ('START', 'Graph start; defaults to initial cycle point') ), COP.optional(( 'STOP', 'Graph stop point or interval; defaults to 3 points from START' )) ] ) parser.add_option( '-g', '--group', help="task family to group. Can be used multiple times. " "Use '' to specify all families above root.", action='append', default=[], dest='grouping') parser.add_option( '-c', '--cycles', help='Group tasks by cycle point.', action='store_true', default=False, ) parser.add_option( '-t', '--transpose', help='Transpose graph.', action='store_true', default=False, ) parser.add_option( '-o', help=( 'Output the graph to a file. The file extension determines the' ' format. E.G. "graph.png", "graph.svg", "graph.dot".' ), action='store', dest='output' ) parser.add_option( '-n', '--namespaces', help='Plot the workflow namespace inheritance hierarchy ' '(task run time properties).', action='store_true', default=False, dest='namespaces') parser.add_option( '-r', '--reference', help='Output in a sorted plain text format for comparison purposes. ' 'If not given, assume --output-file=-.', action='store_true', default=False, dest='reference') parser.add_option( '--show-suicide', help='Show suicide triggers. Not shown by default.', action='store_true', default=False, dest='show_suicide') parser.add_option( '--flatten-icp', help=( 'Flatten dependence on tasks at the initial cycle point.' ' Workflows often contain repeated dependencies between the' ' initial cycle point and later cycles, which can make graph' ' layout messy. Instead of drawing arrows from R1 leaf tasks' ' (i.e. those R1 tasks that connect to the rest of the graph) to' ' every later cycle, this option duplicates them into the later' ' cycles, for a cleaner graph layout.' ), action='store_true', default=False, dest='flatten_icp_dependence', ) parser.add_option(icp_option) parser.add_option( '--diff', help='Show the difference between two workflows (implies --reference)', action='store', ) parser.add_option( *AGAINST_SOURCE_OPTION.args, **AGAINST_SOURCE_OPTION.kwargs) parser.add_cylc_rose_options() return parser @cli_function(get_option_parser) def main( parser: COP, opts: 'Values', workflow_id: str, start: Optional[str] = None, stop: Optional[str] = None ) -> None: result = asyncio.run(_main(opts, workflow_id, start, stop)) sys.exit(result) async def _main( opts: 'Values', workflow_id: str, start: Optional[str] = None, stop: Optional[str] = None ) -> int: """Implement ``cylc graph``.""" if opts.grouping and opts.namespaces: raise InputError('Cannot combine --group and --namespaces.') if opts.cycles and opts.namespaces: raise InputError('Cannot combine --cycles and --namespaces.') workflow_id, _, flow_file = await parse_id_async( workflow_id, src=True, constraint='workflows', ) # Save the location of the existing workflow run dir in the # against source option: if opts.against_source: opts.against_source = Path(get_workflow_run_dir(workflow_id)) if opts.diff: return await graph_diff( opts, workflow_id, opts.diff, start, stop, flow_file) if opts.reference: return graph_reference( opts, workflow_id, start, stop, flow_file) return graph_render(opts, workflow_id, start, stop, flow_file) cylc-flow-8.7.0/cylc/flow/scripts/validate.py0000775000175000017500000001470515264420307021361 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc validate [OPTIONS] ARGS Validate a workflow configuration. If the workflow definition uses include-files, reported line numbers will correspond to the inlined version seen by the parser; use 'cylc view -i,--inline WORKFLOW' for comparison. """ import asyncio from ansimarkup import parse as cparse from copy import deepcopy from pathlib import Path import sys from typing import TYPE_CHECKING from cylc.flow import LOG, __version__ as CYLC_VERSION from cylc.flow.config import WorkflowConfig from cylc.flow.exceptions import ( WorkflowConfigError, TaskProxySequenceBoundsError, TriggerExpressionError ) import cylc.flow.flags from cylc.flow.id import Tokens from cylc.flow.id_cli import parse_id_async from cylc.flow.loggingutil import set_timestamps from cylc.flow.option_parsers import ( AGAINST_SOURCE_OPTION, WORKFLOW_ID_OR_PATH_ARG_DOC, CylcOptionParser as COP, OptionSettings, Options, ICP_OPTION, ) from cylc.flow.profiler import Profiler from cylc.flow.scheduler_cli import RUN_MODE from cylc.flow.task_proxy import TaskProxy from cylc.flow.templatevars import get_template_vars from cylc.flow.terminal import cli_function from cylc.flow.run_modes import RunMode from cylc.flow.workflow_files import check_flow_file, get_workflow_run_dir if TYPE_CHECKING: from cylc.flow.option_parsers import Values VALIDATE_RUN_MODE = deepcopy(RUN_MODE) VALIDATE_RUN_MODE.sources = {'validate'} VALIDATE_ICP_OPTION = deepcopy(ICP_OPTION) VALIDATE_ICP_OPTION.sources = {'validate'} VALIDATE_AGAINST_SOURCE_OPTION = deepcopy(AGAINST_SOURCE_OPTION) VALIDATE_AGAINST_SOURCE_OPTION.sources = {'validate'} VALIDATE_OPTIONS = [ OptionSettings( ["--check-circular"], help=( "Check for circular dependencies in graphs when the number of" " tasks is greater than 100 (smaller graphs are always" " checked). This can be slow when the number of" " tasks is high."), action="store_true", default=False, dest="check_circular", sources={'validate'} ), OptionSettings( ["--output", "-o"], help="Specify a file name to dump the processed flow.cylc.", metavar="FILENAME", action="store", dest="output", sources={'validate'} ), OptionSettings( ["--profile"], help="Output profiling (performance) information", action="store_true", default=False, dest="profile_mode", sources={'validate'} ), VALIDATE_RUN_MODE, VALIDATE_ICP_OPTION, ] def get_option_parser(): parser = COP( __doc__, jset=True, argdoc=[WORKFLOW_ID_OR_PATH_ARG_DOC], ) for option in [ *parser.get_cylc_rose_options(), *VALIDATE_OPTIONS, VALIDATE_AGAINST_SOURCE_OPTION, ]: parser.add_option(*option.args, **option.kwargs) parser.set_defaults(is_validate=True) return parser ValidateOptions = Options( get_option_parser(), # defaults { 'check_circular': False, 'profile_mode': False, 'run_mode': RunMode.LIVE.value } ) @cli_function(get_option_parser) def main(parser: COP, options: 'Values', workflow_id: str) -> None: asyncio.run(run(options, workflow_id)) async def run( options: 'Values', workflow_id: str ) -> None: """cylc validate CLI.""" profiler = Profiler(None, options.profile_mode) profiler.start() if cylc.flow.flags.verbosity < 2: set_timestamps(LOG, False) workflow_id, _, flow_file = await parse_id_async( workflow_id, src=True, constraint='workflows', ) # BACK COMPAT: check_flow_file (to disallow suite.rc) # REMOVE AT: 8.11.0 check_flow_file(flow_file.parent, allow_suite_rc=False) # Save the location of the existing workflow run dir in the # against source option: if getattr(options, 'against_source', False): options.against_source = Path(get_workflow_run_dir(workflow_id)) cfg = WorkflowConfig( workflow_id, flow_file, options, get_template_vars(options), output_fname=options.output, mem_log_func=profiler.log_memory ) # Instantiate tasks and force evaluation of trigger expressions. # (Taken from config.py to avoid circular import problems.) # TODO - This is not exhaustive, it only uses the initial cycle point. if cylc.flow.flags.verbosity > 0: print('Instantiating tasks to check trigger expressions') for name, taskdef in cfg.taskdefs.items(): try: itask = TaskProxy( Tokens(workflow_id), taskdef, cfg.start_point, ) except TaskProxySequenceBoundsError: # Should already failed above mesg = 'Task out of bounds for %s: %s\n' % (cfg.start_point, name) if cylc.flow.flags.verbosity > 0: sys.stderr.write(' + %s\n' % mesg) continue except Exception as exc: raise WorkflowConfigError( 'failed to instantiate task %s: %s' % (name, exc) ) from None # force trigger evaluation now try: itask.state.prerequisites_eval_all() except TriggerExpressionError: raise WorkflowConfigError("ERROR: bad trigger") from None except Exception as exc: print(str(exc), file=sys.stderr) raise WorkflowConfigError( '%s: failed to evaluate triggers.' % name ) from None if cylc.flow.flags.verbosity > 0: print(' + %s ok' % itask.identity) print(cparse('Valid for cylc-%s' % CYLC_VERSION)) profiler.stop() cylc-flow-8.7.0/cylc/flow/scripts/list.py0000775000175000017500000002003015264420307020527 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc list [OPTIONS] ARGS List tasks and families defined in a workflow. Print runtime namespace names (tasks and families), the first-parent inheritance graph, or actual tasks for a given cycle range. The first-parent inheritance graph determines the primary task family groupings that are collapsible in cylc visualisation tools. To visualize the full multiple inheritance hierarchy use: $ cylc graph -n """ import asyncio from copy import copy import os from pathlib import Path import sys from typing import TYPE_CHECKING from cylc.flow.config import WorkflowConfig from cylc.flow.exceptions import InputError from cylc.flow.id_cli import parse_id_async from cylc.flow.option_parsers import ( AGAINST_SOURCE_OPTION, WORKFLOW_ID_OR_PATH_ARG_DOC, CylcOptionParser as COP, icp_option, ) from cylc.flow.print_tree import print_tree from cylc.flow.workflow_files import get_workflow_run_dir from cylc.flow.templatevars import get_template_vars from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values def get_option_parser(): parser = COP( __doc__, jset=True, argdoc=[WORKFLOW_ID_OR_PATH_ARG_DOC], ) parser.add_option( "-a", "--all-tasks", help="Print all tasks, not just those used in the graph.", action="store_true", default=False, dest="all_tasks") parser.add_option( "-n", "--all-namespaces", help="Print all runtime namespaces, not just tasks.", action="store_true", default=False, dest="all_namespaces") parser.add_option( "-m", "--mro", help="Print the linear \"method resolution order\" for each namespace " "(the multiple-inheritance precedence order as determined by the " "C3 linearization algorithm).", action="store_true", default=False, dest="mro") parser.add_option( "-t", "--tree", help="Print the first-parent inheritance hierarchy in tree form.", action="store_true", default=False, dest="tree") parser.add_option( "-b", "--box", help="With -t/--tree, using unicode box characters. Your terminal " "must be able to display unicode characters.", action="store_true", default=False, dest="box") parser.add_option( "-w", "--with-titles", help="Print namespaces titles too.", action="store_true", default=False, dest="titles") parser.add_option( "-p", "--points", help="Print task IDs from [START] to [STOP] cycle points. Both bounds " "are optional and STOP can be an interval from START (or from the " "initial cycle point, by default). Use '-p , ' for the default range.", metavar="[START],[STOP]", action="store", default=None, dest="prange") parser.add_option( *AGAINST_SOURCE_OPTION.args, **AGAINST_SOURCE_OPTION.kwargs) parser.add_option(icp_option) parser.add_cylc_rose_options() return parser @cli_function(get_option_parser) def main(parser: COP, options: 'Values', workflow_id: str) -> None: asyncio.run(_main(options, workflow_id)) def define_inheritance_tree(tree, hierarchy): """Combine inheritance hierarchies into a tree structure.""" for rt_ in hierarchy: hier = copy(hierarchy[rt_]) hier.reverse() cur_tree = tree for item in hier: if item not in cur_tree: cur_tree[item] = {} cur_tree = cur_tree[item] def print_first_parent_tree(config, pretty=False, titles=False): # find task namespaces (no descendants) tasks = [] for ns in config.cfg['runtime']: if ns not in config.runtime['descendants']: tasks.append(ns) pruned_ancestors = config.get_first_parent_ancestors(pruned=True) tree = {} define_inheritance_tree(tree, pruned_ancestors) padding = '' if titles: config.add_tree_titles(tree) # compute pre-title padding maxlen = 0 for namespace in pruned_ancestors: items = copy(pruned_ancestors[namespace]) items.reverse() for itt, item in enumerate(items): tmp = 2 * itt + 1 + len(item) if itt == 0: tmp -= 1 if tmp > maxlen: maxlen = tmp padding = maxlen * ' ' print_tree(tree, padding=padding, use_unicode=pretty) async def _main(options: 'Values', workflow_id: str) -> None: workflow_id, _, flow_file = await parse_id_async( workflow_id, src=True, constraint='workflows', ) template_vars = get_template_vars(options) if options.all_tasks and options.all_namespaces: raise InputError("Choose either -a or -n") if (options.all_tasks or options.all_namespaces) and options.prange: raise InputError( '--points cannot be used with --all-tasks or --all-namespaces' ) if options.box and not options.tree: options.tree = True if options.all_tasks: which = "all tasks" elif options.all_namespaces: which = "all namespaces" elif options.prange: which = "prange" if options.prange == ",": tr_start = None tr_stop = None elif options.prange.endswith(","): tr_start = options.prange[:-1] tr_stop = None elif options.prange.startswith(","): tr_start = None tr_stop = options.prange[1:] else: tr_start, tr_stop = options.prange.split(',') else: which = "graphed tasks" if options.tree and os.environ['LANG'] == 'C' and options.box: print("WARNING, ignoring -t/--tree: $LANG=C", file=sys.stderr) options.tree = False if options.titles and options.mro: raise InputError("Please choose --mro or --title, not both") if options.tree and any( [options.all_tasks, options.all_namespaces, options.mro]): print("WARNING: -t chosen, ignoring non-tree options.", file=sys.stderr) # Save the location of the existing workflow run dir in the # against source option: if options.against_source: options.against_source = Path(get_workflow_run_dir(workflow_id)) config = WorkflowConfig( workflow_id, flow_file, options, template_vars ) if options.tree: print_first_parent_tree( config, pretty=options.box, titles=options.titles, ) elif options.prange: for node in sorted(config.get_node_labels(tr_start, tr_stop)): print(node) else: result = config.get_namespace_list(which) namespaces = list(result) namespaces.sort() if (options.mro or options.titles): # compute padding maxlen = 0 for ns in namespaces: if len(ns) > maxlen: maxlen = len(ns) padding = maxlen * ' ' for ns in namespaces: if options.mro: print(ns, padding[0:len(padding) - len(ns)], end=' ') print(' '.join(config.get_mro(ns))) elif options.titles: print(ns, padding[0:len(padding) - len(ns)], end=' ') print(result[ns]) else: print(ns) cylc-flow-8.7.0/cylc/flow/scripts/completion_server.py0000664000175000017500000005771515264420307023334 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc completion-server [OPTIONS] A server for providing Cylc CLI completion results. This server accepts requests containing the command line and returns responses containing possible completions for the last word on the command line. See `cylc get-resources` for available completion scripts using this server. """ # NOTE: Developers # # This server is launched automatically in the user's shell. If it dies it # gets restarted. It's important to handle error cases very carefully as # we don't want these servers getting caught in crash/restart loops or causing # hassle to users. # # This file contains: # * Completion functions: # Which "complete" words e.g. 'a' might match 'a1', 'a2' but not 'b1'. # * Listing functions: # Which provide possible values to the completion functions. import asyncio from contextlib import suppress import inspect import os from pathlib import Path import selectors import sys import typing as t from packaging.specifiers import SpecifierSet from packaging.version import parse as parse_version from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.exceptions import CylcError from cylc.flow.id import ( IDTokens, Tokens, tokenise, ) from cylc.flow.network.scan import scan from cylc.flow.option_parsers import CylcOptionParser as COP from cylc.flow.pathutil import get_workflow_run_job_dir from cylc.flow.resources import ( RESOURCE_DIR, list_resources as list_resources_, ) from cylc.flow.scripts.cylc import ( ALIASES, COMMANDS, ) from cylc.flow.scripts.scan import ( FLOW_STATES, ScanOptions, get_pipe, ) from cylc.flow.terminal import cli_function from cylc.flow.workflow_files import infer_latest_run_from_id if t.TYPE_CHECKING: from optparse import Values # do not list this command in `cylc help all` output INTERNAL = True # set the compatibility range for completion scripts with this server # I.E. if we change the server interface, change this compatibility range. # User's will be presented with an upgrade notice if this happens. REQUIRED_SCRIPT_VERSION = '>=1.0.0, <2.0.0' # register the pseudo "help" and "version" commands COMMAND_LIST = list(COMMANDS) + ['help', 'version'] def stdin(timeout: int = 2) -> t.Iterator[str]: """Yield lines from stdin, stop on configured read timeout. Args: timeout: Read timeout in seconds, if no new input is received within this period the generator will exit. Yields: The file line by line. """ selector = selectors.DefaultSelector() selector.register(sys.stdin, selectors.EVENT_READ) while selector.select(timeout): # wait for input => yield yield sys.stdin.readline() # timeout => stop iterating selector.close() async def server( listener: t.Callable, responder: t.Callable, once: bool = False, write: t.Callable[[str], None] = print, timeout: int = 60, ) -> None: """The bit that handles incoming requests. * Accepts requests in the format `cylc|trigger|workfl`. * Writes responses in the format cylc trigger workflow//`. Args: listener: Generator function which yields requests. Must accept a `timeout` (seconds) argument. responder: Function which yields a list of possible responses. responder(commandline: List[str]) => List[str]. once: Return after sending the first response. write: The function which sends the response back to the originator. timeout: Max time to wait for new requests. Note: The server will not play nicely with completions which contain spaces (they will show as separate completions). Completions should not contain spaces, workflows/cycles/tasks/jobs cannot contain spaces (due to validation rules) so this is ok. """ for line in listener(timeout=timeout): try: write(' '.join( item for item in sorted(await responder(*line[:-1].split('|'))) )) except Exception: # catch everything! If an error occurs swallow it, we don't want # it to reach the CLI or to cause the server to get caught in # a crash/restart loop write('') # send an empty reply if once: return sys.stdout.flush() async def complete_cylc(_root: str, *items: str) -> t.List[str]: """Return Cylc CLI completions for the provided args. Args: _root: The root command i.e. `cylc`. items: The command items e.g. ['trigger', 'workfl'] Returns: List of possible completions e.g. ['workflow_a//', 'workflow_b//'] """ length: int = len(items) # the word to complete partial: t.Optional[str] = None if length > 0: partial = items[-1] # the previous word previous: t.Optional[str] = None if length > 1: previous = items[-2] # complete the cylc sub-command if length == 0: return await complete_command() if length == 1: return await complete_command(partial) command: str = ALIASES.get(items[0], items[0]) # resolve command aliases # special logic for the pseudo "help" and "version" commands if command == 'help': return complete(partial, ['id', 'all']) if command == 'version': return complete(partial, ['--long']) # complete --options if partial and partial.startswith('-'): ret = await complete_option(command, partial) if ret is not None: return ret if previous and previous.startswith('-'): ret = await complete_option_value( command, previous, partial, items=items, ) if ret is not None: return ret # complete positional arguments return await complete_argument(command, partial) def complete( partial: t.Optional[str], words: t.Iterable[str] ) -> t.List[str]: """Return only matching words. Args: partial: The string to match. words: The list of words to match against. Returns: The filtered list of matching words. Examples: >>> complete('w', ['beans', 'walrus', 'wellington']) ['walrus', 'wellington'] """ if partial is None: return list(words) return [ word for word in words if word.startswith(partial) ] async def complete_command(partial: t.Optional[str] = None) -> t.List[str]: """Complete Cylc commands.""" return complete(partial, COMMAND_LIST) async def complete_option( command: str, partial: t.Optional[str] = None ) -> t.Optional[t.List[str]]: """Complete --options.""" if partial and '=' in partial: option, partial = partial.split('=', 1) values = await complete_option_value(command, option, partial) if values is None: return None return [ # filtering already performed by complete_option_value f'{option}={value}' for value in values ] return complete(partial, list_options(command)) async def complete_option_value( command: str, option: str, partial: t.Optional[str] = None, items: t.Optional[t.Iterable[str]] = None, ) -> t.Optional[t.List[str]]: """Complete values for --options.""" vals = await list_option_values(command, option, partial, items=items) if vals is not None: return complete(partial, vals) return None async def complete_argument( command: str, partial: t.Optional[str] = None, ) -> t.List[str]: """Complete arguments for commands.""" coro = COMMAND_MAP.get(command, list_cylc_id) if coro is None: # argument completion disabled for this command return [] return complete( partial, await coro(partial), ) async def list_cylc_id(partial: str) -> t.List[str]: """List Cylc IDs (workflows//cycles/tasks/jobs).""" partial = partial or '' try: tokens = tokenise(partial) except ValueError: tokens = Tokens() # strip off any partial IDs # e.g. workflow//12 => workflow// # e.g. workflow//1234/ta => workflow//1234 if not partial.endswith('/') and not tokens.is_null: tokens.pop_token() if tokens.is_null: # if there are no tokens, list all workflows ids = await list_workflows() else: # if there are tokens, list things within that workflow ids = await list_in_workflow(tokens) return ids def list_options(command: str) -> t.List[str]: """List CLI options from a Cylc command. E.G. ['--help', '--debug', '--color', ....] Note: This provides the long formats of options e.g. `--help` not `-h`. """ if command in COMMAND_OPTION_MAP: return COMMAND_OPTION_MAP[command] try: entry_point = COMMANDS[command].load() except KeyError: return [] parser = entry_point.parser_function() if getattr(parser, 'auto_add', None): parser.add_std_options() return [ option.get_opt_string() for option in parser.option_list ] async def list_option_values( command: str, option: str, partial: t.Optional[str] = '', items: t.Optional[t.Iterable[str]] = None, ) -> t.Optional[t.List[str]]: """List values for an option in a Cylc command. Args: command: The Cylc sub-command. option: The --option to list possible values for. partial: The part of the command the user is completing. items: The CLI context, i.e. everything that has been typed on the CLI before the partial. E.G. --flow ['new', 'none'] """ if option in OPTION_MAP: list_option = OPTION_MAP[option] if not list_option: # do not perform completion for this option return [] kwargs = {} if 'tokens_list' in inspect.getfullargspec(list_option).args: # the function requires information about tokens already specified # on the CLI # (e.g. the workflow//cycle/task the command is operating on) tokens_list = [] for item in items or []: # pull out things from the command which look like IDs if '//' in item: with suppress(ValueError): tokens_list.append(Tokens(item)) continue with suppress(ValueError): tokens_list.append(Tokens(item, relative=True)) kwargs['tokens_list'] = tokens_list return await list_option(partial, **kwargs) return None async def list_workflows(states: t.Optional[t.Set[str]] = None) -> t.List[str]: """List workflows from run directories.""" pipe = get_pipe(ScanOptions(states=states or FLOW_STATES), 'None') ids = [] async for flow in pipe: ids.append(cli_detokenise(Tokens(workflow=flow['name']))) return ids async def list_src_workflows(_partial: str) -> t.List[str]: """List workflow source directories from "source dirs".""" ret = [] for src_dir in glbl_cfg().get(['install', 'source dirs']): async for src_flow in scan(run_dir=Path(src_dir).expanduser()): ret.append(src_flow['name']) return ret async def list_in_workflow(tokens: Tokens, infer_run=True) -> t.List[str]: """List cycles/tasks/jobs from within a workflow.""" if not tokens.get('workflow'): return [] parts = [] if tokens.get('cycle'): parts.append(tokens['cycle']) if tokens.get('task'): parts.append(tokens['task']) if tokens.get('job'): parts.append(tokens['job']) input_workflow = tokens['workflow'] # workflow ID as provided on the CLI if infer_run: # workflow ID after run name inference inferred_workflow = infer_latest_run_from_id(input_workflow) else: inferred_workflow = input_workflow job_dir = Path(get_workflow_run_job_dir(inferred_workflow, *parts)) if not job_dir.exists(): # no job dir (e.g. workflow has not been run yet) return [] token = next_token(tokens) if token: return [ # list possible IDs cli_detokenise( tokens.duplicate( # use the workflow ID provided on the CLI to allow # run name inference workflow=input_workflow, **{token: path.name}, ) ) for path in job_dir.iterdir() ] return [] async def list_resources(_partial: str) -> t.List[str]: """Return the list of resources accessible through cylc get-resources.""" resources: t.List[str] = [] list_resources_(resources.append, headers=False) return [ resource.split('#')[0].strip() for resource in resources ] async def list_dir( partial: t.Optional[str] ) -> t.List[str]: """List an arbitrary dir on the filesystem. Relative dirs are assumed relative to `$PWD`. """ if partial: path = Path(partial) if not partial.endswith('/'): path = path.parent if path.is_absolute(): return list_abs_dir(path) return list_rel_dir(path, Path('.')) return list_rel_dir(Path('.'), Path('.')) def list_abs_dir(path: Path) -> t.List[str]: """List an absolute directory.""" if not path.is_dir(): return [str(path)] return [ f'{sub_path}/' if sub_path.is_dir() else str(sub_path) for sub_path in path.iterdir() ] def list_rel_dir(path: Path, base: Path) -> t.List[str]: """List a relative directory. Args: path: The directory to list. base: Results are given relative to this. """ if not path.is_dir(): return [str(path)] return [ f'{sub_path.relative_to(base)}/' if sub_path.is_dir() else str(sub_path.relative_to(base)) for sub_path in path.iterdir() ] async def list_flows( _partial: t.Optional[str] ) -> t.List[str]: """List values for the --flow option.""" return ['none', 'new'] async def list_colours( _partial: t.Optional[str] ) -> t.List[str]: """List values for the --color option.""" return ['never', 'auto', 'always'] async def list_outputs( _partial: t.Optional[str], tokens_list: t.Optional[t.List[Tokens]], ): """List task outputs.""" return (await _list_prereqs_and_outputs(tokens_list))[1] async def list_prereqs( _partial: t.Optional[str], tokens_list: t.Optional[t.List[Tokens]], ): """List task prerequisites.""" return (await _list_prereqs_and_outputs(tokens_list))[0] + ['all'] async def _list_prereqs_and_outputs( tokens_list: t.Optional[t.List[Tokens]], ) -> t.Tuple[t.List[str], t.List[str]]: """List task prerequisites and outputs. Returns: tuple - (prereqs, outputs) """ if not tokens_list: # no context information available on the CLI # we can't list prereqs/outputs return ([], []) # dynamic import for this relatively unlikely case to avoid slowing down # server startup unnecessarily from cylc.flow.network.client_factory import get_client from cylc.flow.scripts.show import prereqs_and_outputs_query from types import SimpleNamespace workflows: t.Dict[str, t.List[Tokens]] = {} current_workflow = None for tokens in tokens_list: workflow = tokens['workflow'] task = tokens['task'] if workflow: workflows.setdefault(workflow, []) current_workflow = workflow if current_workflow and task: workflows[current_workflow].append(tokens.task) clients = {} for workflow in workflows: with suppress(CylcError): clients[workflow] = get_client(workflow) if not workflows: return ([], []) json: dict = {} await asyncio.gather(*( prereqs_and_outputs_query( workflow, workflows[workflow], pclient, SimpleNamespace(json=True), json, ) for workflow, pclient in clients.items() )) if not json: return ([], []) return ( [ f"{cond['taskId']}:{cond['reqState']}" for value in json.values() for prerequisite in value['prerequisites'] for cond in prerequisite['conditions'] ], [ output['label'] for value in json.values() for output in value['outputs'] ], ) # non-exhaustive list of Cylc commands which take non-workflow arguments COMMAND_MAP: t.Dict[str, t.Optional[t.Callable]] = { # register commands which have special positional arguments 'get-resources': list_resources, 'install': list_src_workflows, 'vip': list_src_workflows, # commands for which we should not attempt to complete arguments 'cycle-point': None, 'hub': None, 'message': None, 'scan': None, } # commands we can't list options for in the standard way COMMAND_OPTION_MAP = { 'gui': [ '--debug', '--help', '--new', '--no-browser', ], 'hub': [ '--debug', '--help', ], } # non-exhaustive list of Cylc CLI options # (the ideal solution would inspect the meta-variables for each option) OPTION_MAP: t.Dict[str, t.Optional[t.Callable]] = { # register --options which need special treatment '--set-file': list_dir, '--file': list_dir, '--output': list_dir, '--flow': list_flows, '--colour': list_colours, '--color': list_colours, '--out': list_outputs, '--pre': list_prereqs, # options for which we should not attempt to complete values for '--rm': None, '--run-name': None, '--initial-cycle-point': None, '--final-cycle-point': None, '--start-cycle-point': None, '--stop-cycle-point': None, '--start-task': None, '--hold-cycle-point': None, '--host': None, '--format': None, '--main-loop': None, } def cli_detokenise(tokens: Tokens, relative=False) -> str: """Format tokens for use on the command line. I.E. add the trailing slash[es] onto the end. """ if tokens.is_null: # shouldn't happen but prevents possible error return '' if relative: id_ = tokens.relative_id else: id_ = tokens.id if tokens.lowest_token == IDTokens.Workflow.value: return f'{id_}//' return f'{id_}/' def next_token(tokens: Tokens) -> t.Optional[str]: """Return the first unspecified token. Examples: >>> next_token(Tokens()) 'workflow' >>> next_token(Tokens(workflow='a')) 'cycle' >>> next_token(Tokens(workflow='a', cycle='1')) 'task' >>> next_token(Tokens(workflow='a', cycle='1', task='x')) 'job' >>> next_token(Tokens(workflow='a', cycle='1', task='x', job='01')) """ if tokens.is_null: return IDTokens.Workflow.value lowest_token = IDTokens(tokens.lowest_token) next_ = False for token in IDTokens: if next_: return token.value if token == lowest_token: next_ = True return None def get_completion_script_file(completion_lang: str) -> t.Optional[Path]: """Return the path of the completion script for the specified language. Returns None if there is no completion script for the provided language. """ completion_script = (RESOURCE_DIR / f'cylc-completion.{completion_lang}') if not completion_script.exists(): return None return completion_script def get_current_completion_script_version( completion_script: Path, completion_lang: str, ) -> t.Optional[str]: """Extract the script version from the provided script. Returns None if the script version cannot be determined. """ if completion_lang == 'bash': with open(completion_script, 'r') as script: for line in script: if line.startswith('export CYLC_COMPLETION_SCRIPT_VERSION='): return line.replace( 'export CYLC_COMPLETION_SCRIPT_VERSION=', '' ) return None def check_completion_script_compatibility( completion_lang: str, completion_script_version: str, ) -> bool: """Check if a completion script is compatible with this server. Prints upgrade advice to `sys.stderr` as appropriate. Args: completion_lang: The language the completion script is implemented for e.g. `Bash`. completion_script_version: The completion script version e.g. `1.0.0`. Returns: True if the completion script is compatible, else False. """ is_compatible = True # get the version of the completion script bundled with this version of # Cylc completion_script = get_completion_script_file(completion_lang) if completion_script is None: return False current_version = get_current_completion_script_version( completion_script, completion_lang, ) if current_version is None: return True current_version = parse_version(current_version) installed_version = parse_version(completion_script_version) # check that the installed completion script is compabile with this # completion server version if installed_version not in SpecifierSet(REQUIRED_SCRIPT_VERSION): is_compatible = False print( f'The Cylc {completion_lang} script needs to be updated to' ' work with this version of Cylc.', file=sys.stderr, ) # check for completion script updates if installed_version < current_version: print( f'A new version of the Cylc {completion_lang} script is available.' f'\nGet it with: cylc get-resources {completion_script.name} PATH', file=sys.stderr, ) return is_compatible def get_option_parser() -> COP: parser = COP( __doc__, ) parser.add_option( '--timeout', type='int', default=900, # PT15M help=( 'The maximum idle time before the server shuts down in seconds.' ) ) parser.add_option( '--once', action='store_true', default=False, help=( 'Exit after returning the first response (for testing purposes).' ) ) return parser @cli_function(get_option_parser) def main(_, opts: 'Values') -> None: completion_lang = os.environ.get('CYLC_COMPLETION_LANG') completion_script_version = os.environ.get( 'CYLC_COMPLETION_SCRIPT_VERSION' ) if ( # if we are being called by a completion script, check the script # is compatible with this server version before proceeding completion_lang and completion_script_version and not check_completion_script_compatibility( completion_lang, completion_script_version, ) ): # this server version is not compatible with the script version return asyncio.run( server( stdin, complete_cylc, once=opts.once, timeout=opts.timeout ) ) cylc-flow-8.7.0/cylc/flow/scripts/ext_trigger.py0000775000175000017500000001061615264420307022110 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc ext-trigger [OPTIONS] ARGS Report an external event message to a scheduler. External triggers allow any program to send a message to the Cylc scheduler. Cylc can use this message as a signal that an external prerequisite has been satisfied and trigger the task accordingly. The WORKFLOW argument should be unique to each external trigger event. When an incoming message satisfies a task's external trigger the message TRIGGER_ID is broadcast to all downstream tasks in the cycle point as ``$CYLC_EXT_TRIGGER_ID``. Tasks can use ``$CYLC_EXT_TRIGGER_ID``, for example, to identify a new data file that the external triggering system is responding to. Use the retry options in case the target workflow is down or out of contact. Note: to manually trigger a task use 'cylc trigger', not this command. """ from time import sleep from typing import TYPE_CHECKING from cylc.flow import LOG from cylc.flow.exceptions import CylcError, ClientError from cylc.flow.id_cli import parse_id from cylc.flow.network.client_factory import get_client from cylc.flow.option_parsers import ( WORKFLOW_ID_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values MAX_N_TRIES = 5 RETRY_INTVL_SECS = 10.0 MSG_SEND_FAILED = "Send message: try %s of %s failed" MSG_SEND_RETRY = "Retrying in %s seconds, timeout is %s" MSG_SEND_SUCCEED = "Send message: try %s of %s succeeded" MUTATION = ''' mutation ( $wFlows: [WorkflowID]!, $eventMsg: String!, $eventId: String! ) { extTrigger ( workflows: $wFlows, message: $eventMsg, id: $eventId ) { result } } ''' def get_option_parser() -> COP: parser = COP( __doc__, comms=True, argdoc=[ WORKFLOW_ID_ARG_DOC, ("MSG", "External trigger message"), ("TRIGGER_ID", "Unique trigger ID"), ], ) parser.add_option( "--max-tries", help=r"Maximum number of send attempts (default: %default).", metavar="INT", action="store", default=MAX_N_TRIES, dest="max_n_tries" ) parser.add_option( "--retry-interval", help=r"Delay in seconds before retrying (default: %default).", metavar="SEC", action="store", default=RETRY_INTVL_SECS, dest="retry_intvl_secs" ) return parser @cli_function(get_option_parser) def main( parser: COP, options: 'Values', workflow_id: str, event_msg: str, event_id: str ) -> None: workflow_id, *_ = parse_id( workflow_id, constraint='workflows', ) LOG.info( 'Send to workflow %s: "%s" (%s)', workflow_id, event_msg, event_id ) pclient = get_client(workflow_id, timeout=options.comms_timeout) max_n_tries = int(options.max_n_tries) retry_intvl_secs = float(options.retry_intvl_secs) mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [workflow_id], 'eventMsg': event_msg, 'eventId': event_id, } } for i_try in range(max_n_tries): try: pclient('graphql', mutation_kwargs) except ClientError as exc: LOG.exception(exc) LOG.info(MSG_SEND_FAILED, i_try + 1, max_n_tries) if i_try == max_n_tries - 1: # final attempt raise CylcError('send failed') from None LOG.info(MSG_SEND_RETRY, retry_intvl_secs, options.comms_timeout) sleep(retry_intvl_secs) else: if i_try > 0: LOG.info(MSG_SEND_SUCCEED, i_try + 1, max_n_tries) break cylc-flow-8.7.0/cylc/flow/scripts/lint.py0000775000175000017500000014026315264420307020535 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Cylc configuration linter. Checks code style, deprecated syntax and other issues. """ # NOTE: docstring needed for `cylc help all` output and docs # (if editing check this still comes out as expected) COP_DOC = """cylc lint [OPTIONS] ARGS Check .cylc and .rc files for code style, deprecated syntax and other issues. By default, suggestions are written to stdout. In-place mode ("-i, --inplace") writes suggestions into the file as comments. Commit to version control before using this, in case you want to back out. A non-zero return code will be returned if any issues are identified. This can be overridden by providing the "--exit-zero" flag. """ NOQA = """ Individual errors can be ignored using the ``# noqa`` line comment. It is good practice to specify specific errors you wish to ignore using ``# noqa: S002 S007 U999`` """ TOMLDOC = """ pyproject.toml configuration: [tool.cylc.lint] ignore = ['S001', 'S002'] # List of rules to ignore exclude = ['etc/foo.cylc'] # List of files to ignore rulesets = ['style', '728'] # Sets default rulesets to check max-line-length = 130 # Max line length for linting """ from collections import Counter import functools from pathlib import Path import pkgutil import re import shutil import sys from tomllib import ( # type: ignore[no-redef] TOMLDecodeError, loads as toml_loads, ) from typing import ( TYPE_CHECKING, Any, Callable, Counter as CounterType, Dict, Iterator, List, Literal, Optional, Union, ) from ansimarkup import parse as cparse from cylc.flow import ( LOG, job_runner_handlers, ) from cylc.flow.cfgspec.workflow import ( SPEC, upg, ) from cylc.flow.exceptions import CylcError import cylc.flow.flags from cylc.flow.id_cli import parse_id from cylc.flow.job_runner_mgr import JobRunnerManager from cylc.flow.loggingutil import set_timestamps from cylc.flow.option_parsers import ( WORKFLOW_ID_OR_PATH_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.parsec.config import ParsecConfig from cylc.flow.scripts.cylc import DEAD_ENDS from cylc.flow.task_outputs import ( TASK_OUTPUT_FAILED, TASK_OUTPUT_SUCCEEDED, ) from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values LINT_TABLE = ['tool', 'cylc', 'lint'] LINT_SECTION = '.'.join(LINT_TABLE) # BACK COMPAT: DEPR_LINT_SECTION # url: https://github.com/cylc/cylc-flow/issues/5811 # from: 8.1.0 # to: 8.3.0 # remove at: 8.8 DEPR_LINT_SECTION = 'cylc-lint' IGNORE = 'ignore' EXCLUDE = 'exclude' RULESETS = 'rulesets' WARN = 'warn' MAX_LINE_LENGTH = 'max-line-length' DEPRECATED_ENV_VARS = { 'CYLC_SUITE_HOST': 'CYLC_WORKFLOW_HOST', 'CYLC_SUITE_OWNER': 'CYLC_WORKFLOW_OWNER', 'CYLC_SUITE_SHARE_DIR': 'CYLC_WORKFLOW_SHARE_DIR', 'CYLC_SUITE_SHARE_PATH': 'CYLC_WORKFLOW_SHARE_PATH', 'CYLC_SUITE_NAME': 'CYLC_WORKFLOW_ID', 'CYLC_SUITE_LOG_DIR': 'CYLC_WORKFLOW_LOG_DIR', 'CYLC_SUITE_INITIAL_CYCLE_POINT': 'CYLC_WORKFLOW_INITIAL_CYCLE_POINT', 'CYLC_SUITE_INITIAL_CYCLE_TIME': 'CYLC_WORKFLOW_INITIAL_CYCLE_TIME', 'CYLC_SUITE_FINAL_CYCLE_POINT': 'CYLC_WORKFLOW_FINAL_CYCLE_POINT', 'CYLC_SUITE_FINAL_CYCLE_TIME': 'CYLC_WORKFLOW_FINAL_CYCLE_TIME', 'CYLC_SUITE_WORK_DIR': 'CYLC_WORKFLOW_WORK_DIR', 'CYLC_SUITE_UUID': 'CYLC_WORKFLOW_UUID', 'CYLC_SUITE_RUN_DIR': 'CYLC_WORKFLOW_RUN_DIR', } OBSOLETE_ENV_VARS = { 'CYLC_SUITE_DEF_PATH', 'CYLC_SUITE_DEF_PATH_ON_SUITE_HOST' } DEPRECATED_STRING_TEMPLATES = { 'suite': ['workflow'], 'suite_uuid': ['uuid'], 'batch_sys_name': ['job_runner_name'], 'batch_sys_job_id': ['job_id'], 'user@host': ['platform_name'], 'task_url': ['URL', '(if set in :cylc:conf:`[meta]URL`)'], 'workflow_url': [ 'workflow_URL', '(if set in :cylc:conf:`[runtime][][meta]URL`)', ], } LIST_ITEM = '\n * ' FUNCTION = 'function' # Don't remove deprecated check. REMOVED_CHECKS = ['U008'] deprecated_string_templates = { key: ( re.compile(r'%\(' + key + r'\)s'), value ) for key, value in DEPRECATED_STRING_TEMPLATES.items() } def get_wallclock_directives(): """Get a set of directives equivalent to execution time limit""" job_runner_manager = JobRunnerManager() directives = {} for module in pkgutil.iter_modules(job_runner_handlers.__path__): directive = getattr( job_runner_manager._get_sys(module.name), 'TIME_LIMIT_DIRECTIVE', None ) if directive and directive == '-W': # LSF directive -W needs to have a particular form to # avoid matching PBS directive -W: directives[module.name] = re.compile( r'^-W\s*=?\s*(\d+:)?\d+[^/]*$') elif directive: directives[module.name] = re.compile(rf'^{directive}.*') return directives WALLCLOCK_DIRECTIVES = get_wallclock_directives() def check_wallclock_directives(line: str) -> Dict[str, str]: """Check for job runner specific directives equivalent to execution time limit. It's recommended that users prefer execution time limit because it gives the Cylc scheduler awareness should communications with a remote job runner be lost. Examples: >>> this = check_wallclock_directives >>> this(' -W 42:22') {'directive': '-W 42:22'} >>> this(' -W 42:22/hostname') # Legit LSF use case {} >>> this(' -W 422') {'directive': '-W 422'} >>> this(' -W foo=42') # Legit PBS use case. {} >>> this(' -W foo="Hello World"') # Legit PBS use case. {} >>> this(' -l walltime whatever') {'directive': '-l walltime whatever'} """ for directive in set(WALLCLOCK_DIRECTIVES.values()): if directive.findall(line.strip()): return {'directive': line.strip()} return {} def check_jinja2_no_shebang( line: str, file: Path, function: Callable, jinja_shebang: bool = False, **kwargs ): """Check ONLY top level workflow files for jinja without shebangs. Examples: >>> func = re.compile(r'{{').findall >>> check_jinja2_no_shebang( ... '{{FOO}}', ... function=func, jinja_shebang=True, file=Path('foo.cylc')) False >>> check_jinja2_no_shebang( ... '{{FOO}}', ... function=func, jinja_shebang=False, file=Path('suite.rc')) ['{{'] """ if ( jinja_shebang or file.name not in {'flow.cylc', 'suite.rc'} ): return False return function(line) def check_if_jinja2( line: str, jinja_shebang: bool, function: Callable, **kwargs ): """Run function if Jinja2 switched on. Examples: >>> func = re.compile('foo').findall >>> check_if_jinja2('barfoo', jinja_shebang=False, function=func) False >>> check_if_jinja2('foofoo', jinja_shebang=True, function=func) ['foo', 'foo'] """ if jinja_shebang: return function(line) return False def check_dead_ends(line: str) -> bool: """Check for dead end cylc scripts as defined in cylc.flow.scripts.cylc Examples: # Context: # [runtime] # [[task]] # script = \"\"\" >>> check_dead_ends(' cylc check-software') True >>> check_dead_ends(' cylc log') False """ return any( f'cylc {dead_end}' in line for dead_end in DEAD_ENDS ) def check_for_deprecated_environment_variables( line: str ) -> dict[str, str] | dict[str, list[str]] | Literal[False]: """Warn that environment variables with SUITE in are deprecated Examples: >>> check_for_deprecated_environment_variables('CYLC_SUITE_HOST') {'vars': ['CYLC_SUITE_HOST: CYLC_WORKFLOW_HOST']} """ vars_found = [ f'{k}: {v}' for k, v in DEPRECATED_ENV_VARS.items() if k in line ] if len(vars_found) == 1: return {'vars': vars_found} elif vars_found: return {'vars': '\n * ' + '\n * '.join(vars_found)} return False def check_for_obsolete_environment_variables( line: str, ) -> dict[str, list[str]]: """Warn that environment variables are obsolete. Examples: >>> this = check_for_obsolete_environment_variables >>> this('script = echo $CYLC_SUITE_DEF_PATH') {'vars': ['CYLC_SUITE_DEF_PATH']} >>> this('script = echo "irrelevant"') {} """ vars_found = [i for i in OBSOLETE_ENV_VARS if i in line] if vars_found: return {'vars': vars_found} return {} def check_for_deprecated_task_event_template_vars( line: str ) -> Optional[Dict[str, str]]: """Look for string variables which are no longer supported Examples: >>> this = check_for_deprecated_task_event_template_vars >>> this('hello = "My name is %(suite)s"') {'suggest': '\\n * %(suite)s ⇒ %(workflow)s'} >>> this('x = %(user@host)s, %(suite)') {'suggest': '\\n * %(user@host)s ⇒ %(platform_name)s'} >>> this('x = %(task_url)s') {'suggest': '\\n * %(task_url)s ⇒ %(URL)s (if set in ...)'} """ result = [ f"%({key})s ⇒ %({replacement})s {' '.join(extra)}".rstrip() for key, ( regex, (replacement, *extra), ) in deprecated_string_templates.items() if regex.findall(line) ] if result: return {'suggest': LIST_ITEM + LIST_ITEM.join(result)} return None BAD_SKIP_OUTS = re.compile(r'outputs\s*=\s*(.*)') def check_skip_mode_outputs(line: str) -> Dict: """Ensure skip mode output setting doesn't include succeeded _and_ failed, as they are mutually exclusive. n.b. This should be separable from ``[[outputs]]`` because it's a key value pair not a section heading. Examples: >>> this = check_skip_mode_outputs >>> this('outputs = succeeded, failed') {'description': 'are ... together', 'outputs': 'failed...succeeded'} """ outputs = BAD_SKIP_OUTS.findall(line) if outputs: outputs = [i.strip() for i in outputs[0].split(',')] if TASK_OUTPUT_FAILED in outputs and TASK_OUTPUT_SUCCEEDED in outputs: return { 'description': 'are mutually exclusive and cannot be used together', 'outputs': f'{TASK_OUTPUT_FAILED} and {TASK_OUTPUT_SUCCEEDED}' } return {} INDENTATION = re.compile(r'^(\s*)(.*)') def check_indentation(line: str) -> bool: """The key value pair is not indented 4*X spaces n.b. We test for trailing whitespace and incorrect section indenting elsewhere Examples: >>> check_indentation('') False >>> check_indentation(' ') False >>> check_indentation(' [') False >>> check_indentation('baz') False >>> check_indentation(' qux') False >>> check_indentation(' foo') True >>> check_indentation(' bar') True """ match = INDENTATION.findall(line)[0] if not match[0] or not match[1] or match[1].startswith('['): return False return bool(len(match[0]) % 4 != 0) INHERIT_REGEX = re.compile(r'\s*inherit\s*=\s*(.*)') FAM_NAME_IGNORE_REGEX = re.compile( # Stuff we want to ignore when checking for lowercase in family names r''' # comments (?]+> # or Jinja2 | {{.*?}} | {%.*?%} | {\#.*?\#} ''', re.X ) LOWERCASE_REGEX = re.compile(r'[a-z]') def check_lowercase_family_names(line: str) -> bool: """Check for lowercase in family names. Examples: >>> check_lowercase_family_names(' inherit = FOO') False >>> check_lowercase_family_names(' inherit = foo') True """ match = INHERIT_REGEX.match(line) if not match: return False # Replace stuff we want to ignore with a neutral char (tilde will do): content = FAM_NAME_IGNORE_REGEX.sub('~', match.group(1)) return any( LOWERCASE_REGEX.search(i) for i in content.split(',') if i.strip(' \'"') not in {'None', 'none', 'root'} ) CHECK_FOR_OLD_VARS = re.compile( r'CYLC_VERSION\s*=\s*\{\{\s*CYLC_VERSION\s*\}\}' r'|ROSE_VERSION\s*=\s*\{\{\s*ROSE_VERSION\s*\}\}' r'|FCM_VERSION\s*=\s*\{\{\s*FCM_VERSION\s*\}\}' ) def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]: """Take a line and a check function and return a Dict if there is a result, None otherwise. Returns Dict, in for {'vars': Error string} """ result = check(line) if result: return {'vars': '\n * '.join(result)} return None STYLE_GUIDE = ( 'https://cylc.github.io/cylc-doc/stable/html/workflow-design-guide/' 'style-guide.html#' ) SECTION2 = r'\[\[\s*{}\s*\]\]' SECTION3 = r'\[\[\[\s*{}\s*\]\]\]' FILEGLOBS = ['*.rc', '*.cylc'] JINJA2_SHEBANG = '#!jinja2' DEPENDENCY_SECTION_MSG = { 'text': ( '"[scheduling][dependencies][]graph =" -> ' '"[scheduling][graph] ="' ), 'rst': ( '``[scheduling][dependencies][]graph =`` -> ' '``[scheduling][graph] =``' ) } JOBANDREMOTE_SECTION_MSG = { 'text': ( 'settings in "[runtime][][{}]" have been moved to ' '"[runtime][]" and "global.cylc[platforms]' '[]"' ), 'rst': ( 'settings in ``[runtime][][{}]`` have been moved to ' '``[runtime][]`` and ``global.cylc[platforms]' '[]``' ) } JINJA2_FOUND_WITHOUT_SHEBANG = 'jinja2 found: no shebang (#!jinja2)' CHECKS_DESC = { 'U': '7 to 8 upgrades', 'A': 'Auto Generated 7 to 8 upgrades', 'S': 'Style' } LINE_LEN_NO = 'S012' # Checks Dictionary fields: # TODO: Consider making the checks an object. # Key: A unique reference number. # - short: A short description of the issue. # - url: A link to a fuller description. # - function: A function to use to run the check. # - kwargs: We want to pass a set of common kwargs to the check function. # - evaluate commented lines: Run this check on commented lines. # - rst: An rst description, for use in the Cylc docs. STYLE_CHECKS = { "S001": { 'short': 'Use multiple spaces, not tabs', 'url': STYLE_GUIDE + 'tab-characters', FUNCTION: re.compile(r'^\t').findall }, "S002": { 'short': 'Item not indented.', # Non-indented items should be sections: 'url': STYLE_GUIDE + 'indentation', FUNCTION: re.compile(r'^[^%\{\[|\s]').findall }, "S003": { 'short': 'Top level sections should not be indented.', 'url': STYLE_GUIDE + 'indentation', FUNCTION: re.compile(r'^\s+\[[^\[.]*\]').findall }, "S004": { 'short': ( 'Second level sections should be indented exactly ' '4 spaces.' ), 'url': STYLE_GUIDE + 'indentation', FUNCTION: re.compile(r'^(|\s|\s{2,3}|\s{5,})\[\[[^\[.]*\]\]').findall }, "S005": { 'short': ( 'Third level sections should be indented exactly ' '8 spaces.' ), 'url': STYLE_GUIDE + 'indentation', FUNCTION: re.compile(r'^(|\s{1,7}|\s{9,})\[\[\[[^\[.]*\]\]\]').findall }, "S006": { 'short': 'trailing whitespace.', 'url': STYLE_GUIDE + 'trailing-whitespace', FUNCTION: re.compile(r'[ \t]$').findall }, "S007": { 'short': 'Family name contains lowercase characters.', 'url': STYLE_GUIDE + 'task-naming-conventions', FUNCTION: check_lowercase_family_names, }, "S008": { 'short': JINJA2_FOUND_WITHOUT_SHEBANG, 'url': '', 'kwargs': True, FUNCTION: functools.partial( check_jinja2_no_shebang, function=re.compile(r'{[{%]').findall ) }, "S009": { 'short': 'Host Selection Script may be redundant with platform', 'url': ( 'https://cylc.github.io/cylc-doc/stable/html/7-to-8/' 'major-changes/platforms.html' ), FUNCTION: re.compile(r'platform\s*=\s*\$\(.*?\)').findall, }, "S010": { 'short': 'Using backticks to invoke subshell is deprecated', 'url': 'https://github.com/cylc/cylc-flow/issues/3825', FUNCTION: re.compile(r'platform\s*=\s*(`.*?`)').findall, }, "S011": { 'short': 'Cylc will process commented Jinja2!', 'url': '', 'kwargs': True, 'evaluate commented lines': True, FUNCTION: functools.partial( check_if_jinja2, function=re.compile(r'(?]execution " "time limit` rather than directly specifying a timeout " "directive, otherwise Cylc has no way of knowing when the job " "should have finished. Cylc automatically translates the " "execution time limit to the correct timeout directive for the " "particular job runner.\n\n" "See :ref:`design-guide.execution-time-limit`." ), FUNCTION: check_wallclock_directives, }, 'S015': { 'short': ( '``=>`` implies line continuation without ``\\``.' ), FUNCTION: re.compile(r'=>\s*\\').findall }, 'S016': { 'short': 'Task outputs {outputs}: {description}.', FUNCTION: check_skip_mode_outputs }, 'S017': { 'short': 'Run mode is not live: This task will only appear to run.', FUNCTION: re.compile(r'run mode\s*=\s*[^l][^i][^v][^e]$').findall }, } # Subset of deprecations which are tricky (impossible?) to scrape from the # upgrader. MANUAL_DEPRECATIONS = { "U001": { 'short': ( DEPENDENCY_SECTION_MSG['text'] + ' (``[dependencies]`` detected)' ), 'url': '', 'rst': ( DEPENDENCY_SECTION_MSG['rst'] + ' (``[dependencies]`` detected)' ), FUNCTION: re.compile(SECTION2.format('dependencies')).findall, }, "U002": { 'short': DEPENDENCY_SECTION_MSG['text'] + ' (``graph =`` detected)', 'url': '', 'rst': DEPENDENCY_SECTION_MSG['rst'] + ' (``graph =`` detected)', FUNCTION: re.compile(r'graph\s*=\s*').findall, }, "U003": { 'short': JOBANDREMOTE_SECTION_MSG['text'].format('remote'), 'url': '', 'rst': JOBANDREMOTE_SECTION_MSG['rst'].format('remote'), FUNCTION: re.compile(SECTION3.format('remote')).findall }, "U004": { 'short': JOBANDREMOTE_SECTION_MSG['text'].format('job'), 'url': '', 'rst': JOBANDREMOTE_SECTION_MSG['rst'].format('job'), FUNCTION: re.compile(SECTION3.format('job')).findall, }, "U005": { 'short': ( 'flow.cylc[runtime][][job]batch system -> ' 'global.cylc[platforms][]job runner' ), 'url': '', 'rst': ( '``flow.cylc[runtime][][job]batch system`` -> ' '``global.cylc[platforms][]job runner``' ), FUNCTION: re.compile(r'batch system\s*=\s*').findall, }, "U006": { 'short': 'Using backticks to invoke subshell will fail at Cylc 8.', 'url': 'https://github.com/cylc/cylc-flow/issues/3825', FUNCTION: re.compile(r'host\s*=\s*(`.*?`)').findall, }, 'U007': { 'short': ( 'Use built in platform selection instead of rose host-select.'), 'url': ''' https://cylc.github.io/cylc-doc/stable/html/7-to-8/major-changes/platforms.html ''', FUNCTION: re.compile(r'platform\s*=\s*\$\(\s*rose host-select').findall }, 'U009': { 'short': 'This line contains an obsolete Cylc CLI command.', 'url': '', FUNCTION: check_dead_ends }, 'U010': { 'short': 'rose suite-hook is deprecated at Rose 2,', 'url': '', FUNCTION: lambda line: 'rose suite-hook' in line, }, 'U011': { 'short': 'Leading zeros are no longer valid for Jinja2 integers.', 'url': ''' https://cylc.github.io/cylc-doc/stable/html/7-to-8/major-changes/python-2-3.html#jinja2-integers-with-leading-zeros ''', 'kwargs': True, FUNCTION: functools.partial( check_if_jinja2, function=re.compile(r'\{%\s*set\s*.+?\s*=\s*0\d+\s*%\}').findall ) }, 'U012': { 'short': ( 'Deprecated environment variables: {vars}'), 'rst': ( 'The following environment variables are deprecated:\n\n' '.. list-table::' '\n :header-rows: 1' '\n\n * - Deprecated Variable' '\n - New Variable' ) + ''.join( [ f'\n * - ``{old}``\n - ``{new}``' for old, new in DEPRECATED_ENV_VARS.items() ] ), 'url': ''' https://cylc.github.io/cylc-doc/stable/html/reference/job-script-vars/index.html ''', FUNCTION: check_for_deprecated_environment_variables, }, 'U013': { 'short': ( 'Obsolete environment variables: {vars}'), 'rst': ( 'The following environment variables are obsolete:\n\n' + ''.join([f'\n * ``{old}``' for old in OBSOLETE_ENV_VARS]) ), 'url': ''' https://cylc.github.io/cylc-doc/stable/html/reference/job-script-vars/index.html ''', FUNCTION: check_for_obsolete_environment_variables, }, 'U014': { 'short': 'Use "isodatetime [ref]" instead of "rose date [-c]"', 'rst': ( 'For datetime operations in task scripts:\n\n' ' * Use ``isodatetime`` instead of ``rose date``\n' ' * Use ``isodatetime ref`` instead of ``rose date -c`` for ' 'the current cycle point\n' ), 'url': ''' https://cylc.github.io/cylc-doc/stable/html/7-to-8/cheat-sheet.html#datetime-operations ''', FUNCTION: re.compile(r'rose +date').findall, }, 'U015': { 'short': ( 'Deprecated template variables: {suggest}'), 'rst': ( "The following deprecated template variables, mostly used in " "event handlers, should be replaced:\n" + ''.join( f"\n * ``{old}`` ⇒ ``{new}`` {' '.join(extra)}".rstrip() for old, (new, *extra) in DEPRECATED_STRING_TEMPLATES.items() ) ), 'url': ''' https://cylc.github.io/cylc-doc/stable/html/user-guide/writing-workflows/runtime.html#task-event-template-variables ''', FUNCTION: check_for_deprecated_task_event_template_vars, }, 'U016': { 'short': 'Deprecated template vars: {vars}', 'rst': ( 'It is no longer necessary to configure the environment variables ' '``CYLC_VERSION``, ``ROSE_VERSION`` or ``FCM_VERSION``.' ), 'url': ''' https://cylc.github.io/cylc-doc/stable/html/plugins/cylc-rose.html#special-variables ''', FUNCTION: functools.partial( list_wrapper, check=CHECK_FOR_OLD_VARS.findall), }, 'U017': { 'short': ( '``&`` and ``|`` imply line continuation without ``\\``' ), FUNCTION: re.compile(r'[&|]\s*\\').findall }, 'U018': { 'short': ( 'clock-trigger is deprecated, please use @wall_clock' ), 'url': ''' https://cylc.github.io/cylc-doc/stable/html/user-guide/writing-workflows/external-triggers.html#built-in-clock-triggers ''', FUNCTION: re.compile(r'clock-trigger*=*').findall }, } ALL_RULESETS = ['728', 'style', 'all'] EXTRA_TOML_VALIDATION = { IGNORE: { lambda x: re.match(r'[A-Z]\d\d\d', x): '{item} not valid: Ignore codes should be in the form X001', lambda x: x in parse_checks(['728', 'style']) or x in REMOVED_CHECKS: '{item} is a not a known linter code.', lambda x: WARN if x in REMOVED_CHECKS else True: '{item} is a deprecated linter code', }, RULESETS: { lambda item: item in ALL_RULESETS: '{item} not valid: Rulesets can be ' '\'728\', \'style\' or \'all\'.' }, MAX_LINE_LENGTH: { lambda x: isinstance(x, int): 'max-line-length must be an integer.' }, # consider checking that item is file? EXCLUDE: {} } def parse_ruleset_option(ruleset: str) -> List[str]: if ruleset in {'all', ''}: return ['728', 'style'] return [ruleset] def get_url(check_meta: Dict) -> str: """Get URL from check data. If the URL doesn't start with http then prepend with address of the 7-to-8 upgrade guide. Examples: >>> get_url({'no': 'url key'}) '' >>> get_url({'url': ''}) '' >>> get_url({'url': 'https://www.h2g2.com/'}) 'https://www.h2g2.com/' >>> get_url({'url': 'cheat-sheet.html'}) 'https://cylc.github.io/cylc-doc/stable/html/7-to-8/cheat-sheet.html' """ url = check_meta.get('url', '').strip() if url and not url.startswith('http'): url = ( "https://cylc.github.io/cylc-doc/stable/html/7-to-8/" + check_meta['url'] ) return url def validate_toml_items(tomldata): """Check that all tomldata items are lists of strings Explicitly checks and raises if tomldata not: - str in EXTRA_TOML_VALIDATION.keys() - item is not a list of strings. Plus additional validation for each set of values. """ for key, items in tomldata.items(): # Key should only be one of the allowed keys: if key not in EXTRA_TOML_VALIDATION.keys(): raise CylcError( f'Only {[*EXTRA_TOML_VALIDATION.keys()]} ' f'allowed as toml sections but you used "{key}"' ) if key != MAX_LINE_LENGTH: # Item should be a list... if not isinstance(items, list): raise CylcError( f'{key} should be a list, but was: {items}') # ... of strings for item in items: if not isinstance(item, str): raise CylcError( f'Config {item} should be a string but ' f'is {str(type(item))}' ) for check, message in EXTRA_TOML_VALIDATION[key].items(): if not check(item): raise CylcError( message.format(item=item) ) elif check(item) == WARN: LOG.warning(message.format(item=item)) return True def get_pyproject_toml(dir_: Path) -> dict[str, Any]: """if a pyproject.toml file is present open it and return settings. """ tomlfile = dir_ / 'pyproject.toml' tomldata: dict[str, list[str] | int | None] = { RULESETS: [], IGNORE: [], EXCLUDE: [], MAX_LINE_LENGTH: None, } if tomlfile.is_file(): try: loadeddata = toml_loads(tomlfile.read_text()) except TOMLDecodeError as exc: raise CylcError(f'pyproject.toml did not load: {exc}') from None _tool, _cylc, _lint = LINT_TABLE try: data = loadeddata[_tool][_cylc][_lint] except KeyError: if DEPR_LINT_SECTION in loadeddata: LOG.warning( f"The [{DEPR_LINT_SECTION}] section in pyproject.toml is " "deprecated and support will be removed in Cylc 8.8. " f"Use [{LINT_SECTION}] instead." ) data = loadeddata.get(DEPR_LINT_SECTION, {}) tomldata.update(data) validate_toml_items(tomldata) return tomldata def merge_cli_with_tomldata(target: Path, options: 'Values') -> Dict[str, Any]: """Get a list of checks based on the checking options Args: target: Location being linted, in which we might find a pyproject.toml file. options: Cli Options This has not been merged with merged with the logic in _merge_cli_with_tomldata to keep the testing of file-system touching and pure logic separate. """ ruleset_default = (options.ruleset == '') options.ruleset = parse_ruleset_option(options.ruleset) tomlopts = get_pyproject_toml(target) return _merge_cli_with_tomldata( { EXCLUDE: [], IGNORE: options.ignores, RULESETS: options.ruleset }, tomlopts, ruleset_default ) def _merge_cli_with_tomldata( clidata: Dict[str, Any], tomldata: Dict[str, Any], override_cli_default_rules: bool = False ) -> Dict[str, Any]: """Merge options set by pyproject.toml with CLI options. rulesets: CLI should override toml. ignore: CLI and toml should be combined with no duplicates. exclude: No CLI equivalent, return toml if any. Args: override_cli_default_rules: If user doesn't specify a ruleset use the rules from the tomlfile - i.e: if we've set 'rulesets': 'style' we probably don't want to get warnings about 728 upgrades by default, but only if we ask for it on the CLI. Examples: >>> result = _merge_cli_with_tomldata( ... {'rulesets': ['foo'], 'ignore': ['R101'], 'exclude': []}, ... {'rulesets': ['bar'], 'ignore': ['R100'], 'exclude': ['*.bk']}) >>> result['ignore'] ['R100', 'R101'] >>> result['rulesets'] ['foo'] >>> result['exclude'] ['*.bk'] """ if isinstance(clidata[RULESETS][0], list): clidata[RULESETS] = clidata[RULESETS][0] output: dict[str, Any] = {} # Combine 'ignore' sections: output[IGNORE] = sorted(set(clidata[IGNORE] + tomldata[IGNORE])) # Replace 'rulesets' from toml with those from CLI if they exist: if override_cli_default_rules: output[RULESETS] = tomldata[RULESETS] or clidata[RULESETS] else: output[RULESETS] = clidata[RULESETS] or tomldata[RULESETS] # Return 'exclude' and 'max-line-length' for the tomldata: output[EXCLUDE] = tomldata[EXCLUDE] output[MAX_LINE_LENGTH] = tomldata.get(MAX_LINE_LENGTH, None) return output def list_to_config(path_, is_section=False): """Prettify a config list Args: path_: list forming address in the config. is_section: Is this item a section? Examples: >>> list_to_config(['foo', 'bar'], False) '[foo]bar' >>> list_to_config(['foo', 'bar'], True) '[foo][bar]' """ output = '' for item in path_[:-1]: # All but the last item must be sections: output += f'[{item}]' if is_section: # Last item is section: output += f'[{path_[-1]}]' else: # Last item not a section: output += path_[-1] return output def get_upgrader_info(): """Extract info about obseletions and deprecations from Parsec Objects.""" conf = ParsecConfig(SPEC, upg) upgrades = conf.upgrader(conf.dense, '').upgrades deprecations = {} for upgrades_for_version in upgrades.values(): for index, upgrade in enumerate(upgrades_for_version): # Set a flag indicating that a variable has been moved. is_dep, is_obs = False, False if upgrade['new'] is None: section_name = list_to_config( upgrade["old"], upgrade["is_section"]) short = f'{section_name} - not available at Cylc 8' rst = f'``{section_name}`` is not available at Cylc 8' is_obs = True elif upgrade["old"][-1] == upgrade['new'][-1]: # Where an item with the same name has been moved # a 1 line regex isn't going to work. continue else: old = list_to_config( upgrade["old"], upgrade["is_section"]) new = list_to_config( upgrade["new"], upgrade["is_section"]) short = f'{old} -> {new}' rst = f'``{old}`` is now ``{new}``' is_dep = True # Check whether upgrade is section: if upgrade['is_section'] is True: section_depth = len(upgrade['old']) start = r'\[' * section_depth end = r'\]' * section_depth name = upgrade["old"][-1] expr = fr'{start}\s*{name}\s*{end}\s*$' else: name = upgrade["old"][-1] expr = rf'^\s*{name}\s*=\s*.*' deprecations[f'A{index:03d}'] = { 'short': short, 'url': '', 'rst': rst, FUNCTION: re.compile(expr).findall, 'is_obs': is_obs, 'is_dep': is_dep, } return deprecations PURPOSE_FILTER_MAP = { 'style': 'S', '728': 'UA', } def parse_checks(check_args, ignores=None, max_line_len=None, reference=False): """Prepare dictionary of checks. Args: check_arg: list of types of checks to run, currently expecting '728' and/or 'style' ignores: list of codes to ignore. max_line_len: Adds a specific style warning for lines longer than this. (If None, rule not enforced) reference: Function is being used to get a reference. If true max-line-length will have a generic message, rather than using any specific value. """ ignores = ignores or [] parsedchecks = {} purpose_filters = [ purpose for arg in check_args for purpose in PURPOSE_FILTER_MAP[arg] ] checks = { 'S': STYLE_CHECKS, 'U': MANUAL_DEPRECATIONS, 'A': get_upgrader_info(), } for purpose, ruleset in checks.items(): if purpose in purpose_filters: # Run through the rest of the config items. for index, meta in ruleset.items(): meta.update({'purpose': purpose}) if f'{index}' not in ignores: parsedchecks.update({index: meta}) if 'S' in purpose and LINE_LEN_NO not in ignores: # Special handling for max line length: if not max_line_len: max_line_len = 130 regex = r"^.{" + str(max_line_len) + r"}" if reference: msg = ( 'line > ```` characters. Max line ' ' length set in pyproject.toml (default 130)' ) else: msg = f'line > {max_line_len} characters.' parsedchecks[LINE_LEN_NO] = { 'short': msg, 'url': STYLE_GUIDE + 'line-length-and-continuation', FUNCTION: re.compile(regex).findall, 'purpose': 'S' } return parsedchecks def get_index_str(meta: dict, index: str) -> str: """Printable purpose string - mask useless numbers for auto-generated upgrades.""" if meta.get('is_dep', None): return 'U998' elif meta.get('is_obs', None): return 'U999' else: return f'{index}' def check_cylc_file( file: Path, file_rel: Path, checks: Dict[str, dict], counter: CounterType[str], modify: bool = False, ): """Check A Cylc File for Cylc 7 Config""" with open(file, 'r') as cylc_file: # generator which reads and lints one line at a time linter = lint( file_rel, cylc_file, checks, counter, modify, ) if modify: # write modifications into a ".temp" file modify_file_path = file.parent / f'{file.name}.temp' with open(modify_file_path, 'w+') as modify_file: for line in linter: modify_file.write(line) # replace the original with the ".temp" file shutil.move(str(modify_file_path), file) else: for _line in linter: pass def no_qa(line: str, index: str): """This line has a no-qa comment. Examples: # No comment, no exception: >>> no_qa('foo = bar', 'S001') False # Comment, no error codes, no checking: >>> no_qa('foo = bar # noqa', 'S001') True # Comment, no relevant error codes, no checking: >>> no_qa('foo = bar # noqa: S999, 997', 'S001') False # Comment, relevant error codes, checking: >>> no_qa('foo = bar # noqa: S001 S003', 'S001') True """ NOQA = re.compile(r'.*#\s*[Nn][Oo][Qq][Aa]:?(.*)') noqa = NOQA.findall(line) if noqa and (noqa[0] == '' or index in noqa[0]): return True return False def lint( file_rel: Path, lines: Iterator[str], checks: Dict[str, dict], counter: CounterType[str], modify: bool = False, write: Callable = print ) -> Iterator[str]: """Lint text, one line at a time. Arguments: file_rel: The filepath relative to the workflow configuration directory (used in messages). lines: Iterator which produces one line of text at a time e.g. open(file) or iter(['foo\n', 'bar\n', 'baz\n']. counter: Counter for counting lint hits per category. modify: If True, this generator will yield the file one line at a time with comments inserted to help users fix their lint. write: A function for reporting lint messages. Yields: The original file with added comments when `modify is True`. """ # get the first line line_no = 1 line = next(lines) # check if it is a jinja2 shebang jinja_shebang = line.strip().lower() == JINJA2_SHEBANG while True: # run lint checks against the current line for index, check_meta in checks.items(): # Skip commented line unless check says not to. index_str = get_index_str(check_meta, index) if ( ( line.strip().startswith('#') and not check_meta.get('evaluate commented lines', False) ) or no_qa(line, index_str) ): continue if check_meta.get('kwargs', False): # Use a more complex function with keywords: check_function = functools.partial( check_meta['function'], check_meta=check_meta, file=file_rel, jinja_shebang=jinja_shebang, ) else: # Just going to pass the line to the check function: check_function = check_meta['function'] # Run the check: check = check_function(line) if check: # we have lint! if isinstance(check, dict): msg = check_meta['short'].format(**check) else: msg = check_meta['short'] counter[check_meta['purpose']] += 1 if modify: # insert a command to help the user url = get_url(check_meta) yield ( f'# [{index_str}]: ' f'{msg}\n' f'# - see {url}\n' ) else: # write a message to inform the user write(cparse( '' f'[{index_str}] {file_rel}:{line_no}: {msg}' '' )) if modify: yield line try: # get the next line line = next(lines) except StopIteration: # end of iterator return line_no += 1 def get_cylc_files( base: Path, exclusions: Union[list, None] = None ) -> Iterator[Path]: """Given a directory yield paths to check.""" exclusions = [] if exclusions is None else exclusions except_these_files = [ file for exclusion in exclusions for file in base.rglob(exclusion)] for rglob in FILEGLOBS: for path in base.rglob(rglob): # Exclude log directory: if ( path.relative_to(base).parts[0] != 'log' and path not in except_these_files ): yield path def get_reference(ruleset: str, output_type: 'Literal["text", "rst"]') -> str: """Fill out a template with all the issues Cylc Lint looks for. """ checks = parse_checks( parse_ruleset_option(ruleset), reference=True ) issue_heading_template = ( '\n{url}_\n{underline}\n{summary}\n\n' if output_type == 'rst' else '\n{check}:\n {summary}\n {url}\n\n' ) output = '' current_checkset = '' for index, meta in checks.items(): # Check if the purpose has changed - if so create a new # section heading: if meta['purpose'] != current_checkset: current_checkset = meta['purpose'] title = CHECKS_DESC[meta["purpose"]] output += '\n{title}\n{underline}\n'.format( title=title, underline="-" * len(title) ) if current_checkset == 'A': output += ( 'U998 and U999 represent automatically generated' ' sets of deprecations and upgrades.' ) # Fill a template with info about the issue. if output_type == 'rst': summary = meta.get("rst", meta['short']) elif output_type == 'text': summary = meta.get("short").replace('``', '') if current_checkset == 'A': # Condensed check summary for auto-generated lint items. if output_type == 'rst': output += '\n' output += '\n* ' + summary else: check = get_index_str(meta, index) template = issue_heading_template url = get_url(meta) if output_type == 'rst': url = f'`{check} <{url}>`' if url else f'{check}' msg = template.format( title=index, check=check, summary=summary, url=url, underline=(len(url) + 1) * '^' ) output += msg output += '\n' return output def target_version_check( target: Path, quiet: 'Values', mergedopts: Dict[str, Any] ) -> List: """ Check whether target is an upgraded Cylc 8 workflow. If it isn't then we shouldn't run the 7-to-8 checks upon it. If it isn't and the only ruleset requested by the user is '728' we should exit with an error code unless the user has specifically disabled thatr with --exit-zero. """ cylc8 = (target / 'flow.cylc').exists() if not cylc8 and mergedopts[RULESETS] == ['728']: LOG.error( f'{target} not a Cylc 8 workflow: ' 'Lint after renaming ' '"suite.rc" to "flow.cylc"' ) sys.exit(not quiet) elif not cylc8 and '728' in mergedopts[RULESETS]: check_names = mergedopts[RULESETS] check_names.remove('728') else: check_names = mergedopts[RULESETS] return check_names def get_option_parser() -> COP: parser = COP( ( COP_DOC + NOQA.replace('``', '"') + TOMLDOC ), argdoc=[ COP.optional(WORKFLOW_ID_OR_PATH_ARG_DOC) ], ) parser.add_option( '--inplace', '-i', help=( 'Modify files in place, adding comments to files. ' 'If not set, the script will work as a linter' ), action='store_true', default=False, ) parser.add_option( '--ruleset', '-r', help=( 'Set of rules to use: ' '("728", "style", "all")' ), default='', choices=["728", "style", "all", ''], dest='ruleset' ) parser.add_option( '--list-codes', help=( 'List all linter codes.' ), action='store_true', default=False, dest='ref_mode' ) parser.add_option( '--ignore', '-n', help=( 'Ignore this check number.' ), action='append', default=[], dest='ignores', metavar="CODE", choices=list(STYLE_CHECKS.keys()) + [LINE_LEN_NO] ) parser.add_option( '--exit-zero', help='Exit with status code "0" even if there are issues.', action='store_true', default=False, dest='exit_zero' ) return parser @cli_function(get_option_parser) def main(parser: COP, options: 'Values', target=None) -> None: if cylc.flow.flags.verbosity < 2: set_timestamps(LOG, False) if options.ref_mode: print(get_reference(options.ruleset, 'text')) sys.exit(0) # If target not given assume we are looking at PWD: if target is None: target = str(Path.cwd()) # make sure the target is a src/run directory: _, _, target = parse_id( target, src=True, constraint='workflows', ) # We want target to be the containing folder, not the flow.cylc # file identified by parse_id: target = target.parent mergedopts = merge_cli_with_tomldata(target, options) check_names = target_version_check( target=target, quiet=options.exit_zero, mergedopts=mergedopts) # Get the checks object. checks = parse_checks( check_names, ignores=mergedopts[IGNORE], max_line_len=mergedopts[MAX_LINE_LENGTH] ) # Check each file matching a pattern: counter: CounterType[str] = Counter() for file in get_cylc_files(target, mergedopts[EXCLUDE]): LOG.debug(f'Checking {file}') check_cylc_file( file, file.relative_to(target), checks, counter, options.inplace, ) if counter: total_lint_hits = sum(counter.values()) msg = cparse( '\n' f'Checked {target} against {check_names} ' f'rules and found {total_lint_hits} issue' f'{"s" if total_lint_hits > 1 else ""}.' '' ) else: msg = cparse( '' f'Checked {target} against {check_names} rules and ' 'found no issues.' '' ) print(msg) # Exit with an error code if there were warnings and # if --exit-zero was not set. # Return codes: sys.exit(True) == 1, sys.exit(False) == 0 sys.exit(bool(counter) and not options.exit_zero) # NOTE: use += so that this works with __import__ # (docstring needed for `cylc help all` output) __doc__ += NOQA __doc__ += get_reference('all', 'rst') cylc-flow-8.7.0/cylc/flow/scripts/subscribe.py0000775000175000017500000000714515264420307021551 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc subscribe [OPTIONS] ARGS (This command is for internal use.) Invoke workflow subscriber to receive published workflow output. """ import json import sys import time from google.protobuf.json_format import MessageToDict from cylc.flow.exceptions import ClientError from cylc.flow.option_parsers import ( WORKFLOW_ID_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.network import get_location from cylc.flow.network.subscriber import WorkflowSubscriber, process_delta_msg from cylc.flow.terminal import cli_function from cylc.flow.data_store_mgr import DELTAS_MAP INTERNAL = True def print_message(topic, data, subscriber=None, once=False): """Print protobuf message.""" print(f'Received: {topic}') if topic == 'shutdown': print(data.decode('utf-8')) subscriber.stop() return sys.stdout.write( json.dumps(MessageToDict(data), indent=4) + '\n') if once and subscriber is not None: subscriber.stop() def get_option_parser() -> COP: """Augment options parser to current context.""" parser = COP( __doc__, argdoc=[WORKFLOW_ID_ARG_DOC], comms=True ) delta_keys = list(DELTAS_MAP) pb_topics = ("Directly published data-store topics include: '" + ("', '").join(delta_keys[:-1]) + "' and '" + delta_keys[-1] + "'.") parser.add_option( "-T", "--topics", help="Specify a comma delimited list of subscription topics. " + pb_topics, action="store", dest="topics", default='workflow') parser.add_option( "-o", "--once", help="Show a single publish then exit.", action="store_true", default=False, dest="once") return parser @cli_function(get_option_parser) def main(_, options, *args): workflow_id = args[0] try: while True: try: host, _, port, _ = get_location(workflow_id) except (ClientError, IOError, TypeError, ValueError) as exc: print(exc) time.sleep(3) continue break except KeyboardInterrupt: sys.exit() print(f'Connecting to tcp://{host}:{port}') topic_set = set() topic_set.add(b'shutdown') for topic in options.topics.split(','): topic_set.add(topic.encode('utf-8')) subscriber = WorkflowSubscriber( workflow_id, host=host, port=port, topics=topic_set ) subscriber.loop.create_task( subscriber.subscribe( process_delta_msg, func=print_message, subscriber=subscriber, once=options.once ) ) # run Python run try: subscriber.loop.run_forever() except (KeyboardInterrupt, SystemExit): print('\nDisconnecting') subscriber.stop() sys.exit() cylc-flow-8.7.0/cylc/flow/scripts/release.py0000775000175000017500000000736615264420307021215 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc release [OPTIONS] ARGS Release held tasks in a workflow. Examples: # Release mytask at cycle 1234 in my_workflow $ cylc release my_workflow//1234/mytask # Release all active tasks at cycle 1234 in my_workflow $ cylc release 'my_workflow//1234/*' # Release all active instances of mytask in my_workflow $ cylc release 'my_workflow//*/mytask' # Release all held tasks and remove the hold point $ cylc release my_workflow --all Held tasks do not submit their jobs even if ready to run. See also 'cylc hold'. """ from functools import partial import sys from typing import TYPE_CHECKING from cylc.flow.exceptions import InputError from cylc.flow.network.client_factory import get_client from cylc.flow.network.multi import call_multi from cylc.flow.option_parsers import ( FULL_ID_MULTI_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values RELEASE_MUTATION = ''' mutation ( $wFlows: [WorkflowID]!, $tasks: [NamespaceIDGlob]! ) { release ( workflows: $wFlows, tasks: $tasks, ) { result } } ''' RELEASE_HOLD_POINT_MUTATION = ''' mutation ( $wFlows: [WorkflowID]! ) { releaseHoldPoint ( workflows: $wFlows ) { result } } ''' def get_option_parser() -> COP: parser = COP( __doc__, comms=True, multitask=True, multiworkflow=True, argdoc=[FULL_ID_MULTI_ARG_DOC], ) parser.add_option( "--all", help=( "Release all held tasks and remove the 'hold after cycle point', " "if set."), action="store_true", dest="release_all") return parser def _validate(options: 'Values', *tokens_list: str) -> None: """Check combination of options and task globs is valid.""" if options.release_all: if tokens_list: raise InputError("Cannot combine --all with Cycle/Task IDs") else: if not tokens_list: raise InputError( "Must define Cycles/Tasks. See `cylc release --help`." ) async def run(options: 'Values', workflow_id, *tokens_list): _validate(options, *tokens_list) pclient = get_client(workflow_id, timeout=options.comms_timeout) if options.release_all: mutation = RELEASE_HOLD_POINT_MUTATION args = {'tasks': ['*/*']} else: mutation = RELEASE_MUTATION args = { 'tasks': [ tokens.relative_id_with_selectors for tokens in tokens_list ] } mutation_kwargs = { 'request_string': mutation, 'variables': { 'wFlows': [workflow_id], **args } } return await pclient.async_request('graphql', mutation_kwargs) @cli_function(get_option_parser) def main(parser: COP, options: 'Values', *ids): rets = call_multi( partial(run, options), *ids, constraint='mixed', ) sys.exit(all(rets.values()) is False) cylc-flow-8.7.0/cylc/flow/scripts/broadcast.py0000775000175000017500000004220615264420307021527 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc broadcast [OPTIONS] ARGS Override "[runtime]" configurations in a running workflow. Uses for broadcast include making temporary changes to task behaviour, and task-to-downstream-task communication via environment variables. See also "cylc reload" which reads in the flow.cylc file. A broadcast can set/override any "[runtime]" configuration for all cycles or for a specific cycle. If a task is affected by specific-cycle and all-cycle broadcasts at the same time, the specific takes precedence. Broadcasts can also target all tasks, specific tasks or families of tasks. If a task is affected by broadcasts to multiple ancestor namespaces (tasks it inherits from), the result is determined by normal "[runtime]" inheritance. Broadcasts are applied at the time of job submission. Broadcasts persist, even across restarts. Broadcasts made to specific cycle points will expire when the cycle point is older than the oldest active cycle point in the workflow. Active broadcasts can be revoked using the "clear" mode. Any broadcasts matching the specified cycle points and namespaces will be revoked. Note: a "clear" broadcast for a specific cycle or namespace does *not* clear all-cycle or all-namespace broadcasts. Examples: # To broadcast a variable to all tasks (quote items with internal spaces): $ cylc broadcast -s "[environment]VERSE = the quick brown fox" WORKFLOW # To do the same with a file: $ cat >'broadcast.cylc' <<'__FLOW__' > [environment] > VERSE = the quick brown fox > __FLOW__ $ cylc broadcast -F 'broadcast.cylc' WORKFLOW_ID # view active broadcasts $ cylc broadcast --display WORKFLOW_ID # To cancel the same broadcast: $ cylc broadcast --cancel "[environment]VERSE" WORKFLOW_ID # If -F FILE was used, the same file can be used to cancel the broadcast: $ cylc broadcast -G 'broadcast.cylc' WORKFLOW_ID # Use broadcast with multiple workflows $ cylc broadcast [options] WORKFLOW_ID_1// WORKFLOW_ID_2// Use -d/--display to see active broadcasts. Multiple --cancel options or multiple --set and --set-file options can be used on the same command line. Multiple --set and --set-file options are cumulative. The --set-file=FILE option can be used when broadcasting multiple values, or when the value contains newline or other metacharacters. If FILE is "-", read from standard input. Broadcast cannot change [runtime] inheritance. """ from ansimarkup import parse as cparse import asyncio from copy import deepcopy from functools import partial import os.path import re import sys from tempfile import NamedTemporaryFile from typing import Any, Dict, TYPE_CHECKING from cylc.flow.broadcast_report import ( get_broadcast_bad_options_report, get_broadcast_change_report, ) from cylc.flow.cfgspec.workflow import SPEC, upg from cylc.flow.exceptions import InputError from cylc.flow.network.client_factory import get_client from cylc.flow.network.multi import call_multi_async from cylc.flow.option_parsers import ( WORKFLOW_ID_MULTI_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.parsec.config import ParsecConfig from cylc.flow.parsec.util import listjoin from cylc.flow.parsec.validate import cylc_config_validate from cylc.flow.print_tree import get_tree from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values RAW_DEPR_MSG = ( "DEPRECATED: the --raw option will be removed at Cylc 8.9; " "use --format=raw instead." ) REC_ITEM = re.compile(r'^\[([^\]]*)\](.*)$') MUTATION = ''' mutation ( $wFlows: [WorkflowID]!, $bMode: BroadcastMode!, $cPoints: [BroadcastCyclePoint], $nSpaces: [NamespaceName], $bSettings: [BroadcastSetting], $bCutoff: CyclePoint ) { broadcast ( workflows: $wFlows, mode: $bMode, cyclePoints: $cPoints, namespaces: $nSpaces, settings: $bSettings, cutoff: $bCutoff ) { result } } ''' QUERY = ''' query ( $wFlows: [ID], $nIds: [ID] ) { workflows ( ids: $wFlows, ) { id broadcasts (ids: $nIds) } } ''' def get_padding(settings, level=0, padding=0): """Return the left padding for displaying a setting.""" level += 1 for key, val in settings.items(): tmp = level * 2 + len(key) if tmp > padding: padding = tmp if isinstance(val, dict): padding = get_padding(val, level, padding) return padding def get_rdict(left, right=None, for_cancel_broadcast=False): """Check+transform left=right into a nested dict. left can be key, [key], [key1]key2, [key1][key2], [key1][key2]key3, etc. """ if left == "inherit": raise InputError( "Inheritance cannot be changed by broadcast") rdict = {} cur_dict = rdict tail = left while tail: match = REC_ITEM.match(tail) if match: sect, tail = match.groups() if tail: # [sect]... = right cur_dict[sect.strip()] = {} cur_dict = cur_dict[sect.strip()] else: # [sect] = right cur_dict[sect.strip()] = right else: # item = right cur_dict[tail.strip()] = right tail = None upg( {'runtime': {'__MANY__': rdict}}, 'test', broadcast='cancel' if for_cancel_broadcast else True, ) rdict = strip_empty_sections(rdict) # Perform validation, but don't coerce the original (deepcopy). cylc_config_validate(deepcopy(rdict), SPEC['runtime']['__MANY__']) return rdict def strip_empty_sections(data): """Recursively strip data structure of empty dicts. This is needed post-upgrade as the upgrader can leave empty sections behind. >>> strip_empty_sections( ... {'job': {}, 'thing limit': 'PT2S', 'whatever': ''}) {'thing limit': 'PT2S', 'whatever': ''} >>> strip_empty_sections({'section': {'subsection': {}}}) {} """ if not isinstance(data, dict): return data return { key: stripped for key, val in data.items() if (stripped := strip_empty_sections(val)) != {} } def files_to_settings(settings, setting_files, cancel_mode=False): """Parse setting files, and append to settings.""" cfg = ParsecConfig( SPEC['runtime']['__MANY__'], validator=cylc_config_validate) for setting_file in setting_files: if setting_file == '-': with NamedTemporaryFile() as handle: handle.write(sys.stdin.read().encode()) handle.seek(0, 0) cfg.loadcfg(handle.name) else: cfg.loadcfg(os.path.abspath(setting_file)) stack = [([], cfg.get(sparse=True))] while stack: keys, item = stack.pop() if isinstance(item, dict): for key, value in item.items(): stack.append((keys + [key], value)) else: settings.append({}) cur_setting = settings[-1] while keys: key = keys.pop(0) if keys: cur_setting[key] = {} cur_setting = cur_setting[key] elif cancel_mode: cur_setting[key] = None else: if isinstance(item, list): item = listjoin(item) else: item = str(item) cur_setting[key] = item def report_bad_options(bad_options, is_set=False): bad_opts = get_broadcast_bad_options_report(bad_options, is_set=is_set) if bad_opts is not None: return cparse(f'{bad_opts}') return bad_opts def get_option_parser() -> COP: """CLI for "cylc broadcast".""" parser = COP( __doc__, comms=True, multiworkflow=True, argdoc=[WORKFLOW_ID_MULTI_ARG_DOC], ) parser.add_option( "-p", "--point", metavar="CYCLE_POINT", help="Target cycle point. More than one can be added. " "Defaults to '*' with --set and --cancel, " "and nothing with --clear.", action="append", dest="point_strings", default=[]) parser.add_option( "-n", "--namespace", metavar="NAME", help="Target namespace. Defaults to 'root' with " "--set and --cancel, and nothing with --clear.", action="append", dest="namespaces", default=[]) parser.add_option( "-s", "--set", metavar="[SEC]ITEM=VALUE", help="A [runtime] config item and value to broadcast.", action="append", dest="settings", default=[]) parser.add_option( "-F", "--set-file", "--file", metavar="FILE", help="File with config to broadcast. Can be used multiple times.", action="append", dest="setting_files", default=[]) parser.add_option( "-c", "--cancel", metavar="[SEC]ITEM", help="An item-specific broadcast to cancel.", action="append", dest="cancel", default=[]) parser.add_option( "-G", "--cancel-file", metavar="FILE", help="File with broadcasts to cancel. Can be used multiple times.", action="append", dest="cancel_files", default=[]) parser.add_option( "-C", "--clear", help="Cancel all broadcasts, or with -p/--point, " "-n/--namespace, cancel all broadcasts to targeted " "namespaces and/or cycle points. Use \"-C -p '*'\" " "to cancel all all-cycle broadcasts without canceling " "all specific-cycle broadcasts.", action="store_true", dest="clear", default=False) parser.add_option( "-e", "--expire", metavar="CYCLE_POINT", help="Cancel any broadcasts that target cycle " "points earlier than, but not inclusive of, CYCLE_POINT.", action="store", default=None, dest="expire") parser.add_option( "-d", "--display", help="Display active broadcasts.", action="store_true", default=False, dest="show") parser.add_option( "-k", "--display-task", metavar="TASK_ID_GLOB", help=( "Print active broadcasts for a given task " "(in the format cycle/task)." ), action="store", default=None, dest="showtask") parser.add_option( "-b", "--box", help="Use unicode box characters with -d, -k.", action="store_true", default=False, dest="unicode") # BACK COMPAT: --raw # From: < 8.5.1 # To: 8.5.1 # Remove at: 8.9 parser.add_option( "-r", "--raw", help=( "With -d/--display or -k/--display-task, write out " "the broadcast config structure in raw Python form. " f"{RAW_DEPR_MSG}" ), action="store_true", default=False, dest="raw") parser.add_option( '--format', help=( "With -d/--display or -k/--display-task, write out " "the broadcast config structure in one of the following formats: " "tree, json, or raw (like json but as a Python dictionary). " r"Default: %default." ), action='store', dest='format', choices=('tree', 'json', 'raw'), default='tree', ) return parser async def run(options: 'Values', workflow_id): """Implement cylc broadcast.""" pclient = get_client(workflow_id, timeout=options.comms_timeout) # remove any duplicate namespaces # see https://github.com/cylc/cylc-flow/issues/6334 namespaces = list(set(options.namespaces)) ret: Dict[str, Any] = { 'stdout': [], 'stderr': [], 'exit': 0, } mutation_kwargs: Dict[str, Any] = { 'request_string': MUTATION, 'variables': { 'wFlows': [workflow_id], 'bMode': 'Set', 'cPoints': options.point_strings, 'nSpaces': namespaces, 'bSettings': options.settings, 'bCutoff': options.expire, } } query_kwargs: Dict[str, Any] = { 'request_string': QUERY, 'variables': { 'wFlows': [workflow_id], 'nIds': [] } } if options.raw: ret['stderr'].append(cparse(f"{RAW_DEPR_MSG}")) if options.show or options.showtask: if options.showtask: try: query_kwargs['variables']['nIds'] = [options.showtask] except ValueError: # TODO validate showtask? raise InputError( 'TASK_ID_GLOB must be in the format: cycle/task' ) from None result = await pclient.async_request('graphql', query_kwargs) for wflow in result['workflows']: settings = wflow['broadcasts'] padding = get_padding(settings) * ' ' if options.format == 'json': import json ret['stdout'].append( json.dumps(settings, indent=2, sort_keys=True) ) elif options.raw or options.format == 'raw': ret['stdout'].append(str(settings)) else: ret['stdout'].extend( get_tree(settings, padding, options.unicode) ) return ret report_cancel = True report_set = False if options.clear: mutation_kwargs['variables']['bMode'] = 'Clear' if options.expire: mutation_kwargs['variables']['bMode'] = 'Expire' # implement namespace and cycle point defaults here if not namespaces: namespaces = ["root"] point_strings = options.point_strings if not point_strings: point_strings = ["*"] if options.cancel or options.cancel_files: settings = [] for option_item in options.cancel: if "=" in option_item: raise InputError( "--cancel=[SEC]ITEM does not take a value") option_item = option_item.strip() if setting := get_rdict(option_item, for_cancel_broadcast=True): settings.append(setting) files_to_settings(settings, options.cancel_files, options.cancel) if not settings: raise InputError("No valid settings to broadcast.") mutation_kwargs['variables'].update( { 'bMode': 'Clear', 'cPoints': point_strings, 'nSpaces': namespaces, 'bSettings': settings, } ) if options.settings or options.setting_files: settings = [] for option_item in options.settings: if "=" not in option_item: raise InputError( "--set=[SEC]ITEM=VALUE requires a value") lhs, rhs = [s.strip() for s in option_item.split("=", 1)] if setting := get_rdict(lhs, rhs): settings.append(setting) files_to_settings(settings, options.setting_files) if not settings: raise InputError("No valid settings to broadcast.") mutation_kwargs['variables'].update( { 'bMode': 'Set', 'cPoints': point_strings, 'nSpaces': namespaces, 'bSettings': settings, } ) report_cancel = False report_set = True results = await pclient.async_request('graphql', mutation_kwargs) try: bad_options = {} for result in results['broadcast']['result']: modified_settings, bad_options = result['response'] if modified_settings: ret['stdout'].append( get_broadcast_change_report( modified_settings, is_cancel=report_cancel, ) ) bad_result = report_bad_options(bad_options, is_set=report_set) except (TypeError, ValueError): # Catch internal API server errors bad_result = cparse(f'{results}') if bad_result: ret['stderr'].append(f'ERROR: {bad_result}') ret['exit'] = 1 return ret def report(response): return ( '\n'.join(response['stdout']), '\n'.join(line for line in response['stderr'] if line is not None), response['exit'] == 0, ) @cli_function(get_option_parser) def main(_, options: 'Values', *ids) -> None: rets = asyncio.run(_main(options, *ids)) sys.exit(all(rets.values()) is False) async def _main(options: 'Values', *ids): return await call_multi_async( partial(run, options), *ids, constraint='workflows', report=report, ) cylc-flow-8.7.0/cylc/flow/scripts/validate_install_play.py0000664000175000017500000000712615264420307024130 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc validate-install-play [OPTIONS] ARGS Validate install and play a single workflow. This script is equivalent to: $ cylc validate /path/to/myworkflow $ cylc install /path/to/myworkflow $ cylc play myworkflow """ import asyncio import sys from cylc.flow import LOG from cylc.flow.scheduler_cli import PLAY_OPTIONS from cylc.flow.loggingutil import set_timestamps from cylc.flow.option_parsers import ( CylcOptionParser as COP, combine_options, cleanup_sysargv, log_subcommand, ) from cylc.flow.scheduler_cli import cylc_play from cylc.flow.scripts.validate import ( VALIDATE_OPTIONS, run as cylc_validate, ) from cylc.flow.scripts.install import ( INSTALL_OPTIONS, install_cli as cylc_install, get_source_location, ) from cylc.flow.terminal import cli_function from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from optparse import Values CYLC_ROSE_OPTIONS = COP.get_cylc_rose_options() VIP_OPTIONS = combine_options( VALIDATE_OPTIONS, INSTALL_OPTIONS, PLAY_OPTIONS, CYLC_ROSE_OPTIONS, modify={'cylc-rose': 'validate, install'} ) def get_option_parser() -> COP: parser = COP( __doc__, comms=True, jset=True, argdoc=[ COP.optional(( 'SOURCE_NAME | PATH', 'Workflow source name or path to source directory' )) ] ) for option in VIP_OPTIONS: # Make a special exception for option against_source which makes # no sense in a VIP context. if option.kwargs.get('dest') != 'against_source': parser.add_option(*option.args, **option.kwargs) parser.set_defaults(is_validate=True) return parser @cli_function(get_option_parser) def main(parser: COP, options: 'Values', workflow_id: Optional[str] = None): """Run Cylc validate - install - play in sequence.""" # NOTE: We call each of the stages in its own event loop because there is a # strange interaction whereby the cylc.flow.network.scan coroutine pipe can # cause sys.exit to hang on the original process post-daemonization if not workflow_id: workflow_id = '.' orig_source = workflow_id source = get_source_location(workflow_id) log_subcommand('validate', source) asyncio.run(cylc_validate(options, str(source))) # Unset is validate after validation. del options.is_validate log_subcommand('install', source) _, workflow_id = asyncio.run(cylc_install(options, workflow_id)) cleanup_sysargv( 'play', workflow_id, options, compound_script_opts=VIP_OPTIONS, script_opts=(*PLAY_OPTIONS, *parser.get_std_options()), source=orig_source, ) set_timestamps(LOG, options.log_timestamp) log_subcommand(*sys.argv[1:]) cylc_play(options, workflow_id) cylc-flow-8.7.0/cylc/flow/scripts/get_resources.py0000775000175000017500000000535715264420307022444 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc get-resources [OPTIONS] ARGS Extract resources from the cylc.flow package. If the directory is omitted the resource will be copied either to your current working directory for regular resources, or to the first configured "source" directory (~/cylc-src by default) for the tutorials. Examples: # list all resources $ cylc get-resources --list # copy the Cylc wrapper script to the current directory: $ cylc get-resources cylc # copy the Cylc wrapper script to a/b/c: $ cylc get-resources cylc a/b/c # copy the "runtime-tutorial" to your "source" directory: $ cylc get-resources tutorial/runtime-tutorial # copy all of the tutorials to your "source" directory: $ cylc get-resources tutorial """ import sys from cylc.flow import LOG from cylc.flow.exceptions import CylcError import cylc.flow.flags from cylc.flow.loggingutil import set_timestamps from cylc.flow.option_parsers import CylcOptionParser as COP from cylc.flow.resources import get_resources, list_resources from cylc.flow.terminal import cli_function def get_option_parser(): parser = COP( __doc__, argdoc=[ COP.optional( ('RESOURCE', 'Resource to extract.') ), COP.optional( ('DIR', 'Target directory.') ) ] ) parser.add_option( '--list', help="List available resources.", default=False, action='store_true', ) return parser @cli_function(get_option_parser) def main(parser, opts, resource=None, tgt_dir=None): # Intercept requests for syntax/cylc.vim: if resource == "syntax/cylc.vim": raise CylcError( 'syntax/cylc.vim has been replaced by ' 'https://github.com/cylc/cylc.vim' ) if cylc.flow.flags.verbosity < 2: set_timestamps(LOG, False) if not resource or opts.list: list_resources() sys.exit(0) get_resources(resource, tgt_dir) cylc-flow-8.7.0/cylc/flow/scripts/kill.py0000775000175000017500000000505415264420307020520 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc kill [OPTIONS] ARGS Kill running or submitted jobs. Examples: # kill a specific task in my_workflow $ cylc kill my_workflow//1/a # kill multiple tasks in my_workflow $ cylc kill myflow// //1/a //1/b //1/c # kill all active tasks in the my_workflow $ cylc kill 'my_workflow//*' """ from functools import partial import sys from typing import TYPE_CHECKING from cylc.flow.network.client_factory import get_client from cylc.flow.network.multi import call_multi from cylc.flow.option_parsers import ( FULL_ID_MULTI_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values from cylc.flow.id import Tokens MUTATION = ''' mutation ( $wFlows: [WorkflowID]!, $tasks: [NamespaceIDGlob]!, ) { kill ( workflows: $wFlows, tasks: $tasks, ) { result } } ''' def get_option_parser() -> 'COP': parser = COP( __doc__, comms=True, multitask=True, multiworkflow=True, argdoc=[FULL_ID_MULTI_ARG_DOC], ) return parser async def run(options: 'Values', workflow_id: str, *tokens_list: 'Tokens'): pclient = get_client(workflow_id, timeout=options.comms_timeout) mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [workflow_id], 'tasks': [ tokens.relative_id_with_selectors for tokens in tokens_list ] } } return await pclient.async_request('graphql', mutation_kwargs) @cli_function(get_option_parser) def main(parser: 'COP', options: 'Values', *ids: str): """CLI of "cylc kill".""" rets = call_multi( partial(run, options), *ids, ) sys.exit(all(rets.values()) is False) cylc-flow-8.7.0/cylc/flow/scripts/diff.py0000775000175000017500000001357315264420307020502 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc diff [OPTIONS] ARGS Compare two workflow configurations and display any differences. Differencing is done after parsing the flow.cylc files so it takes account of default values that are not explicitly defined, it disregards the order of configuration items, and it sees any include-file content after inlining has occurred. Files in the workflow bin directory and other sub-directories of the run directory are not currently differenced. """ import sys from typing import TYPE_CHECKING from cylc.flow.id_cli import parse_id from cylc.flow.option_parsers import ( WORKFLOW_ID_OR_PATH_ARG_DOC, CylcOptionParser as COP, icp_option, ) from cylc.flow.config import WorkflowConfig from cylc.flow.templatevars import get_template_vars from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values n_oone = 0 n_otwo = 0 n_diff = 0 def diffdict(one, two, oone, otwo, diff): global n_oone, n_otwo, n_diff # Recursively difference two dictionaries in which any element # may be another dictionary, keeping items that appear only # in one or the other, and items that appear in both but differ. for key in one: if key not in two: oone[key] = one[key] n_oone += 1 elif one[key] != two[key]: if isinstance(one[key], dict): for item in oone, otwo, diff: if key not in item: item[key] = {} diffdict(one[key], two[key], oone[key], otwo[key], diff[key]) else: diff[key] = (one[key], two[key]) n_diff += 1 for key in two: if key not in one: otwo[key] = two[key] n_otwo += 1 def prdict(dct, arrow='<', section='', level=0, diff=False, nested=False): """Recursively print, in pseudo 'diff' format, the contents of one of the three dictionaries populated by the diffdict() function above (any element may itself be a dictionary). """ if section != '': prfx = section + ' ' else: prfx = '' if section == '': sctn = '(top)' else: sctn = section foo = False for key in dct: if isinstance(dct[key], dict): lvl = level + 1 if nested: pre = prfx + '\n' + ' ' * lvl else: pre = prfx prdict(dct[key], arrow, pre + '[' * lvl + str(key) + ']' * lvl, lvl, diff, nested) else: if not foo: if nested: print(' ', sctn) else: print('\n ', sctn) foo = True if diff: print(' < ', key, '=', dct[key][0]) print(' > ', key, '=', dct[key][1]) else: print(' ' + arrow + ' ', key, '=', dct[key]) def get_option_parser() -> COP: parser = COP( __doc__, jset=True, argdoc=[ (f'WORKFLOW_{n}', WORKFLOW_ID_OR_PATH_ARG_DOC[1]) for n in (1, 2) ] ) parser.add_option( "-n", "--nested", help="print flow.cylc section headings in nested form.", action="store_true", default=False, dest="nested" ) parser.add_option(icp_option) return parser @cli_function(get_option_parser) def main(parser: COP, options: 'Values', workflow_id1: str, workflow_id2: str): workflow_id_1, _, workflow_file_1_ = parse_id( workflow_id1, src=True, constraint='workflows', ) workflow_id_2, _, workflow_file_2_ = parse_id( workflow_id2, src=True, constraint='workflows', ) if workflow_file_1_ == workflow_file_2_: parser.error("You can't diff a single workflow.") print(f"Parsing {workflow_id_1} ({workflow_file_1_})") template_vars = get_template_vars(options) config1 = WorkflowConfig( workflow_id_1, workflow_file_1_, options, template_vars ).cfg print(f"Parsing {workflow_id_2} ({workflow_file_2_})") config2 = WorkflowConfig( workflow_id_2, workflow_file_2_, options, template_vars ).cfg if config1 == config2: print( f"Workflow definitions {workflow_id_1} and {workflow_id_2} are " f"identical" ) sys.exit(0) print(f"Workflow definitions {workflow_id_1} and {workflow_id_2} differ") workflow1_only = {} # type: ignore workflow2_only = {} # type: ignore diff_1_2 = {} # type: ignore # TODO: this whole file could do with refactoring at some point diffdict(config1, config2, workflow1_only, workflow2_only, diff_1_2) if n_oone > 0: print(f'\n{n_oone} items only in {workflow_id_1} (<)') prdict(workflow1_only, '<', nested=options.nested) if n_otwo > 0: print(f'\n{n_otwo} items only in {workflow_id_2} (>)') prdict(workflow2_only, '>', nested=options.nested) if n_diff > 0: print(f'\n{n_diff} common items differ {workflow_id_1}(<) ' f'{workflow_id_2}(>)') prdict(diff_1_2, '', diff=True, nested=options.nested) cylc-flow-8.7.0/cylc/flow/scripts/client.py0000775000175000017500000000525615264420307021047 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc client [OPTIONS] ARGS (This command is for internal use.) Invoke workflow runtime client, expect JSON from STDIN for keyword arguments. Use the -n option if client function requires no keyword arguments. """ from google.protobuf.json_format import MessageToDict import json import sys from typing import TYPE_CHECKING, cast from cylc.flow.id_cli import parse_id from cylc.flow.option_parsers import ( WORKFLOW_ID_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.network.client import WorkflowRuntimeClient from cylc.flow.network.server import PB_METHOD_MAP from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values from google.protobuf.message import Message INTERNAL = True def get_option_parser(): parser = COP( __doc__, comms=True, argdoc=[ WORKFLOW_ID_ARG_DOC, ('METHOD', 'Network API function name') ] ) parser.add_option( '-n', '--no-input', help='Do not read from STDIN, assume null input', action='store_true', dest='no_input') return parser @cli_function(get_option_parser) def main(_, options: 'Values', workflow_id: str, func: str) -> None: workflow_id, *_ = parse_id( workflow_id, constraint='workflows', ) pclient = WorkflowRuntimeClient(workflow_id, timeout=options.comms_timeout) if options.no_input: kwargs = {} else: kwargs = json.load(sys.stdin) sys.stdin.close() res = pclient(func, kwargs) if func in PB_METHOD_MAP: pb_msg: Message if 'element_type' in kwargs: pb_msg = PB_METHOD_MAP[func][kwargs['element_type']]() else: pb_msg = PB_METHOD_MAP[func]() pb_msg.ParseFromString(cast('bytes', res)) res_msg: object = MessageToDict(pb_msg) else: res_msg = res sys.stdout.write(json.dumps(res_msg, indent=4) + '\n') cylc-flow-8.7.0/cylc/flow/scripts/get_workflow_version.py0000775000175000017500000000455715264420307024052 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc get-workflow-version [OPTIONS] ARGS Find out what version of Cylc a running workflow is using. To find the version you've invoked at the command line see "cylc version". """ from functools import partial from typing import TYPE_CHECKING import sys from cylc.flow.network.client_factory import get_client from cylc.flow.network.multi import call_multi from cylc.flow.option_parsers import ( WORKFLOW_ID_MULTI_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values QUERY = ''' query ($wFlows: [ID]) { workflows(ids: $wFlows) { id name owner cylcVersion } } ''' def get_option_parser(): parser = COP( __doc__, comms=True, multiworkflow=True, argdoc=[WORKFLOW_ID_MULTI_ARG_DOC], ) return parser async def run(options, workflow_id, *_): pclient = get_client(workflow_id, timeout=options.comms_timeout) query_kwargs = { 'request_string': QUERY, 'variables': {'wFlows': [workflow_id]} } result = await pclient.async_request('graphql', query_kwargs) for workflow in result['workflows']: return workflow['cylcVersion'] @cli_function(get_option_parser) def main(parser: COP, options: 'Values', workflow_id: str) -> None: rets = call_multi( partial(run, options), workflow_id, report=lambda x: (x, None, True), # we need the mixed format for call_multi but don't want any tasks constraint='mixed', max_tasks=0, ) sys.exit(all(rets.values()) is False) cylc-flow-8.7.0/cylc/flow/scripts/jobs_poll.py0000775000175000017500000000327115264420307021547 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc jobs-poll [OPTIONS] ARGS (This command is for internal use. Users should use "cylc poll".) Read job status files to obtain the statuses of the jobs. If necessary, Invoke the relevant job runner commands to ask the job runners for more statuses. """ from cylc.flow.job_runner_mgr import JobRunnerManager from cylc.flow.option_parsers import CylcOptionParser as COP from cylc.flow.terminal import cli_function INTERNAL = True def get_option_parser() -> COP: parser = COP( __doc__, argdoc=[ ("JOB-LOG-ROOT", "The log/job sub-directory for the workflow"), COP.optional( ("JOB-LOG-DIR ...", "A point/name/submit_num sub-directory") ) ], ) return parser @cli_function(get_option_parser) def main(parser, options, job_log_root, *job_log_dirs): """CLI main.""" JobRunnerManager().jobs_poll(job_log_root, job_log_dirs) cylc-flow-8.7.0/cylc/flow/scripts/pause.py0000664000175000017500000000462115264420307020676 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc pause [OPTIONS] ARGS Suspend automatic job submission. Manual triggering can still run tasks immediately, even if the workflow is paused. Examples: # pause my_workflow $ cylc pause my_workflow # resume my_workflow $ cylc play my_workflow (Not to be confused with `cylc hold` which suspends submission of individual tasks within a workflow). """ from functools import partial from typing import TYPE_CHECKING import sys from cylc.flow.option_parsers import ( WORKFLOW_ID_MULTI_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.network.client import WorkflowRuntimeClient from cylc.flow.network.multi import call_multi from cylc.flow.terminal import cli_function if TYPE_CHECKING: from optparse import Values MUTATION = ''' mutation ( $wFlows: [WorkflowID]! ) { pause ( workflows: $wFlows ) { result } } ''' def get_option_parser() -> COP: parser = COP( __doc__, comms=True, multitask=False, multiworkflow=True, argdoc=[WORKFLOW_ID_MULTI_ARG_DOC], ) return parser async def run(options: 'Values', workflow_id: str): pclient = WorkflowRuntimeClient(workflow_id, timeout=options.comms_timeout) mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [workflow_id], } } return await pclient.async_request('graphql', mutation_kwargs) @cli_function(get_option_parser) def main(parser: COP, options: 'Values', *ids) -> None: rets = call_multi( partial(run, options), *ids, constraint='workflows', ) sys.exit(all(rets.values()) is False) cylc-flow-8.7.0/cylc/flow/scripts/set.py0000775000175000017500000002055315264420307020361 0ustar alastairalastair#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) Earth Sciences New Zealand & British Crown (Met Office) # & Contributors. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """cylc set [OPTIONS] ARGS Manually complete task outputs, and satisfy task and xtrigger prerequisites. By default this command completes required outputs, plus the "submitted", "started", and "succeeded" outputs even if they are optional. Setting prerequisites promotes target tasks to the n=0 active window; setting outputs also sets the prerequisites of any tasks that depend on those outputs. Note: state selectors such as ":failed" determine which tasks to target, not the outputs to set - see examples below. Note: see `cylc trigger` command help if you want rerun a sub-graph of tasks. Setting Outputs: Completing outputs may affect task state, and it will satisfy prerequisites of any other tasks that depend on those outputs. Format: * --out= # output name (not message) of the target task Completing a final output ("succeeded", "failed", or "expired") sets the state of the target task accordingly. Completing "started", "submitted", or custom outputs does not affect state because no job is actually running. Implied outputs are completed automatically: - "started" implies "submitted" - "succeeded" and "failed" imply "started" - custom outputs and "expired" do not imply other outputs For custom outputs, use the output name, not the associated task message. In [runtime][my-task][outputs]: # = x = "file x completed" Setting Task Prerequisites: Satisfying a task's prerequisite contributes to its readiness to run. Satisfying any prerequisite of an inactive task promotes it to the active window where xtrigger checking commences (if the task has any xtriggers). The --pre=all option promotes even parentless tasks to the active window. Format: * --pre=/[:output] # satisfy a single prerequisite * --pre=all # satisfy all task (not xtrigger) prerequisites Setting Xtrigger Prerequisites: To satisfy an xtrigger prerequisite use --pre with the word "xtrigger" in place of the cycle point, and the xtrigger name in place of the task name. Format: * --pre=xtrigger/ # satisfy one xtrigger prerequisite * --pre=xtrigger/all # satisfy all xtrigger prerequisites (The full format is "xtrigger/[:succeeded]" but "succeeded" is the only valid xtrigger output and the default, so it can be omitted.) Flow numbers of tasks affected by the set command are determined as follows Active tasks (n=0) already have existing flow numbers. * default: merge existing active flow numbers * --flow=INT or "new": merge existing and new flow numbers * --flow="none": ERROR (not valid for already-active tasks) Inactive tasks (n>0) do not have flow numbers assigned: * default: assign all active flow numbers * --flow=INT or "new": assign the given flow numbers * --flow="none": action the command with no flow numbers, which may result in a no-flow task running (activity will not flow on downstream of it). CLI Completion: Cylc can auto-complete prerequisite and output names for tasks in the n=0 window, if you type the task name before attempting TAB-completion. Examples: # complete all required outputs of 3/bar: $ cylc set my_workflow//3/bar # or: $ cylc set --out=required my_workflow//3/bar # complete the succeeded output of 3/bar: $ cylc set --out=succeeded my_workflow//3/bar # complete the outputs defined in [runtime][task][skip] $ cylc set --out=skip my_workflow//3/bar # satisfy the 3/foo:succeeded prerequisite of 3/bar: $ cylc set --pre=3/foo my_workflow//3/bar # or, $ cylc set --pre=3/foo:succeeded my_workflow//3/bar # satisfy all prerequisites (if any) of 3/bar: $ cylc set --pre=all my_workflow//3/bar # satisfy clock trigger prerequisite @clock1 3000/bar: $ cylc set --pre=xtrigger/clock1 my_worklfow//3000/bar # satisfy the "3/bar:file1" prerequisite of 3/qux: $ cylc set --pre=3/bar:file1 my_workflow//3/qux # complete multiple outputs at once: $ cylc set --out=a --out=b,c my_workflow//3/bar # satisfy multiple prerequisites at once: $ cylc set --pre=3/foo:x --pre=3/foo:y,3/foo:z my_workflow//3/bar # set required outputs of all failed tasks in the n=0 window at cycle 3: $ cylc set my_workflow//3/*:failed # set the succeeded output of all failed tasks in the n=0 window at cycle 3: $ cylc set my_workflow//3/*:failed --output=succeeded """ from functools import partial import sys from typing import TYPE_CHECKING from cylc.flow.network.client_factory import get_client from cylc.flow.network.multi import call_multi from cylc.flow.option_parsers import ( FULL_ID_MULTI_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.terminal import ( cli_function, flatten_cli_lists ) from cylc.flow.flow_mgr import add_flow_opts_for_trigger_and_set if TYPE_CHECKING: from optparse import Values from cylc.flow.id import Tokens # For setting xtriggers with --pre: XTRIGGER_PREREQ_PREFIX = "xtrigger" MUTATION = ''' mutation ( $wFlows: [WorkflowID]!, $tasks: [NamespaceIDGlob]!, $prerequisites: [PrerequisiteString], $outputs: [OutputLabel], $flow: [Flow!], $flowWait: Boolean, $flowDescr: String, ) { set ( workflows: $wFlows, tasks: $tasks, prerequisites: $prerequisites, outputs: $outputs, flow: $flow, flowWait: $flowWait, flowDescr: $flowDescr ) { result } } ''' def get_option_parser() -> COP: parser = COP( __doc__, comms=True, multitask=True, multiworkflow=True, argdoc=[FULL_ID_MULTI_ARG_DOC], ) parser.add_option( "-o", "--out", "--output", metavar="OUTPUT(s)", help=( "Complete task outputs. For multiple outputs re-use the" " option, or give a comma-separated list of outputs." " Use '--out=required' to complete all required outputs." " Use '--out=skip' to complete outputs defined in the task's" " [skip] configuration." " OUTPUT format: trigger names as used in the graph." ), action="append", default=None, dest="outputs" ) parser.add_option( "-p", "--pre", "--prerequisite", metavar="PREREQUISITE(s)", help=( "Satisfy task prerequisites. For multiple prerequisites" " re-use the option, or give a comma-separated list, or" ' use "--pre=all" to satisfy all prerequisites, if any.' " PREREQUISITE format:" " '/[:]' or 'all'" " where defaults to succeeded; or" " xtrigger/