pax_global_header00006660000000000000000000000064147270253710014522gustar00rootroot0000000000000052 comment=652bc8d2bc1e5d83e0761438b82bd402f6cc9659 aio-libs-janus-3158ccb/000077500000000000000000000000001472702537100147515ustar00rootroot00000000000000aio-libs-janus-3158ccb/.coveragerc000066400000000000000000000001341472702537100170700ustar00rootroot00000000000000[run] branch = True source = janus, tests omit = site-packages [html] directory = coverage aio-libs-janus-3158ccb/.github/000077500000000000000000000000001472702537100163115ustar00rootroot00000000000000aio-libs-janus-3158ccb/.github/FUNDING.yml000066400000000000000000000016561472702537100201360ustar00rootroot00000000000000# These are supported funding model platforms github: asvetlov # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry polar: # Replace with a single Polar username buy_me_a_coffee: # Replace with a single Buy Me a Coffee username thanks_dev: gh/asvetlov # Replace with a single thanks.dev username custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] aio-libs-janus-3158ccb/.github/dependabot.yml000066400000000000000000000003061472702537100211400ustar00rootroot00000000000000version: 2 updates: - package-ecosystem: pip directory: "/" schedule: interval: daily - package-ecosystem: "github-actions" directory: "/" schedule: interval: daily aio-libs-janus-3158ccb/.github/workflows/000077500000000000000000000000001472702537100203465ustar00rootroot00000000000000aio-libs-janus-3158ccb/.github/workflows/auto-merge.yml000066400000000000000000000011401472702537100231320ustar00rootroot00000000000000name: Dependabot auto-merge on: pull_request_target permissions: pull-requests: write contents: write jobs: dependabot: runs-on: ubuntu-latest if: ${{ github.actor == 'dependabot[bot]' }} steps: - name: Dependabot metadata id: metadata uses: dependabot/fetch-metadata@v2.2.0 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs run: gh pr merge --auto --squash "$PR_URL" env: PR_URL: ${{github.event.pull_request.html_url}} GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} aio-libs-janus-3158ccb/.github/workflows/ci.yml000066400000000000000000000136451472702537100214750ustar00rootroot00000000000000name: CI on: push: branches: [ master ] tags: [ 'v*' ] pull_request: branches: [ master ] schedule: - cron: '0 6 * * *' # Daily 6AM UTC build env: COLOR: >- # Supposedly, pytest or coveragepy use this yes FORCE_COLOR: 1 # Request colored output from CLI tools supporting it MYPY_FORCE_COLOR: 1 # MyPy's color enforcement PIP_DISABLE_PIP_VERSION_CHECK: 1 PIP_NO_PYTHON_VERSION_WARNING: 1 PIP_NO_WARN_SCRIPT_LOCATION: 1 PRE_COMMIT_COLOR: always PROJECT_NAME: janus PY_COLORS: 1 # Recognized by the `py` package, dependency of `pytest` PYTHONIOENCODING: utf-8 PYTHONUTF8: 1 jobs: lint: name: Linter runs-on: ubuntu-latest timeout-minutes: 5 steps: - name: Checkout uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v5 with: python-version: 3.13 - name: Cache PyPI uses: actions/cache@v4 with: key: pip-lint-${{ hashFiles('requirements-dev.txt') }} path: ~/.cache/pip restore-keys: | pip-lint- - name: Install dependencies uses: py-actions/py-dependency-install@v4.1.0 with: path: requirements-dev.txt - name: Run linters run: | make lint unit: name: Unit strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] os: [ubuntu] fail-fast: false runs-on: ${{ matrix.os }}-latest timeout-minutes: 15 steps: - name: Checkout uses: actions/checkout@v4 - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Get pip cache dir id: pip-cache run: | echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT # - name: Cache shell: bash - name: Cache PyPI uses: actions/cache@v4 with: key: pip-ci-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements-dev.txt') }} path: ${{ steps.pip-cache.outputs.dir }} restore-keys: | pip-ci-${{ runner.os }}-${{ matrix.python-version }}- - name: Install dependencies uses: py-actions/py-dependency-install@v4.1.0 with: path: requirements-dev.txt - name: Run unittests env: COLOR: 'yes' run: | pytest --cov=janus --cov=tests --cov-report=term --cov-report=xml:coverage.xml - name: Upload coverage artifact uses: aio-libs/prepare-coverage@v24.9.2 benchmark: name: Benchmark runs-on: ubuntu-latest timeout-minutes: 15 steps: - name: Checkout uses: actions/checkout@v4 - name: Setup Python 3.13 uses: actions/setup-python@v5 with: python-version: 3.13 - name: Get pip cache dir id: pip-cache run: | echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT # - name: Cache shell: bash - name: Cache PyPI uses: actions/cache@v4 with: key: pip-ci-ubuntu-3.13-${{ hashFiles('requirements-dev.txt') }} path: ${{ steps.pip-cache.outputs.dir }} restore-keys: | pip-ci-ubuntu-3.13- - name: Install dependencies uses: py-actions/py-dependency-install@v4.1.0 with: path: requirements-dev.txt - name: Run benchmarks uses: CodSpeedHQ/action@v3 with: token: ${{ secrets.CODSPEED_TOKEN }} run: python -Im pytest --no-cov -vvvvv --codspeed check: # The branch protection check if: always() needs: [lint, unit, benchmark] runs-on: ubuntu-latest steps: - name: Decide whether the needed jobs succeeded or failed uses: re-actors/alls-green@release/v1 with: jobs: ${{ toJSON(needs) }} - name: Checkout uses: actions/checkout@v4 - name: Upload coverage uses: aio-libs/upload-coverage@v24.10.1 with: token: ${{ secrets.CODECOV_TOKEN }} deploy: name: Deploy needs: check runs-on: ubuntu-latest # Run only on pushing a tag if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') permissions: contents: write # IMPORTANT: mandatory for making GitHub Releases id-token: write # IMPORTANT: mandatory for trusted publishing & sigstore environment: name: pypi url: https://pypi.org/p/${{ env.PROJECT_NAME }} steps: - name: Checkout uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v5 with: python-version: 3.13 - name: Install dependencies uses: py-actions/py-dependency-install@v4.1.0 with: path: requirements-dev.txt - name: Install builder run: | python -m pip install wheel build - name: Make dists run: | python -m build - name: Login run: | echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token - name: Make Release uses: aio-libs/create-release@v1.6.6 with: changes_file: CHANGES.rst version_file: ${{ env.PROJECT_NAME }}/__init__.py github_token: ${{ secrets.GITHUB_TOKEN }} head_line: >- {version} \({date}\)\n # fix_issue_regex: >- # :issue:`(\d+)` # fix_issue_repl: >- # #\1 - name: >- Publish 🐍📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - name: Sign the dists with Sigstore uses: sigstore/gh-action-sigstore-python@v3.0.0 with: inputs: >- ./dist/*.tar.gz ./dist/*.whl - name: Upload artifact signatures to GitHub Release # Confusingly, this action also supports updating releases, not # just creating them. This is what we want here, since we've manually # created the release above. uses: softprops/action-gh-release@v2 with: # dist/ contains the built packages, which smoketest-artifacts/ # contains the signatures and certificates. files: dist/** aio-libs-janus-3158ccb/.gitignore000066400000000000000000000013761472702537100167500ustar00rootroot00000000000000# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *,cover # Translations *.mo *.pot # Django stuff: *.log # Sphinx documentation docs/_build/ # PyBuilder target/ coverage .pytest_cache/ .mypy_cache/ # pyenv .python-version aio-libs-janus-3158ccb/.style.yapf000066400000000000000000000000361472702537100170470ustar00rootroot00000000000000[style] based_on_style = pep8 aio-libs-janus-3158ccb/CHANGES.rst000066400000000000000000000065441472702537100165640ustar00rootroot00000000000000Changes ======= .. You should *NOT* be adding new change log entries to this file, this file is managed by towncrier. You *may* edit previous change logs to fix problems like typo corrections or such. To add a new change log entry, please see https://pip.pypa.io/en/latest/development/#adding-a-news-entry we named the news folder "changes". WARNING: Don't drop the next directive! .. towncrier release notes start 2.0.0 (2024-12-13) ------------------ - Implement ``.shutdown(immediate=False)`` for both sync and async APIs #720 The change is not fully backward compatible: 1. If the queue is closed, ``janus.AsyncQueueShutDown`` and ``janus.SyncQueueShutDown`` exceptions are raised instead of ``RuntimeError``. 2. Both sync and async ``.task_done()`` and ``.join()`` don't raise any exception on queue shutdown/closing anymore; it is compatible with shutdown behavior of stdlib sync and async queues. 1.2.0 (2024-12-12) ------------------ - Optimize internal implementation for a little speedup #699 - Make not-full and not-empty notifications faster #703 - Add ``.aclose()`` async method #709 - Reduce notifications for a minor speedup #704 - Allow ``janus.Queue()`` instantiation without running asyncio event loop #710 - Remove sync notifiers for a major speedup #714 - Fix hang in ``AsyncQueue.join()`` #716 1.1.0 (2024-10-30) ------------------ - Drop Python 3.7 and 3.8 support - janus now works on Python 3.9-3.13 - Reexport SyncQueueEmpty, SyncQueueFull, AsyncQueueEmpty, and AsyncQueueFull names #680 1.0.0 (2021-12-17) ------------------ - Drop Python 3.6 support 0.7.0 (2021-11-24) ------------------ - Add SyncQueue and AsyncQueue Protocols to provide type hints for sync and async queues #374 0.6.2 (2021-10-24) ------------------ - Fix Python 3.10 compatibility #358 0.6.1 (2020-10-26) ------------------ - Raise RuntimeError on queue.join() after queue closing. #295 - Replace ``timeout`` type from ``Optional[int]`` to ``Optional[float]`` #267 0.6.0 (2020-10-10) ------------------ - Drop Python 3.5, the minimal supported version is Python 3.6 - Support Python 3.9 - Refomat with ``black`` 0.5.0 (2020-04-23) ------------------ - Remove explicit loop arguments and forbid creating queues outside event loops #246 0.4.0 (2018-07-28) ------------------ - Add ``py.typed`` macro #89 - Drop python 3.4 support and fix minimal version python3.5.3 #88 - Add property with that indicates if queue is closed #86 0.3.2 (2018-07-06) ------------------ - Fixed python 3.7 support #97 0.3.1 (2018-01-30) ------------------ - Fixed bug with join() in case tasks are added by sync_q.put() #75 0.3.0 (2017-02-21) ------------------ - Expose `unfinished_tasks` property #34 0.2.4 (2016-12-05) ------------------ - Restore tarball deploying 0.2.3 (2016-07-12) ------------------ - Fix exception type 0.2.2 (2016-07-11) ------------------ - Update asyncio.async() to use asyncio.ensure_future() #6 0.2.1 (2016-03-24) ------------------ - Fix `python setup.py test` command #4 0.2.0 (2015-09-20) ------------------ - Support Python 3.5 0.1.5 (2015-07-24) ------------------ - Use loop.time() instead of time.monotonic() 0.1.1 (2015-06-12) ------------------ - Fix some typos in README and setup.py - Add addtional checks for loop closing - Mention DataRobot 0.1.0 (2015-06-11) ------------------ - Initial release aio-libs-janus-3158ccb/LICENSE000066400000000000000000000261501472702537100157620ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2015-2018 Andrew Svetlov and aio-libs team Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. aio-libs-janus-3158ccb/MANIFEST.in000066400000000000000000000002071472702537100165060ustar00rootroot00000000000000include LICENSE include CHANGES.rst include README.rst include Makefile graft janus graft tests global-exclude *.pyc prune docs/_build aio-libs-janus-3158ccb/Makefile000066400000000000000000000007661472702537100164220ustar00rootroot00000000000000develop: python setup.py develop lint flake: checkrst pyroma bandit mypy flake8 janus tests test: flake develop pytest tests vtest: flake develop pytest -v tests fmt: isort -rc janus tests setup.py black janus tests setup.py cov: flake develop pytest --cov=janus --cov=tests --cov-report=term --cov-report=html @echo "open file://`pwd`/coverage/index.html" checkrst: python setup.py check --restructuredtext pyroma: pyroma -d . bandit: bandit -r ./janus mypy: mypy janus --strict aio-libs-janus-3158ccb/README.rst000066400000000000000000000073061472702537100164460ustar00rootroot00000000000000======= janus ======= .. image:: https://github.com/aio-libs/janus/actions/workflows/ci.yml/badge.svg :target: https://github.com/aio-libs/janus/actions/workflows/ci.yml .. image:: https://codecov.io/gh/aio-libs/janus/branch/master/graph/badge.svg :target: https://codecov.io/gh/aio-libs/janus .. image:: https://img.shields.io/pypi/v/janus.svg :target: https://pypi.python.org/pypi/janus .. image:: https://badges.gitter.im/Join%20Chat.svg :target: https://gitter.im/aio-libs/Lobby :alt: Chat on Gitter Mixed sync-async queue, supposed to be used for communicating between classic synchronous (threaded) code and asynchronous (in terms of `asyncio `_) one. Like `Janus god `_ the queue object from the library has two faces: synchronous and asynchronous interface. Synchronous is fully compatible with `standard queue `_, asynchronous one follows `asyncio queue design `_. Usage ===== Three queues are available: * ``Queue`` * ``LifoQueue`` * ``PriorityQueue`` Each has two properties: ``sync_q`` and ``async_q``. Use the first to get synchronous interface and the second to get asynchronous one. Example ------- .. code:: python import asyncio import janus def threaded(sync_q: janus.SyncQueue[int]) -> None: for i in range(100): sync_q.put(i) sync_q.join() async def async_coro(async_q: janus.AsyncQueue[int]) -> None: for i in range(100): val = await async_q.get() assert val == i async_q.task_done() async def main() -> None: queue: janus.Queue[int] = janus.Queue() loop = asyncio.get_running_loop() fut = loop.run_in_executor(None, threaded, queue.sync_q) await async_coro(queue.async_q) await fut await queue.aclose() asyncio.run(main()) Limitations =========== This library is built using a classic thread-safe design. The design is time-tested, but has some limitations. * Once you are done working with a queue, you must properly close it using ``aclose()``. This is because this library creates new tasks to notify other threads. If you do not properly close the queue, `asyncio may generate error messages `_. * The library has quite good performance only when used as intended, that is, for communication between synchronous code and asynchronous one. For sync-only and async-only cases, use queues from `queue `_ and `asyncio queue `_ modules, otherwise `the slowdown can be significant `_. * You cannot use queues for communicating between two different event loops because, like all asyncio primitives, they bind to the current one. Development status is production/stable. The ``janus`` library is maintained to support the latest versions of Python and fixes, but no major changes will be made. If your application is performance-sensitive, or if you need any new features such as ``anyio`` support, try the experimental `culsans `_ library as an alternative. Communication channels ====================== GitHub Discussions: https://github.com/aio-libs/janus/discussions Feel free to post your questions and ideas here. *gitter chat* https://gitter.im/aio-libs/Lobby License ======= ``janus`` library is offered under Apache 2 license. Thanks ====== The library development is sponsored by DataRobot (https://datarobot.com) aio-libs-janus-3158ccb/janus/000077500000000000000000000000001472702537100160715ustar00rootroot00000000000000aio-libs-janus-3158ccb/janus/__init__.py000066400000000000000000000633521472702537100202130ustar00rootroot00000000000000import asyncio import sys import threading from asyncio import QueueEmpty as AsyncQueueEmpty from asyncio import QueueFull as AsyncQueueFull from collections import deque from heapq import heappop, heappush from queue import Empty as SyncQueueEmpty from queue import Full as SyncQueueFull from time import monotonic from typing import Callable, Generic, Optional, Protocol, TypeVar if sys.version_info >= (3, 13): from asyncio import QueueShutDown as AsyncQueueShutDown from queue import ShutDown as SyncQueueShutDown else: class QueueShutDown(Exception): pass AsyncQueueShutDown = QueueShutDown class ShutDown(Exception): pass SyncQueueShutDown = ShutDown __version__ = "2.0.0" __all__ = ( "Queue", "PriorityQueue", "LifoQueue", "SyncQueue", "SyncQueueEmpty", "SyncQueueFull", "SyncQueueShutDown", "AsyncQueue", "AsyncQueueEmpty", "AsyncQueueFull", "AsyncQueueShutDown", "BaseQueue", ) T = TypeVar("T") OptFloat = Optional[float] class BaseQueue(Protocol[T]): @property def maxsize(self) -> int: ... @property def closed(self) -> bool: ... def task_done(self) -> None: ... def qsize(self) -> int: ... @property def unfinished_tasks(self) -> int: ... def empty(self) -> bool: ... def full(self) -> bool: ... def put_nowait(self, item: T) -> None: ... def get_nowait(self) -> T: ... def shutdown(self, immediate: bool = False) -> None: ... class SyncQueue(BaseQueue[T], Protocol[T]): def put(self, item: T, block: bool = True, timeout: OptFloat = None) -> None: ... def get(self, block: bool = True, timeout: OptFloat = None) -> T: ... def join(self) -> None: ... class AsyncQueue(BaseQueue[T], Protocol[T]): async def put(self, item: T) -> None: ... async def get(self) -> T: ... async def join(self) -> None: ... class Queue(Generic[T]): _loop: Optional[asyncio.AbstractEventLoop] = None def __init__(self, maxsize: int = 0) -> None: if sys.version_info < (3, 10): self._loop = asyncio.get_running_loop() self._maxsize = maxsize self._is_shutdown = False self._init(maxsize) self._unfinished_tasks = 0 self._sync_mutex = threading.Lock() self._sync_not_empty = threading.Condition(self._sync_mutex) self._sync_not_empty_waiting = 0 self._sync_not_full = threading.Condition(self._sync_mutex) self._sync_not_full_waiting = 0 self._sync_tasks_done = threading.Condition(self._sync_mutex) self._sync_tasks_done_waiting = 0 self._async_mutex = asyncio.Lock() if sys.version_info[:3] == (3, 10, 0): # Workaround for Python 3.10 bug, see #358: getattr(self._async_mutex, "_get_loop", lambda: None)() self._async_not_empty = asyncio.Condition(self._async_mutex) self._async_not_empty_waiting = 0 self._async_not_full = asyncio.Condition(self._async_mutex) self._async_not_full_waiting = 0 self._async_tasks_done = asyncio.Condition(self._async_mutex) self._async_tasks_done_waiting = 0 self._pending: deque[asyncio.Future[None]] = deque() self._sync_queue = _SyncQueueProxy(self) self._async_queue = _AsyncQueueProxy(self) def _get_loop(self) -> asyncio.AbstractEventLoop: # Warning! # The function should be called when self._sync_mutex is locked, # otherwise the code is not thread-safe loop = asyncio.get_running_loop() if self._loop is None: self._loop = loop if loop is not self._loop: raise RuntimeError(f"{self!r} is bound to a different event loop") return loop def shutdown(self, immediate: bool = False) -> None: """Shut-down the queue, making queue gets and puts raise an exception. By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. All blocked callers of put() and get() will be unblocked. If 'immediate', a task is marked as done for each item remaining in the queue, which may unblock callers of join(). The raise exception is SyncQueueShutDown for sync api and AsyncQueueShutDown for async one. """ with self._sync_mutex: self._is_shutdown = True if immediate: while self._qsize(): self._get() if self._unfinished_tasks > 0: self._unfinished_tasks -= 1 # release all blocked threads in `join()` if self._sync_tasks_done_waiting: self._sync_tasks_done.notify_all() if self._async_tasks_done_waiting: self._notify_async(self._async_tasks_done.notify_all) # All getters need to re-check queue-empty to raise ShutDown if self._sync_not_empty_waiting: self._sync_not_empty.notify_all() if self._sync_not_full_waiting: self._sync_not_full.notify_all() if self._async_not_empty_waiting: self._notify_async(self._async_not_empty.notify_all) if self._async_not_full_waiting: self._notify_async(self._async_not_full.notify_all) def close(self) -> None: """Close the queue. The method is a shortcut for .shutdown(immediate=True) """ self.shutdown(immediate=True) async def wait_closed(self) -> None: """Wait for finishing all pending activities""" # should be called from loop after close(). # Nobody should put/get at this point, # so lock acquiring is not required if not self._is_shutdown: raise RuntimeError("Waiting for non-closed queue") # give a chance for the task-done callbacks # of async tasks created inside # _notify_async() # methods to be executed. await asyncio.sleep(0) if not self._pending: return await asyncio.wait(self._pending) async def aclose(self) -> None: """Shutdown the queue and wait for actual shutting down""" self.close() await self.wait_closed() @property def closed(self) -> bool: return self._is_shutdown and not self._pending @property def maxsize(self) -> int: return self._maxsize @property def sync_q(self) -> "_SyncQueueProxy[T]": return self._sync_queue @property def async_q(self) -> "_AsyncQueueProxy[T]": return self._async_queue # Override these methods to implement other queue organizations # (e.g. stack or priority queue). # These will only be called with appropriate locks held def _init(self, maxsize: int) -> None: self._queue: deque[T] = deque() def _qsize(self) -> int: return len(self._queue) # Put a new item in the queue def _put(self, item: T) -> None: self._queue.append(item) # Get an item from the queue def _get(self) -> T: return self._queue.popleft() def _put_internal(self, item: T) -> None: self._put(item) self._unfinished_tasks += 1 async def _do_async_notifier(self, method: Callable[[], None]) -> None: async with self._async_mutex: method() def _setup_async_notifier( self, loop: asyncio.AbstractEventLoop, method: Callable[[], None] ) -> None: task = loop.create_task(self._do_async_notifier(method)) task.add_done_callback(self._pending.remove) self._pending.append(task) def _notify_async(self, method: Callable[[], None]) -> None: # Warning! # The function should be called when self._sync_mutex is locked, # otherwise the code is not thread-safe loop = self._loop if loop is None or loop.is_closed(): # async API is not available, nothing to notify return loop.call_soon_threadsafe(self._setup_async_notifier, loop, method) class _SyncQueueProxy(SyncQueue[T]): """Create a queue object with a given maximum size. If maxsize is <= 0, the queue size is infinite. """ def __init__(self, parent: Queue[T]): self._parent = parent @property def maxsize(self) -> int: return self._parent._maxsize @property def closed(self) -> bool: return self._parent.closed def task_done(self) -> None: """Indicate that a formerly enqueued task is complete. Used by Queue consumer threads. For each get() used to fetch a task, a subsequent call to task_done() tells the queue that the processing on the task is complete. If a join() is currently blocking, it will resume when all items have been processed (meaning that a task_done() call was received for every item that had been put() into the queue). Raises a ValueError if called more times than there were items placed in the queue. """ parent = self._parent with parent._sync_tasks_done: unfinished = parent._unfinished_tasks - 1 if unfinished <= 0: if unfinished < 0: raise ValueError("task_done() called too many times") if parent._sync_tasks_done_waiting: parent._sync_tasks_done.notify_all() if parent._async_tasks_done_waiting: parent._notify_async(parent._async_tasks_done.notify_all) parent._unfinished_tasks = unfinished def join(self) -> None: """Blocks until all items in the Queue have been gotten and processed. The count of unfinished tasks goes up whenever an item is added to the queue. The count goes down whenever a consumer thread calls task_done() to indicate the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, join() unblocks. """ parent = self._parent with parent._sync_tasks_done: while parent._unfinished_tasks: parent._sync_tasks_done_waiting += 1 try: parent._sync_tasks_done.wait() finally: parent._sync_tasks_done_waiting -= 1 def qsize(self) -> int: """Return the approximate size of the queue (not reliable!).""" return self._parent._qsize() @property def unfinished_tasks(self) -> int: """Return the number of unfinished tasks.""" return self._parent._unfinished_tasks def empty(self) -> bool: """Return True if the queue is empty, False otherwise (not reliable!). This method is likely to be removed at some point. Use qsize() == 0 as a direct substitute, but be aware that either approach risks a race condition where a queue can grow before the result of empty() or qsize() can be used. To create code that needs to wait for all queued tasks to be completed, the preferred technique is to use the join() method. """ return not self._parent._qsize() def full(self) -> bool: """Return True if the queue is full, False otherwise (not reliable!). This method is likely to be removed at some point. Use qsize() >= n as a direct substitute, but be aware that either approach risks a race condition where a queue can shrink before the result of full() or qsize() can be used. """ parent = self._parent return 0 < parent._maxsize <= parent._qsize() def put(self, item: T, block: bool = True, timeout: OptFloat = None) -> None: """Put an item into the queue. If optional args 'block' is true and 'timeout' is None (the default), block if necessary until a free slot is available. If 'timeout' is a non-negative number, it blocks at most 'timeout' seconds and raises the Full exception if no free slot was available within that time. Otherwise ('block' is false), put an item on the queue if a free slot is immediately available, else raise the Full exception ('timeout' is ignored in that case). """ parent = self._parent with parent._sync_not_full: if parent._is_shutdown: raise SyncQueueShutDown if parent._maxsize > 0: if not block: if parent._qsize() >= parent._maxsize: raise SyncQueueFull elif timeout is None: while parent._qsize() >= parent._maxsize: parent._sync_not_full_waiting += 1 try: parent._sync_not_full.wait() finally: parent._sync_not_full_waiting -= 1 if parent._is_shutdown: raise SyncQueueShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: endtime = monotonic() + timeout while parent._qsize() >= parent._maxsize: remaining = endtime - monotonic() if remaining <= 0.0: raise SyncQueueFull parent._sync_not_full_waiting += 1 try: parent._sync_not_full.wait(remaining) finally: parent._sync_not_full_waiting -= 1 if parent._is_shutdown: raise SyncQueueShutDown parent._put_internal(item) if parent._sync_not_empty_waiting: parent._sync_not_empty.notify() if parent._async_not_empty_waiting: parent._notify_async(parent._async_not_empty.notify) def get(self, block: bool = True, timeout: OptFloat = None) -> T: """Remove and return an item from the queue. If optional args 'block' is true and 'timeout' is None (the default), block if necessary until an item is available. If 'timeout' is a non-negative number, it blocks at most 'timeout' seconds and raises the Empty exception if no item was available within that time. Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored in that case). """ parent = self._parent with parent._sync_not_empty: if parent._is_shutdown and not parent._qsize(): raise SyncQueueShutDown if not block: if not parent._qsize(): raise SyncQueueEmpty elif timeout is None: while not parent._qsize(): parent._sync_not_empty_waiting += 1 try: parent._sync_not_empty.wait() finally: parent._sync_not_empty_waiting -= 1 if parent._is_shutdown and not parent._qsize(): raise SyncQueueShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: endtime = monotonic() + timeout while not parent._qsize(): remaining = endtime - monotonic() if remaining <= 0.0: raise SyncQueueEmpty parent._sync_not_empty_waiting += 1 try: parent._sync_not_empty.wait(remaining) finally: parent._sync_not_empty_waiting -= 1 if parent._is_shutdown and not parent._qsize(): raise SyncQueueShutDown item = parent._get() if parent._sync_not_full_waiting: parent._sync_not_full.notify() if parent._async_not_full_waiting: parent._notify_async(parent._async_not_full.notify) return item def put_nowait(self, item: T) -> None: """Put an item into the queue without blocking. Only enqueue the item if a free slot is immediately available. Otherwise raise the Full exception. """ return self.put(item, block=False) def get_nowait(self) -> T: """Remove and return an item from the queue without blocking. Only get an item if one is immediately available. Otherwise raise the Empty exception. """ return self.get(block=False) def shutdown(self, immediate: bool = False) -> None: """Shut-down the queue, making queue gets and puts raise an exception. By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. All blocked callers of put() and get() will be unblocked. If 'immediate', a task is marked as done for each item remaining in the queue, which may unblock callers of join(). The raise exception is SyncQueueShutDown for sync api and AsyncQueueShutDown for async one. """ self._parent.shutdown(immediate) class _AsyncQueueProxy(AsyncQueue[T]): """Create a queue object with a given maximum size. If maxsize is <= 0, the queue size is infinite. """ def __init__(self, parent: Queue[T]): self._parent = parent @property def closed(self) -> bool: parent = self._parent return parent.closed def qsize(self) -> int: """Number of items in the queue.""" parent = self._parent return parent._qsize() @property def unfinished_tasks(self) -> int: """Return the number of unfinished tasks.""" parent = self._parent return parent._unfinished_tasks @property def maxsize(self) -> int: """Number of items allowed in the queue.""" parent = self._parent return parent._maxsize def empty(self) -> bool: """Return True if the queue is empty, False otherwise.""" return self.qsize() == 0 def full(self) -> bool: """Return True if there are maxsize items in the queue. Note: if the Queue was initialized with maxsize=0 (the default), then full() is never True. """ parent = self._parent if parent._maxsize <= 0: return False else: return parent._qsize() >= parent._maxsize async def put(self, item: T) -> None: """Put an item into the queue. Put an item into the queue. If the queue is full, wait until a free slot is available before adding item. This method is a coroutine. """ parent = self._parent async with parent._async_not_full: with parent._sync_mutex: if parent._is_shutdown: raise AsyncQueueShutDown parent._get_loop() # check the event loop while 0 < parent._maxsize <= parent._qsize(): parent._async_not_full_waiting += 1 parent._sync_mutex.release() try: await parent._async_not_full.wait() finally: parent._sync_mutex.acquire() parent._async_not_full_waiting -= 1 if parent._is_shutdown: raise AsyncQueueShutDown parent._put_internal(item) if parent._async_not_empty_waiting: parent._async_not_empty.notify() if parent._sync_not_empty_waiting: parent._sync_not_empty.notify() def put_nowait(self, item: T) -> None: """Put an item into the queue without blocking. If no free slot is immediately available, raise QueueFull. """ parent = self._parent with parent._sync_mutex: if parent._is_shutdown: raise AsyncQueueShutDown parent._get_loop() if 0 < parent._maxsize <= parent._qsize(): raise AsyncQueueFull parent._put_internal(item) if parent._async_not_empty_waiting: parent._notify_async(parent._async_not_empty.notify) if parent._sync_not_empty_waiting: parent._sync_not_empty.notify() async def get(self) -> T: """Remove and return an item from the queue. If queue is empty, wait until an item is available. This method is a coroutine. """ parent = self._parent async with parent._async_not_empty: with parent._sync_mutex: if parent._is_shutdown and not parent._qsize(): raise AsyncQueueShutDown parent._get_loop() # check the event loop while not parent._qsize(): parent._async_not_empty_waiting += 1 parent._sync_mutex.release() try: await parent._async_not_empty.wait() finally: parent._sync_mutex.acquire() parent._async_not_empty_waiting -= 1 if parent._is_shutdown and not parent._qsize(): raise AsyncQueueShutDown item = parent._get() if parent._async_not_full_waiting: parent._async_not_full.notify() if parent._sync_not_full_waiting: parent._sync_not_full.notify() return item def get_nowait(self) -> T: """Remove and return an item from the queue. Return an item if one is immediately available, else raise QueueEmpty. """ parent = self._parent with parent._sync_mutex: if parent._is_shutdown and not parent._qsize(): raise AsyncQueueShutDown if not parent._qsize(): raise AsyncQueueEmpty parent._get_loop() item = parent._get() if parent._async_not_full_waiting: parent._notify_async(parent._async_not_full.notify) if parent._sync_not_full_waiting: parent._sync_not_full.notify() return item def task_done(self) -> None: """Indicate that a formerly enqueued task is complete. Used by queue consumers. For each get() used to fetch a task, a subsequent call to task_done() tells the queue that the processing on the task is complete. If a join() is currently blocking, it will resume when all items have been processed (meaning that a task_done() call was received for every item that had been put() into the queue). Raises ValueError if called more times than there were items placed in the queue. """ parent = self._parent with parent._sync_tasks_done: if parent._unfinished_tasks <= 0: raise ValueError("task_done() called too many times") parent._unfinished_tasks -= 1 if parent._unfinished_tasks == 0: if parent._async_tasks_done_waiting: parent._notify_async(parent._async_tasks_done.notify_all) if parent._sync_tasks_done_waiting: parent._sync_tasks_done.notify_all() async def join(self) -> None: """Block until all items in the queue have been gotten and processed. The count of unfinished tasks goes up whenever an item is added to the queue. The count goes down whenever a consumer calls task_done() to indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, join() unblocks. """ parent = self._parent async with parent._async_tasks_done: with parent._sync_mutex: parent._get_loop() # check the event loop while parent._unfinished_tasks: parent._async_tasks_done_waiting += 1 parent._sync_mutex.release() try: await parent._async_tasks_done.wait() finally: parent._sync_mutex.acquire() parent._async_tasks_done_waiting -= 1 def shutdown(self, immediate: bool = False) -> None: """Shut-down the queue, making queue gets and puts raise an exception. By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. All blocked callers of put() and get() will be unblocked. If 'immediate', a task is marked as done for each item remaining in the queue, which may unblock callers of join(). The raise exception is SyncQueueShutDown for sync api and AsyncQueueShutDown for async one. """ self._parent.shutdown(immediate) class PriorityQueue(Queue[T]): """Variant of Queue that retrieves open entries in priority order (lowest first). Entries are typically tuples of the form: (priority number, data). """ def _init(self, maxsize: int) -> None: self._heap_queue: list[T] = [] def _qsize(self) -> int: return len(self._heap_queue) def _put(self, item: T) -> None: heappush(self._heap_queue, item) def _get(self) -> T: return heappop(self._heap_queue) class LifoQueue(Queue[T]): """Variant of Queue that retrieves most recently added entries first.""" def _qsize(self) -> int: return len(self._queue) def _put(self, item: T) -> None: self._queue.append(item) def _get(self) -> T: return self._queue.pop() aio-libs-janus-3158ccb/janus/py.typed000066400000000000000000000000151472702537100175640ustar00rootroot00000000000000# Placeholderaio-libs-janus-3158ccb/pyproject.toml000066400000000000000000000001441472702537100176640ustar00rootroot00000000000000[build-system] requires = ["setuptools>=51", "wheel>=0.36"] build-backend = "setuptools.build_meta" aio-libs-janus-3158ccb/requirements-dev.txt000066400000000000000000000004311472702537100210070ustar00rootroot00000000000000-e . backports.asyncio.runner==1.1.0; python_version < "3.11" black==24.10.0 bandit==1.8.0 coverage==7.6.9 docutils==0.21.2 flake8==7.1.1 mypy==1.13.0 pyroma==4.2 pytest-cov==6.0.0 pytest==8.3.4 pytest-asyncio==0.24.0 pytest-codspeed==3.1.0 isort==5.13.2 tox==4.23.2 wheel==0.45.1 aio-libs-janus-3158ccb/setup.cfg000066400000000000000000000037701472702537100166010ustar00rootroot00000000000000[metadata] name = janus version = attr: janus.__version__ url = https://github.com/aio-libs/janus project_urls = Chat: Gitter = https://gitter.im/aio-libs/Lobby CI: GitHub Actions = https://github.com/aio-libs/janus/actions/workflows/ci.yml Coverage: codecov = https://codecov.io/github/aio-libs/janus GitHub: issues = https://github.com/aio-libs/janus/issues GitHub: repo = https://github.com/aio-libs/janus description = Mixed sync-async queue to interoperate between asyncio tasks and classic threads long_description = file: README.rst long_description_content_type = text/x-rst author = Andrew Svetlov author_email = andrew.svetlov@gmail.com license = Apache 2 license_files = LICENSE classifiers = Development Status :: 5 - Production/Stable Framework :: AsyncIO Intended Audience :: Developers License :: OSI Approved :: Apache Software License Operating System :: POSIX Operating System :: MacOS :: MacOS X Operating System :: Microsoft :: Windows Programming Language :: Python Programming Language :: Python :: 3 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 Programming Language :: Python :: 3.13 Topic :: Software Development :: Libraries keywords= janus, queue, asyncio [options] python_requires = >=3.9 packages = find: # https://setuptools.readthedocs.io/en/latest/setuptools.html#setting-the-zip-safe-flag zip_safe = True include_package_data = True [flake8] exclude = .git,.env,__pycache__,.eggs max-line-length = 88 [tool:pytest] addopts= --cov-branch --cov-report xml log_cli=false log_level=INFO junit_family=xunit2 asyncio_mode=strict asyncio_default_fixture_loop_scope=function filterwarnings=error # pytest-asyncio 0.24.0 does't close the event loop for some reason # it is the source of unclosed loop and socket warnings ignore:unclosed event loop:ResourceWarning ignore:unclosed :ResourceWarning aio-libs-janus-3158ccb/setup.py000066400000000000000000000000461472702537100164630ustar00rootroot00000000000000from setuptools import setup setup() aio-libs-janus-3158ccb/tests/000077500000000000000000000000001472702537100161135ustar00rootroot00000000000000aio-libs-janus-3158ccb/tests/test_async.py000066400000000000000000000402361472702537100206460ustar00rootroot00000000000000"""Tests for queues.py""" import asyncio import time import re import pytest import janus async def close(_q): for i in range(5): time.sleep(0.001) if not _q._sync_mutex.locked(): break else: assert not _q._sync_mutex.locked() await _q.aclose() assert _q.closed assert _q.sync_q.closed assert _q.async_q.closed class TestQueueBasic: async def _test_repr_or_str(self, fn, expect_id): """Test Queue's repr or str. fn is repr or str. expect_id is True if we expect the Queue's id to appear in fn(Queue()). """ _q = janus.Queue() q = _q.async_q assert fn(q).startswith("= (3, 11): from asyncio import Runner else: from backports.asyncio.runner import Runner def test_bench_sync_put_async_get(benchmark): q: janus.Queue async def init(): nonlocal q q = janus.Queue() def threaded(): for i in range(5): q.sync_q.put(i) async def go(): for i in range(100): f = asyncio.get_running_loop().run_in_executor(None, threaded) for i in range(5): val = await q.async_q.get() assert val == i await f assert q.async_q.empty() async def finish(): q.close() await q.wait_closed() with Runner(debug=True) as runner: runner.run(init()) @benchmark def _run(): runner.run(go()) runner.run(finish()) def test_bench_sync_put_async_join(benchmark): q: janus.Queue async def init(): nonlocal q q = janus.Queue() async def go(): for i in range(100): for i in range(5): q.sync_q.put(i) async def do_work(): await asyncio.sleep(0.01) while not q.async_q.empty(): await q.async_q.get() q.async_q.task_done() task = asyncio.create_task(do_work()) await q.async_q.join() await task async def finish(): q.close() await q.wait_closed() with Runner(debug=True) as runner: runner.run(init()) @benchmark def _run(): runner.run(go()) runner.run(finish()) def test_bench_async_put_sync_get(benchmark): q: janus.Queue async def init(): nonlocal q q = janus.Queue() def threaded(): for i in range(5): val = q.sync_q.get() assert val == i async def go(): for i in range(100): f = asyncio.get_running_loop().run_in_executor(None, threaded) for i in range(5): await q.async_q.put(i) await f assert q.async_q.empty() async def finish(): q.close() await q.wait_closed() with Runner(debug=True) as runner: runner.run(init()) @benchmark def _run(): runner.run(go()) runner.run(finish()) def test_sync_join_async_done(benchmark): q: janus.Queue async def init(): nonlocal q q = janus.Queue() def threaded(): for i in range(5): q.sync_q.put(i) q.sync_q.join() async def go(): for i in range(100): f = asyncio.get_running_loop().run_in_executor(None, threaded) for i in range(5): val = await q.async_q.get() assert val == i q.async_q.task_done() await f assert q.async_q.empty() async def finish(): q.close() await q.wait_closed() with Runner(debug=True) as runner: runner.run(init()) @benchmark def _run(): runner.run(go()) runner.run(finish()) aio-libs-janus-3158ccb/tests/test_mixed.py000066400000000000000000000243021472702537100206330ustar00rootroot00000000000000import asyncio import sys from concurrent.futures import ThreadPoolExecutor import pytest import janus class TestMixedMode: @pytest.mark.skipif( sys.version_info >= (3, 10), reason="Python 3.10+ supports delayed initialization", ) def test_ctor_noloop(self): with pytest.raises(RuntimeError): janus.Queue() @pytest.mark.asyncio async def test_get_loop_ok(self): q = janus.Queue() loop = asyncio.get_running_loop() assert q._get_loop() is loop assert q._loop is loop @pytest.mark.asyncio async def test_get_loop_different_loop(self): q = janus.Queue() # emulate binding another loop loop = q._loop = asyncio.new_event_loop() with pytest.raises(RuntimeError, match="is bound to a different event loop"): q._get_loop() loop.close() @pytest.mark.asyncio async def test_maxsize(self): q = janus.Queue(5) assert 5 == q.maxsize @pytest.mark.asyncio async def test_maxsize_named_param(self): q = janus.Queue(maxsize=7) assert 7 == q.maxsize @pytest.mark.asyncio async def test_maxsize_default(self): q = janus.Queue() assert 0 == q.maxsize @pytest.mark.asyncio async def test_unfinished(self): q = janus.Queue() assert q.sync_q.unfinished_tasks == 0 assert q.async_q.unfinished_tasks == 0 q.sync_q.put(1) assert q.sync_q.unfinished_tasks == 1 assert q.async_q.unfinished_tasks == 1 q.sync_q.get() assert q.sync_q.unfinished_tasks == 1 assert q.async_q.unfinished_tasks == 1 q.sync_q.task_done() assert q.sync_q.unfinished_tasks == 0 assert q.async_q.unfinished_tasks == 0 q.close() await q.wait_closed() @pytest.mark.asyncio async def test_sync_put_async_get(self): loop = asyncio.get_running_loop() q = janus.Queue() def threaded(): for i in range(5): q.sync_q.put(i) async def go(): f = loop.run_in_executor(None, threaded) for i in range(5): val = await q.async_q.get() assert val == i assert q.async_q.empty() await f for i in range(3): await go() q.close() await q.wait_closed() @pytest.mark.asyncio async def test_sync_put_async_join(self): loop = asyncio.get_running_loop() q = janus.Queue() for i in range(5): q.sync_q.put(i) async def do_work(): await asyncio.sleep(0.1) while not q.async_q.empty(): await q.async_q.get() q.async_q.task_done() task = loop.create_task(do_work()) async def wait_for_empty_queue(): await q.async_q.join() await task await wait_for_empty_queue() q.close() await q.wait_closed() @pytest.mark.asyncio async def test_async_put_sync_get(self): loop = asyncio.get_running_loop() q = janus.Queue() def threaded(): for i in range(5): val = q.sync_q.get() assert val == i async def go(): f = loop.run_in_executor(None, threaded) for i in range(5): await q.async_q.put(i) await f assert q.async_q.empty() for i in range(3): await go() q.close() await q.wait_closed() @pytest.mark.asyncio async def test_sync_join_async_done(self): loop = asyncio.get_running_loop() q = janus.Queue() def threaded(): for i in range(5): q.sync_q.put(i) q.sync_q.join() async def go(): f = loop.run_in_executor(None, threaded) for i in range(5): val = await q.async_q.get() assert val == i q.async_q.task_done() assert q.async_q.empty() await f for i in range(3): await go() q.close() await q.wait_closed() @pytest.mark.asyncio async def test_async_join_async_done(self): loop = asyncio.get_running_loop() q = janus.Queue() def threaded(): for i in range(5): val = q.sync_q.get() assert val == i q.sync_q.task_done() async def go(): f = loop.run_in_executor(None, threaded) for i in range(5): await q.async_q.put(i) await q.async_q.join() await f assert q.async_q.empty() for i in range(3): await go() q.close() await q.wait_closed() @pytest.mark.asyncio async def test_wait_without_closing(self): q = janus.Queue() with pytest.raises(RuntimeError, match="Waiting for non-closed queue"): await q.wait_closed() q.close() await q.wait_closed() @pytest.mark.asyncio async def test_modifying_forbidden_after_closing(self): q = janus.Queue() q.close() with pytest.raises( janus.SyncQueueShutDown ): q.sync_q.put(5) with pytest.raises( janus.SyncQueueShutDown ): q.sync_q.get() with pytest.raises( janus.AsyncQueueShutDown ): await q.async_q.put(5) with pytest.raises( janus.AsyncQueueShutDown ): q.async_q.put_nowait(5) with pytest.raises( janus.AsyncQueueShutDown ): q.async_q.get_nowait() await q.wait_closed() @pytest.mark.asyncio async def test_double_closing(self): q = janus.Queue() q.close() q.close() await q.wait_closed() @pytest.mark.asyncio async def test_closed(self): q = janus.Queue() assert not q.closed assert not q.async_q.closed assert not q.sync_q.closed q.close() await q.wait_closed() assert q.closed assert q.async_q.closed assert q.sync_q.closed @pytest.mark.asyncio async def test_async_join_after_closing(self): q = janus.Queue() q.close() await asyncio.wait_for(q.async_q.join(), timeout=0.1) await q.wait_closed() @pytest.mark.asyncio async def test_close_after_async_join(self): q = janus.Queue() q.sync_q.put(1) task = asyncio.create_task(q.async_q.join()) await asyncio.sleep(0.01) # ensure tasks are blocking q.close() await asyncio.wait_for(task, timeout=0.1) await q.wait_closed() @pytest.mark.asyncio async def test_sync_join_after_closing(self): loop = asyncio.get_running_loop() q = janus.Queue() q.sync_q.put(1) q.close() await asyncio.wait_for(loop.run_in_executor(None, q.sync_q.join), timeout=0.1) await q.wait_closed() @pytest.mark.asyncio async def test_close_after_sync_join(self): loop = asyncio.get_running_loop() q = janus.Queue() q.sync_q.put(1) fut = loop.run_in_executor(None, q.sync_q.join) await asyncio.sleep(0.1) # ensure tasks are blocking q.close() await asyncio.wait_for(fut, timeout=0.1) await q.wait_closed() @pytest.mark.asyncio async def test_put_notifies_sync_not_empty(self): loop = asyncio.get_running_loop() q = janus.Queue() with ThreadPoolExecutor(4) as executor: for _ in range(4): executor.submit(q.sync_q.get) while q._sync_not_empty_waiting != 4: await asyncio.sleep(0.001) q.sync_q.put_nowait(1) q.async_q.put_nowait(2) await loop.run_in_executor(executor, q.sync_q.put, 3) await q.async_q.put(4) assert q.sync_q.empty() await q.aclose() @pytest.mark.asyncio async def test_put_notifies_async_not_empty(self): loop = asyncio.get_running_loop() q = janus.Queue() tasks = [loop.create_task(q.async_q.get()) for _ in range(4)] while q._async_not_empty_waiting != 4: await asyncio.sleep(0) q.sync_q.put_nowait(1) q.async_q.put_nowait(2) await loop.run_in_executor(None, q.sync_q.put, 3) await q.async_q.put(4) await asyncio.gather(*tasks) assert q.sync_q.empty() await q.aclose() @pytest.mark.asyncio async def test_get_notifies_sync_not_full(self): loop = asyncio.get_running_loop() q = janus.Queue(2) q.sync_q.put_nowait(1) q.sync_q.put_nowait(2) with ThreadPoolExecutor(4) as executor: for _ in range(4): executor.submit(q.sync_q.put, object()) while q._sync_not_full_waiting != 4: await asyncio.sleep(0.001) q.sync_q.get_nowait() q.async_q.get_nowait() await loop.run_in_executor(executor, q.sync_q.get) await q.async_q.get() assert q.sync_q.qsize() == 2 await q.aclose() @pytest.mark.asyncio async def test_get_notifies_async_not_full(self): loop = asyncio.get_running_loop() q = janus.Queue(2) q.sync_q.put_nowait(1) q.sync_q.put_nowait(2) tasks = [loop.create_task(q.async_q.put(object())) for _ in range(4)] while q._async_not_full_waiting != 4: await asyncio.sleep(0) q.sync_q.get_nowait() q.async_q.get_nowait() await loop.run_in_executor(None, q.sync_q.get) await q.async_q.get() await asyncio.gather(*tasks) assert q.sync_q.qsize() == 2 await q.aclose() @pytest.mark.asyncio async def test_wait_closed_with_pending_tasks(self): q = janus.Queue() async def getter(): await q.async_q.get() task = asyncio.create_task(getter()) await asyncio.sleep(0.01) q.shutdown() # q._pending is not empty now await q.wait_closed() with pytest.raises(janus.AsyncQueueShutDown): await task aio-libs-janus-3158ccb/tests/test_sync.py000066400000000000000000000440711472702537100205060ustar00rootroot00000000000000# Some simple queue module tests, plus some failure conditions # to ensure the Queue locks remain stable. import asyncio import queue import re import sys import threading import time from unittest.mock import patch import pytest import janus QUEUE_SIZE = 5 def qfull(q): return q._parent._maxsize > 0 and q.qsize() == q._parent._maxsize # A thread to run a function that unclogs a blocked Queue. class _TriggerThread(threading.Thread): def __init__(self, fn, args): self.fn = fn self.args = args self.startedEvent = threading.Event() threading.Thread.__init__(self) def run(self): # The sleep isn't necessary, but is intended to give the blocking # function in the main thread a chance at actually blocking before # we unclog it. But if the sleep is longer than the timeout-based # tests wait in their blocking functions, those tests will fail. # So we give them much longer timeout values compared to the # sleep here (I aimed at 10 seconds for blocking functions -- # they should never actually wait that long - they should make # progress as soon as we call self.fn()). time.sleep(0.1) self.startedEvent.set() self.fn(*self.args) # Execute a function that blocks, and in a separate thread, a function that # triggers the release. Returns the result of the blocking function. Caution: # block_func must guarantee to block until trigger_func is called, and # trigger_func must guarantee to change queue state so that block_func can make # enough progress to return. In particular, a block_func that just raises an # exception regardless of whether trigger_func is called will lead to # timing-dependent sporadic failures, and one of those went rarely seen but # undiagnosed for years. Now block_func must be unexceptional. If block_func # is supposed to raise an exception, call do_exceptional_blocking_test() # instead. class BlockingTestMixin: def do_blocking_test(self, block_func, block_args, trigger_func, trigger_args): self.t = _TriggerThread(trigger_func, trigger_args) self.t.start() self.result = block_func(*block_args) # If block_func returned before our thread made the call, we failed! if not self.t.startedEvent.is_set(): pytest.fail("blocking function '%r' appeared not to block" % block_func) self.t.join(10) # make sure the thread terminates if self.t.is_alive(): pytest.fail("trigger function '%r' appeared to not return" % trigger_func) return self.result # Call this instead if block_func is supposed to raise an exception. def do_exceptional_blocking_test( self, block_func, block_args, trigger_func, trigger_args, expected_exception_class, ): self.t = _TriggerThread(trigger_func, trigger_args) self.t.start() try: try: block_func(*block_args) except expected_exception_class: raise else: pytest.fail("expected exception of kind %r" % expected_exception_class) finally: self.t.join(10) # make sure the thread terminates if self.t.is_alive(): pytest.fail( "trigger function '%r' appeared to not return" % trigger_func ) if not self.t.startedEvent.is_set(): pytest.fail("trigger thread ended but event never set") class BaseQueueTestMixin(BlockingTestMixin): cum = 0 cumlock = threading.Lock() def simple_queue_test(self, _q): q = _q.sync_q if q.qsize(): raise RuntimeError("Call this function with an empty queue") assert q.empty() assert not q.full() # I guess we better check things actually queue correctly a little :) q.put(111) q.put(333) q.put(222) target_order = dict( Queue=[111, 333, 222], LifoQueue=[222, 333, 111], PriorityQueue=[111, 222, 333], ) actual_order = [q.get(), q.get(), q.get()] assert actual_order == target_order[_q.__class__.__name__] for i in range(QUEUE_SIZE - 1): q.put(i) assert q.qsize() assert not qfull(q) last = 2 * QUEUE_SIZE full = 3 * 2 * QUEUE_SIZE q.put(last) assert qfull(q) assert not q.empty() assert q.full() try: q.put(full, block=0) pytest.fail("Didn't appear to block with a full queue") except queue.Full: pass try: q.put(full, timeout=0.01) pytest.fail("Didn't appear to time-out with a full queue") except queue.Full: pass # Test a blocking put self.do_blocking_test(q.put, (full,), q.get, ()) self.do_blocking_test(q.put, (full, True, 10), q.get, ()) # Empty it for i in range(QUEUE_SIZE): q.get() assert not q.qsize() try: q.get(block=0) pytest.fail("Didn't appear to block with an empty queue") except queue.Empty: pass try: q.get(timeout=0.01) pytest.fail("Didn't appear to time-out with an empty queue") except queue.Empty: pass # Test a blocking get self.do_blocking_test(q.get, (), q.put, ("empty",)) self.do_blocking_test(q.get, (True, 10), q.put, ("empty",)) def worker(self, q): try: while True: x = q.get() if x < 0: q.task_done() return with self.cumlock: self.cum += x q.task_done() except Exception as ex: from traceback import print_exc print_exc(ex) def queue_join_test(self, q): self.cum = 0 for i in (0, 1): threading.Thread(target=self.worker, args=(q,)).start() for i in range(100): q.put(i) q.join() assert self.cum == sum(range(100)) for i in (0, 1): q.put(-1) # instruct the threads to close q.join() # verify that you can join twice @pytest.mark.asyncio async def test_queue_task_done(self): # Test to make sure a queue task completed successfully. _q = self.type2test() q = _q.sync_q with pytest.raises( ValueError, match=re.escape("task_done() called too many times") ): q.task_done() _q.close() await _q.wait_closed() @pytest.mark.asyncio async def test_queue_join(self): # Test that a queue join()s successfully, and before anything else # (done twice for insurance). _q = self.type2test() q = _q.sync_q self.queue_join_test(q) self.queue_join_test(q) with pytest.raises( ValueError, match=re.escape("task_done() called too many times") ): q.task_done() _q.close() await _q.wait_closed() @pytest.mark.asyncio async def test_simple_queue(self): # Do it a couple of times on the same queue. # Done twice to make sure works with same instance reused. _q = self.type2test(QUEUE_SIZE) self.simple_queue_test(_q) self.simple_queue_test(_q) _q.close() await _q.wait_closed() @pytest.mark.asyncio async def test_negative_timeout_raises_exception(self): _q = self.type2test(QUEUE_SIZE) q = _q.sync_q with pytest.raises(ValueError, match="timeout' must be a non-negative number"): q.put(1, timeout=-1) with pytest.raises(ValueError, match="timeout' must be a non-negative number"): q.get(1, timeout=-1) _q.close() await _q.wait_closed() @pytest.mark.asyncio async def test_nowait(self): _q = self.type2test(QUEUE_SIZE) q = _q.sync_q for i in range(QUEUE_SIZE): q.put_nowait(1) with pytest.raises(queue.Full): q.put_nowait(1) for i in range(QUEUE_SIZE): q.get_nowait() with pytest.raises(queue.Empty): q.get_nowait() _q.close() await _q.wait_closed() @pytest.mark.asyncio async def test_shrinking_queue(self): # issue 10110 _q = self.type2test(3) q = _q.sync_q q.put(1) q.put(2) q.put(3) with pytest.raises(queue.Full): q.put_nowait(4) assert q.qsize() == 3 q._maxsize = 2 # shrink the queue with pytest.raises(queue.Full): q.put_nowait(4) _q.close() await _q.wait_closed() @pytest.mark.asyncio async def test_maxsize(self): # Test to make sure a queue task completed successfully. _q = self.type2test(5) q = _q.sync_q assert q.maxsize == 5 _q.close() await _q.wait_closed() class TestQueue(BaseQueueTestMixin): type2test = janus.Queue class TestLifoQueue(BaseQueueTestMixin): type2test = janus.LifoQueue class TestPriorityQueue(BaseQueueTestMixin): type2test = janus.PriorityQueue # A Queue subclass that can provoke failure at a moment's notice :) class FailingQueueException(Exception): pass class FailingQueue(janus.Queue): def __init__(self, *args, **kwargs): self.fail_next_put = False self.fail_next_get = False super().__init__(*args, **kwargs) def _put(self, item): if self.fail_next_put: self.fail_next_put = False raise FailingQueueException("You Lose") return super()._put(item) def _get(self): if self.fail_next_get: self.fail_next_get = False raise FailingQueueException("You Lose") return super()._get() class TestFailingQueue(BlockingTestMixin): def failing_queue_test(self, _q): q = _q.sync_q if q.qsize(): raise RuntimeError("Call this function with an empty queue") for i in range(QUEUE_SIZE - 1): q.put(i) # Test a failing non-blocking put. _q.fail_next_put = True try: q.put("oops", block=0) pytest.fail("The queue didn't fail when it should have") except FailingQueueException: pass _q.fail_next_put = True try: q.put("oops", timeout=0.1) pytest.fail("The queue didn't fail when it should have") except FailingQueueException: pass q.put("last") assert qfull(q) # Test a failing blocking put _q.fail_next_put = True try: self.do_blocking_test(q.put, ("full",), q.get, ()) pytest.fail("The queue didn't fail when it should have") except FailingQueueException: pass # Check the Queue isn't damaged. # put failed, but get succeeded - re-add q.put("last") # Test a failing timeout put _q.fail_next_put = True try: self.do_exceptional_blocking_test( q.put, ("full", True, 10), q.get, (), FailingQueueException ) pytest.fail("The queue didn't fail when it should have") except FailingQueueException: pass # Check the Queue isn't damaged. # put failed, but get succeeded - re-add q.put("last") assert qfull(q) q.get() assert not qfull(q) q.put("last") assert qfull(q) # Test a blocking put self.do_blocking_test(q.put, ("full",), q.get, ()) # Empty it for i in range(QUEUE_SIZE): q.get() assert not q.qsize() q.put("first") _q.fail_next_get = True try: q.get() pytest.fail("The queue didn't fail when it should have") except FailingQueueException: pass assert q.qsize() _q.fail_next_get = True try: q.get(timeout=0.1) pytest.fail("The queue didn't fail when it should have") except FailingQueueException: pass assert q.qsize() q.get() assert not q.qsize() _q.fail_next_get = True try: self.do_exceptional_blocking_test( q.get, (), q.put, ("empty",), FailingQueueException ) pytest.fail("The queue didn't fail when it should have") except FailingQueueException: pass # put succeeded, but get failed. assert q.qsize() q.get() assert not q.qsize() @pytest.mark.asyncio async def test_failing_queue(self): # Test to make sure a queue is functioning correctly. # Done twice to the same instance. q = FailingQueue(QUEUE_SIZE) self.failing_queue_test(q) self.failing_queue_test(q) q.close() await q.wait_closed() @pytest.mark.asyncio async def test_closed_loop_non_failing(self): loop = asyncio.get_running_loop() _q = janus.Queue(QUEUE_SIZE) q = _q.sync_q # we are pacthing loop to follow setUp/tearDown agreement with patch.object(loop, "is_closed") as func: func.return_value = True task = loop.create_task(_q.async_q.get()) await asyncio.sleep(0) try: q.put_nowait(1) finally: task.cancel() with pytest.raises(asyncio.CancelledError): await task assert func.call_count == 1 _q.close() await _q.wait_closed() @pytest.mark.skipif( sys.version_info < (3, 10), reason="Python 3.10+ is required", ) def test_sync_only_api(): q = janus.Queue() q.sync_q.put(1) assert q.sync_q.get() == 1 class TestQueueShutdown: @pytest.mark.asyncio async def test_shutdown_empty(self): _q = janus.Queue() q = _q.sync_q q.shutdown() with pytest.raises(janus.SyncQueueShutDown): q.put("data") with pytest.raises(janus.SyncQueueShutDown): q.get() with pytest.raises(janus.SyncQueueShutDown): q.get_nowait() @pytest.mark.asyncio async def test_shutdown_nonempty(self): _q = janus.Queue() q = _q.sync_q q.put("data") q.shutdown() q.get() with pytest.raises(janus.SyncQueueShutDown): q.get() @pytest.mark.asyncio async def test_shutdown_nonempty_get_nowait(self): _q = janus.Queue() q = _q.sync_q q.put("data") q.shutdown() q.get_nowait() with pytest.raises(janus.SyncQueueShutDown): q.get_nowait() @pytest.mark.asyncio async def test_shutdown_immediate(self): _q = janus.Queue() q = _q.sync_q q.put("data") q.shutdown(immediate=True) with pytest.raises(janus.SyncQueueShutDown): q.get() with pytest.raises(janus.SyncQueueShutDown): q.get_nowait() @pytest.mark.asyncio async def test_shutdown_immediate_with_undone_tasks(self): _q = janus.Queue() q = _q.sync_q q.put(1) q.put(2) # artificial .task_done() without .get() for covering specific codeline # in .shutdown(True) q.task_done() q.shutdown(True) @pytest.mark.asyncio async def test_shutdown_putter(self): loop = asyncio.get_running_loop() _q = janus.Queue(maxsize=1) q = _q.sync_q q.put(1) def putter(): q.put(2) fut = loop.run_in_executor(None, putter) # wait for the task start await asyncio.sleep(0.01) q.shutdown() with pytest.raises(janus.SyncQueueShutDown): await fut await _q.aclose() @pytest.mark.asyncio async def test_shutdown_many_putters(self): loop = asyncio.get_running_loop() _q = janus.Queue(maxsize=1) q = _q.sync_q q.put(1) def putter(n): q.put(n) futs = [] for i in range(2): futs.append(loop.run_in_executor(None, putter, i)) # wait for the task start await asyncio.sleep(0.01) q.shutdown() for fut in futs: with pytest.raises(janus.SyncQueueShutDown): await fut await _q.aclose() @pytest.mark.asyncio async def test_shutdown_many_putters_with_timeout(self): loop = asyncio.get_running_loop() _q = janus.Queue(maxsize=1) q = _q.sync_q q.put(1) def putter(n): q.put(n, timeout=60) futs = [] for i in range(2): futs.append(loop.run_in_executor(None, putter, i)) # wait for the task start await asyncio.sleep(0.01) q.shutdown() for fut in futs: with pytest.raises(janus.SyncQueueShutDown): await fut await _q.aclose() @pytest.mark.asyncio async def test_shutdown_getter(self): loop = asyncio.get_running_loop() _q = janus.Queue() q = _q.sync_q def getter(): q.get() fut = loop.run_in_executor(None, getter) # wait for the task start await asyncio.sleep(0.01) q.shutdown() with pytest.raises(janus.SyncQueueShutDown): await fut await _q.aclose() @pytest.mark.asyncio async def test_shutdown_getter_with_timeout(self): loop = asyncio.get_running_loop() _q = janus.Queue() q = _q.sync_q def getter(): q.get(timeout=60) fut = loop.run_in_executor(None, getter) # wait for the task start await asyncio.sleep(0.01) q.shutdown() with pytest.raises(janus.SyncQueueShutDown): await fut await _q.aclose() @pytest.mark.asyncio async def test_shutdown_early_getter(self): _q = janus.Queue() q = _q.sync_q q.shutdown() with pytest.raises(janus.SyncQueueShutDown): q.get() await _q.aclose()