pax_global_header00006660000000000000000000000064152427723460014526gustar00rootroot0000000000000052 comment=6d0e6a02547dae56c5cbbb11a2496b5a80c12686 scrapy-queuelib-6d0e6a0/000077500000000000000000000000001524277234600152475ustar00rootroot00000000000000scrapy-queuelib-6d0e6a0/.git-blame-ignore-revs000066400000000000000000000001241524277234600213440ustar00rootroot00000000000000# applying pre-commit hooks to the project 7e8be052f6c49034d94571151b0d53d5d64717ce scrapy-queuelib-6d0e6a0/.github/000077500000000000000000000000001524277234600166075ustar00rootroot00000000000000scrapy-queuelib-6d0e6a0/.github/dependabot.yml000066400000000000000000000003331524277234600214360ustar00rootroot00000000000000version: 2 updates: - package-ecosystem: github-actions directory: "/" schedule: interval: monthly groups: github-actions: patterns: - "*" cooldown: default-days: 7 scrapy-queuelib-6d0e6a0/.github/workflows/000077500000000000000000000000001524277234600206445ustar00rootroot00000000000000scrapy-queuelib-6d0e6a0/.github/workflows/checks.yml000066400000000000000000000022151524277234600226270ustar00rootroot00000000000000name: Checks permissions: contents: read on: push: branches: - master pull_request: jobs: checks: runs-on: ubuntu-latest strategy: fail-fast: false matrix: include: - python-version: 3.14 env: TOXENV: pylint - python-version: 3.14 env: TOXENV: mypy - python-version: 3.14 env: TOXENV: twinecheck steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} - name: Run check env: ${{ matrix.env }} run: | pip install -U pip pip install -U tox tox pre-commit: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 scrapy-queuelib-6d0e6a0/.github/workflows/publish.yml000066400000000000000000000025271524277234600230430ustar00rootroot00000000000000name: Publish permissions: contents: read on: push: tags: - "v[0-9]+.[0-9]+.[0-9]+" jobs: build: name: Build distribution runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.14" - name: Install pypa/build run: python3 -m pip install build --user - name: Build a binary wheel and a source tarball run: python3 -m build - name: Store the distribution packages uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: python-package-distributions path: dist/ publish: name: Publish to PyPI needs: - build runs-on: ubuntu-latest environment: name: pypi url: https://pypi.org/p/queuelib permissions: id-token: write steps: - name: Download all the dists uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: python-package-distributions path: dist/ - name: Publish distribution to PyPI uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 scrapy-queuelib-6d0e6a0/.github/workflows/tests.yml000066400000000000000000000021541524277234600225330ustar00rootroot00000000000000name: Tests permissions: contents: read on: push: branches: - master pull_request: env: FORCE_COLOR: 1 jobs: tests: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15", "pypy3.11"] os: [windows-latest, macos-latest, ubuntu-latest] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} allow-prereleases: true - name: Run tests run: | pip install -U tox tox -e py - name: Upload coverage report uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - name: Upload test results if: ${{ !cancelled() }} uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: report_type: test_results scrapy-queuelib-6d0e6a0/.gitignore000066400000000000000000000001201524277234600172300ustar00rootroot00000000000000htmlcov/ coverage.xml .coverage .tox *egg-info *.pyc /.tox/ build/ dist/ .idea/ scrapy-queuelib-6d0e6a0/.pre-commit-config.yaml000066400000000000000000000007721524277234600215360ustar00rootroot00000000000000repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.16.4 hooks: - id: ruff-check args: [ --fix ] - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/rhysd/actionlint rev: v1.7.12 hooks: - id: actionlint - repo: https://github.com/zizmorcore/zizmor-pre-commit rev: v1.29.0 hooks: - id: zizmor args: [--no-progress, --fix] scrapy-queuelib-6d0e6a0/LICENSE000066400000000000000000000027651524277234600162660ustar00rootroot00000000000000Copyright (c) w3lib and Scrapy developers. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of Scrapy nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. scrapy-queuelib-6d0e6a0/NEWS000066400000000000000000000032511524277234600157470ustar00rootroot00000000000000Queuelib release notes ====================== Version 1.10.0 --------------- (released on 2026-08-24) * Added ``clear()``, to remove every item from a queue while keeping it open * Fixed ``RoundRobinQueue.pop()`` dropping falsy items * Added support for Python 3.15 * Improved linting and CI configuration Version 1.9.0 ------------- (released on 2026-01-29) * Added support for Python 3.14 * Removed support for Python 3.9 and PyPy 3.10 * Improved linting and CI configuration Version 1.8.0 ------------- (released on 2025-03-31) * Added support for Python 3.13 and PyPy 3.11 * Removed support for Python 3.8 * Queue classes now accept ``os.PathLike`` paths * Fixed test failures on Windows * Switched the build system to ``hatchling`` * Improved linting and CI configuration Version 1.7.0 ------------- (released on 2024-05-04) No functionality changes with respect to 1.6.2 * Added support for Python 3.10-3.12 and PyPy 3.10 * Removed support for Python 3.5-3.7 * Improved type annotations and added the ``py.typed`` file * Added ``pre-commit`` configuration * Improved linting and CI configuration Version 1.6.2 ------------- (released on 2021-08-26) No functionality changes with respect to 1.6.1 * Added `python_requires>=3.5` to `setup.py` * Formatted the codebase with `black` * Added type annotations * Added CI checks for typing, security and linting Version 1.6.1 ------------- (released on 2021-04-21) No code changes with respect to 1.6 * Migrate CI to GitHub actions * Fix release Version 1.6 ----------- (released on 2021-04-21) * Add peek support * Remove py2 support * Full test coverage Version 1.0 ----------- (released on 2013-04-23) First release of Queuelib. scrapy-queuelib-6d0e6a0/README.rst000066400000000000000000000226321524277234600167430ustar00rootroot00000000000000======== queuelib ======== .. image:: https://img.shields.io/pypi/v/queuelib.svg :target: https://pypi.python.org/pypi/queuelib .. image:: https://img.shields.io/pypi/pyversions/queuelib.svg :target: https://pypi.python.org/pypi/queuelib .. image:: https://github.com/scrapy/queuelib/actions/workflows/tests-ubuntu.yml/badge.svg :target: https://github.com/scrapy/queuelib/actions/workflows/tests-ubuntu.yml .. image:: https://img.shields.io/codecov/c/github/scrapy/queuelib/master.svg :target: http://codecov.io/github/scrapy/queuelib?branch=master :alt: Coverage report Queuelib is a Python library that implements object collections which are stored in memory or persisted to disk, provide a simple API, and run fast. Queuelib provides collections for queues_ (FIFO), stacks_ (LIFO), queues sorted by priority and queues that are emptied in a round-robin_ fashion. .. note:: Queuelib collections are not thread-safe. Queuelib supports Python 3.10+ and has no dependencies. .. _queues: https://en.wikipedia.org/wiki/FIFO_(computing_and_electronics) .. _round-robin: https://en.wikipedia.org/wiki/Round-robin_scheduling .. _stacks: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) Installation ============ You can install Queuelib either via the Python Package Index (PyPI) or from source. To install using pip:: $ pip install queuelib To install using easy_install:: $ easy_install queuelib If you have downloaded a source tarball you can install it by running the following (as root):: # python setup.py install FIFO/LIFO disk queues ===================== Queuelib provides FIFO and LIFO queue implementations. Here is an example usage of the FIFO queue:: >>> from queuelib import FifoDiskQueue >>> q = FifoDiskQueue("queuefile") >>> q.push(b'a') >>> q.push(b'b') >>> q.push(b'c') >>> q.pop() b'a' >>> q.close() >>> q = FifoDiskQueue("queuefile") >>> q.pop() b'b' >>> q.pop() b'c' >>> q.pop() >>> The LIFO queue is identical (API-wise), but importing ``LifoDiskQueue`` instead. PriorityQueue ============= A discrete-priority queue implemented by combining multiple FIFO/LIFO queues (one per priority). First, select the type of queue to be used per priority (FIFO or LIFO):: >>> from queuelib import FifoDiskQueue >>> qfactory = lambda priority: FifoDiskQueue('queue-dir-%s' % priority) Then instantiate the Priority Queue with it:: >>> from queuelib import PriorityQueue >>> pq = PriorityQueue(qfactory) And use it:: >>> pq.push(b'a', 3) >>> pq.push(b'b', 1) >>> pq.push(b'c', 2) >>> pq.push(b'd', 2) >>> pq.pop() b'b' >>> pq.pop() b'c' >>> pq.pop() b'd' >>> pq.pop() b'a' RoundRobinQueue =============== Has nearly the same interface and implementation as a Priority Queue except that each element must be pushed with a (mandatory) key. Popping from the queue cycles through the keys "round robin". Instantiate the Round Robin Queue similarly to the Priority Queue:: >>> from queuelib import RoundRobinQueue >>> rr = RoundRobinQueue(qfactory) And use it:: >>> rr.push(b'a', '1') >>> rr.push(b'b', '1') >>> rr.push(b'c', '2') >>> rr.push(b'd', '2') >>> rr.pop() b'a' >>> rr.pop() b'c' >>> rr.pop() b'b' >>> rr.pop() b'd' Clearing a queue ================ ``clear()`` removes every item from a queue, freeing the disk space that they used, and leaves the queue open and usable:: >>> q.clear() >>> len(q) 0 ``PriorityQueue`` and ``RoundRobinQueue`` also close their internal queues, the same way that ``pop()`` does when one of them becomes empty. Disk persistence ================ ``FifoDiskQueue`` and ``LifoDiskQueue`` write their items to the path they get on instantiation, so that a queue can be resumed later, even by a different process. Each class uses that path differently: - ``FifoDiskQueue`` uses a directory, which it creates, together with any missing parent directory. Items go into chunk files (``q00000``, ``q00001``, etc.), each holding up to ``chunksize`` items, and the queue also keeps an ``info.json`` file there for its own bookkeeping. - ``LifoDiskQueue`` uses a single file, whose parent directory must already exist. The layout and the contents of those files are an implementation detail that may change in any release. Do not read or write them yourself, and do not expect a queue written by one version of Queuelib to be readable by a different one. Always close disk queues ------------------------ While a disk queue is open, its bookkeeping (number of items, read and write positions) only lives in memory, and ``close()`` is what writes it to disk. Queuelib never calls ``fsync()`` either, and ``LifoDiskQueue`` writes items through a buffered file object, so the most recent items may not have reached the disk at all. Calling ``close()`` is hence mandatory:: from contextlib import closing with closing(FifoDiskQueue("queuedir")) as q: q.push(b'a') If a process ends without calling ``close()``, the queue on disk keeps the bookkeeping that the last ``close()`` call wrote, which no longer matches the files. Items pushed since then become unreachable, and using the queue again is unsafe: it may report a wrong length, return items that had already been popped, delete files that still contain items, or raise ``OSError``. Queuelib offers no way to repair or to recover such a queue. Empty queues delete their files ------------------------------- ``close()`` on an empty queue deletes its file, or, in the case of ``FifoDiskQueue``, its chunk files and its ``info.json`` file, and also its directory if nothing else remains in it. Using that same path again creates a new, empty queue. FifoDiskQueue frees disk space one chunk at a time -------------------------------------------------- ``FifoDiskQueue`` deletes a chunk file once every item in it has been popped. Until then, popped items keep using disk space, so a queue uses up to ``chunksize`` items worth of disk space on top of the items that it holds. Lower ``chunksize`` to lower that overhead, at the cost of more chunk files and more file operations. For example, a queue that holds 400 items of 1 MB each uses about 100 GB of disk space with the default ``chunksize`` of 100000, and about 800 MB with a ``chunksize`` of 400. Reopening a FifoDiskQueue keeps its chunk size ---------------------------------------------- ``FifoDiskQueue`` stores its ``chunksize`` when creating a queue, and reuses the stored value when reopening one, ignoring the ``chunksize`` parameter. Use one queue object per path at a time --------------------------------------- Queuelib does not lock the files that it uses. On top of not being thread-safe, a given path must not be used by more than one open queue object at a time, in the same process or not. Such queue objects overwrite each other's items and bookkeeping; for example, two ``FifoDiskQueue`` objects on the same directory return the same items, and their ``close()`` calls may raise ``FileNotFoundError``. Persisting a PriorityQueue or a RoundRobinQueue ----------------------------------------------- ``PriorityQueue`` and ``RoundRobinQueue`` do not write anything to disk themselves; their persistence comes entirely from the queues that ``qfactory`` builds, and it is up to ``qfactory`` to map a priority or a key to a valid path. Their ``close()`` method returns the priorities or keys whose underlying queue was not empty. Storing that value is your responsibility, and so is passing it back as ``startprios`` or ``start_domains`` on the next run:: >>> import json >>> from queuelib import FifoDiskQueue, PriorityQueue >>> qfactory = lambda priority: FifoDiskQueue('queue-dir-%s' % priority) >>> pq = PriorityQueue(qfactory) >>> pq.push(b'a', 3) >>> active = pq.close() >>> with open('active.json', 'w') as f: ... json.dump(active, f) ... >>> with open('active.json') as f: ... startprios = json.load(f) ... >>> pq = PriorityQueue(qfactory, startprios) >>> pq.pop() b'a' Priorities and keys that you do not pass back are not detected, and the items in their queues stay on disk, unreachable. Bug tracker =========== If you have any suggestions, bug reports or annoyances please report them to our issue tracker at: http://github.com/scrapy/queuelib/issues/ Contributing ============ Development of Queuelib happens at GitHub: http://github.com/scrapy/queuelib You are highly encouraged to participate in the development. If you don't like GitHub (for some reason) you're welcome to send regular patches. All changes require tests to be merged. Tests ===== Tests are located in `queuelib/tests` directory. They can be run using `nosetests`_ with the following command:: nosetests The output should be something like the following:: $ nosetests ............................................................................. ---------------------------------------------------------------------- Ran 77 tests in 0.145s OK License ======= This software is licensed under the BSD License. See the LICENSE file in the top distribution directory for the full license text. Versioning ========== This software follows `Semantic Versioning`_ .. _Scrapy framework: http://scrapy.org .. _scrapy-users: http://groups.google.com/group/scrapy-users .. _Semantic Versioning: http://semver.org/ .. _nosetests: https://nose.readthedocs.org/en/latest/ scrapy-queuelib-6d0e6a0/pyproject.toml000066400000000000000000000115651524277234600201730ustar00rootroot00000000000000[build-system] requires = ["hatchling"] build-backend = "hatchling.build" [project] name = "queuelib" description = "Collection of persistent (disk-based) and non-persistent (memory-based) queues" readme = "README.rst" license = "BSD-3-Clause" license-files = ["LICENSE"] authors = [{ name = "Scrapy project", email = "info@scrapy.org" }] classifiers = [ "Development Status :: 5 - Production/Stable", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Programming Language :: Python :: 3.15", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] requires-python = ">=3.10" dynamic = ["version"] [project.urls] Homepage = "https://github.com/scrapy/queuelib" Source = "https://github.com/scrapy/queuelib" Issues = "https://github.com/scrapy/queuelib/issues" Docs = "https://github.com/scrapy/queuelib/blob/master/README.rst" "Release notes" = "https://github.com/scrapy/queuelib/blob/master/NEWS" [tool.hatch.version] path = "queuelib/__init__.py" [tool.hatch.build.targets.sdist] include = [ "/queuelib", "/NEWS", ] [tool.bumpversion] current_version = "1.10.0" commit = true tag = true tag_name = "v{new_version}" [[tool.bumpversion.files]] filename = "queuelib/__init__.py" [[tool.bumpversion.files]] filename = "NEWS" search = "^\\(unreleased\\)$" replace = "(released on {now:%Y-%m-%d})" regex = true [tool.coverage.run] branch = true omit = [ "queuelib/tests/*", ] [tool.mypy] strict = true [[tool.mypy.overrides]] module = "queuelib.tests.*" allow_untyped_defs = true check_untyped_defs = false [tool.pylint.MASTER] persistent = "no" [tool.pylint."MESSAGES CONTROL"] enable = [ "useless-suppression", ] disable = [ "consider-using-with", "duplicate-code", "invalid-name", "line-too-long", "missing-class-docstring", "missing-function-docstring", "missing-module-docstring", "too-few-public-methods", "unspecified-encoding", ] [tool.ruff.lint] extend-select = [ # flake8-builtins "A", # flake8-async "ASYNC", # flake8-bugbear "B", # flake8-comprehensions "C4", # flake8-commas "COM", # pydocstyle "D", # flake8-future-annotations "FA", # flynt "FLY", # refurb "FURB", # isort "I", # flake8-implicit-str-concat "ISC", # flake8-logging "LOG", # Perflint "PERF", # pygrep-hooks "PGH", # flake8-pie "PIE", # pylint "PL", # flake8-pytest-style "PT", # flake8-use-pathlib "PTH", # flake8-pyi "PYI", # flake8-quotes "Q", # flake8-return "RET", # flake8-raise "RSE", # Ruff-specific rules "RUF", # flake8-bandit "S", # flake8-simplify "SIM", # flake8-slots "SLOT", # flake8-debugger "T10", # flake8-type-checking "TC", # pyupgrade "UP", # pycodestyle warnings "W", # flake8-2020 "YTT", ] ignore = [ # Trailing comma missing "COM812", # Missing docstring in public module "D100", # Missing docstring in public class "D101", # Missing docstring in public method "D102", # Missing docstring in public function "D103", # Missing docstring in public package "D104", # Missing docstring in magic method "D105", # Missing docstring in public nested class "D106", # Missing docstring in __init__ "D107", # One-line docstring should fit on one line with quotes "D200", # No blank lines allowed after function docstring "D202", # 1 blank line required between summary line and description "D205", # Multi-line docstring closing quotes should be on a separate line "D209", # First line should end with a period "D400", # First line should be in imperative mood; try rephrasing "D401", # First line should not be the function's "signature" "D402", # First word of the first line should be properly capitalized "D403", # Too many return statements "PLR0911", # Too many branches "PLR0912", # Too many arguments in function definition "PLR0913", # Too many statements "PLR0915", # Magic value used in comparison "PLR2004", # String contains ambiguous {}. "RUF001", # Docstring contains ambiguous {}. "RUF002", # Comment contains ambiguous {}. "RUF003", # Mutable class attributes should be annotated with `typing.ClassVar` "RUF012", # Use of `assert` detected "S101", ] [tool.ruff.lint.isort] split-on-trailing-comma = false [tool.ruff.lint.pydocstyle] convention = "pep257" scrapy-queuelib-6d0e6a0/queuelib/000077500000000000000000000000001524277234600170625ustar00rootroot00000000000000scrapy-queuelib-6d0e6a0/queuelib/__init__.py000066400000000000000000000004141524277234600211720ustar00rootroot00000000000000__version__ = "1.10.0" from queuelib.pqueue import PriorityQueue from queuelib.queue import FifoDiskQueue, LifoDiskQueue from queuelib.rrqueue import RoundRobinQueue __all__ = [ "FifoDiskQueue", "LifoDiskQueue", "PriorityQueue", "RoundRobinQueue", ] scrapy-queuelib-6d0e6a0/queuelib/pqueue.py000066400000000000000000000053311524277234600207420ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING, Any from queuelib.queue import _clear if TYPE_CHECKING: from collections.abc import Callable, Iterable from queuelib.queue import BaseQueue class PriorityQueue: """A priority queue implemented using multiple internal queues (typically, FIFO queues). The internal queue must implement the following methods: * push(obj) * pop() * peek() * close() * __len__() Internal queues may also implement clear(); those that do not are emptied by popping. The constructor receives a qfactory argument, which is a callable used to instantiate a new (internal) queue when a new priority is allocated. The qfactory function is called with the priority number as first and only argument. Only integer priorities should be used. Lower numbers are higher priorities. startprios is a sequence of priorities to start with. If the queue was previously closed leaving some priority buckets non-empty, those priorities should be passed in startprios. """ def __init__( self, qfactory: Callable[[int], BaseQueue], startprios: Iterable[int] = () ) -> None: self.queues = {} self.qfactory = qfactory for p in startprios: self.queues[p] = self.qfactory(p) self.curprio = min(startprios) if startprios else None def push(self, obj: Any, priority: int = 0) -> None: if priority not in self.queues: self.queues[priority] = self.qfactory(priority) q = self.queues[priority] q.push(obj) # this may fail (eg. serialization error) if self.curprio is None or priority < self.curprio: self.curprio = priority def pop(self) -> Any | None: if self.curprio is None: return None q = self.queues[self.curprio] m = q.pop() if len(q) == 0: del self.queues[self.curprio] q.close() prios = [p for p, q in self.queues.items() if len(q) > 0] self.curprio = min(prios) if prios else None return m def peek(self) -> Any | None: if self.curprio is None: return None return self.queues[self.curprio].peek() def clear(self) -> None: for q in self.queues.values(): _clear(q) q.close() self.queues.clear() self.curprio = None def close(self) -> list[int]: active = [] for p, q in self.queues.items(): if len(q): active.append(p) q.close() return active def __len__(self) -> int: return sum(len(x) for x in self.queues.values()) if self.queues else 0 scrapy-queuelib-6d0e6a0/queuelib/py.typed000066400000000000000000000000001524277234600205470ustar00rootroot00000000000000scrapy-queuelib-6d0e6a0/queuelib/queue.py000066400000000000000000000243461524277234600205710ustar00rootroot00000000000000from __future__ import annotations import json import os import sqlite3 import struct from abc import abstractmethod from collections import deque from contextlib import suppress from pathlib import Path from typing import Any, BinaryIO, Literal, cast def _clear(queue: Any) -> None: """Remove every item from *queue*, popping them one by one if it does not implement ``clear()``.""" clear = getattr(queue, "clear", None) if clear is not None: clear() return while queue.pop() is not None: pass class _BaseQueueMeta(type): """ Metaclass to check queue classes against the necessary interface """ def __instancecheck__(cls, instance: Any) -> bool: return cls.__subclasscheck__( # pylint: disable=no-value-for-parameter type(instance) ) def __subclasscheck__(cls, subclass: Any) -> bool: return ( hasattr(subclass, "push") and callable(subclass.push) and hasattr(subclass, "pop") and callable(subclass.pop) and hasattr(subclass, "peek") and callable(subclass.peek) and hasattr(subclass, "close") and callable(subclass.close) and hasattr(subclass, "__len__") and callable(subclass.__len__) ) class BaseQueue(metaclass=_BaseQueueMeta): @abstractmethod def push(self, obj: Any) -> None: raise NotImplementedError @abstractmethod def pop(self) -> Any | None: raise NotImplementedError @abstractmethod def peek(self) -> Any | None: raise NotImplementedError @abstractmethod def __len__(self) -> int: raise NotImplementedError def close(self) -> None: pass class FifoMemoryQueue: """In-memory FIFO queue, API compliant with FifoDiskQueue.""" def __init__(self) -> None: self.q: deque[Any] = deque() def push(self, obj: Any) -> None: self.q.append(obj) def pop(self) -> Any | None: return self.q.popleft() if self.q else None def peek(self) -> Any | None: return self.q[0] if self.q else None def clear(self) -> None: self.q.clear() def close(self) -> None: pass def __len__(self) -> int: return len(self.q) class LifoMemoryQueue(FifoMemoryQueue): """In-memory LIFO queue, API compliant with LifoDiskQueue.""" def pop(self) -> Any | None: return self.q.pop() if self.q else None def peek(self) -> Any | None: return self.q[-1] if self.q else None class FifoDiskQueue: """Persistent FIFO queue.""" szhdr_format = ">L" szhdr_size = struct.calcsize(szhdr_format) def __init__(self, path: str | os.PathLike[str], chunksize: int = 100000) -> None: self.path = str(path) if not Path(path).exists(): Path(path).mkdir(parents=True) self.info = self._loadinfo(chunksize) self.chunksize = self.info["chunksize"] self.headf = self._openchunk(self.info["head"][0], "ab+") self.tailf = self._openchunk(self.info["tail"][0]) os.lseek(self.tailf.fileno(), self.info["tail"][2], os.SEEK_SET) def push(self, string: bytes) -> None: if not isinstance(string, bytes): raise TypeError(f"Unsupported type: {type(string).__name__}") hnum, hpos = self.info["head"] hpos += 1 szhdr = struct.pack(self.szhdr_format, len(string)) os.write(self.headf.fileno(), szhdr + string) if hpos == self.chunksize: hpos = 0 hnum += 1 self.headf.close() self.headf = self._openchunk(hnum, "ab+") self.info["size"] += 1 self.info["head"] = [hnum, hpos] def _openchunk(self, number: int, mode: Literal["rb", "ab+"] = "rb") -> BinaryIO: return Path(self.path, f"q{number:05d}").open(mode) def pop(self) -> bytes | None: tnum, tcnt, toffset = self.info["tail"] if [tnum, tcnt] >= self.info["head"]: return None tfd = self.tailf.fileno() szhdr = os.read(tfd, self.szhdr_size) if not szhdr: return None (size,) = struct.unpack(self.szhdr_format, szhdr) data = os.read(tfd, size) tcnt += 1 toffset += self.szhdr_size + size if tcnt == self.chunksize and tnum <= self.info["head"][0]: tcnt = toffset = 0 tnum += 1 self.tailf.close() Path(self.tailf.name).unlink() self.tailf = self._openchunk(tnum) self.info["size"] -= 1 self.info["tail"] = [tnum, tcnt, toffset] return data def peek(self) -> bytes | None: tnum, tcnt, _ = self.info["tail"] if [tnum, tcnt] >= self.info["head"]: return None tfd = self.tailf.fileno() tfd_initial_pos = os.lseek(tfd, 0, os.SEEK_CUR) szhdr = os.read(tfd, self.szhdr_size) if not szhdr: return None (size,) = struct.unpack(self.szhdr_format, szhdr) data = os.read(tfd, size) os.lseek(tfd, tfd_initial_pos, os.SEEK_SET) return data def clear(self) -> None: self.headf.close() self.tailf.close() for chunk in Path(self.path).glob("q*"): chunk.unlink() self.info["size"] = 0 self.info["tail"] = [0, 0, 0] self.info["head"] = [0, 0] self.headf = self._openchunk(0, "ab+") self.tailf = self._openchunk(0) def close(self) -> None: self.headf.close() self.tailf.close() self._saveinfo(self.info) if len(self) == 0: self._cleanup() def __len__(self) -> int: return cast("int", self.info["size"]) def _loadinfo(self, chunksize: int) -> dict[str, Any]: infopath = self._infopath() if infopath.exists(): info = cast("dict[str, Any]", json.loads(infopath.read_text())) else: info = { "chunksize": chunksize, "size": 0, "tail": [0, 0, 0], "head": [0, 0], } return info def _saveinfo(self, info: dict[str, Any]) -> None: self._infopath().write_text(json.dumps(info)) def _infopath(self) -> Path: return Path(self.path, "info.json") def _cleanup(self) -> None: for x in Path(self.path).glob("q*"): x.unlink() Path(self.path, "info.json").unlink() with suppress(OSError): Path(self.path).rmdir() class LifoDiskQueue: """Persistent LIFO queue.""" SIZE_FORMAT = ">L" SIZE_SIZE = struct.calcsize(SIZE_FORMAT) def __init__(self, path: str | os.PathLike[str]) -> None: self.size: int self.path = str(path) if Path(path).exists(): self.f = Path(path).open("rb+") # noqa: SIM115 qsize = self.f.read(self.SIZE_SIZE) (self.size,) = struct.unpack(self.SIZE_FORMAT, qsize) self.f.seek(0, os.SEEK_END) else: self.f = Path(path).open("wb+") # noqa: SIM115 self.f.write(struct.pack(self.SIZE_FORMAT, 0)) self.size = 0 def push(self, string: bytes) -> None: if not isinstance(string, bytes): raise TypeError(f"Unsupported type: {type(string).__name__}") self.f.write(string) ssize = struct.pack(self.SIZE_FORMAT, len(string)) self.f.write(ssize) self.size += 1 def pop(self) -> bytes | None: if not self.size: return None self.f.seek(-self.SIZE_SIZE, os.SEEK_END) (size,) = struct.unpack(self.SIZE_FORMAT, self.f.read()) self.f.seek(-size - self.SIZE_SIZE, os.SEEK_END) data = self.f.read(size) self.f.seek(-size, os.SEEK_CUR) self.f.truncate() self.size -= 1 return data def peek(self) -> bytes | None: if not self.size: return None self.f.seek(-self.SIZE_SIZE, os.SEEK_END) (size,) = struct.unpack(self.SIZE_FORMAT, self.f.read()) self.f.seek(-size - self.SIZE_SIZE, os.SEEK_END) return self.f.read(size) def clear(self) -> None: self.f.seek(0) self.f.write(struct.pack(self.SIZE_FORMAT, 0)) self.f.truncate() self.size = 0 def close(self) -> None: if self.size: self.f.seek(0) self.f.write(struct.pack(self.SIZE_FORMAT, self.size)) self.f.close() if not self.size: Path(self.path).unlink() def __len__(self) -> int: return self.size class FifoSQLiteQueue: _sql_create = "CREATE TABLE IF NOT EXISTS queue (id INTEGER PRIMARY KEY AUTOINCREMENT, item BLOB)" _sql_size = "SELECT COUNT(*) FROM queue" _sql_push = "INSERT INTO queue (item) VALUES (?)" _sql_pop = "SELECT id, item FROM queue ORDER BY id LIMIT 1" _sql_del = "DELETE FROM queue WHERE id = ?" _sql_clear = "DELETE FROM queue" def __init__(self, path: str | os.PathLike[str]) -> None: self._path = Path(path).resolve() self._db = sqlite3.Connection(self._path, timeout=60) self._db.text_factory = bytes with self._db as conn: conn.execute(self._sql_create) def push(self, item: bytes) -> None: if not isinstance(item, bytes): raise TypeError(f"Unsupported type: {type(item).__name__}") with self._db as conn: conn.execute(self._sql_push, (item,)) def pop(self) -> bytes | None: with self._db as conn: for id_, item in conn.execute(self._sql_pop): conn.execute(self._sql_del, (id_,)) return cast("bytes", item) return None def peek(self) -> bytes | None: with self._db as conn: for _, item in conn.execute(self._sql_pop): return cast("bytes", item) return None def clear(self) -> None: with self._db as conn: conn.execute(self._sql_clear) def close(self) -> None: size = len(self) self._db.close() if not size: self._path.unlink() def __len__(self) -> int: with self._db as conn: return cast("int", next(conn.execute(self._sql_size))[0]) class LifoSQLiteQueue(FifoSQLiteQueue): _sql_pop = "SELECT id, item FROM queue ORDER BY id DESC LIMIT 1" scrapy-queuelib-6d0e6a0/queuelib/rrqueue.py000066400000000000000000000057261524277234600211360ustar00rootroot00000000000000from __future__ import annotations from collections import deque from typing import TYPE_CHECKING, Any from queuelib.queue import _clear if TYPE_CHECKING: from collections.abc import Callable, Hashable, Iterable from queuelib.queue import BaseQueue class RoundRobinQueue: """A round robin queue implemented using multiple internal queues (typically, FIFO queues). The internal queue must implement the following methods: * push(obj) * pop() * peek() * close() * __len__() Internal queues may also implement clear(); those that do not are emptied by popping. The constructor receives a qfactory argument, which is a callable used to instantiate a new (internal) queue when a new key is allocated. The qfactory function is called with the key number as first and only argument. start_domains is a sequence of domains to initialize the queue with. If the queue was previously closed leaving some domain buckets non-empty, those domains should be passed in start_domains. The queue maintains a fifo queue of keys. The key that went last is popped first and the next queue for that key is then popped. """ def __init__( self, qfactory: Callable[[Hashable], BaseQueue], start_domains: Iterable[Hashable] = (), ) -> None: self.queues = {} self.qfactory = qfactory for key in start_domains: self.queues[key] = self.qfactory(key) self.key_queue = deque(start_domains) def push(self, obj: Any, key: Hashable) -> None: if key not in self.key_queue: self.queues[key] = self.qfactory(key) self.key_queue.appendleft(key) # it's new, might as well pop first q = self.queues[key] q.push(obj) # this may fail (eg. serialization error) def peek(self) -> Any | None: try: key = self.key_queue[-1] except IndexError: return None return self.queues[key].peek() def clear(self) -> None: for q in self.queues.values(): _clear(q) q.close() self.queues.clear() self.key_queue.clear() def pop(self) -> Any | None: # pop until we find a valid object, closing necessary queues while True: try: key = self.key_queue.pop() except IndexError: return None q = self.queues[key] m = q.pop() if len(q) == 0: del self.queues[key] q.close() else: self.key_queue.appendleft(key) if m is not None: return m def close(self) -> list[Hashable]: active = [] for k, q in self.queues.items(): if len(q): active.append(k) q.close() return active def __len__(self) -> int: return sum(len(x) for x in self.queues.values()) if self.queues else 0 scrapy-queuelib-6d0e6a0/queuelib/tests/000077500000000000000000000000001524277234600202245ustar00rootroot00000000000000scrapy-queuelib-6d0e6a0/queuelib/tests/__init__.py000066400000000000000000000027451524277234600223450ustar00rootroot00000000000000from __future__ import annotations import shutil import tempfile import unittest from pathlib import Path from typing import Any class DummyQueue: """Minimal queue implementation, without the optional clear() method.""" def __init__(self) -> None: self.q: list[Any] = [] def push(self, obj: Any) -> None: self.q.append(obj) def pop(self) -> Any | None: return self.q.pop() if self.q else None def peek(self) -> Any | None: return self.q[-1] if self.q else None def close(self) -> None: pass def __len__(self) -> int: return len(self.q) class QueuelibTestCase(unittest.TestCase): def setUp(self) -> None: self.tmpdir = tempfile.mkdtemp(prefix="queuelib-tests-") self.qpath: Path = self.tempfilename() self.qdir = self.mkdtemp() def tearDown(self) -> None: shutil.rmtree(self.qdir) shutil.rmtree(self.tmpdir) def tempfilename(self) -> Path: with tempfile.NamedTemporaryFile(dir=self.tmpdir) as nf: return Path(nf.name) def mkdtemp(self) -> str: return tempfile.mkdtemp(dir=self.tmpdir) def track_closed(cls): """Wraps a queue class to track down if close() method was called""" class TrackingClosed(cls): def __init__(self, *a, **kw): super().__init__(*a, **kw) self.closed = False def close(self): super().close() self.closed = True return TrackingClosed scrapy-queuelib-6d0e6a0/queuelib/tests/test_pqueue.py000066400000000000000000000154421524277234600231470ustar00rootroot00000000000000from pathlib import Path import pytest from queuelib.pqueue import PriorityQueue from queuelib.queue import ( FifoDiskQueue, FifoMemoryQueue, FifoSQLiteQueue, LifoDiskQueue, LifoMemoryQueue, LifoSQLiteQueue, ) from queuelib.tests import DummyQueue, QueuelibTestCase, track_closed class PQueueTestMixin: def setUp(self): QueuelibTestCase.setUp(self) self.q = PriorityQueue(self.qfactory) def qfactory(self, prio): raise NotImplementedError def test_len_nonzero(self): assert not self.q assert len(self.q) == 0 self.q.push(b"a", 3) assert self.q self.q.push(b"b", 1) self.q.push(b"c", 2) self.q.push(b"d", 1) assert len(self.q) == 4 self.q.pop() self.q.pop() self.q.pop() self.q.pop() assert not self.q assert len(self.q) == 0 def test_close(self): self.q.push(b"a", 3) self.q.push(b"b", 1) self.q.push(b"c", 2) self.q.push(b"d", 1) iqueues = self.q.queues.values() assert sorted(self.q.close()) == [1, 2, 3] assert all(q.closed for q in iqueues) def test_close_return_active(self): self.q.push(b"b", 1) self.q.push(b"c", 2) self.q.push(b"a", 3) self.q.pop() assert sorted(self.q.close()) == [2, 3] def test_clear(self): self.q.clear() assert len(self.q) == 0 self.q.push(b"a", 3) self.q.push(b"b", 1) iqueues = list(self.q.queues.values()) self.q.clear() assert len(self.q) == 0 assert self.q.peek() is None assert self.q.pop() is None assert all(q.closed for q in iqueues) self.q.push(b"c", 2) assert self.q.pop() == b"c" assert not self.q.close() def test_popped_internal_queues_closed(self): self.q.push(b"a", 3) self.q.push(b"b", 1) self.q.push(b"c", 2) p1queue = self.q.queues[1] assert self.q.pop() == b"b" self.q.close() assert p1queue.closed class FifoTestMixin: def test_push_pop_peek_noprio(self): assert self.q.peek() is None self.q.push(b"a") self.q.push(b"b") self.q.push(b"c") assert self.q.peek() == b"a" assert self.q.pop() == b"a" assert self.q.peek() == b"b" assert self.q.pop() == b"b" assert self.q.peek() == b"c" assert self.q.pop() == b"c" assert self.q.peek() is None assert self.q.pop() is None def test_push_pop_peek_prio(self): assert self.q.peek() is None self.q.push(b"a", 3) self.q.push(b"b", 1) self.q.push(b"c", 2) self.q.push(b"d", 1) assert self.q.peek() == b"b" assert self.q.pop() == b"b" assert self.q.peek() == b"d" assert self.q.pop() == b"d" assert self.q.peek() == b"c" assert self.q.pop() == b"c" assert self.q.peek() == b"a" assert self.q.pop() == b"a" assert self.q.peek() is None assert self.q.pop() is None class LifoTestMixin: def test_push_pop_peek_noprio(self): assert self.q.peek() is None self.q.push(b"a") self.q.push(b"b") self.q.push(b"c") assert self.q.peek() == b"c" assert self.q.pop() == b"c" assert self.q.peek() == b"b" assert self.q.pop() == b"b" assert self.q.peek() == b"a" assert self.q.pop() == b"a" assert self.q.peek() is None assert self.q.pop() is None def test_push_pop_peek_prio(self): assert self.q.peek() is None self.q.push(b"a", 3) self.q.push(b"b", 1) self.q.push(b"c", 2) self.q.push(b"d", 1) assert self.q.peek() == b"d" assert self.q.pop() == b"d" assert self.q.peek() == b"b" assert self.q.pop() == b"b" assert self.q.peek() == b"c" assert self.q.pop() == b"c" assert self.q.peek() == b"a" assert self.q.pop() == b"a" assert self.q.peek() is None assert self.q.pop() is None class FifoMemoryPriorityQueueTest(PQueueTestMixin, FifoTestMixin, QueuelibTestCase): def qfactory(self, prio): return track_closed(FifoMemoryQueue)() class LifoMemoryPriorityQueueTest(PQueueTestMixin, LifoTestMixin, QueuelibTestCase): def qfactory(self, prio): return track_closed(LifoMemoryQueue)() class DiskTestMixin: def test_nonserializable_object_one(self): with pytest.raises(TypeError): self.q.push(lambda x: x, 0) assert self.q.close() == [] def test_nonserializable_object_many_close(self): self.q.push(b"a", 3) self.q.push(b"b", 1) with pytest.raises(TypeError): self.q.push(lambda x: x, 0) self.q.push(b"c", 2) assert self.q.pop() == b"b" assert sorted(self.q.close()) == [2, 3] def test_nonserializable_object_many_pop(self): self.q.push(b"a", 3) self.q.push(b"b", 1) with pytest.raises(TypeError): self.q.push(lambda x: x, 0) self.q.push(b"c", 2) assert self.q.pop() == b"b" assert self.q.pop() == b"c" assert self.q.pop() == b"a" assert self.q.pop() is None assert self.q.close() == [] def test_reopen_with_prio(self): q1 = PriorityQueue(self.qfactory) q1.push(b"a", 3) q1.push(b"b", 1) q1.push(b"c", 2) active = q1.close() q2 = PriorityQueue(self.qfactory, startprios=active) assert q2.pop() == b"b" assert q2.pop() == b"c" assert q2.pop() == b"a" assert not q2.close() class FifoDiskPriorityQueueTest( PQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase ): def qfactory(self, prio): path = Path(self.qdir, str(prio)) return track_closed(FifoDiskQueue)(path) class LifoDiskPriorityQueueTest( PQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase ): def qfactory(self, prio): path = Path(self.qdir, str(prio)) return track_closed(LifoDiskQueue)(path) class FifoSQLitePriorityQueueTest( PQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase ): def qfactory(self, prio): path = Path(self.qdir, str(prio)) return track_closed(FifoSQLiteQueue)(path) class LifoSQLitePriorityQueueTest( PQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase ): def qfactory(self, prio): path = Path(self.qdir, str(prio)) return track_closed(LifoSQLiteQueue)(path) class DummyPriorityQueueTest(QueuelibTestCase): def test_clear(self): q = PriorityQueue(lambda prio: DummyQueue()) q.push(b"a", 1) q.push(b"b", 2) q.clear() assert len(q) == 0 assert q.pop() is None assert not q.close() scrapy-queuelib-6d0e6a0/queuelib/tests/test_queue.py000066400000000000000000000237621524277234600227730ustar00rootroot00000000000000from __future__ import annotations from abc import abstractmethod from unittest import mock import pytest from queuelib.queue import ( BaseQueue, FifoDiskQueue, FifoMemoryQueue, FifoSQLiteQueue, LifoDiskQueue, LifoMemoryQueue, LifoSQLiteQueue, ) from queuelib.tests import DummyQueue, QueuelibTestCase class InterfaceTest(QueuelibTestCase): def test_queue(self): queue = BaseQueue() with pytest.raises(NotImplementedError): queue.push(b"") with pytest.raises(NotImplementedError): queue.peek() with pytest.raises(NotImplementedError): queue.pop() with pytest.raises(NotImplementedError): len(queue) queue.close() def test_issubclass(self): assert not issubclass(list, BaseQueue) assert not issubclass(int, BaseQueue) assert not issubclass(QueuelibTestCase, BaseQueue) assert issubclass(DummyQueue, BaseQueue) assert issubclass(FifoMemoryQueue, BaseQueue) assert issubclass(LifoMemoryQueue, BaseQueue) assert issubclass(FifoDiskQueue, BaseQueue) assert issubclass(LifoDiskQueue, BaseQueue) assert issubclass(FifoSQLiteQueue, BaseQueue) assert issubclass(LifoSQLiteQueue, BaseQueue) def test_isinstance(self): assert not isinstance(1, BaseQueue) assert not isinstance([], BaseQueue) assert isinstance(DummyQueue(), BaseQueue) assert isinstance(FifoMemoryQueue(), BaseQueue) assert isinstance(LifoMemoryQueue(), BaseQueue) for cls in [FifoDiskQueue, LifoDiskQueue, FifoSQLiteQueue, LifoSQLiteQueue]: queue = cls(self.tempfilename()) assert isinstance(queue, BaseQueue) queue.close() class QueueTestMixin: @abstractmethod def queue(self) -> BaseQueue: raise NotImplementedError def test_empty(self): """Empty queue test""" q = self.queue() assert q.pop() is None q.close() def test_single_pushpop(self): q = self.queue() q.push(b"a") assert q.pop() == b"a" q.close() def test_binary_element(self): elem = ( b"\x80\x02}q\x01(U\x04bodyq\x02U\x00U\t_encodingq\x03U\x05utf-" b"8q\x04U\x07cookiesq\x05}q\x06U\x04metaq\x07}q\x08U\x07header" b"sq\t}U\x03urlq\nX\x15\x00\x00\x00file:///tmp/tmphDJYsgU\x0bd" b"ont_filterq\x0b\x89U\x08priorityq\x0cK\x00U\x08callbackq\rNU" b"\x06methodq\x0eU\x03GETq\x0fU\x07errbackq\x10Nu." ) q = self.queue() q.push(elem) assert q.pop() == elem q.close() def test_len(self): q = self.queue() assert len(q) == 0 q.push(b"a") assert len(q) == 1 q.push(b"b") q.push(b"c") assert len(q) == 3 q.pop() q.pop() q.pop() assert len(q) == 0 q.close() def test_clear(self): q = self.queue() q.clear() assert len(q) == 0 q.push(b"a") q.push(b"b") q.clear() assert len(q) == 0 assert q.peek() is None assert q.pop() is None q.push(b"c") assert q.pop() == b"c" q.close() def test_peek_one_element(self): q = self.queue() assert q.peek() is None q.push(b"a") assert q.peek() == b"a" assert q.pop() == b"a" assert q.peek() is None q.close() class FifoTestMixin: def test_push_pop1(self): """Basic push/pop test""" q = self.queue() q.push(b"a") q.push(b"b") q.push(b"c") assert q.pop() == b"a" assert q.pop() == b"b" assert q.pop() == b"c" assert q.pop() is None q.close() def test_push_pop2(self): """Test interleaved push and pops""" q = self.queue() q.push(b"a") q.push(b"b") q.push(b"c") q.push(b"d") assert q.pop() == b"a" assert q.pop() == b"b" q.push(b"e") assert q.pop() == b"c" assert q.pop() == b"d" assert q.pop() == b"e" q.close() def test_peek_fifo(self): q = self.queue() assert q.peek() is None q.push(b"a") q.push(b"b") q.push(b"c") assert q.peek() == b"a" assert q.peek() == b"a" assert q.pop() == b"a" assert q.peek() == b"b" assert q.peek() == b"b" assert q.pop() == b"b" assert q.peek() == b"c" assert q.peek() == b"c" assert q.pop() == b"c" assert q.peek() is None q.close() class LifoTestMixin: def test_push_pop1(self): """Basic push/pop test""" q = self.queue() q.push(b"a") q.push(b"b") q.push(b"c") assert q.pop() == b"c" assert q.pop() == b"b" assert q.pop() == b"a" assert q.pop() is None q.close() def test_push_pop2(self): """Test interleaved push and pops""" q = self.queue() q.push(b"a") q.push(b"b") q.push(b"c") q.push(b"d") assert q.pop() == b"d" assert q.pop() == b"c" q.push(b"e") assert q.pop() == b"e" assert q.pop() == b"b" assert q.pop() == b"a" q.close() def test_peek_lifo(self): q = self.queue() assert q.peek() is None q.push(b"a") q.push(b"b") q.push(b"c") assert q.peek() == b"c" assert q.peek() == b"c" assert q.pop() == b"c" assert q.peek() == b"b" assert q.peek() == b"b" assert q.pop() == b"b" assert q.peek() == b"a" assert q.peek() == b"a" assert q.pop() == b"a" assert q.peek() is None q.close() class PersistentTestMixin: chunksize = 100000 # Scrapy SerializableQueue doesn't raise TypeError for non-bytes objects, # so this test fails when run for it. @pytest.mark.xfail( reason="Reenable once Scrapy.squeues stops extending from this testsuite" ) def test_non_bytes_raises_typeerror(self): q = self.queue() try: with pytest.raises(TypeError): q.push(0) with pytest.raises(TypeError): q.push("") with pytest.raises(TypeError): q.push(None) with pytest.raises(TypeError): q.push(lambda x: x) finally: q.close() def test_text_in_windows(self): e1 = b"\r\n" q = self.queue() q.push(e1) q.close() q = self.queue() e2 = q.pop() assert e1 == e2 q.close() def test_close_open(self): """Test closing and re-opening keeps state""" q = self.queue() q.push(b"a") q.push(b"b") q.push(b"c") q.push(b"d") q.pop() q.pop() q.close() del q q = self.queue() assert len(q) == 2 q.push(b"e") q.pop() q.pop() q.close() del q q = self.queue() assert q.pop() is not None assert len(q) == 0 q.close() def test_cleanup(self): """Test queue dir is removed if queue is empty""" q = self.queue() values = [b"0", b"1", b"2", b"3", b"4"] assert self.qpath.exists() for x in values: q.push(x) for _ in values: q.pop() q.close() assert not self.qpath.exists() class FifoMemoryQueueTest(FifoTestMixin, QueueTestMixin, QueuelibTestCase): def queue(self): return FifoMemoryQueue() class LifoMemoryQueueTest(LifoTestMixin, QueueTestMixin, QueuelibTestCase): def queue(self): return LifoMemoryQueue() class FifoDiskQueueTest( FifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase ): def queue(self): return FifoDiskQueue(self.qpath, chunksize=self.chunksize) def test_not_szhdr(self): q = self.queue() q.push(b"something") with ( self.tempfilename().open("w+") as empty_file, mock.patch.object(q, "tailf", empty_file), ): assert q.peek() is None assert q.pop() is None q.close() def test_chunks(self): """Test chunks are created and removed""" values = [b"0", b"1", b"2", b"3", b"4"] q = self.queue() for x in values: q.push(x) chunks = list(self.qpath.glob("q*")) assert len(chunks) == 5 // self.chunksize + 1 for _ in values: q.pop() chunks = list(self.qpath.glob("q*")) assert len(chunks) == 1 q.close() def test_clear_removes_chunks(self): q = self.queue() for x in [b"0", b"1", b"2", b"3", b"4"]: q.push(x) q.clear() assert list(self.qpath.glob("q*")) == [self.qpath / "q00000"] q.close() class ChunkSize1FifoDiskQueueTest(FifoDiskQueueTest): chunksize = 1 class ChunkSize2FifoDiskQueueTest(FifoDiskQueueTest): chunksize = 2 class ChunkSize3FifoDiskQueueTest(FifoDiskQueueTest): chunksize = 3 class ChunkSize4FifoDiskQueueTest(FifoDiskQueueTest): chunksize = 4 class LifoDiskQueueTest( LifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase ): def queue(self): return LifoDiskQueue(self.qpath) def test_file_size_shrinks(self): """Test size of queue file shrinks when popping items""" q = self.queue() q.push(b"a") q.push(b"b") q.close() size = self.qpath.stat().st_size q = self.queue() q.pop() q.close() assert self.qpath.stat().st_size, size class FifoSQLiteQueueTest( FifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase ): def queue(self): return FifoSQLiteQueue(self.qpath) class LifoSQLiteQueueTest( LifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase ): def queue(self): return LifoSQLiteQueue(self.qpath) scrapy-queuelib-6d0e6a0/queuelib/tests/test_rrqueue.py000066400000000000000000000173721524277234600233370ustar00rootroot00000000000000from pathlib import Path import pytest from queuelib.queue import ( FifoDiskQueue, FifoMemoryQueue, FifoSQLiteQueue, LifoDiskQueue, LifoMemoryQueue, LifoSQLiteQueue, ) from queuelib.rrqueue import RoundRobinQueue from queuelib.tests import DummyQueue, QueuelibTestCase, track_closed class RRQueueTestMixin: def setUp(self): super().setUp() self.q = RoundRobinQueue(self.qfactory) def qfactory(self, key): raise NotImplementedError def test_len_nonzero(self): assert not self.q assert len(self.q) == 0 self.q.push(b"a", "3") assert self.q self.q.push(b"b", "1") self.q.push(b"c", "2") self.q.push(b"d", "1") assert len(self.q) == 4 self.q.pop() self.q.pop() self.q.pop() self.q.pop() assert not self.q assert len(self.q) == 0 def test_close(self): self.q.push(b"a", "3") self.q.push(b"b", "1") self.q.push(b"c", "2") self.q.push(b"d", "1") iqueues = self.q.queues.values() assert sorted(self.q.close()) == ["1", "2", "3"] assert all(q.closed for q in iqueues) def test_clear(self): self.q.clear() assert len(self.q) == 0 self.q.push(b"a", "3") self.q.push(b"b", "1") iqueues = list(self.q.queues.values()) self.q.clear() assert len(self.q) == 0 assert self.q.peek() is None assert self.q.pop() is None assert all(q.closed for q in iqueues) self.q.push(b"c", "2") assert self.q.pop() == b"c" assert not self.q.close() def test_close_return_active(self): self.q.push(b"b", "1") self.q.push(b"c", "2") self.q.push(b"a", "3") self.q.pop() assert sorted(self.q.close()) == ["2", "3"] def test_pop_returns_falsy_items(self): self.q.push(b"", "a") assert len(self.q) == 1 assert self.q.pop() == b"" assert len(self.q) == 0 assert self.q.pop() is None def test_falsy_items_keep_round_robin_order(self): self.q.push(b"one", "a") self.q.push(b"", "b") self.q.push(b"two", "c") assert [self.q.pop(), self.q.pop(), self.q.pop()] == [b"one", b"", b"two"] class FifoTestMixin: def test_push_pop_peek_key(self): assert self.q.peek() is None self.q.push(b"a", "1") self.q.push(b"b", "1") self.q.push(b"c", "2") self.q.push(b"d", "2") assert self.q.peek() == b"a" assert self.q.pop() == b"a" assert self.q.peek() == b"c" assert self.q.pop() == b"c" assert self.q.peek() == b"b" assert self.q.pop() == b"b" assert self.q.peek() == b"d" assert self.q.pop() == b"d" assert self.q.peek() is None assert self.q.pop() is None class LifoTestMixin: def test_push_pop_peek_key(self): assert self.q.peek() is None self.q.push(b"a", "1") self.q.push(b"b", "1") self.q.push(b"c", "2") self.q.push(b"d", "2") assert self.q.peek() == b"b" assert self.q.pop() == b"b" assert self.q.peek() == b"d" assert self.q.pop() == b"d" assert self.q.peek() == b"a" assert self.q.pop() == b"a" assert self.q.peek() == b"c" assert self.q.pop() == b"c" assert self.q.peek() is None assert self.q.pop() is None class FifoMemoryRRQueueTest(RRQueueTestMixin, FifoTestMixin, QueuelibTestCase): def qfactory(self, key): return track_closed(FifoMemoryQueue)() class LifoMemoryRRQueueTest(RRQueueTestMixin, LifoTestMixin, QueuelibTestCase): def qfactory(self, key): return track_closed(LifoMemoryQueue)() class DiskTestMixin: def test_nonserializable_object_one(self): with pytest.raises(TypeError): self.q.push(lambda x: x, "0") assert self.q.close() == [] def test_nonserializable_object_many_close(self): self.q.push(b"a", "3") self.q.push(b"b", "1") with pytest.raises(TypeError): self.q.push(lambda x: x, "0") self.q.push(b"c", "2") assert self.q.pop() == b"a" assert sorted(self.q.close()) == ["1", "2"] def test_nonserializable_object_many_pop(self): self.q.push(b"a", "3") self.q.push(b"b", "1") with pytest.raises(TypeError): self.q.push(lambda x: x, "0") self.q.push(b"c", "2") assert self.q.pop() == b"a" assert self.q.pop() == b"b" assert self.q.pop() == b"c" assert self.q.pop() is None assert self.q.close() == [] class FifoDiskRRQueueTest( RRQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase ): def qfactory(self, key): path = Path(self.qdir, str(key)) return track_closed(FifoDiskQueue)(path) class LifoDiskRRQueueTest( RRQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase ): def qfactory(self, key): path = Path(self.qdir, str(key)) return track_closed(LifoDiskQueue)(path) class FifoSQLiteRRQueueTest( RRQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase ): def qfactory(self, key): path = Path(self.qdir, str(key)) return track_closed(FifoSQLiteQueue)(path) class LifoSQLiteRRQueueTest( RRQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase ): def qfactory(self, key): path = Path(self.qdir, str(key)) return track_closed(LifoSQLiteQueue)(path) class RRQueueStartDomainsTestMixin: def setUp(self): super().setUp() self.q = RoundRobinQueue(self.qfactory, start_domains=["1", "2"]) def qfactory(self, key): raise NotImplementedError def test_push_pop_peek_key(self): self.q.push(b"c", "1") self.q.push(b"d", "2") assert self.q.peek() == b"d" assert self.q.pop() == b"d" assert self.q.peek() == b"c" assert self.q.pop() == b"c" assert self.q.peek() is None assert self.q.pop() is None def test_push_pop_peek_key_reversed(self): self.q.push(b"d", "2") self.q.push(b"c", "1") assert self.q.peek() == b"d" assert self.q.pop() == b"d" assert self.q.peek() == b"c" assert self.q.pop() == b"c" assert self.q.peek() is None assert self.q.pop() is None class FifoMemoryRRQueueStartDomainsTest(RRQueueStartDomainsTestMixin, QueuelibTestCase): def qfactory(self, key): return track_closed(FifoMemoryQueue)() class LifoMemoryRRQueueStartDomainsTest(RRQueueStartDomainsTestMixin, QueuelibTestCase): def qfactory(self, key): return track_closed(LifoMemoryQueue)() class FifoDiskRRQueueStartDomainsTest(RRQueueStartDomainsTestMixin, QueuelibTestCase): def qfactory(self, key): path = Path(self.qdir, str(key)) return track_closed(FifoDiskQueue)(path) class LifoDiskRRQueueStartDomainsTest(RRQueueStartDomainsTestMixin, QueuelibTestCase): def qfactory(self, key): path = Path(self.qdir, str(key)) return track_closed(LifoDiskQueue)(path) class FifoSQLiteRRQueueStartDomainsTest(RRQueueStartDomainsTestMixin, QueuelibTestCase): def qfactory(self, key): path = Path(self.qdir, str(key)) return track_closed(FifoSQLiteQueue)(path) class LifoSQLiteRRQueueStartDomainsTest(RRQueueStartDomainsTestMixin, QueuelibTestCase): def qfactory(self, key): path = Path(self.qdir, str(key)) return track_closed(LifoSQLiteQueue)(path) class DummyRRQueueTest(QueuelibTestCase): def test_clear(self): q = RoundRobinQueue(lambda key: DummyQueue()) q.push(b"a", "1") q.push(b"b", "2") q.clear() assert len(q) == 0 assert q.pop() is None assert not q.close() scrapy-queuelib-6d0e6a0/tox.ini000066400000000000000000000014271524277234600165660ustar00rootroot00000000000000[tox] envlist = py, pylint, mypy, twinecheck, pre-commit [testenv] deps = pytest pytest-cov >= 7.0.0 commands = py.test --cov=queuelib --cov-report=xml --cov-report=term-missing \ --junitxml=testenv.junit.xml -o junit_family=legacy \ {posargs:queuelib} [testenv:pylint] basepython = python3 deps = {[testenv]deps} pylint==4.0.7 commands = pylint {posargs:queuelib} [testenv:mypy] basepython = python3 deps = mypy==2.3.1 pytest==9.1.1 commands = mypy {posargs:queuelib} [testenv:twinecheck] basepython = python3 deps = twine==7.0.0 build==1.5.0 commands = python -m build --sdist twine check dist/* [testenv:pre-commit] deps = pre-commit commands = pre-commit run --all-files --show-diff-on-failure skip_install = true