pax_global_header00006660000000000000000000000064152706076260014525gustar00rootroot0000000000000052 comment=a76f227f6d0d8fc027b617391540590bf18b1ffa scrapy-itemadapter-78cc818/000077500000000000000000000000001527060762600156725ustar00rootroot00000000000000scrapy-itemadapter-78cc818/.editorconfig000066400000000000000000000002651527060762600203520ustar00rootroot00000000000000[*] trim_trailing_whitespace = true insert_final_newline = true indent_style = space [Makefile] indent_style = tab [*.py] indent_size = 4 charset = utf-8 [*.yml] indent_size = 2 scrapy-itemadapter-78cc818/.git-blame-ignore-revs000066400000000000000000000001241527060762600217670ustar00rootroot00000000000000# applying pre-commit hooks to the project 106a4e0af9e9ac07defef3c9a781d2fe0ac4640f scrapy-itemadapter-78cc818/.github/000077500000000000000000000000001527060762600172325ustar00rootroot00000000000000scrapy-itemadapter-78cc818/.github/dependabot.yml000066400000000000000000000003331527060762600220610ustar00rootroot00000000000000version: 2 updates: - package-ecosystem: github-actions directory: "/" schedule: interval: monthly groups: github-actions: patterns: - "*" cooldown: default-days: 7 scrapy-itemadapter-78cc818/.github/workflows/000077500000000000000000000000001527060762600212675ustar00rootroot00000000000000scrapy-itemadapter-78cc818/.github/workflows/checks.yml000066400000000000000000000024601527060762600232540ustar00rootroot00000000000000name: Checks on: push: branches: - master - '[0-9]+.[0-9]+' pull_request: permissions: {} jobs: checks: runs-on: ubuntu-latest permissions: contents: read strategy: fail-fast: false matrix: include: - python-version: "3.14" env: TOXENV: mypy - python-version: "3.14" env: TOXENV: docs - python-version: "3.14" env: TOXENV: twinecheck - python-version: "3.14" env: TOXENV: pylint steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} - name: Run check env: ${{ matrix.env }} run: | pip install -U pip pip install -U tox tox pre-commit: runs-on: ubuntu-latest permissions: contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 scrapy-itemadapter-78cc818/.github/workflows/codspeed.yml000066400000000000000000000020251527060762600235770ustar00rootroot00000000000000name: CodSpeed on: push: branches: - master pull_request: paths: - itemadapter/** - tests/benchmarks/** - .github/workflows/codspeed.yml - tox.ini workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true permissions: {} jobs: benchmark: runs-on: ubuntu-latest permissions: contents: read id-token: write # OIDC authentication with CodSpeed steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Set up Python 3.14 uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.14" - name: Install dependencies run: | pip install tox tox -n -e benchmark - name: Run benchmarks uses: CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5.6.0 with: mode: simulation run: tox -e benchmark scrapy-itemadapter-78cc818/.github/workflows/publish.yml000066400000000000000000000013141527060762600234570ustar00rootroot00000000000000name: Publish on: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' jobs: publish: runs-on: ubuntu-latest environment: name: pypi url: https://pypi.org/p/${{ github.event.repository.name }} permissions: id-token: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.14" - run: | python -m pip install --upgrade build python -m build - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 scrapy-itemadapter-78cc818/.github/workflows/tests.yml000066400000000000000000000055601527060762600231620ustar00rootroot00000000000000name: Tests on: push: branches: - master - '[0-9]+.[0-9]+' pull_request: permissions: {} jobs: tests-ubuntu: name: "Test: ${{ matrix.env.TOXENV }}, ${{ matrix.python-version }}, Ubuntu" runs-on: ubuntu-latest permissions: contents: read id-token: write # OIDC authentication with Codecov strategy: fail-fast: false matrix: include: - python-version: "3.11" env: TOXENV: min-attrs - python-version: "3.11" env: TOXENV: min-pydantic - python-version: "3.11" env: TOXENV: min-scrapy - python-version: "3.11" env: TOXENV: min-extra - python-version: "3.11" env: TOXENV: py - python-version: "3.12" env: TOXENV: py - python-version: "3.13" env: TOXENV: py - python-version: "3.14" env: TOXENV: py - python-version: "3.14" env: TOXENV: attrs - python-version: "3.14" env: TOXENV: pydantic - python-version: "3.14" env: TOXENV: scrapy - python-version: "3.14" env: TOXENV: extra - python-version: "pypy3.11" env: TOXENV: py # pre-release - python-version: "3.15" env: TOXENV: py - python-version: "3.15" env: TOXENV: extra steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} allow-prereleases: true - name: Install tox run: pip install tox - name: Run tests env: ${{ matrix.env }} run: tox - name: Upload coverage report uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 tests-other-os: name: "Test: py311, ${{ matrix.os }}" runs-on: "${{ matrix.os }}" permissions: contents: read id-token: write # OIDC authentication with Codecov strategy: matrix: os: [macos-latest, windows-latest] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.11" - name: Install tox run: pip install tox - name: Run tests run: tox -e py - name: Upload coverage report uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 scrapy-itemadapter-78cc818/.gitignore000066400000000000000000000001651527060762600176640ustar00rootroot00000000000000*.pyc .~lock* .DS_Store .mypy_cache/ *.egg-info/ .tox/ .coverage htmlcov/ coverage.xml /dist/ .venv/ .idea/ .vscode/ scrapy-itemadapter-78cc818/.pre-commit-config.yaml000066400000000000000000000007671527060762600221650ustar00rootroot00000000000000repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.16.10 hooks: - id: ruff-check args: [ --fix ] - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/rhysd/actionlint rev: v1.7.12 hooks: - id: actionlint - repo: https://github.com/zizmorcore/zizmor-pre-commit rev: v1.30.1 hooks: - id: zizmor args: [--no-progress, --fix] scrapy-itemadapter-78cc818/Changelog.md000066400000000000000000000223001527060762600201000ustar00rootroot00000000000000# Changelog ### 0.14.0 (2026-10-29) Removed support for Python 3.9 and 3.10, and added support for Python 3.15 ([#112](https://github.com/scrapy/itemadapter/pull/112), [#130](https://github.com/scrapy/itemadapter/pull/130), [#137](https://github.com/scrapy/itemadapter/pull/137)). Raised the minimum supported versions of optional dependencies: `attrs` 22.1.0 (was 20.1.0), `pydantic` 2.7 (was 1.8) and `scrapy` 2.7 (was 2.2) ([#137](https://github.com/scrapy/itemadapter/pull/137), [#139](https://github.com/scrapy/itemadapter/pull/139), [#140](https://github.com/scrapy/itemadapter/pull/140)). Removed support for Pydantic 1 models, including `pydantic.v1` models from Pydantic 2, and for `scrapy.item.BaseItem` subclasses that are not `scrapy.Item` subclasses ([#139](https://github.com/scrapy/itemadapter/pull/139)). `ItemAdapter.ADAPTER_CLASSES` is now a `tuple` instead of a `collections.deque`, so it no longer supports `appendleft()` and other in-place changes. To register an adapter class, reassign it instead, e.g. `ItemAdapter.ADAPTER_CLASSES = (MyAdapter, *ItemAdapter.ADAPTER_CLASSES)` ([#118](https://github.com/scrapy/itemadapter/pull/118), [#128](https://github.com/scrapy/itemadapter/pull/128)). Which adapter class handles each item class is now cached, making `ItemAdapter()` and `is_item()` faster. As a result, the `is_item_class()` class method of adapter classes must return the same result every time it is called with the same item class ([#128](https://github.com/scrapy/itemadapter/pull/128)). Overriding the `is_item()` class method in an adapter class is deprecated, override `is_item_class()` instead. Adapter classes that override `is_item()` still work, but trigger a `DeprecationWarning` and disable the cache above ([#128](https://github.com/scrapy/itemadapter/pull/128)). Added support for [`TypedDict`](https://docs.python.org/3/library/typing.html#typing.TypedDict) item classes, including `typing_extensions.TypedDict` ones: field names, field metadata from the first mapping in [`Annotated`](https://docs.python.org/3/library/typing.html#typing.Annotated) type hints, and `get_json_schema()` support, which honors `Required` and `NotRequired` ([#123](https://github.com/scrapy/itemadapter/pull/123)). `get_json_schema()` is much faster on repeated calls, as field docstrings read from source code are now cached ([#141](https://github.com/scrapy/itemadapter/pull/141)). `get_json_schema()` no longer fails on unions of 2 or more non-simple types, e.g. 2 item classes ([#123](https://github.com/scrapy/itemadapter/pull/123)). `get_json_schema()` now maps `int | float` to `number` regardless of the order of the union members; `float | int` already did ([#133](https://github.com/scrapy/itemadapter/pull/133)). `get_json_schema()` now handles `attrs.validators.matches_re()` validators better: patterns are anchored to match their match function (`re.fullmatch()` by default), and patterns with flags that JSON Schema cannot express, e.g. `re.IGNORECASE`, are left out ([#122](https://github.com/scrapy/itemadapter/pull/122)). Improved type hints ([#138](https://github.com/scrapy/itemadapter/pull/138)). ### 0.13.1 (2026-01-08) Fixed `get_json_schema()` to properly format multiline field docstrings using `inspect.cleandoc()`, ensuring consistent indentation and whitespace handling ([#115](https://github.com/scrapy/itemadapter/pull/115)). ### 0.13.0 (2025-12-15) Added support for Python 3.14 and removed support for [PyPy](https://pypy.org/) 3.10. `get_json_schema()` no longer reports all `scrapy.Item` fields as required. ### 0.12.2 (2025-09-02) The return value of `get_json_schema()` is now deterministic (deterministic order of dict keys and list items). ### 0.12.1 (2025-08-08) `get_json_schema()` now supports inherited field docstrings. ### 0.12.0 (2025-07-24) Added support for [PyPy](https://pypy.org/) 3.11 ([#97](https://github.com/scrapy/itemadapter/pull/97)). Added a new `get_json_schema()` class method to `ItemAdapter` and all built-in adapters to output a [JSON Schema](https://json-schema.org/) for a given item class ([#101](https://github.com/scrapy/itemadapter/pull/101)). Modernized the code base, now making full use of [pyproject.toml](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) and using [ruff](https://docs.astral.sh/ruff/) ([#96](https://github.com/scrapy/itemadapter/pull/96), [#98](https://github.com/scrapy/itemadapter/pull/98)). ### 0.11.0 (2025-01-29) Removed functions deprecated in 0.5.0: - `itemadapter.utils.is_attrs_instance()` - `itemadapter.utils.is_dataclass_instance()` - `itemadapter.utils.is_pydantic_instance()` - `itemadapter.utils.is_scrapy_item()` ([#93](https://github.com/scrapy/itemadapter/pull/93)). Added support for Pydantic 2 ([#91](https://github.com/scrapy/itemadapter/pull/91)). Added `__all__` to the top-level module to improve type checks ([#90](https://github.com/scrapy/itemadapter/pull/90)). Improved `pre-commit` and CI configuration ([#91](https://github.com/scrapy/itemadapter/pull/91), [#92](https://github.com/scrapy/itemadapter/pull/92)). ### 0.10.0 (2024-11-29) Dropped Python 3.8 support, added official Python 3.13 and PyPy 3.10 support ([#79](https://github.com/scrapy/itemadapter/pull/79), [#87](https://github.com/scrapy/itemadapter/pull/87)). Fixed the typing check when run with Scrapy 2.12.0+ ([#88](https://github.com/scrapy/itemadapter/pull/88)). Fixed `MANIFEST.in` that was missing some files ([#84](https://github.com/scrapy/itemadapter/pull/84)). Enabled `pre-commit` ([#85](https://github.com/scrapy/itemadapter/pull/85)). ### 0.9.0 (2024-05-07) Dropped Python 3.7 support, added official Python 3.12 support ([#75](https://github.com/scrapy/itemadapter/pull/75), [#77](https://github.com/scrapy/itemadapter/pull/77)). Updated the documentation and the type hint about `ItemAdapter.ADAPTER_CLASSES` to say that subclasses can use any iterable, not just `collections.deque` ([#74](https://github.com/scrapy/itemadapter/pull/74)). Documented that `Pydantic >= 2` is not supported yet ([#73](https://github.com/scrapy/itemadapter/pull/73)). Updated CI configuration ([#77](https://github.com/scrapy/itemadapter/pull/77), [#80](https://github.com/scrapy/itemadapter/pull/80)). ### 0.8.0 (2023-03-30) Dropped Python 3.6 support, and made Python 3.11 support official ([#65](https://github.com/scrapy/itemadapter/pull/65), [#66](https://github.com/scrapy/itemadapter/pull/66), [#69](https://github.com/scrapy/itemadapter/pull/69)). It is now possible to declare custom `ItemAdapter` subclasses with their own `ADAPTER_CLASSES` attribute, allowing to support different item types in different parts of the same code base ([#68](https://github.com/scrapy/itemadapter/pull/68)). Improved type hint support ([#67](https://github.com/scrapy/itemadapter/pull/67)). ### 0.7.0 (2022-08-02) ItemAdapter.get_field_names_from_class ([#64](https://github.com/scrapy/itemadapter/pull/64)) ### 0.6.0 (2022-05-12) Slight performance improvement ([#62](https://github.com/scrapy/itemadapter/pull/62)) ### 0.5.0 (2022-03-18) Improve performance by removing imports inside functions ([#60](https://github.com/scrapy/itemadapter/pull/60)) ### 0.4.0 (2021-08-26) Added `ItemAdapter.is_item_class` and `ItemAdapter.get_field_meta_from_class` ([#54](https://github.com/scrapy/itemadapter/pull/54)) ### 0.3.0 (2021-07-15) Added built-in support for `pydantic` models ([#53](https://github.com/scrapy/itemadapter/pull/53)) ### 0.2.0 (2020-11-06) Adapter interface: added the ability to support arbitrary types, by implementing a MutableMapping-based interface. By way of this change, now any type can be used as a Scrapy item. ### 0.1.1 (2020-09-28) Dropped support for Python 3.5 (#38). The new `get_field_meta_from_class` function offers the same functionality as `ItemAdapter.get_field_meta` but for an item class, as opposed to an item object (#34, #35). `ItemAdapter.__repr__` no longer raises exceptions caused by the underlying item (#31, #41). Minor improvement to the release process (#37), and cleanup of test warnings (#40). ### 0.1.0 (2020-06-10) Added `ItemAdapter.asdict`, which allows converting an item and all of its nested items into `dict` objects (#27, #29). Improved `ItemAdapter` performance by reducing time complexity for lookups and traversals for dataclass and attrs items (#28). ### 0.0.8 (2020-05-22) `ItemAdapter.field_names` now returns a `KeysView` instead of a `list`. Minor CI and test changes. ### 0.0.7 (2020-05-22) `ItemAdapter.get_field_meta` now returns an empty `MappingProxyType` object for items without metadata support, instead of raising `TypeError`. Improved the README and some docstrings. Provided full test coverage, and refactored CI configuration, test configuration and tests themselves. ### 0.0.6 (2020-05-09) Added support for Scrapy’s `BaseItem`. Refactored and extended tests. Code style and documentation fixes. ### 0.0.5 (2020-04-28) Removed support for `MutableMapping`. ### 0.0.4 (2020-04-28) Removed metadata support for arbitrary mutable mappings. ### 0.0.3 (2020-04-27) Rebuild for the Python Package Index. ### 0.0.2 (2020-04-27) Split the implementation into several files for better code organization, and without an impact on the existing API import paths. Also improved the README. ### 0.0.1 (2020-04-25) Initial release. scrapy-itemadapter-78cc818/LICENSE000066400000000000000000000026651527060762600167100ustar00rootroot00000000000000Copyright 2020 Eugenio Lacuesta Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. scrapy-itemadapter-78cc818/README.md000066400000000000000000000533651527060762600171650ustar00rootroot00000000000000# itemadapter [![version](https://img.shields.io/pypi/v/itemadapter.svg)](https://pypi.python.org/pypi/itemadapter) [![pyversions](https://img.shields.io/pypi/pyversions/itemadapter.svg)](https://pypi.python.org/pypi/itemadapter) [![actions](https://github.com/scrapy/itemadapter/workflows/Tests/badge.svg)](https://github.com/scrapy/itemadapter/actions) [![codecov](https://codecov.io/gh/scrapy/itemadapter/branch/master/graph/badge.svg)](https://codecov.io/gh/scrapy/itemadapter) The `ItemAdapter` class is a wrapper for data container objects, providing a common interface to handle objects of different types in an uniform manner, regardless of their underlying implementation. Currently supported types are: * [`scrapy.item.Item`](https://docs.scrapy.org/en/latest/topics/items.html#scrapy.item.Item) * [`dict`](https://docs.python.org/3/library/stdtypes.html#dict), including [`TypedDict`](https://docs.python.org/3/library/typing.html#typing.TypedDict) subclasses * [`dataclass`](https://docs.python.org/3/library/dataclasses.html)-based classes * [`attrs`](https://www.attrs.org)-based classes * [`pydantic`](https://pydantic.dev/docs/validation/)-based classes Additionally, interaction with arbitrary types is supported, by implementing a pre-defined interface (see [extending `itemadapter`](#extending-itemadapter)). --- ## Requirements * Python 3.11+, either the CPython implementation (default) or the PyPy implementation * [`scrapy`](https://scrapy.org/) 2.7+: optional, needed to interact with `scrapy` items * [`attrs`](https://pypi.org/project/attrs/) 22.1.0+: optional, needed to interact with `attrs`-based items * [`pydantic`](https://pypi.org/project/pydantic/) 2.7+: optional, needed to interact with `pydantic`-based items --- ## Installation `itemadapter` is available on [`PyPI`](https://pypi.python.org/pypi/itemadapter), it can be installed with `pip`: ``` pip install itemadapter ``` For `attrs`, `pydantic` and `scrapy` support, install the corresponding extra to ensure that a supported version of the corresponding dependencies is installed. For example: ``` pip install itemadapter[scrapy] ``` Mind that you can install multiple extras as needed. For example: ``` pip install itemadapter[attrs,pydantic,scrapy] ``` --- ## License `itemadapter` is distributed under a [BSD-3](https://opensource.org/licenses/BSD-3-Clause) license. --- ## Basic usage The following is a simple example using a `dataclass` object. Consider the following type definition: ```python >>> from dataclasses import dataclass >>> from itemadapter import ItemAdapter >>> @dataclass ... class InventoryItem: ... name: str ... price: float ... stock: int >>> ``` An `ItemAdapter` object can be treated much like a dictionary: ```python >>> obj = InventoryItem(name='foo', price=20.5, stock=10) >>> ItemAdapter.is_item(obj) True >>> adapter = ItemAdapter(obj) >>> len(adapter) 3 >>> adapter["name"] 'foo' >>> adapter.get("price") 20.5 >>> ``` The wrapped object is modified in-place: ```python >>> adapter["name"] = "bar" >>> adapter.update({"price": 12.7, "stock": 9}) >>> adapter.item InventoryItem(name='bar', price=12.7, stock=9) >>> adapter.item is obj True >>> ``` ### Converting to dict The `ItemAdapter` class provides the `asdict` method, which converts nested items recursively. Consider the following example: ```python >>> from dataclasses import dataclass >>> from itemadapter import ItemAdapter >>> @dataclass ... class Price: ... value: int ... currency: str >>> @dataclass ... class Product: ... name: str ... price: Price >>> ``` ```python >>> item = Product("Stuff", Price(42, "UYU")) >>> adapter = ItemAdapter(item) >>> adapter.asdict() {'name': 'Stuff', 'price': {'value': 42, 'currency': 'UYU'}} >>> ``` Note that just passing an adapter object to the `dict` built-in also works, but it doesn't traverse the object recursively converting nested items: ```python >>> dict(adapter) {'name': 'Stuff', 'price': Price(value=42, currency='UYU')} >>> ``` --- ## API reference ### Built-in adapters The following adapters are included by default: * `itemadapter.adapter.ScrapyItemAdapter`: handles `Scrapy` items * `itemadapter.adapter.DictAdapter`: handles `Python` dictionaries, including `TypedDict` subclasses when used as item classes * `itemadapter.adapter.DataclassAdapter`: handles `dataclass` objects * `itemadapter.adapter.AttrsAdapter`: handles `attrs` objects * `itemadapter.adapter.PydanticAdapter`: handles `pydantic` objects ### class `itemadapter.adapter.ItemAdapter(item: Any)` This is the main entrypoint for the package. Tipically, user code wraps an item using this class, and proceeds to handle it with the provided interface. `ItemAdapter` implements the [`MutableMapping`](https://docs.python.org/3/library/collections.abc.html#collections.abc.MutableMapping) interface, providing a `dict`-like API to manipulate data for the object it wraps (which is modified in-place). **Attributes** #### class attribute `ADAPTER_CLASSES: Iterable` Stores the currently registered adapter classes. The order in which the adapters are registered is important. When an `ItemAdapter` object is created for a specific item, the registered adapters are traversed in order and the first adapter class to return `True` for the `is_item_class` class method is used for all subsequent operations. The default order is the one defined in the [built-in adapters](#built-in-adapters) section. The default implementation is a `tuple`, but any other iterable (e.g. `list`, [`collections.deque`](https://docs.python.org/3/library/collections.html#collections.deque)) will work. Which adapter class handles a given item class is cached, so `is_item_class` must return the same result every time it is called with the same item class. **Methods** #### class method `is_item(item: Any) -> bool` Return `True` if any of the registered adapters can handle the class of the item (i.e. if any of them returns `True` for its `is_item_class` method with `item.__class__` as argument), `False` otherwise. #### class method `is_item_class(item_class: type) -> bool` Return `True` if any of the registered adapters can handle the item class (i.e. if any of them returns `True` for its `is_item_class` method with `item_class` as argument), `False` otherwise. #### class method `get_field_meta_from_class(item_class: type, field_name: str) -> MappingProxyType` Return a [`types.MappingProxyType`](https://docs.python.org/3/library/types.html#types.MappingProxyType) object, which is a read-only mapping with metadata about the given field. If the item class does not support field metadata, or there is no metadata for the given field, an empty object is returned. The returned value is taken from the following sources, depending on the item type: * [`scrapy.item.Field`](https://docs.scrapy.org/en/latest/topics/items.html#item-fields) for `scrapy.item.Item`s * [`dataclasses.field.metadata`](https://docs.python.org/3/library/dataclasses.html#dataclasses.field) for `dataclass`-based items * [`attrs.Attribute.metadata`](https://www.attrs.org/en/stable/examples.html#metadata) for `attrs`-based items * [`pydantic.fields.FieldInfo`](https://pydantic.dev/docs/validation/latest/api/pydantic/fields/#pydantic.fields.FieldInfo) for `pydantic`-based items * the first mapping in the [`typing.Annotated`](https://docs.python.org/3/library/typing.html#typing.Annotated) metadata of the field type hint for `TypedDict`-based items #### class method `get_field_names_from_class(item_class: type) -> Optional[list[str]]` Return a list with the names of all the fields defined for the item class. If an item class doesn't support defining fields upfront, None is returned. #### class method `get_json_schema(item_class: type) -> dict[str, Any]` Return a dict with a [JSON Schema](https://json-schema.org/) representation of the item class. The generated JSON Schema reflects field type hints, attribute docstrings and class and field metadata of any supported item class. It also supports using item classes in field types of other item classes. For example, given: ```python from dataclasses import dataclass import attrs @dataclass class Brand: name: str @attrs.define class Product: name: str """Product name""" brand: Brand | None in_stock: bool = True ``` `ItemAdapter.get_json_schema(Product)` returns: ```python { "type": "object", "additionalProperties": False, "properties": { "name": {"type": "string", "description": "Product name"}, "brand": { "anyOf": [ {"type": "null"}, { "type": "object", "additionalProperties": False, "properties": {"name": {"type": "string"}}, "required": ["name"], }, ] }, "in_stock": {"default": True, "type": "boolean"}, }, "required": ["name", "brand"], } ``` You can also extend or override JSON Schema data at the item class or field level: - Set `json_schema_extra` in field metadata to extend or override the JSON Schema data for that field. For example: ```python >>> from scrapy.item import Item, Field >>> from itemadapter import ItemAdapter >>> class MyItem(Item): ... name: str = Field(json_schema_extra={"minLength": 1}) ... >>> ItemAdapter.get_json_schema(MyItem) {'type': 'object', 'additionalProperties': False, 'properties': {'name': {'minLength': 1, 'type': 'string'}}} ``` - Define a `__json_schema_extra__` class attribute dict to extend or override JSON Schema data for the entire class. For example: ```python >>> from dataclasses import dataclass >>> from itemadapter import ItemAdapter >>> @dataclass ... class MyItem: ... __json_schema_extra__ = {"additionalProperties": True} ... name: str ... >>> ItemAdapter.get_json_schema(MyItem) {'additionalProperties': True, 'type': 'object', 'properties': {'name': {'type': 'string'}}, 'required': ['name']} ``` Note that, for Pydantic items, itemadapter does not use [`model_json_schema()`](https://docs.pydantic.dev/latest/api/base_model/#pydantic.BaseModel.model_json_schema) and instead uses its own implementation. That way, the output JSON Schema is consistent across different item types. It also makes it possible to generate JSON Schemas for Pydantic models that have nested non-Pydantic item classes as fields. The downside is that JSON Schema support in itemadapter may not be as advanced as Pydantic‘s. The following are some known limitations of JSON Schema generation in itemadapter: - Attribute docstrings are read with [`inspect.getsource()`](https://docs.python.org/3/library/inspect.html#inspect.getsource), and may not be readable at run time in some cases. For such cases, define `description` within `json_schema_extra` instead (see below). - String pattern contraints are translated to the regular expression dialect of JSON Schema where possible, and silently ignored where not, e.g. when they are case-insensitive or multiline. - Recursion is silently ignored: if you have an item class that has an attribute with that same item class as a type or as part of its type, a simple `{"type": "object"}` is used to map the nested instances of that item class. #### `get_field_meta(field_name: str) -> MappingProxyType` Return metadata for the given field, if available. Unless overriden in a custom adapter class, by default this method calls the adapter's `get_field_meta_from_class` method, passing the wrapped item's class. #### `field_names() -> collections.abc.KeysView` Return a [keys view](https://docs.python.org/3/library/collections.abc.html#collections.abc.KeysView) with the names of all the defined fields for the item. #### `asdict() -> dict` Return a `dict` object with the contents of the adapter. This works slightly different than calling `dict(adapter)`, because it's applied recursively to nested items (if there are any). ### function `itemadapter.utils.is_item(obj: Any) -> bool` Return `True` if the given object belongs to (at least) one of the supported types, `False` otherwise. This is an alias, using the `itemadapter.adapter.ItemAdapter.is_item` class method is encouraged for better performance. ### function `itemadapter.utils.get_field_meta_from_class(item_class: type, field_name: str) -> types.MappingProxyType` Alias for `itemadapter.adapter.ItemAdapter.get_field_meta_from_class` --- ## Metadata support `scrapy.item.Item`, `dataclass`, `attrs`, `pydantic` and `TypedDict` objects allow the definition of arbitrary field metadata. This can be accessed through a [`MappingProxyType`](https://docs.python.org/3/library/types.html#types.MappingProxyType) object, which can be retrieved from an item instance with `itemadapter.adapter.ItemAdapter.get_field_meta`, or from an item class with the `itemadapter.adapter.ItemAdapter.get_field_meta_from_class` method (or its alias `itemadapter.utils.get_field_meta_from_class`). The source of the data depends on the underlying type (see the docs for `ItemAdapter.get_field_meta_from_class`). #### `scrapy.item.Item` objects ```python >>> from scrapy.item import Item, Field >>> from itemadapter import ItemAdapter >>> class InventoryItem(Item): ... name = Field(serializer=str) ... value = Field(serializer=int, limit=100) ... >>> adapter = ItemAdapter(InventoryItem(name="foo", value=10)) >>> adapter.get_field_meta("name") mappingproxy({'serializer': }) >>> adapter.get_field_meta("value") mappingproxy({'serializer': , 'limit': 100}) >>> ``` #### `dataclass` objects ```python >>> from dataclasses import dataclass, field >>> @dataclass ... class InventoryItem: ... name: str = field(metadata={"serializer": str}) ... value: int = field(metadata={"serializer": int, "limit": 100}) ... >>> adapter = ItemAdapter(InventoryItem(name="foo", value=10)) >>> adapter.get_field_meta("name") mappingproxy({'serializer': }) >>> adapter.get_field_meta("value") mappingproxy({'serializer': , 'limit': 100}) >>> ``` #### `attrs` objects ```python >>> import attrs >>> @attrs.define ... class InventoryItem: ... name: str = attrs.field(metadata={"serializer": str}) ... value: int = attrs.field(metadata={"serializer": int, "limit": 100}) ... >>> adapter = ItemAdapter(InventoryItem(name="foo", value=10)) >>> adapter.get_field_meta("name") mappingproxy({'serializer': }) >>> adapter.get_field_meta("value") mappingproxy({'serializer': , 'limit': 100}) >>> ``` #### `pydantic` objects ```python >>> from pydantic import BaseModel, Field >>> class InventoryItem(BaseModel): ... name: str = Field(json_schema_extra={"serializer": str}) ... value: int = Field(json_schema_extra={"serializer": int, "limit": 100}) ... >>> adapter = ItemAdapter(InventoryItem(name="foo", value=10)) >>> adapter.get_field_meta("name") mappingproxy({'annotation': , 'json_schema_extra': {'serializer': }, 'repr': True}) >>> adapter.get_field_meta("value") mappingproxy({'annotation': , 'json_schema_extra': {'serializer': , 'limit': 100}, 'repr': True}) >>> ``` #### `TypedDict` objects `TypedDict` instances are plain dictionaries at run time, with no reference to the `TypedDict` subclass they were declared as, so metadata can only be read from the item class: ```python >>> from typing import Annotated, TypedDict >>> from itemadapter import ItemAdapter >>> class InventoryItem(TypedDict): ... name: Annotated[str, {"serializer": str}] ... value: Annotated[int, {"serializer": int, "limit": 100}] ... >>> ItemAdapter.get_field_meta_from_class(InventoryItem, "name") mappingproxy({'serializer': }) >>> ItemAdapter.get_field_meta_from_class(InventoryItem, "value") mappingproxy({'serializer': , 'limit': 100}) >>> ``` --- ## Extending `itemadapter` This package allows to handle arbitrary item classes, by implementing an adapter interface: _class `itemadapter.adapter.AdapterInterface(item: Any)`_ Abstract Base Class for adapters. An adapter that handles a specific type of item must inherit from this class and implement the abstract methods defined on it. `AdapterInterface` inherits from [`collections.abc.MutableMapping`](https://docs.python.org/3/library/collections.abc.html#collections.abc.MutableMapping), so all methods from the `MutableMapping` interface must be implemented as well. * _class method `is_item_class(cls, item_class: type) -> bool`_ Return `True` if the adapter can handle the given item class, `False` otherwise. Abstract (mandatory). * _class method `is_item(cls, item: Any) -> bool`_ Return `True` if the adapter can handle the given item, `False` otherwise. The default implementation calls `cls.is_item_class(item.__class__)`. * _class method `get_field_meta_from_class(cls, item_class: type) -> bool`_ Return metadata for the given item class and field name, if available. By default, this method returns an empty `MappingProxyType` object. Please supply your own method definition if you want to handle field metadata based on custom logic. See the [section on metadata support](#metadata-support) for additional information. * _method `get_field_meta(self, field_name: str) -> types.MappingProxyType`_ Return metadata for the given field name, if available. It's usually not necessary to override this method, since the `itemadapter.adapter.AdapterInterface` base class provides a default implementation that calls `ItemAdapter.get_field_meta_from_class` with the wrapped item's class as argument. See the [section on metadata support](#metadata-support) for additional information. * _method `field_names(self) -> collections.abc.KeysView`_: Return a [dynamic view](https://docs.python.org/3/library/collections.abc.html#collections.abc.KeysView) of the item's field names. By default, this method returns the result of calling `keys()` on the current adapter, i.e., its return value depends on the implementation of the methods from the `MutableMapping` interface (more specifically, it depends on the return value of `__iter__`). You might want to override this method if you want a way to get all fields for an item, whether or not they are populated. For instance, Scrapy uses this method to define column names when exporting items to CSV. ### Registering an adapter Add your custom adapter class to the `itemadapter.adapter.ItemAdapter.ADAPTER_CLASSES` class attribute in order to handle custom item classes. **Example** ``` pip install zyte-common-items ``` ```python >>> from itemadapter.adapter import ItemAdapter >>> from zyte_common_items import Item, ZyteItemAdapter >>> >>> ItemAdapter.ADAPTER_CLASSES = (ZyteItemAdapter, *ItemAdapter.ADAPTER_CLASSES) >>> item = Item() >>> adapter = ItemAdapter(item) >>> adapter >>> ``` ### Multiple adapter classes If you need to have different handlers and/or priorities for different cases you can subclass the `ItemAdapter` class and set the `ADAPTER_CLASSES` attribute as needed: **Example** ```python >>> from itemadapter.adapter import ( ... ItemAdapter, ... AttrsAdapter, ... DataclassAdapter, ... DictAdapter, ... PydanticAdapter, ... ScrapyItemAdapter, ... ) >>> from scrapy.item import Item, Field >>> >>> class BuiltinTypesItemAdapter(ItemAdapter): ... ADAPTER_CLASSES = [DictAdapter, DataclassAdapter] ... >>> class ThirdPartyTypesItemAdapter(ItemAdapter): ... ADAPTER_CLASSES = [AttrsAdapter, PydanticAdapter, ScrapyItemAdapter] ... >>> class ScrapyItem(Item): ... foo = Field() ... >>> BuiltinTypesItemAdapter.is_item(dict()) True >>> ThirdPartyTypesItemAdapter.is_item(dict()) False >>> BuiltinTypesItemAdapter.is_item(ScrapyItem(foo="bar")) False >>> ThirdPartyTypesItemAdapter.is_item(ScrapyItem(foo="bar")) True >>> ``` --- ## More examples ### `scrapy.item.Item` objects ```python >>> from scrapy.item import Item, Field >>> from itemadapter import ItemAdapter >>> class InventoryItem(Item): ... name = Field() ... price = Field() ... >>> item = InventoryItem(name="foo", price=10) >>> adapter = ItemAdapter(item) >>> adapter.item is item True >>> adapter["name"] 'foo' >>> adapter["name"] = "bar" >>> adapter["price"] = 5 >>> item {'name': 'bar', 'price': 5} >>> ``` ### `dict` ```python >>> from itemadapter import ItemAdapter >>> item = dict(name="foo", price=10) >>> adapter = ItemAdapter(item) >>> adapter.item is item True >>> adapter["name"] 'foo' >>> adapter["name"] = "bar" >>> adapter["price"] = 5 >>> item {'name': 'bar', 'price': 5} >>> ``` ### `dataclass` objects ```python >>> from dataclasses import dataclass >>> from itemadapter import ItemAdapter >>> @dataclass ... class InventoryItem: ... name: str ... price: int ... >>> item = InventoryItem(name="foo", price=10) >>> adapter = ItemAdapter(item) >>> adapter.item is item True >>> adapter["name"] 'foo' >>> adapter["name"] = "bar" >>> adapter["price"] = 5 >>> item InventoryItem(name='bar', price=5) >>> ``` ### `attrs` objects ```python >>> import attrs >>> from itemadapter import ItemAdapter >>> @attrs.define ... class InventoryItem: ... name: str ... price: int ... >>> item = InventoryItem(name="foo", price=10) >>> adapter = ItemAdapter(item) >>> adapter.item is item True >>> adapter["name"] 'foo' >>> adapter["name"] = "bar" >>> adapter["price"] = 5 >>> item InventoryItem(name='bar', price=5) >>> ``` ### `pydantic` objects ```python >>> from pydantic import BaseModel >>> from itemadapter import ItemAdapter >>> class InventoryItem(BaseModel): ... name: str ... price: int ... >>> item = InventoryItem(name="foo", price=10) >>> adapter = ItemAdapter(item) >>> adapter.item is item True >>> adapter["name"] 'foo' >>> adapter["name"] = "bar" >>> adapter["price"] = 5 >>> item InventoryItem(name='bar', price=5) >>> ``` ## Changelog See the [full changelog](Changelog.md) scrapy-itemadapter-78cc818/itemadapter/000077500000000000000000000000001527060762600201715ustar00rootroot00000000000000scrapy-itemadapter-78cc818/itemadapter/__init__.py000066400000000000000000000003011527060762600222740ustar00rootroot00000000000000from .adapter import ItemAdapter from .utils import get_field_meta_from_class, is_item __version__ = "0.14.0" __all__ = [ "ItemAdapter", "get_field_meta_from_class", "is_item", ] scrapy-itemadapter-78cc818/itemadapter/_imports.py000066400000000000000000000021641527060762600224020ustar00rootroot00000000000000from __future__ import annotations import typing from typing import Any # attempt the following imports only once, # to be imported from itemadapter's submodules _scrapy_item_classes: tuple[type, ...] scrapy: Any try: import scrapy _scrapy_item_classes = (scrapy.item.Item,) except ImportError: scrapy = None _scrapy_item_classes = () typing_extensions: Any try: import typing_extensions except ImportError: typing_extensions = None # typing.is_typeddict() does not recognize typing_extensions.TypedDict # subclasses, and typing.get_type_hints() does not strip # typing_extensions.ReadOnly before Python 3.13. if typing_extensions is None: _is_typeddict = typing.is_typeddict _get_type_hints = typing.get_type_hints else: _is_typeddict = typing_extensions.is_typeddict _get_type_hints = typing_extensions.get_type_hints attrs: Any try: import attrs except ImportError: attrs = None pydantic: Any = None PydanticUndefined: object = None try: import pydantic # noqa: F401 from pydantic_core import PydanticUndefined # noqa: F401 except ImportError: # No pydantic pass scrapy-itemadapter-78cc818/itemadapter/_json_schema.py000066400000000000000000000666161527060762600232120ustar00rootroot00000000000000from __future__ import annotations import ast import dataclasses import inspect import operator import re from collections.abc import Iterable, Iterator, Mapping, Sequence from collections.abc import Set as AbstractSet from copy import copy from enum import Enum from functools import lru_cache from textwrap import dedent from types import MappingProxyType, UnionType from typing import ( TYPE_CHECKING, Any, Protocol, Union, get_args, get_origin, get_type_hints, runtime_checkable, ) from ._imports import PydanticUndefined, _get_type_hints, attrs from ._utils import _split_typed_dict_hint if TYPE_CHECKING: from attrs import Attribute from .adapter import AdapterInterface, ItemAdapter SIMPLE_TYPES = { type(None): "null", bool: "boolean", int: "integer", float: "number", str: "string", } @dataclasses.dataclass class _JsonSchemaState: adapter: type[ItemAdapter | AdapterInterface] """ItemAdapter class or AdapterInterface implementation used on the initial get_json_schema() call. On types for which adapter.is_item_class() returns True, adapter.get_json_schema() is used to get the corresponding, nested JSON Schema. """ containers: set[type] = dataclasses.field(default_factory=set) """Used to keep track of item classes that are being processed, to avoid recursion.""" def dedupe_types(types: Sequence[Any]) -> list[Any]: seen = set() result = [] # JSON Schema's number type includes integers, but not the other way around. numeric_type = float if float in types else int for t in types: key = numeric_type if t in (int, float) else t if key not in seen: seen.add(key) result.append(key) return result def update_prop_from_union(prop: dict[str, Any], prop_type: Any, state: _JsonSchemaState) -> None: prop_types = dedupe_types(get_args(prop_type)) simple_types = [v for k, v in SIMPLE_TYPES.items() if k in prop_types] complex_types = [t for t in prop_types if t not in SIMPLE_TYPES] if not complex_types: prop.setdefault("type", simple_types) return new_any_of: list[dict[str, Any]] = [] any_of = prop.setdefault("anyOf", new_any_of) if any_of is not new_any_of: return if simple_types: any_of.append({"type": simple_types if len(simple_types) > 1 else simple_types[0]}) for complex_type in complex_types: complex_prop: dict[str, Any] = {} update_prop_from_type(complex_prop, complex_type, state) any_of.append(complex_prop) @runtime_checkable class ArrayProtocol(Protocol): def __iter__(self) -> Iterator[Any]: ... def __len__(self) -> int: ... def __contains__(self, item: object) -> bool: ... @runtime_checkable class ObjectProtocol(Protocol): # noqa: PLW1641 def __getitem__(self, key: str) -> Any: ... def __iter__(self) -> Iterator[str]: ... def __len__(self) -> int: ... def __contains__(self, key: object) -> bool: ... def keys(self) -> Iterable[str]: ... def items(self) -> Iterable[tuple[str, Any]]: ... def values(self) -> Iterable[Any]: ... def get(self, key: str, default: Any = ...) -> Any: ... def __eq__(self, other: object) -> bool: ... def __ne__(self, other: object) -> bool: ... # Flags that change the meaning of a pattern in a way that cannot be # expressed in JSON Schema, where patterns are always flagless. re.ASCII is # not one of them: it brings Python semantics closer to JSON Schema ones. INVALID_PATTERN_FLAGS = re.IGNORECASE | re.MULTILINE INLINE_FLAGS = { "a": re.ASCII, "i": re.IGNORECASE, "L": re.LOCALE, "m": re.MULTILINE, "s": re.DOTALL, "u": re.UNICODE, "x": re.VERBOSE, } # Python only allows global inline flags at the start of a pattern. LEADING_INLINE_FLAGS = re.compile(r"\(\?([aiLmsux]+)\)") # Whitespace that re.VERBOSE ignores. VERBOSE_WHITESPACE = frozenset(" \t\n\r\v\f") # Escape sequences that have the same meaning in Python and in ECMA-262. \b # and \B are included even though Python matches word boundaries against # Unicode word characters, while ECMA-262 only matches them against ASCII # ones. SHARED_ESCAPE_LETTERS = frozenset("bdDfnrstvwSW") # Escape sequences with an ECMA-262 equivalent. ECMA-262 patterns are never # multiline, so ^ and $ can only match at the start and at the end of the # string, like \A and \Z do in Python. CONVERTED_ESCAPES = {"A": "^", "Z": "$"} # Escape sequences made of a letter and a fixed number of hexadecimal digits. HEX_ESCAPE_SIZES = {"x": 2, "u": 4} HEX_DIGITS = frozenset("0123456789abcdefABCDEF") DIGITS = frozenset("0123456789") # Groups that have the same meaning in Python and in ECMA-262. SHARED_GROUP_PREFIXES = ("(?:", "(?=", "(?!", "(?<=", "(? tuple[int, str] | None: """Return the ECMA-262 translation of the escape sequence that starts at *index*, along with the index right after it, or ``None`` if that sequence has no ECMA-262 equivalent.""" char = pattern[index + 1 : index + 2] if not char: return None if char in HEX_ESCAPE_SIZES: end = index + 2 + HEX_ESCAPE_SIZES[char] digits = pattern[index + 2 : end] if len(digits) != HEX_ESCAPE_SIZES[char] or not HEX_DIGITS.issuperset(digits): return None return end, pattern[index:end] if char in SHARED_ESCAPE_LETTERS or (char == "B" and not in_class): return index + 2, pattern[index : index + 2] if char in CONVERTED_ESCAPES and not in_class: return index + 2, CONVERTED_ESCAPES[char] if char in DIGITS: # Backreferences have the same meaning, but \0 and multi-digit escapes # may be octal escapes instead, which the 2 dialects number # differently. if in_class or char == "0" or pattern[index + 2 : index + 3] in DIGITS: return None return index + 2, pattern[index : index + 2] if char.isalnum(): return None return index + 2, pattern[index : index + 2] def scan_class(pattern: str, index: int) -> tuple[int, str] | None: """Return the ECMA-262 translation of the character class that starts at *index*, along with the index right after it, or ``None`` if that class has no ECMA-262 equivalent.""" start = index index += 1 if pattern[index : index + 1] == "^": index += 1 if pattern[index : index + 1] == "]": # Python reads a leading ] as a literal, ECMA-262 as the end of an # empty class. return None while index < len(pattern): char = pattern[index] if char == "]": return index + 1, pattern[start : index + 1] if char == "\\": # Escape sequences are never translated within a character class, # so the class can be copied verbatim. escape = scan_escape(pattern, index, in_class=True) if escape is None: return None index = escape[0] continue index += 1 return None def scan_group(pattern: str, index: int) -> tuple[int, str] | None: """Return the ECMA-262 translation of the opening parenthesis of the group that starts at *index*, along with the index right after it, or ``None`` if that group has no ECMA-262 equivalent.""" if pattern[index + 1 : index + 2] != "?": return index + 1, "(" if pattern.startswith("(?#", index): end = pattern.find(")", index) return None if end == -1 else (end + 1, "") if pattern.startswith("(?P<", index): # ECMA-262 group names follow different rules, and JSON Schema never # reads captured groups, so the group is left unnamed. It stays a # capturing group to keep numbered backreferences working. end = pattern.find(">", index) return None if end == -1 else (end + 1, "(") for prefix in SHARED_GROUP_PREFIXES: if pattern.startswith(prefix, index): return index + len(prefix), prefix return None def scan_braces(pattern: str, index: int, *, verbose: bool) -> tuple[int, str] | None: """Return the ECMA-262 translation of the quantifier or literal brace that starts at *index*, along with the index right after it, or ``None`` if it has no ECMA-262 equivalent.""" end = pattern.find("}", index) if end == -1: return index + 1, "{" if pattern[end + 1 : end + 2] == "+": return None # possessive quantifier if verbose and not VERBOSE_WHITESPACE.isdisjoint(pattern[index:end]): # Python drops the whitespace and reads the result as a literal, # ECMA-262 reads it as a quantifier. return None return end + 1, pattern[index : end + 1] def translate_pattern(pattern: str, flags: int) -> str | None: """Return *pattern*, a Python regular expression with *flags*, as an ECMA-262 one, the regular expression dialect of JSON Schema patterns, or ``None`` if it has no ECMA-262 equivalent. See https://ecma-international.org/publications-and-standards/standards/ecma-262/ """ verbose = bool(flags & re.VERBOSE) output = [] index = 0 while index < len(pattern): char = pattern[index] if char == "\\": result = scan_escape(pattern, index, in_class=False) elif char == "[": result = scan_class(pattern, index) elif char == "(": result = scan_group(pattern, index) elif char == "{": result = scan_braces(pattern, index, verbose=verbose) elif char in "*+?" and pattern[index + 1 : index + 2] == "+": return None # possessive quantifier elif verbose and char in VERBOSE_WHITESPACE: result = (index + 1, "") elif verbose and char == "#": end = pattern.find("\n", index) result = (len(pattern) if end == -1 else end + 1, "") elif char == "." and flags & re.DOTALL: result = (index + 1, r"[\s\S]") else: result = (index + 1, char) if result is None: return None index, text = result output.append(text) return "".join(output) def json_schema_pattern(pattern: str | re.Pattern[str]) -> str | None: """Return *pattern* as a JSON Schema pattern, or ``None`` if it cannot be expressed as one.""" flags = 0 if isinstance(pattern, re.Pattern): flags = pattern.flags pattern = pattern.pattern match = LEADING_INLINE_FLAGS.match(pattern) if match: for letter in match[1]: flags |= INLINE_FLAGS[letter] pattern = pattern[match.end() :] if flags & INVALID_PATTERN_FLAGS: return None return translate_pattern(pattern, flags) def array_type(type_hint: Any) -> Any: """Given the type hint of a Python type that maps to a JSON Schema array, such as a list, a tuple or a set, return the type of the items in that array.""" args = get_args(type_hint) if not args: return Any if args[-1] is Ellipsis: args = args[:-1] unique_args = set(args) if len(unique_args) == 1: return next(iter(unique_args)) return Union[tuple(unique_args)] # noqa: UP007 # JSON Schema patterns are unanchored, i.e. they behave like re.search(), so # patterns meant for re.match() or re.fullmatch() need explicit anchors. MATCH_FUNC_ANCHORS = { "search": ("", ""), "match": ("^(?:", ")"), "fullmatch": ("^(?:", ")$"), } def update_prop_from_pattern(prop: dict[str, Any], pattern: str | re.Pattern[str]) -> None: json_schema_value = json_schema_pattern(pattern) if json_schema_value is not None: prop.setdefault("pattern", json_schema_value) UNION_TYPES = {Union, UnionType} def update_prop_from_origin( prop: dict[str, Any], origin: Any, prop_type: Any, state: _JsonSchemaState ) -> None: if isinstance(origin, type): if issubclass(origin, (Sequence, AbstractSet)): prop.setdefault("type", "array") if issubclass(origin, AbstractSet): prop.setdefault("uniqueItems", True) had_items = "items" in prop items = prop.setdefault("items", {}) item_type = array_type(prop_type) update_prop_from_type(items, item_type, state) if not items and not had_items: del prop["items"] return if issubclass(origin, Mapping): prop.setdefault("type", "object") args = get_args(prop_type) if args: assert len(args) == 2 value_type = args[1] props = prop.setdefault("additionalProperties", {}) update_prop_from_type(props, value_type, state) return if origin in UNION_TYPES: update_prop_from_union(prop, prop_type, state) def update_prop_from_type(prop: dict[str, Any], prop_type: Any, state: _JsonSchemaState) -> None: if (origin := get_origin(prop_type)) is not None: update_prop_from_origin(prop, origin, prop_type, state) return if isinstance(prop_type, type): if state.adapter.is_item_class(prop_type): if prop_type in state.containers: prop.setdefault("type", "object") return state.containers.add(prop_type) subschema = state.adapter.get_json_schema( prop_type, _state=state, ) state.containers.remove(prop_type) for k, v in subschema.items(): prop.setdefault(k, v) return if issubclass(prop_type, Enum): values = [item.value for item in prop_type] value_types = tuple({type(v) for v in values}) prop_type = value_types[0] if len(value_types) == 1 else Union[value_types] # noqa: UP007 update_prop_from_type(prop, prop_type, state) prop.setdefault("enum", values) return if not issubclass(prop_type, str): if isinstance(prop_type, ObjectProtocol): prop.setdefault("type", "object") return if isinstance(prop_type, ArrayProtocol): prop.setdefault("type", "array") if issubclass(prop_type, AbstractSet): prop.setdefault("uniqueItems", True) return json_schema_type = SIMPLE_TYPES.get(prop_type) if json_schema_type is not None: prop.setdefault("type", json_schema_type) def _setdefault_attribute_types_on_json_schema( schema: dict[str, Any], item_class: type, state: _JsonSchemaState ) -> None: """Inspect the type hints of the class attributes of the item class and, for any matching JSON Schema property that has no type set, set the type based on the type hint.""" props = schema.get("properties", {}) attribute_type_hints = get_type_hints(item_class) for prop_name, prop in props.items(): if prop_name not in attribute_type_hints: continue prop_type = attribute_type_hints[prop_name] update_prop_from_type(prop, prop_type, state) def iter_docstrings(item_class: type) -> Iterator[tuple[str, str]]: try: source = inspect.getsource(item_class) except (OSError, TypeError): return tree = ast.parse(dedent(source)) class_node = None for node in tree.body: if isinstance(node, ast.ClassDef) and node.name == item_class.__name__: class_node = node break if class_node is None: # pragma: no cover # This can be reproduced with the doctests of the README, but the # coverage data does not seem to include those. return assert isinstance(class_node, ast.ClassDef) for child in ast.iter_child_nodes(class_node): if isinstance(child, ast.Assign) and isinstance(child.targets[0], ast.Name): attr_name = child.targets[0].id elif isinstance(child, ast.AnnAssign) and isinstance(child.target, ast.Name): attr_name = child.target.id else: continue next_idx = class_node.body.index(child) + 1 if next_idx >= len(class_node.body): continue next_node = class_node.body[next_idx] if ( isinstance(next_node, ast.Expr) and isinstance(next_node.value, ast.Constant) and isinstance(next_node.value.value, str) ): yield attr_name, next_node.value.value @lru_cache(maxsize=1024) def get_attr_docstrings(item_class: type) -> Mapping[str, str]: """Return the docstrings of the attributes defined in the source of the given class, by attribute name.""" docstrings: dict[str, str] = {} for attr_name, doc in iter_docstrings(item_class): docstrings.setdefault(attr_name, doc) return MappingProxyType(docstrings) def get_inherited_attr_docstring(item_class: type, attr_name: str) -> str | None: """Recursively search the MRO for a docstring for the given attribute name.""" for cls in item_class.__mro__: docstrings = get_attr_docstrings(cls) if attr_name in docstrings: return docstrings[attr_name] return None def _setdefault_attribute_docstrings_on_json_schema( schema: dict[str, Any], item_class: type ) -> None: """Inspect the docstrings after each class attribute of the item class and its bases and, for any matching JSON Schema property that has no description set, set the description to the contents of the docstring.""" props = schema.get("properties", {}) attr_names = set(props) if not attr_names: return for attr_name in attr_names: prop = props.setdefault(attr_name, {}) if "description" not in prop: doc = get_inherited_attr_docstring(item_class, attr_name) if doc: prop["description"] = inspect.cleandoc(doc) def base_json_schema_from_item_class(item_class: type) -> dict[str, Any]: json_schema_extra = getattr(item_class, "__json_schema_extra__", {}) schema = copy(json_schema_extra) schema.setdefault("type", "object") schema.setdefault("additionalProperties", False) return schema def _json_schema_from_item_class( adapter: type[AdapterInterface], item_class: type, state: _JsonSchemaState | None = None ) -> dict[str, Any]: state = state or _JsonSchemaState(adapter=adapter, containers={item_class}) schema = base_json_schema_from_item_class(item_class) fields_meta = { field_name: adapter.get_field_meta_from_class(item_class, field_name) for field_name in adapter.get_field_names_from_class(item_class) or () } if not fields_meta: return schema schema["properties"] = { field_name: copy(field_meta.get("json_schema_extra", {})) for field_name, field_meta in fields_meta.items() } required = [ field_name for field_name, field_data in schema["properties"].items() if "default" not in field_data ] if required: schema.setdefault("required", required) return schema def update_required_fields( schema: dict[str, Any], optional_fields: set[str] | None = None ) -> None: optional_fields = optional_fields or set() if "required" in schema: return required = [ field for field, metadata in schema["properties"].items() if field not in optional_fields and "default" not in metadata ] if required: schema["required"] = required def _json_schema_from_attrs(item_class: type, state: _JsonSchemaState) -> dict[str, Any]: schema = base_json_schema_from_item_class(item_class) fields = attrs.fields(item_class) if not fields: return schema attrs.resolve_types(item_class) # Ensure field.type annotations are resolved schema["properties"] = { field.name: copy(field.metadata.get("json_schema_extra", {})) for field in fields } default_factory_fields: set[str] = set() for field in fields: prop = schema["properties"][field.name] _update_attrs_prop(prop, field, state, default_factory_fields) update_required_fields(schema, default_factory_fields) _setdefault_attribute_docstrings_on_json_schema(schema, item_class) return schema def _update_attrs_prop( prop: dict[str, Any], field: Attribute[Any], state: _JsonSchemaState, default_factory_fields: set[str], ) -> None: update_prop_from_type(prop, field.type, state) if isinstance(field.default, attrs.Factory): default_factory_fields.add(field.name) elif field.default is not attrs.NOTHING: prop.setdefault("default", field.default) _update_attrs_prop_validation(prop, field) ATTRS_NUMBER_VALIDATORS = { operator.ge: "minimum", operator.gt: "exclusiveMinimum", operator.le: "maximum", operator.lt: "exclusiveMaximum", } def _update_attrs_prop_validation( prop: dict[str, Any], field: Attribute[Any], ) -> None: if not field.validator: return if type(field.validator).__name__ == "_AndValidator": validators = field.validator._validators # type: ignore[attr-defined] else: validators = [field.validator] for validator in validators: validator_type_name = type(validator).__name__ if validator_type_name == "_NumberValidator": key = ATTRS_NUMBER_VALIDATORS.get(validator.compare_func) if not key: # pragma: no cover continue prop.setdefault(key, validator.bound) elif validator_type_name == "_InValidator": prop.setdefault("enum", list(validator.options)) elif validator_type_name == "_MinLengthValidator": key = "minLength" if field.type is str else "minItems" prop.setdefault(key, validator.min_length) elif validator_type_name == "_MaxLengthValidator": key = "maxLength" if field.type is str else "maxItems" prop.setdefault(key, validator.max_length) elif validator_type_name == "_MatchesReValidator": pattern = json_schema_pattern(validator.pattern) if pattern is not None: prefix, suffix = MATCH_FUNC_ANCHORS[validator.match_func.__name__] prop.setdefault("pattern", f"{prefix}{pattern}{suffix}") def _json_schema_from_typed_dict(item_class: type, state: _JsonSchemaState) -> dict[str, Any]: schema = base_json_schema_from_item_class(item_class) type_hints = _get_type_hints(item_class) if not type_hints: return schema extra_type_hints = _get_type_hints(item_class, include_extras=True) schema["properties"] = {} required = [] required_keys = item_class.__required_keys__ # type: ignore[attr-defined] for field_name, field_type in type_hints.items(): field_metadata, field_required = _split_typed_dict_hint(extra_type_hints[field_name]) prop = copy(field_metadata.get("json_schema_extra", {})) update_prop_from_type(prop, field_type, state) schema["properties"][field_name] = prop if field_required is None: field_required = field_name in required_keys if field_required: required.append(field_name) if required: schema.setdefault("required", required) _setdefault_attribute_docstrings_on_json_schema(schema, item_class) return schema def _json_schema_from_dataclass(item_class: type, state: _JsonSchemaState) -> dict[str, Any]: schema = base_json_schema_from_item_class(item_class) fields = dataclasses.fields(item_class) resolved_field_types = get_type_hints(item_class) default_factory_fields = set() if fields: schema["properties"] = { field.name: copy(field.metadata.get("json_schema_extra", {})) for field in fields } for field in fields: prop = schema["properties"][field.name] update_prop_from_type(prop, resolved_field_types.get(field.name), state) if field.default_factory is not dataclasses.MISSING: default_factory_fields.add(field.name) elif field.default is not dataclasses.MISSING: prop.setdefault("default", field.default) update_required_fields(schema, default_factory_fields) _setdefault_attribute_docstrings_on_json_schema(schema, item_class) return schema def _json_schema_from_pydantic( adapter: type[AdapterInterface], item_class: type, state: _JsonSchemaState | None = None ) -> dict[str, Any]: state = state or _JsonSchemaState(adapter=adapter, containers={item_class}) schema: dict[str, Any] = copy( item_class.model_config.get("json_schema_extra", {}) # type: ignore[attr-defined] ) extra = item_class.model_config.get("extra") # type: ignore[attr-defined] schema.setdefault("type", "object") if extra == "forbid": schema.setdefault("additionalProperties", False) fields = { name: adapter.get_field_meta_from_class(item_class, name) for name in adapter.get_field_names_from_class(item_class) or () } if not fields: return schema schema["properties"] = { name: copy(metadata.get("json_schema_extra", {})) for name, metadata in fields.items() } default_factory_fields: set[str] = set() for name, metadata in fields.items(): prop = schema["properties"][name] _update_pydantic_prop(prop, name, metadata, state, default_factory_fields) update_required_fields(schema, default_factory_fields) _setdefault_attribute_docstrings_on_json_schema(schema, item_class) return schema def _update_pydantic_prop( prop: dict[str, Any], name: str, metadata: Mapping[str, Any], _state: _JsonSchemaState, default_factory_fields: set[str], ) -> None: field_type = metadata.get("annotation") update_prop_from_type(prop, field_type, _state) if "default_factory" in metadata: default_factory_fields.add(name) elif "default" in metadata and metadata["default"] is not PydanticUndefined: prop.setdefault("default", metadata["default"]) if "metadata" in metadata: _update_pydantic_prop_validation(prop, metadata["metadata"], field_type) for metadata_key, json_schema_field in ( ("title", "title"), ("description", "description"), ("examples", "examples"), ): if metadata_key in metadata: prop.setdefault(json_schema_field, metadata[metadata_key]) if "deprecated" in metadata: prop.setdefault("deprecated", bool(metadata["deprecated"])) def _update_pydantic_prop_validation( prop: dict[str, Any], metadata: Sequence[Any], field_type: object, ) -> None: for metadata_item in metadata: metadata_item_type = type(metadata_item).__name__ if metadata_item_type == "_PydanticGeneralMetadata": if "pattern" in metadata_item.__dict__: pattern = metadata_item.__dict__["pattern"] update_prop_from_pattern(prop, pattern) elif metadata_item_type == "MinLen": key = "minLength" if field_type is str else "minItems" prop.setdefault(key, metadata_item.min_length) elif metadata_item_type == "MaxLen": key = "maxLength" if field_type is str else "maxItems" prop.setdefault(key, metadata_item.max_length) else: for metadata_key, json_schema_field in ( ("ge", "minimum"), ("gt", "exclusiveMinimum"), ("le", "maximum"), ("lt", "exclusiveMaximum"), ): if metadata_item_type == metadata_key.capitalize(): prop.setdefault(json_schema_field, getattr(metadata_item, metadata_key)) scrapy-itemadapter-78cc818/itemadapter/_utils.py000066400000000000000000000074741527060762600220560ustar00rootroot00000000000000from __future__ import annotations import typing from collections.abc import Mapping from types import MappingProxyType from typing import Annotated, Any, NotRequired, Required, get_args, get_origin from itemadapter._imports import ( PydanticUndefined, _get_type_hints, attrs, pydantic, typing_extensions, ) def _is_attrs_class(obj: type) -> bool: if attrs is None: return False return bool(attrs.has(obj)) def _is_pydantic_model(obj: type) -> bool: if pydantic is None: return False return issubclass(obj, pydantic.BaseModel) def _get_pydantic_model_metadata(item_model: Any, field_name: str) -> MappingProxyType[str, Any]: metadata = {} field = item_model.model_fields[field_name] for attribute in [ "alias_priority", "alias", "allow_inf_nan", "annotation", "coerce_numbers_to_str", "decimal_places", "default_factory", "deprecated", "description", "discriminator", "examples", "exclude", "fail_fast", "field_title_generator", "frozen", "ge", "gt", "init_var", "init", "json_schema_extra", "kw_only", "le", "lt", "max_digits", "max_length", "min_length", "multiple_of", "pattern", "repr", "serialization_alias", "strict", "title", "union_mode", "validate_default", "validation_alias", ]: if hasattr(field, attribute) and (value := getattr(field, attribute)) is not None: metadata[attribute] = value for attribute, default_value in [ ("default", PydanticUndefined), ("metadata", []), ]: if hasattr(field, attribute) and (value := getattr(field, attribute)) != default_value: metadata[attribute] = value return MappingProxyType(metadata) _EMPTY_METADATA: MappingProxyType[str, Any] = MappingProxyType({}) _READ_ONLY = {getattr(typing, "ReadOnly", None), getattr(typing_extensions, "ReadOnly", None)} _READ_ONLY.discard(None) def _split_typed_dict_hint(type_hint: Any) -> tuple[MappingProxyType[str, Any], bool | None]: """Return the metadata and the requiredness of a ``TypedDict`` field, given its type hint including extras. The metadata is the first mapping in the metadata of an :data:`~typing.Annotated` type hint. The requiredness is ``None`` unless the type hint is wrapped in :data:`~typing.Required` or :data:`~typing.NotRequired`. """ metadata = None required = None while True: origin = get_origin(type_hint) if origin is Annotated: if metadata is None: metadata = next( (entry for entry in type_hint.__metadata__ if isinstance(entry, Mapping)), None, ) type_hint = type_hint.__origin__ # Required and NotRequired are missing from __required_keys__ and # __optional_keys__ when annotations are postponed, hence the need to # read requiredness from type hints as well. elif origin is Required or origin is NotRequired: required = origin is Required type_hint = get_args(type_hint)[0] elif origin in _READ_ONLY: type_hint = get_args(type_hint)[0] else: break return MappingProxyType(metadata) if metadata is not None else _EMPTY_METADATA, required def _get_typed_dict_field_metadata( item_class: type, field_name: str ) -> MappingProxyType[str, Any]: type_hints = _get_type_hints(item_class, include_extras=True) if field_name not in type_hints: raise KeyError(f"{item_class.__name__} does not support field: {field_name}") return _split_typed_dict_hint(type_hints[field_name])[0] scrapy-itemadapter-78cc818/itemadapter/adapter.py000066400000000000000000000443751527060762600222000ustar00rootroot00000000000000from __future__ import annotations import dataclasses import warnings from abc import ABCMeta, abstractmethod from collections.abc import Iterable, Iterator, KeysView, MutableMapping from functools import lru_cache from types import MappingProxyType from typing import Any, ClassVar, cast from itemadapter._imports import _is_typeddict, _scrapy_item_classes, attrs from itemadapter._json_schema import ( _json_schema_from_attrs, _json_schema_from_dataclass, _json_schema_from_item_class, _json_schema_from_pydantic, _json_schema_from_typed_dict, _JsonSchemaState, _setdefault_attribute_docstrings_on_json_schema, _setdefault_attribute_types_on_json_schema, ) from itemadapter._utils import ( _get_pydantic_model_metadata, _get_typed_dict_field_metadata, _is_attrs_class, _is_pydantic_model, ) __all__ = [ "AdapterInterface", "AttrsAdapter", "DataclassAdapter", "DictAdapter", "ItemAdapter", "PydanticAdapter", "ScrapyItemAdapter", ] class AdapterInterface(MutableMapping[str, Any], metaclass=ABCMeta): """Abstract Base Class for adapters. An adapter that handles a specific type of item should inherit from this class and implement the abstract methods defined here, plus the abtract methods inherited from the MutableMapping base class. """ def __init__(self, item: Any) -> None: self.item = item @classmethod @abstractmethod def is_item_class(cls, item_class: type) -> bool: """Return True if the adapter can handle the given item class, False otherwise.""" raise NotImplementedError @classmethod def is_item(cls, item: object) -> bool: """Return True if the adapter can handle the given item, False otherwise.""" return cls.is_item_class(item.__class__) @classmethod def get_field_meta_from_class( cls, item_class: type, field_name: str ) -> MappingProxyType[str, Any]: return MappingProxyType({}) @classmethod def get_field_names_from_class(cls, item_class: type) -> list[str] | None: """Return a list of fields defined for ``item_class``. If a class doesn't support fields, None is returned.""" return None @classmethod def get_json_schema( cls, item_class: type, *, _state: _JsonSchemaState | None = None ) -> dict[str, Any]: return _json_schema_from_item_class(cls, item_class, _state) def get_field_meta(self, field_name: str) -> MappingProxyType[str, Any]: """Return metadata for the given field name, if available.""" return self.get_field_meta_from_class(self.item.__class__, field_name) def field_names(self) -> KeysView[str]: """Return a dynamic view of the item's field names.""" return self.keys() class _MixinAttrsDataclassAdapter: _fields_dict: dict[str, Any] item: Any def get_field_meta(self, field_name: str) -> MappingProxyType[str, Any]: return cast("MappingProxyType[str, Any]", self._fields_dict[field_name].metadata) def field_names(self) -> KeysView[str]: return KeysView(self._fields_dict) def __getitem__(self, field_name: str) -> Any: if field_name in self._fields_dict: try: return getattr(self.item, field_name) except AttributeError as ex: raise KeyError(field_name) from ex raise KeyError(field_name) def __setitem__(self, field_name: str, value: Any) -> None: if field_name in self._fields_dict: setattr(self.item, field_name, value) else: raise KeyError(f"{self.item.__class__.__name__} does not support field: {field_name}") def __delitem__(self, field_name: str) -> None: if field_name in self._fields_dict: try: if hasattr(self.item, field_name): delattr(self.item, field_name) else: raise AttributeError except AttributeError as ex: raise KeyError(field_name) from ex else: raise KeyError(f"{self.item.__class__.__name__} does not support field: {field_name}") def __iter__(self) -> Iterator[str]: return iter(attr for attr in self._fields_dict if hasattr(self.item, attr)) def __len__(self) -> int: return len(list(iter(self))) class AttrsAdapter(_MixinAttrsDataclassAdapter, AdapterInterface): def __init__(self, item: Any) -> None: super().__init__(item) if attrs is None: raise RuntimeError("attrs module is not available") # store a reference to the item's fields to avoid O(n) lookups and O(n^2) traversals self._fields_dict = attrs.fields_dict(self.item.__class__) @classmethod def is_item_class(cls, item_class: type) -> bool: return _is_attrs_class(item_class) @classmethod def get_field_meta_from_class( cls, item_class: type, field_name: str ) -> MappingProxyType[str, Any]: if attrs is None: raise RuntimeError("attrs module is not available") try: return cast( "MappingProxyType[str, Any]", attrs.fields_dict(item_class)[field_name].metadata ) except KeyError as ex: raise KeyError(f"{item_class.__name__} does not support field: {field_name}") from ex @classmethod def get_field_names_from_class(cls, item_class: type) -> list[str] | None: if attrs is None: raise RuntimeError("attrs module is not available") return [a.name for a in attrs.fields(item_class)] @classmethod def get_json_schema( cls, item_class: type, *, _state: _JsonSchemaState | None = None ) -> dict[str, Any]: _state = _state or _JsonSchemaState(adapter=cls, containers={item_class}) return _json_schema_from_attrs(item_class, _state) class DataclassAdapter(_MixinAttrsDataclassAdapter, AdapterInterface): def __init__(self, item: Any) -> None: super().__init__(item) # store a reference to the item's fields to avoid O(n) lookups and O(n^2) traversals self._fields_dict = {field.name: field for field in dataclasses.fields(self.item)} @classmethod def is_item_class(cls, item_class: type) -> bool: return dataclasses.is_dataclass(item_class) @classmethod def get_field_meta_from_class( cls, item_class: type, field_name: str ) -> MappingProxyType[str, Any]: for field in dataclasses.fields(item_class): if field.name == field_name: return field.metadata raise KeyError(f"{item_class.__name__} does not support field: {field_name}") @classmethod def get_field_names_from_class(cls, item_class: type) -> list[str] | None: return [a.name for a in dataclasses.fields(item_class)] @classmethod def get_json_schema( cls, item_class: type, *, _state: _JsonSchemaState | None = None ) -> dict[str, Any]: _state = _state or _JsonSchemaState(adapter=cls, containers={item_class}) return _json_schema_from_dataclass(item_class, _state) class PydanticAdapter(AdapterInterface): @classmethod def is_item_class(cls, item_class: type) -> bool: return _is_pydantic_model(item_class) @classmethod def get_field_meta_from_class( cls, item_class: type, field_name: str ) -> MappingProxyType[str, Any]: try: return _get_pydantic_model_metadata(item_class, field_name) except KeyError as ex: raise KeyError(f"{item_class.__name__} does not support field: {field_name}") from ex @classmethod def get_field_names_from_class(cls, item_class: type) -> list[str] | None: return list(item_class.model_fields.keys()) # type: ignore[attr-defined] @classmethod def get_json_schema( cls, item_class: type, *, _state: _JsonSchemaState | None = None ) -> dict[str, Any]: return _json_schema_from_pydantic(cls, item_class, _state) def field_names(self) -> KeysView[str]: return KeysView(self.item.__class__.model_fields) def __getitem__(self, field_name: str) -> Any: if field_name not in self.item.__class__.model_fields: raise KeyError(field_name) try: return getattr(self.item, field_name) except AttributeError as ex: raise KeyError(field_name) from ex def __setitem__(self, field_name: str, value: Any) -> None: if field_name not in self.item.__class__.model_fields: raise KeyError(f"{self.item.__class__.__name__} does not support field: {field_name}") setattr(self.item, field_name, value) def __delitem__(self, field_name: str) -> None: if field_name not in self.item.__class__.model_fields: raise KeyError(f"{self.item.__class__.__name__} does not support field: {field_name}") try: if hasattr(self.item, field_name): delattr(self.item, field_name) return raise AttributeError except AttributeError as ex: raise KeyError(field_name) from ex def __iter__(self) -> Iterator[str]: return iter(attr for attr in self.item.__class__.model_fields if hasattr(self.item, attr)) def __len__(self) -> int: return len(list(iter(self))) class _MixinDictScrapyItemAdapter: item: Any def __getitem__(self, field_name: str) -> Any: return self.item[field_name] def __setitem__(self, field_name: str, value: Any) -> None: self.item[field_name] = value def __delitem__(self, field_name: str) -> None: del self.item[field_name] def __iter__(self) -> Iterator[str]: return iter(self.item) def __len__(self) -> int: return len(self.item) class DictAdapter(_MixinDictScrapyItemAdapter, AdapterInterface): """Adapter for :class:`dict` items. :class:`~typing.TypedDict` subclasses are also supported as item classes. Their instances are plain dictionaries at run time, so they are handled like any other dictionary. """ @classmethod def is_item_class(cls, item_class: type) -> bool: return issubclass(item_class, dict) @classmethod def get_field_meta_from_class( cls, item_class: type, field_name: str ) -> MappingProxyType[str, Any]: if _is_typeddict(item_class): return _get_typed_dict_field_metadata(item_class, field_name) return MappingProxyType({}) @classmethod def get_field_names_from_class(cls, item_class: type) -> list[str] | None: if _is_typeddict(item_class): return list(item_class.__annotations__) return None @classmethod def get_json_schema( cls, item_class: type, *, _state: _JsonSchemaState | None = None ) -> dict[str, Any]: if _is_typeddict(item_class): _state = _state or _JsonSchemaState(adapter=cls, containers={item_class}) return _json_schema_from_typed_dict(item_class, _state) return {"type": "object"} def field_names(self) -> KeysView[str]: return KeysView(self.item) class ScrapyItemAdapter(_MixinDictScrapyItemAdapter, AdapterInterface): @classmethod def is_item_class(cls, item_class: type) -> bool: return issubclass(item_class, _scrapy_item_classes) @classmethod def get_field_meta_from_class( cls, item_class: type, field_name: str ) -> MappingProxyType[str, Any]: return MappingProxyType(item_class.fields[field_name]) # type: ignore[attr-defined] @classmethod def get_field_names_from_class(cls, item_class: type) -> list[str] | None: return list(item_class.fields.keys()) # type: ignore[attr-defined] @classmethod def get_json_schema( cls, item_class: type, *, _state: _JsonSchemaState | None = None ) -> dict[str, Any]: _state = _state or _JsonSchemaState(adapter=cls, containers={item_class}) schema = super().get_json_schema(item_class, _state=_state) _setdefault_attribute_types_on_json_schema(schema, item_class, _state) _setdefault_attribute_docstrings_on_json_schema(schema, item_class) schema.pop("required", None) return schema def field_names(self) -> KeysView[str]: return KeysView(self.item.fields) @lru_cache def _has_item_overrides(adapter_classes: tuple[type[AdapterInterface], ...]) -> bool: """Return True if any of the given adapter classes overrides is_item(), i.e. decides on an item basis, and hence cannot be matched to an item by item class alone.""" base_is_item = AdapterInterface.is_item.__func__ # type: ignore[attr-defined] overriding = [ adapter_class for adapter_class in adapter_classes if getattr(adapter_class.is_item, "__func__", adapter_class.is_item) is not base_is_item ] if overriding: names = ", ".join(adapter_class.__qualname__ for adapter_class in overriding) warnings.warn( f"The following adapter classes override the is_item() class method: {names}. " f"Support for that will be removed, and items will then be matched to an " f"adapter class by item class alone. Override is_item_class() instead.", DeprecationWarning, stacklevel=4, ) return bool(overriding) @lru_cache(maxsize=1024) def _find_adapter_class( adapter_classes: tuple[type[AdapterInterface], ...], item_class: type ) -> tuple[type[AdapterInterface] | None, bool]: """Return the first of the given adapter classes that handles the given item class, or None, along with the return value of _has_item_overrides().""" for adapter_class in adapter_classes: if adapter_class.is_item_class(item_class): return adapter_class, _has_item_overrides(adapter_classes) return None, _has_item_overrides(adapter_classes) class ItemAdapter(MutableMapping[str, Any]): """Wrapper class to interact with data container objects. It provides a common interface to extract and set data without having to take the object's type into account. """ ADAPTER_CLASSES: ClassVar[Iterable[type[AdapterInterface]]] = ( ScrapyItemAdapter, DictAdapter, DataclassAdapter, AttrsAdapter, PydanticAdapter, ) def __init__(self, item: Any) -> None: adapter_classes = tuple(self.ADAPTER_CLASSES) adapter_class, item_overrides = _find_adapter_class(adapter_classes, item.__class__) if item_overrides: adapter_class = next( ( candidate_class for candidate_class in adapter_classes if candidate_class.is_item(item) ), None, ) if adapter_class is None: raise TypeError(f"No adapter found for objects of type: {type(item)} ({item})") self.adapter = adapter_class(item) @classmethod def is_item(cls, item: Any) -> bool: adapter_classes = tuple(cls.ADAPTER_CLASSES) adapter_class, item_overrides = _find_adapter_class(adapter_classes, item.__class__) if item_overrides: return any(adapter_class.is_item(item) for adapter_class in adapter_classes) return adapter_class is not None @classmethod def is_item_class(cls, item_class: type) -> bool: return _find_adapter_class(tuple(cls.ADAPTER_CLASSES), item_class)[0] is not None @classmethod def _get_adapter_class(cls, item_class: type) -> type[AdapterInterface]: adapter_class = _find_adapter_class(tuple(cls.ADAPTER_CLASSES), item_class)[0] if adapter_class is None: raise TypeError(f"{item_class} is not a valid item class") return adapter_class @classmethod def get_field_meta_from_class( cls, item_class: type, field_name: str ) -> MappingProxyType[str, Any]: adapter_class = cls._get_adapter_class(item_class) return adapter_class.get_field_meta_from_class(item_class, field_name) @classmethod def get_field_names_from_class(cls, item_class: type) -> list[str] | None: adapter_class = cls._get_adapter_class(item_class) return adapter_class.get_field_names_from_class(item_class) @classmethod def get_json_schema( cls, item_class: type, *, _state: _JsonSchemaState | None = None ) -> dict[str, Any]: _state = _state or _JsonSchemaState(adapter=cls, containers={item_class}) adapter_class = cls._get_adapter_class(item_class) return adapter_class.get_json_schema(item_class, _state=_state) @property def item(self) -> Any: return self.adapter.item def __repr__(self) -> str: values = ", ".join([f"{key}={value!r}" for key, value in self.items()]) return f"<{self.__class__.__name__} for {self.item.__class__.__name__}({values})>" def __getitem__(self, field_name: str) -> Any: return self.adapter.__getitem__(field_name) def __setitem__(self, field_name: str, value: Any) -> None: self.adapter.__setitem__(field_name, value) def __delitem__(self, field_name: str) -> None: self.adapter.__delitem__(field_name) def __iter__(self) -> Iterator[str]: return self.adapter.__iter__() def __len__(self) -> int: return self.adapter.__len__() def get_field_meta(self, field_name: str) -> MappingProxyType[str, Any]: """Return metadata for the given field name.""" return self.adapter.get_field_meta(field_name) def field_names(self) -> KeysView[str]: """Return read-only key view with the names of all the defined fields for the item.""" return self.adapter.field_names() def asdict(self) -> dict[str, Any]: """Return a dict object with the contents of the adapter. This works slightly different than calling `dict(adapter)`: it's applied recursively to nested items (if there are any). """ return {key: self._asdict(value) for key, value in self.items()} @classmethod def _asdict(cls, obj: object) -> object: if isinstance(obj, dict): return {key: cls._asdict(value) for key, value in obj.items()} if isinstance(obj, (list, set, tuple)): return obj.__class__(cls._asdict(x) for x in obj) if isinstance(obj, cls): return obj.asdict() if cls.is_item(obj): return cls(obj).asdict() return obj scrapy-itemadapter-78cc818/itemadapter/py.typed000066400000000000000000000000001527060762600216560ustar00rootroot00000000000000scrapy-itemadapter-78cc818/itemadapter/utils.py000066400000000000000000000025611527060762600217070ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING, Any from itemadapter.adapter import ItemAdapter if TYPE_CHECKING: from types import MappingProxyType __all__ = ["get_field_meta_from_class", "is_item"] def is_item(obj: object) -> bool: """Return True if the given object belongs to one of the supported types, False otherwise. Alias for ItemAdapter.is_item """ return ItemAdapter.is_item(obj) def get_field_meta_from_class(item_class: type, field_name: str) -> MappingProxyType[str, Any]: """Return a read-only mapping with metadata for the given field name, within the given item class. If there is no metadata for the field, or the item class does not support field metadata, an empty object is returned. Field metadata is taken from different sources, depending on the item type: * scrapy.item.Item: corresponding scrapy.item.Field object * dataclass items: "metadata" attribute for the corresponding field * attrs items: "metadata" attribute for the corresponding field * pydantic models: corresponding pydantic.fields.FieldInfo object The returned value is an instance of types.MappingProxyType, i.e. a dynamic read-only view of the original mapping, which gets automatically updated if the original mapping changes. """ return ItemAdapter.get_field_meta_from_class(item_class, field_name) scrapy-itemadapter-78cc818/pyproject.toml000066400000000000000000000136201527060762600206100ustar00rootroot00000000000000[build-system] requires = ["hatchling"] build-backend = "hatchling.build" [project] name = "itemadapter" description = "Common interface for data container classes" authors = [ { name = "Eugenio Lacuesta", email = "eugenio.lacuesta@gmail.com" }, ] readme = "README.md" license = "BSD-3-Clause" license-files = ["LICENSE"] requires-python = ">=3.11" classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Programming Language :: Python :: 3.15", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Operating System :: OS Independent", "Framework :: Scrapy", "Topic :: Internet :: WWW/HTTP", "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", ] dynamic = ["version"] [project.optional-dependencies] attrs = [ "attrs>=22.1.0", ] pydantic = [ "pydantic>=2.7", ] scrapy = [ "scrapy>=2.7", ] [project.urls] Homepage = "https://github.com/scrapy/itemadapter" Source = "https://github.com/scrapy/itemadapter" Tracker = "https://github.com/scrapy/itemadapter/issues" "Release notes" = "https://github.com/scrapy/itemadapter/blob/master/Changelog.md" [tool.hatch.version] path = "itemadapter/__init__.py" [tool.hatch.build.targets.sdist] include = [ "/itemadapter", "/tests", "/Changelog.md", "/tox.ini", ] [tool.bumpversion] current_version = "0.14.0" commit = true tag = true [[tool.bumpversion.files]] filename = "Changelog.md" search = "\\(unreleased\\)$" replace = "({now:%Y-%m-%d})" regex = true [[tool.bumpversion.files]] filename = "itemadapter/__init__.py" [tool.coverage.run] branch = true [tool.mypy] strict = true [tool.pylint.MASTER] persistent = "no" load-plugins=[ "pylint_per_file_ignores", ] [tool.pylint."MESSAGES CONTROL"] enable = [ "useless-suppression", ] disable = [ "cyclic-import", "duplicate-code", "import-error", "import-outside-toplevel", "invalid-name", "line-too-long", # https://docs.astral.sh/ruff/rules/line-too-long/ "missing-class-docstring", "missing-function-docstring", "missing-module-docstring", "not-callable", "pointless-statement", "protected-access", "too-few-public-methods", "too-many-public-methods", "too-many-return-statements", "unused-argument", ] per-file-ignores = [ "tests/*:disallowed-name,too-many-instance-attributes", "itemadapter/_imports.py:unused-import,ungrouped-imports", ] [tool.pylint.FORMAT] expected-line-ending-format = "LF" max-line-length = 99 [tool.pylint.IMPORTS] allow-any-import-level = [ "attrs", "dataclasses", "scrapy" ] [tool.ruff] line-length = 99 [tool.ruff.lint] extend-select = [ # flake8-builtins "A", # flake8-async "ASYNC", # flake8-bugbear "B", # flake8-comprehensions "C4", # flake8-commas "COM", # pydocstyle "D", # flake8-future-annotations "FA", # flynt "FLY", # refurb "FURB", # isort "I", # flake8-implicit-str-concat "ISC", # flake8-logging "LOG", # Perflint "PERF", # pygrep-hooks "PGH", # flake8-pie "PIE", # pylint "PL", # flake8-pytest-style "PT", # flake8-use-pathlib "PTH", # flake8-pyi "PYI", # flake8-quotes "Q", # flake8-return "RET", # flake8-raise "RSE", # Ruff-specific rules "RUF", # flake8-bandit "S", # flake8-simplify "SIM", # flake8-slots "SLOT", # flake8-debugger "T10", # flake8-type-checking "TC", # flake8-tidy-imports "TID", # pyupgrade "UP", # pycodestyle warnings "W", # flake8-2020 "YTT", ] ignore = [ # Trailing comma missing "COM812", # Missing docstring in public module "D100", # Missing docstring in public class "D101", # Missing docstring in public method "D102", # Missing docstring in public function "D103", # Missing docstring in public package "D104", # Missing docstring in magic method "D105", # Missing docstring in public nested class "D106", # Missing docstring in __init__ "D107", # One-line docstring should fit on one line with quotes "D200", # No blank lines allowed after function docstring "D202", # 1 blank line required between summary line and description "D205", # Multi-line docstring closing quotes should be on a separate line "D209", # First line should end with a period "D400", # First line should be in imperative mood; try rephrasing "D401", # First line should not be the function's "signature" "D402", # First word of the first line should be properly capitalized "D403", # `import` should be at the top-level of a file "PLC0415", # Too many return statements "PLR0911", # Too many branches "PLR0912", # Too many arguments in function definition "PLR0913", # Too many statements "PLR0915", # Too many positional arguments "PLR0917", # Magic value used in comparison "PLR2004", # String contains ambiguous {}. "RUF001", # Docstring contains ambiguous {}. "RUF002", # Comment contains ambiguous {}. "RUF003", # Mutable class attributes should be annotated with `typing.ClassVar` "RUF012", # Use of `assert` detected "S101", # Yoda condition detected "SIM300", ] [tool.ruff.lint.flake8-type-checking] runtime-evaluated-decorators = ["attrs.define"] [tool.ruff.lint.isort] split-on-trailing-comma = false [tool.ruff.lint.pydocstyle] convention = "pep257" scrapy-itemadapter-78cc818/tests/000077500000000000000000000000001527060762600170345ustar00rootroot00000000000000scrapy-itemadapter-78cc818/tests/__init__.py000066400000000000000000000251401527060762600211470ustar00rootroot00000000000000from __future__ import annotations import importlib import sys from contextlib import contextmanager from dataclasses import dataclass, field from enum import Enum from typing import TYPE_CHECKING, Any import pytest from itemadapter import ItemAdapter from itemadapter._imports import pydantic if TYPE_CHECKING: from collections.abc import Callable, Generator def make_mock_import(block_name: str) -> Callable: def mock_import(name: str, *args, **kwargs): """Prevent importing a specific module, let everything else pass.""" if name.split(".", maxsplit=1)[0] == block_name: raise ImportError(name) return importlib.__import__(name, *args, **kwargs) return mock_import @contextmanager def clear_itemadapter_imports() -> Generator[None]: backup = {} for key in sys.modules.copy(): if key.startswith("itemadapter"): backup[key] = sys.modules.pop(key) try: yield finally: sys.modules.update(backup) class Color(Enum): RED = "red" GREEN = "green" BLUE = "blue" @dataclass class DataClassItem: name: str = field(default_factory=lambda: None, metadata={"serializer": str}) value: int = field(default_factory=lambda: None, metadata={"serializer": int}) @dataclass class DataClassItemNested: nested: DataClassItem adapter: ItemAdapter dict_: dict list_: list set_: set tuple_: tuple int_: int @dataclass(init=False) class DataClassWithoutInit: name: str = field(metadata={"serializer": str}) value: int = field(metadata={"serializer": int}) @dataclass class DataClassItemSubclassed(DataClassItem): subclassed: bool = True @dataclass class DataClassItemEmpty: pass @dataclass class DataClassItemJsonSchemaNested: is_nested: bool = True @dataclass class DataClassItemJsonSchema: __json_schema_extra__ = { "llmHint": "Hi model!", } name: str = field(metadata={"json_schema_extra": {"title": "Name"}}) """Display name""" color: Color answer: str | float | int | None numbers: list[float] aliases: dict[str, str] nested: DataClassItemJsonSchemaNested nested_list: list[DataClassItemJsonSchemaNested] nested_dict: dict[str, DataClassItemJsonSchemaNested] nested_dict_list: list[dict[str, DataClassItemJsonSchemaNested]] value: Any = None produced: bool = field(default_factory=lambda: True) try: import attr import attrs except ImportError: AttrsItem = None AttrsItemNested = None AttrsItemWithoutInit = None AttrsItemSubclassed = None AttrsItemEmpty = None AttrsItemJsonSchema = None AttrsItemJsonSchemaNested = None AttrsClassicItem = None AttrsClassicItemNested = None AttrsClassicItemWithoutInit = None AttrsClassicItemSubclassed = None AttrsClassicItemEmpty = None AttrsClassicItemJsonSchema = None AttrsClassicItemJsonSchemaNested = None else: @attrs.define class AttrsItem: name = attrs.field(default=None, metadata={"serializer": str}) value = attrs.field(default=None, metadata={"serializer": int}) @attrs.define class AttrsItemNested: nested: AttrsItem adapter: ItemAdapter dict_: dict list_: list set_: set tuple_: tuple int_: int @attrs.define(init=False) class AttrsItemWithoutInit: name = attrs.field(default=None, metadata={"serializer": str}) value = attrs.field(default=None, metadata={"serializer": int}) @attrs.define(init=False) class AttrsItemSubclassed(AttrsItem): subclassed: bool = True @attrs.define class AttrsItemEmpty: pass @attrs.define class AttrsItemJsonSchemaNested: is_nested: bool = True @attrs.define class AttrsItemJsonSchema: __json_schema_extra__ = { "llmHint": "Hi model!", } name: str = attrs.field(metadata={"json_schema_extra": {"title": "Name"}}) """Display name""" color: Color answer: str | float | int | None numbers: list[float] aliases: dict[str, str] nested: AttrsItemJsonSchemaNested nested_list: list[AttrsItemJsonSchemaNested] nested_dict: dict[str, AttrsItemJsonSchemaNested] nested_dict_list: list[dict[str, AttrsItemJsonSchemaNested]] value: Any = None produced: bool = attrs.field(factory=lambda: True) # Classic API (attr.s, attr.ib): dict-based classes, unlike the slotted # ones created by attrs.define. @attr.s class AttrsClassicItem: name = attr.ib(default=None, metadata={"serializer": str}) value = attr.ib(default=None, metadata={"serializer": int}) @attr.s class AttrsClassicItemNested: nested = attr.ib(type=AttrsClassicItem) adapter = attr.ib(type=ItemAdapter) dict_ = attr.ib(type=dict) list_ = attr.ib(type=list) set_ = attr.ib(type=set) tuple_ = attr.ib(type=tuple) int_ = attr.ib(type=int) @attr.s(init=False) class AttrsClassicItemWithoutInit: name = attr.ib(default=None, metadata={"serializer": str}) value = attr.ib(default=None, metadata={"serializer": int}) @attr.s(init=False) class AttrsClassicItemSubclassed(AttrsClassicItem): subclassed = attr.ib(default=True, type=bool) @attr.s class AttrsClassicItemEmpty: pass @attr.s class AttrsClassicItemJsonSchemaNested: is_nested: bool = attr.ib(default=True) @attr.s class AttrsClassicItemJsonSchema: __json_schema_extra__ = { "llmHint": "Hi model!", } name: str = attr.ib(metadata={"json_schema_extra": {"title": "Name"}}) """Display name""" color: Color = attr.ib() answer: str | float | int | None = attr.ib() numbers: list[float] = attr.ib() aliases: dict[str, str] = attr.ib() nested: AttrsClassicItemJsonSchemaNested = attr.ib() nested_list: list[AttrsClassicItemJsonSchemaNested] = attr.ib() nested_dict: dict[str, AttrsClassicItemJsonSchemaNested] = attr.ib() nested_dict_list: list[dict[str, AttrsClassicItemJsonSchemaNested]] = attr.ib() value: Any = attr.ib(default=None) produced: bool = attr.ib(factory=lambda: True) if pydantic is None: PydanticModel = None PydanticSpecialCasesModel = None PydanticModelNested = None PydanticModelSubclassed = None PydanticModelEmpty = None PydanticModelJsonSchema = None PydanticModelJsonSchemaNested = None else: class PydanticModel(pydantic.BaseModel): name: str | None = pydantic.Field( default_factory=lambda: None, json_schema_extra={"serializer": str}, ) value: int | None = pydantic.Field( default_factory=lambda: None, json_schema_extra={"serializer": int}, ) class PydanticSpecialCasesModel(pydantic.BaseModel): special_cases: int | None = pydantic.Field( default_factory=lambda: None, alias="special_cases", frozen=True, ) model_config = { "validate_assignment": True, } class PydanticModelNested(pydantic.BaseModel): nested: PydanticModel adapter: ItemAdapter dict_: dict list_: list set_: set tuple_: tuple int_: int model_config = { "arbitrary_types_allowed": True, } class PydanticModelSubclassed(PydanticModel): subclassed: bool = pydantic.Field( default_factory=lambda: True, ) class PydanticModelEmpty(pydantic.BaseModel): pass class PydanticModelJsonSchemaNested(pydantic.BaseModel): is_nested: bool = True class PydanticModelJsonSchema(pydantic.BaseModel): name: str = pydantic.Field(description="Display name", title="Name") value: Any = None color: Color produced: bool = pydantic.Field(default_factory=lambda: True) answer: str | float | int | None numbers: list[float] aliases: dict[str, str] nested: PydanticModelJsonSchemaNested nested_list: list[PydanticModelJsonSchemaNested] nested_dict: dict[str, PydanticModelJsonSchemaNested] nested_dict_list: list[dict[str, PydanticModelJsonSchemaNested]] model_config = { "json_schema_extra": { "llmHint": "Hi model!", }, } try: from scrapy.item import Field from scrapy.item import Item as ScrapyItem except ImportError: ScrapyItem = None ScrapySubclassedItem = None ScrapySubclassedItemNested = None ScrapySubclassedItemSubclassed = None ScrapySubclassedItemEmpty = None ScrapySubclassedItemJsonSchema = None ScrapySubclassedItemJsonSchemaNested = None else: class ScrapySubclassedItem(ScrapyItem): name = Field(serializer=str) value = Field(serializer=int) class ScrapySubclassedItemNested(ScrapyItem): nested = Field() adapter = Field() dict_ = Field() list_ = Field() set_ = Field() tuple_ = Field() int_ = Field() class ScrapySubclassedItemSubclassed(ScrapySubclassedItem): subclassed = Field() class ScrapySubclassedItemEmpty(ScrapyItem): pass class ScrapySubclassedItemJsonSchemaNested(ScrapyItem): is_nested: bool = Field( json_schema_extra={ "default": True, }, ) class ScrapySubclassedItemJsonSchema(ScrapyItem): __json_schema_extra__ = { "llmHint": "Hi model!", } name: str = Field( json_schema_extra={ "title": "Name", }, ) """Display name""" value = Field( json_schema_extra={ "default": None, }, ) color: Color = Field() produced = Field() answer: str | float | int | None = Field() numbers: list[float] = Field() aliases: dict[str, str] = Field() nested: ScrapySubclassedItemJsonSchemaNested = Field() nested_list: list[ScrapySubclassedItemJsonSchemaNested] = Field() nested_dict: dict[str, ScrapySubclassedItemJsonSchemaNested] = Field() nested_dict_list: list[dict[str, ScrapySubclassedItemJsonSchemaNested]] = Field() requires_attrs = pytest.mark.skipif(AttrsItem is None, reason="attrs module is not available") requires_pydantic = pytest.mark.skipif( PydanticModel is None, reason="pydantic module is not available" ) requires_scrapy = pytest.mark.skipif( ScrapySubclassedItem is None, reason="scrapy module is not available" ) scrapy-itemadapter-78cc818/tests/benchmarks/000077500000000000000000000000001527060762600211515ustar00rootroot00000000000000scrapy-itemadapter-78cc818/tests/benchmarks/__init__.py000066400000000000000000000001431527060762600232600ustar00rootroot00000000000000import pytest pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed") scrapy-itemadapter-78cc818/tests/benchmarks/test_is_item.py000066400000000000000000000062041527060762600242150ustar00rootroot00000000000000from __future__ import annotations from dataclasses import field, make_dataclass from typing import TYPE_CHECKING, Any import pytest import itemadapter from itemadapter import ItemAdapter from itemadapter._imports import attrs, pydantic, scrapy if TYPE_CHECKING: from collections.abc import Callable, Sequence from pytest_codspeed import BenchmarkFixture # Number of calls per benchmark. Kept the same for every scenario so that the # results of different scenarios can be compared to each other. CALLS = 100 def _dict_class(index: int) -> type: return type(f"DictItem{index}", (dict,), {}) def _dataclass_class(index: int) -> type: return make_dataclass( f"DataClassItem{index}", [("name", str, field(default=None)), ("value", int, field(default=None))], ) def _non_item_class(index: int) -> type: return type(f"NonItem{index}", (), {}) def _attrs_class(index: int) -> type: return attrs.make_class( f"AttrsItem{index}", {"name": attrs.field(default=None), "value": attrs.field(default=None)}, ) def _pydantic_class(index: int) -> type: return pydantic.create_model(f"PydanticModel{index}", name=(str, None), value=(int, None)) def _scrapy_class(index: int) -> type: return type(scrapy.Item)( f"ScrapyItem{index}", (scrapy.Item,), {"name": scrapy.Field(), "value": scrapy.Field()}, ) ITEM_CLASS_FACTORIES: dict[str, Callable[[int], type]] = { "dict": _dict_class, "dataclass": _dataclass_class, "non-item": _non_item_class, } if attrs is not None: ITEM_CLASS_FACTORIES["attrs"] = _attrs_class if pydantic is not None: ITEM_CLASS_FACTORIES["pydantic"] = _pydantic_class if scrapy is not None: ITEM_CLASS_FACTORIES["scrapy"] = _scrapy_class item_class_factory = pytest.mark.parametrize( "factory", ITEM_CLASS_FACTORIES.values(), ids=list(ITEM_CLASS_FACTORIES), ) is_item_function = pytest.mark.parametrize( "func", [ItemAdapter.is_item, itemadapter.is_item], ids=["ItemAdapter.is_item", "is_item"], ) @is_item_function @item_class_factory def test_shared_class( benchmark: BenchmarkFixture, factory: Callable[[int], type], func: Callable[[Any], bool], ) -> None: """Items all share a single class, as in most real usage.""" item_class = factory(0) items = [item_class() for _ in range(CALLS)] @benchmark def _() -> None: for item in items: func(item) @is_item_function @item_class_factory def test_distinct_classes( benchmark: BenchmarkFixture, factory: Callable[[int], type], func: Callable[[Any], bool], ) -> None: """Every item has a class that no earlier call has seen. Item classes are built by a setup callback, which runs before every round and outside the measured window, so that their creation is not measured and the measured calls cannot benefit from earlier ones. """ def setup() -> tuple[tuple[Any, ...], dict[str, Any]]: return ([factory(index)() for index in range(CALLS)],), {} def call(items: Sequence[Any]) -> None: for item in items: func(item) benchmark.pedantic(call, setup=setup) scrapy-itemadapter-78cc818/tests/conftest.py000066400000000000000000000012601527060762600212320ustar00rootroot00000000000000import pytest from itemadapter._json_schema import get_attr_docstrings from itemadapter.adapter import _find_adapter_class, _has_item_overrides @pytest.fixture(autouse=True) def _clear_adapter_class_cache(): """Keep tests that change which item types are supported, e.g. by patching an optional import out, from seeing results cached by earlier tests.""" _find_adapter_class.cache_clear() _has_item_overrides.cache_clear() @pytest.fixture(autouse=True) def _clear_attr_docstring_cache(): """Keep tests that mock reading class sources from seeing attribute docstrings cached by earlier tests, and the other way around.""" get_attr_docstrings.cache_clear() scrapy-itemadapter-78cc818/tests/test_adapter.py000066400000000000000000000436511527060762600220760ustar00rootroot00000000000000from __future__ import annotations from collections.abc import KeysView from types import MappingProxyType import pytest from packaging.version import Version from itemadapter.adapter import ItemAdapter, PydanticAdapter from tests import ( AttrsClassicItem, AttrsClassicItemEmpty, AttrsClassicItemJsonSchema, AttrsClassicItemNested, AttrsClassicItemSubclassed, AttrsClassicItemWithoutInit, AttrsItem, AttrsItemEmpty, AttrsItemJsonSchema, AttrsItemNested, AttrsItemSubclassed, AttrsItemWithoutInit, DataClassItem, DataClassItemEmpty, DataClassItemJsonSchema, DataClassItemNested, DataClassItemSubclassed, DataClassWithoutInit, PydanticModel, PydanticModelEmpty, PydanticModelJsonSchema, PydanticModelNested, PydanticModelSubclassed, ScrapySubclassedItem, ScrapySubclassedItemEmpty, ScrapySubclassedItemJsonSchema, ScrapySubclassedItemNested, ScrapySubclassedItemSubclassed, requires_attrs, requires_pydantic, requires_scrapy, ) from tests.test_json_schema import check_schemas try: import scrapy SCRAPY_VERSION = Version(scrapy.__version__) except ImportError: SCRAPY_VERSION = None class TestItemAdapterRepr: def test_repr_dict(self): item = {"name": "asdf", "value": 1234} adapter = ItemAdapter(item) assert repr(adapter) == "" @requires_scrapy def test_repr_scrapy_item(self): item = ScrapySubclassedItem(name="asdf", value=1234) adapter = ItemAdapter(item) assert repr(adapter) == "" def test_repr_dataclass(self): item = DataClassItem(name="asdf", value=1234) adapter = ItemAdapter(item) assert repr(adapter) == "" def test_repr_dataclass_init_false(self): item = DataClassWithoutInit() adapter = ItemAdapter(item) assert repr(adapter) == "" adapter["name"] = "set after init" assert repr(adapter) == "" @requires_attrs def test_repr_attrs(self): item = AttrsItem(name="asdf", value=1234) adapter = ItemAdapter(item) assert repr(adapter) == "" @requires_attrs def test_repr_attrs_init_false(self): item = AttrsItemWithoutInit() adapter = ItemAdapter(item) assert repr(adapter) == "" adapter["name"] = "set after init" assert repr(adapter) == "" @requires_attrs def test_repr_attrs_classic(self): item = AttrsClassicItem(name="asdf", value=1234) adapter = ItemAdapter(item) assert repr(adapter) == "" @requires_attrs def test_repr_attrs_classic_init_false(self): item = AttrsClassicItemWithoutInit() adapter = ItemAdapter(item) assert repr(adapter) == "" adapter["name"] = "set after init" assert ( repr(adapter) == "" ) @requires_pydantic def test_repr_pydantic(self): item = PydanticModel(name="asdf", value=1234) adapter = ItemAdapter(item) assert repr(adapter) == "" def test_non_item() -> None: with pytest.raises(TypeError): ItemAdapter(ScrapySubclassedItem) with pytest.raises(TypeError): ItemAdapter(dict) with pytest.raises(TypeError): ItemAdapter(1234) class BaseTestMixin: item_class = None item_class_nested = None item_class_json_schema = None def test_get_set_value(self): item = self.item_class() adapter = ItemAdapter(item) assert adapter.get("name") is None assert adapter.get("value") is None adapter["name"] = "asdf" adapter["value"] = 1234 assert adapter.get("name") == "asdf" assert adapter.get("value") == 1234 assert adapter["name"] == "asdf" assert adapter["value"] == 1234 item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) assert adapter.get("name") == "asdf" assert adapter.get("value") == 1234 assert adapter["name"] == "asdf" assert adapter["value"] == 1234 def test_get_value_keyerror(self): item = self.item_class() adapter = ItemAdapter(item) with pytest.raises(KeyError): adapter["undefined_field"] def test_deleted_field_keyerror(self): adapter = ItemAdapter(self.item_class(name="asdf", value=1234)) del adapter["name"] with pytest.raises(KeyError) as exc_info: adapter["name"] assert exc_info.value.args == ("name",) def test_deleted_field_get_default(self): adapter = ItemAdapter(self.item_class(name="asdf", value=1234)) del adapter["name"] assert adapter.get("name", "fallback") == "fallback" assert adapter.pop("name", "fallback") == "fallback" def test_deleted_field_membership(self): adapter = ItemAdapter(self.item_class(name="asdf", value=1234)) del adapter["name"] assert "name" not in adapter adapter["name"] = "restored" assert "name" in adapter assert adapter["name"] == "restored" def test_as_dict(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) assert dict(adapter) == {"name": "asdf", "value": 1234} def test_as_dict_nested(self): item = self.item_class_nested( nested=self.item_class(name="asdf", value=1234), adapter=ItemAdapter({"foo": "bar", "nested_list": [1, 2, 3, 4, 5]}), dict_={"foo": "bar", "answer": 42, "nested_dict": {"a": "b"}}, list_=[1, 2, 3], set_={1, 2, 3}, tuple_=(1, 2, 3), int_=123, ) adapter = ItemAdapter(item) assert adapter.asdict() == { "nested": {"name": "asdf", "value": 1234}, "adapter": {"foo": "bar", "nested_list": [1, 2, 3, 4, 5]}, "dict_": {"foo": "bar", "answer": 42, "nested_dict": {"a": "b"}}, "list_": [1, 2, 3], "set_": {1, 2, 3}, "tuple_": (1, 2, 3), "int_": 123, } def test_field_names(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) assert isinstance(adapter.field_names(), KeysView) assert sorted(adapter.field_names()) == ["name", "value"] def test_json_schema(self): item_class = self.item_class_json_schema actual = ItemAdapter.get_json_schema(item_class) check_schemas(actual, self.expected_json_schema) def test_json_schema_empty(self): actual = ItemAdapter.get_json_schema(self.item_class_empty) expected = {"type": "object"} if self.item_class_empty is not dict and not PydanticAdapter.is_item_class( self.item_class_empty ): expected["additionalProperties"] = False check_schemas(actual, expected) _NESTED_JSON_SCHEMA = { "type": "object", "additionalProperties": False, "properties": { "is_nested": {"type": "boolean", "default": True}, }, } class NonDictTestMixin(BaseTestMixin): item_class_subclassed = None item_class_empty = None expected_json_schema = { "llmHint": "Hi model!", "type": "object", "additionalProperties": False, "properties": { "name": { "type": "string", "title": "Name", "description": "Display name", }, "value": { "default": None, }, "color": { "type": "string", "enum": ["red", "green", "blue"], }, "produced": {"type": "boolean"}, "answer": { "type": ["null", "number", "string"], }, "numbers": {"type": "array", "items": {"type": "number"}}, "aliases": { "type": "object", "additionalProperties": {"type": "string"}, }, "nested": _NESTED_JSON_SCHEMA, "nested_list": { "type": "array", "items": _NESTED_JSON_SCHEMA, }, "nested_dict": { "type": "object", "additionalProperties": _NESTED_JSON_SCHEMA, }, "nested_dict_list": { "type": "array", "items": { "type": "object", "additionalProperties": _NESTED_JSON_SCHEMA, }, }, }, "required": [ "name", "color", "answer", "numbers", "aliases", "nested", "nested_list", "nested_dict", "nested_dict_list", ], } def test_set_value_keyerror(self): item = self.item_class() adapter = ItemAdapter(item) with pytest.raises(KeyError): adapter["undefined_field"] = "some value" def test_metadata_common(self): adapter = ItemAdapter(self.item_class()) assert isinstance(adapter.get_field_meta("name"), MappingProxyType) assert isinstance(adapter.get_field_meta("value"), MappingProxyType) with pytest.raises(KeyError): adapter.get_field_meta("undefined_field") def test_get_field_meta_defined_fields(self): adapter = ItemAdapter(self.item_class()) assert adapter.get_field_meta("name") == MappingProxyType({"serializer": str}) assert adapter.get_field_meta("value") == MappingProxyType({"serializer": int}) def test_delitem_len_iter(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) assert len(adapter) == 2 assert sorted(iter(adapter)) == ["name", "value"] del adapter["name"] assert len(adapter) == 1 assert sorted(iter(adapter)) == ["value"] del adapter["value"] assert len(adapter) == 0 assert sorted(iter(adapter)) == [] with pytest.raises(KeyError): del adapter["name"] with pytest.raises(KeyError): del adapter["value"] with pytest.raises(KeyError): del adapter["undefined_field"] def test_field_names_from_class(self): field_names = ItemAdapter.get_field_names_from_class(self.item_class) assert isinstance(field_names, list) assert sorted(field_names) == ["name", "value"] def test_field_names_from_class_nested(self): field_names = ItemAdapter.get_field_names_from_class(self.item_class_subclassed) assert isinstance(field_names, list) assert sorted(field_names) == ["name", "subclassed", "value"] def test_field_names_from_class_empty(self): field_names = ItemAdapter.get_field_names_from_class(self.item_class_empty) assert isinstance(field_names, list) assert field_names == [] class TestDict(BaseTestMixin): item_class = dict item_class_nested = dict item_class_json_schema = dict item_class_empty = dict expected_json_schema = {"type": "object"} def test_get_value_keyerror_item_dict(self): """Instantiate without default values.""" adapter = ItemAdapter(self.item_class()) with pytest.raises(KeyError): adapter["name"] def test_empty_metadata(self): adapter = ItemAdapter(self.item_class(name="foo", value=5)) for field_name in ("name", "value", "undefined_field"): assert adapter.get_field_meta(field_name) == MappingProxyType({}) def test_field_names_updated(self): item = self.item_class(name="asdf") field_names = ItemAdapter(item).field_names() assert sorted(field_names) == ["name"] item["value"] = 1234 assert sorted(field_names) == ["name", "value"] def test_field_names_from_class(self): assert ItemAdapter.get_field_names_from_class(dict) is None _SCRAPY_NESTED_JSON_SCHEMA = { "type": "object", "additionalProperties": False, "properties": { "is_nested": {"default": True, "type": "boolean"}, }, } _SCRAPY_JSON_SCHEMA_PROPERTIES = { **NonDictTestMixin.expected_json_schema["properties"], # Title is set through json_schema_extra, so it comes first. "name": {"title": "Name", "type": "string", "description": "Display name"}, "nested": _SCRAPY_NESTED_JSON_SCHEMA, "nested_list": { "type": "array", "items": _SCRAPY_NESTED_JSON_SCHEMA, }, "nested_dict": { "type": "object", "additionalProperties": _SCRAPY_NESTED_JSON_SCHEMA, }, "nested_dict_list": { "type": "array", "items": { "type": "object", "additionalProperties": _SCRAPY_NESTED_JSON_SCHEMA, }, }, # No type, since none was specified in json_schema_extra. "produced": {}, } if SCRAPY_VERSION is not None and SCRAPY_VERSION < Version("2.17"): # Scrapy < 2.17 sorts item fields alphabetically. # https://github.com/scrapy/scrapy/issues/7015 _SCRAPY_JSON_SCHEMA_PROPERTIES = dict(sorted(_SCRAPY_JSON_SCHEMA_PROPERTIES.items())) @requires_scrapy class TestScrapySubclassedItem(NonDictTestMixin): item_class = ScrapySubclassedItem item_class_nested = ScrapySubclassedItemNested item_class_subclassed = ScrapySubclassedItemSubclassed item_class_empty = ScrapySubclassedItemEmpty item_class_json_schema = ScrapySubclassedItemJsonSchema expected_json_schema = { "llmHint": "Hi model!", "type": "object", "additionalProperties": False, "properties": _SCRAPY_JSON_SCHEMA_PROPERTIES, } def test_get_value_keyerror_item_dict(self): """Instantiate without default values.""" adapter = ItemAdapter(self.item_class()) with pytest.raises(KeyError): adapter["name"] _PYDANTIC_NESTED_JSON_SCHEMA = { k: v for k, v in _NESTED_JSON_SCHEMA.items() if k != "additionalProperties" } @requires_pydantic class TestPydanticModel(NonDictTestMixin): item_class = PydanticModel item_class_nested = PydanticModelNested item_class_subclassed = PydanticModelSubclassed item_class_empty = PydanticModelEmpty item_class_json_schema = PydanticModelJsonSchema expected_json_schema = { **{ k: v for k, v in NonDictTestMixin.expected_json_schema.items() if k != "additionalProperties" }, "properties": { **{ k: v for k, v in NonDictTestMixin.expected_json_schema["properties"].items() if k not in {"nested", "nested_list", "nested_dict", "nested_dict_list"} }, "nested": _PYDANTIC_NESTED_JSON_SCHEMA, "nested_list": { "type": "array", "items": _PYDANTIC_NESTED_JSON_SCHEMA, }, "nested_dict": { "type": "object", "additionalProperties": _PYDANTIC_NESTED_JSON_SCHEMA, }, "nested_dict_list": { "type": "array", "items": { "type": "object", "additionalProperties": _PYDANTIC_NESTED_JSON_SCHEMA, }, }, }, "required": NonDictTestMixin.expected_json_schema["required"], } def test_get_field_meta_defined_fields(self): adapter = ItemAdapter(self.item_class()) assert adapter.get_field_meta("name")["json_schema_extra"] == MappingProxyType( {"serializer": str} ) assert adapter.get_field_meta("value")["json_schema_extra"] == MappingProxyType( {"serializer": int} ) class TestDataClassItem(NonDictTestMixin): item_class = DataClassItem item_class_nested = DataClassItemNested item_class_subclassed = DataClassItemSubclassed item_class_empty = DataClassItemEmpty item_class_json_schema = DataClassItemJsonSchema expected_json_schema = { **NonDictTestMixin.expected_json_schema, "properties": { **{ k: v for k, v in NonDictTestMixin.expected_json_schema["properties"].items() if k not in {"value", "produced"} }, # Title is set through json_schema_extra, so it comes first. "name": {"title": "Name", "type": "string", "description": "Display name"}, # value and produced come last because they have a default value, # and dataclass does not support values without a default after # values with a default. "value": NonDictTestMixin.expected_json_schema["properties"]["value"], "produced": NonDictTestMixin.expected_json_schema["properties"]["produced"], }, "required": NonDictTestMixin.expected_json_schema["required"], } @requires_attrs class TestAttrsItem(NonDictTestMixin): item_class = AttrsItem item_class_nested = AttrsItemNested item_class_subclassed = AttrsItemSubclassed item_class_empty = AttrsItemEmpty item_class_json_schema = AttrsItemJsonSchema expected_json_schema = TestDataClassItem.expected_json_schema @requires_attrs class TestAttrsClassicItem(NonDictTestMixin): item_class = AttrsClassicItem item_class_nested = AttrsClassicItemNested item_class_subclassed = AttrsClassicItemSubclassed item_class_empty = AttrsClassicItemEmpty item_class_json_schema = AttrsClassicItemJsonSchema expected_json_schema = TestDataClassItem.expected_json_schema scrapy-itemadapter-78cc818/tests/test_adapter_attrs.py000066400000000000000000000173211527060762600233060ustar00rootroot00000000000000import re from types import MappingProxyType import pytest from itemadapter.adapter import ItemAdapter from itemadapter.utils import get_field_meta_from_class from tests import ( AttrsClassicItem, AttrsItem, DataClassItem, PydanticModel, ScrapyItem, ScrapySubclassedItem, clear_itemadapter_imports, make_mock_import, requires_attrs, ) from tests.test_json_schema import check_schemas class TestAttrs: def test_false(self): from itemadapter.adapter import AttrsAdapter assert not AttrsAdapter.is_item(int) assert not AttrsAdapter.is_item(sum) assert not AttrsAdapter.is_item(1234) assert not AttrsAdapter.is_item(object()) assert not AttrsAdapter.is_item(DataClassItem()) assert not AttrsAdapter.is_item("a string") assert not AttrsAdapter.is_item(b"some bytes") assert not AttrsAdapter.is_item({"a": "dict"}) assert not AttrsAdapter.is_item(["a", "list"]) assert not AttrsAdapter.is_item(("a", "tuple")) assert not AttrsAdapter.is_item({"a", "set"}) assert not AttrsAdapter.is_item(AttrsItem) if PydanticModel is not None: assert not AttrsAdapter.is_item(PydanticModel()) try: import scrapy # noqa: F401 # pylint: disable=unused-import except ImportError: pass else: assert not AttrsAdapter.is_item(ScrapyItem()) assert not AttrsAdapter.is_item(ScrapySubclassedItem()) @requires_attrs def test_module_import_error(self, monkeypatch: pytest.MonkeyPatch) -> None: with monkeypatch.context() as m, clear_itemadapter_imports(): m.setattr("builtins.__import__", make_mock_import("attrs")) from itemadapter import utils from itemadapter.adapter import AttrsAdapter assert not AttrsAdapter.is_item(AttrsItem(name="asdf", value=1234)) with pytest.raises(RuntimeError, match="attrs module is not available"): AttrsAdapter(AttrsItem(name="asdf", value=1234)) with pytest.raises(RuntimeError, match="attrs module is not available"): AttrsAdapter.get_field_meta_from_class(AttrsItem, "name") with pytest.raises(RuntimeError, match="attrs module is not available"): AttrsAdapter.get_field_names_from_class(AttrsItem) with pytest.raises(TypeError, match=r"'tests.AttrsItem'\> is not a valid item class"): utils.get_field_meta_from_class(AttrsItem, "name") @requires_attrs def test_module_not_available(self, monkeypatch: pytest.MonkeyPatch) -> None: from itemadapter.adapter import AttrsAdapter monkeypatch.setattr("itemadapter._utils.attrs", None) assert not AttrsAdapter.is_item(AttrsItem(name="asdf", value=1234)) with pytest.raises(TypeError, match=r"'tests.AttrsItem'\> is not a valid item class"): get_field_meta_from_class(AttrsItem, "name") @requires_attrs def test_true(self): from itemadapter.adapter import AttrsAdapter assert AttrsAdapter.is_item(AttrsItem()) assert AttrsAdapter.is_item(AttrsItem(name="asdf", value=1234)) # field metadata assert get_field_meta_from_class(AttrsItem, "name") == MappingProxyType( {"serializer": str} ) assert get_field_meta_from_class(AttrsItem, "value") == MappingProxyType( {"serializer": int} ) with pytest.raises(KeyError, match="AttrsItem does not support field: non_existent"): get_field_meta_from_class(AttrsItem, "non_existent") @requires_attrs def test_true_classic(self): from itemadapter.adapter import AttrsAdapter assert AttrsAdapter.is_item(AttrsClassicItem()) assert AttrsAdapter.is_item(AttrsClassicItem(name="asdf", value=1234)) assert not AttrsAdapter.is_item(AttrsClassicItem) # field metadata assert get_field_meta_from_class(AttrsClassicItem, "name") == MappingProxyType( {"serializer": str} ) assert get_field_meta_from_class(AttrsClassicItem, "value") == MappingProxyType( {"serializer": int} ) with pytest.raises( KeyError, match="AttrsClassicItem does not support field: non_existent" ): get_field_meta_from_class(AttrsClassicItem, "non_existent") @requires_attrs def test_json_schema_validators(self): import attrs from attrs import validators @attrs.define class ItemClass: # String with min/max length and regex pattern name: str = attrs.field( validator=[ validators.min_len(3), validators.max_len(10), validators.matches_re(r"^[A-Za-z]+$"), ], ) # Integer with minimum, maximum, exclusive minimum, exclusive maximum age: int = attrs.field( validator=[ validators.ge(18), validators.le(99), validators.gt(17), validators.lt(100), ], ) # Enum (membership) color: str = attrs.field(validator=validators.in_(["red", "green", "blue"])) # Unsupported pattern [(?i)] year: str = attrs.field( validator=[ validators.matches_re(r"(?i)\bY\d{4}\b"), ], ) # Unsupported flags month: str = attrs.field( validator=[ validators.matches_re(r"\bM\d{2}\b", flags=re.IGNORECASE), ], ) # Unanchored pattern day: str = attrs.field( validator=[ validators.matches_re(r"\bD\d{2}\b", func=re.search), ], ) # Prefix-anchored pattern hour: str = attrs.field( validator=[ validators.matches_re(r"H\d{2}", func=re.match), ], ) # Len limits on sequences/sets. tags: set[str] = attrs.field( validator=validators.max_len(50), ) actual = ItemAdapter.get_json_schema(ItemClass) expected = { "type": "object", "additionalProperties": False, "properties": { "name": { "type": "string", "minLength": 3, "maxLength": 10, "pattern": "^(?:^[A-Za-z]+$)$", }, "age": { "type": "integer", "minimum": 18, "maximum": 99, "exclusiveMinimum": 17, "exclusiveMaximum": 100, }, "color": {"type": "string", "enum": ["red", "green", "blue"]}, "year": { "type": "string", }, "month": { "type": "string", }, "day": { "type": "string", "pattern": r"\bD\d{2}\b", }, "hour": { "type": "string", "pattern": r"^(?:H\d{2})", }, "tags": { "type": "array", "uniqueItems": True, "items": { "type": "string", }, "maxItems": 50, }, }, "required": ["name", "age", "color", "year", "month", "day", "hour", "tags"], } check_schemas(actual, expected) scrapy-itemadapter-78cc818/tests/test_adapter_dataclasses.py000066400000000000000000000042001527060762600244300ustar00rootroot00000000000000from types import MappingProxyType import pytest from itemadapter.utils import get_field_meta_from_class from tests import AttrsItem, DataClassItem, PydanticModel, ScrapyItem, ScrapySubclassedItem class TestDataclass: def test_false(self): from itemadapter.adapter import DataclassAdapter assert not DataclassAdapter.is_item(int) assert not DataclassAdapter.is_item(sum) assert not DataclassAdapter.is_item(1234) assert not DataclassAdapter.is_item(object()) assert not DataclassAdapter.is_item("a string") assert not DataclassAdapter.is_item(b"some bytes") assert not DataclassAdapter.is_item({"a": "dict"}) assert not DataclassAdapter.is_item(["a", "list"]) assert not DataclassAdapter.is_item(("a", "tuple")) assert not DataclassAdapter.is_item({"a", "set"}) assert not DataclassAdapter.is_item(DataClassItem) try: import attrs # noqa: F401 # pylint: disable=unused-import except ImportError: pass else: assert not DataclassAdapter.is_item(AttrsItem()) if PydanticModel is not None: assert not DataclassAdapter.is_item(PydanticModel()) try: import scrapy # noqa: F401 # pylint: disable=unused-import except ImportError: pass else: assert not DataclassAdapter.is_item(ScrapyItem()) assert not DataclassAdapter.is_item(ScrapySubclassedItem()) def test_true(self): from itemadapter.adapter import DataclassAdapter assert DataclassAdapter.is_item(DataClassItem()) assert DataclassAdapter.is_item(DataClassItem(name="asdf", value=1234)) # field metadata assert get_field_meta_from_class(DataClassItem, "name") == MappingProxyType( {"serializer": str} ) assert get_field_meta_from_class(DataClassItem, "value") == MappingProxyType( {"serializer": int} ) with pytest.raises(KeyError, match="DataClassItem does not support field: non_existent"): get_field_meta_from_class(DataClassItem, "non_existent") scrapy-itemadapter-78cc818/tests/test_adapter_pydantic.py000066400000000000000000000175641527060762600237750ustar00rootroot00000000000000from __future__ import annotations from types import MappingProxyType import pytest from itemadapter.adapter import ItemAdapter from itemadapter.utils import get_field_meta_from_class from tests import ( AttrsItem, DataClassItem, PydanticModel, PydanticSpecialCasesModel, ScrapyItem, ScrapySubclassedItem, clear_itemadapter_imports, make_mock_import, requires_pydantic, ) from tests.test_json_schema import check_schemas class TestPydantic: def test_false(self): from itemadapter.adapter import PydanticAdapter assert not PydanticAdapter.is_item(int) assert not PydanticAdapter.is_item(sum) assert not PydanticAdapter.is_item(1234) assert not PydanticAdapter.is_item(object()) assert not PydanticAdapter.is_item(DataClassItem()) assert not PydanticAdapter.is_item("a string") assert not PydanticAdapter.is_item(b"some bytes") assert not PydanticAdapter.is_item({"a": "dict"}) assert not PydanticAdapter.is_item(["a", "list"]) assert not PydanticAdapter.is_item(("a", "tuple")) assert not PydanticAdapter.is_item({"a", "set"}) assert not PydanticAdapter.is_item(PydanticModel) try: import attrs # noqa: F401 # pylint: disable=unused-import except ImportError: pass else: assert not PydanticAdapter.is_item(AttrsItem()) try: import scrapy # noqa: F401 # pylint: disable=unused-import except ImportError: pass else: assert not PydanticAdapter.is_item(ScrapyItem()) assert not PydanticAdapter.is_item(ScrapySubclassedItem()) @requires_pydantic def test_module_import_error(self, monkeypatch: pytest.MonkeyPatch) -> None: with monkeypatch.context() as m, clear_itemadapter_imports(): m.setattr("builtins.__import__", make_mock_import("pydantic")) from itemadapter import utils from itemadapter.adapter import PydanticAdapter assert not PydanticAdapter.is_item(PydanticModel(name="asdf", value=1234)) with pytest.raises( TypeError, match=r"tests.PydanticModel'\> is not a valid item class" ): utils.get_field_meta_from_class(PydanticModel, "name") @requires_pydantic def test_module_not_available(self, monkeypatch: pytest.MonkeyPatch) -> None: from itemadapter.adapter import PydanticAdapter monkeypatch.setattr("itemadapter._utils.pydantic", None) assert not PydanticAdapter.is_item(PydanticModel(name="asdf", value=1234)) with pytest.raises(TypeError, match=r"tests.PydanticModel'\> is not a valid item class"): get_field_meta_from_class(PydanticModel, "name") @requires_pydantic def test_true(self): from itemadapter.adapter import PydanticAdapter assert PydanticAdapter.is_item(PydanticModel()) assert PydanticAdapter.is_item(PydanticModel(name="asdf", value=1234)) # field metadata mapping_proxy_type = get_field_meta_from_class(PydanticModel, "name") assert mapping_proxy_type == MappingProxyType( { "annotation": str | None, "default_factory": mapping_proxy_type["default_factory"], "json_schema_extra": {"serializer": str}, "repr": True, } ) mapping_proxy_type = get_field_meta_from_class(PydanticModel, "value") assert get_field_meta_from_class(PydanticModel, "value") == MappingProxyType( { "annotation": int | None, "default_factory": mapping_proxy_type["default_factory"], "json_schema_extra": {"serializer": int}, "repr": True, } ) mapping_proxy_type = get_field_meta_from_class(PydanticSpecialCasesModel, "special_cases") assert mapping_proxy_type == MappingProxyType( { "annotation": int | None, "alias": "special_cases", "alias_priority": 2, "default_factory": mapping_proxy_type["default_factory"], "validation_alias": "special_cases", "serialization_alias": "special_cases", "frozen": True, "repr": True, } ) with pytest.raises(KeyError, match="PydanticModel does not support field: non_existent"): get_field_meta_from_class(PydanticModel, "non_existent") @requires_pydantic def test_json_schema_forbid(self): from itemadapter._imports import pydantic class Item(pydantic.BaseModel): foo: str model_config = { "extra": "forbid", } actual = ItemAdapter.get_json_schema(Item) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": {"type": "string"}, }, "required": ["foo"], } check_schemas(actual, expected) @requires_pydantic def test_json_schema_field_deprecated_bool(self): from itemadapter._imports import pydantic class Item(pydantic.BaseModel): foo: str = pydantic.Field(deprecated=True) actual = ItemAdapter.get_json_schema(Item) expected = { "type": "object", "properties": { "foo": {"type": "string", "deprecated": True}, }, "required": ["foo"], } check_schemas(actual, expected) @requires_pydantic def test_json_schema_field_deprecated_str(self): from itemadapter._imports import pydantic class Item(pydantic.BaseModel): foo: str = pydantic.Field(deprecated="Use something else") actual = ItemAdapter.get_json_schema(Item) expected = { "type": "object", "properties": { "foo": {"type": "string", "deprecated": True}, }, "required": ["foo"], } check_schemas(actual, expected) @requires_pydantic def test_json_schema_validators(self): from itemadapter._imports import pydantic class Model(pydantic.BaseModel): # String with min/max length and regex pattern name: str = pydantic.Field( min_length=3, max_length=10, pattern=r"^[A-Za-z]+$", ) # Integer with minimum, maximum, exclusive minimum, exclusive maximum age: int = pydantic.Field( ge=18, le=99, gt=17, lt=100, ) # Unsupported pattern [(?i)] year: str = pydantic.Field(pattern=r"(?i)\bY\d{4}\b") # Sequence with max_items tags: set[str] = pydantic.Field(max_length=50) actual = ItemAdapter.get_json_schema(Model) expected = { "type": "object", "properties": { "name": { "type": "string", "minLength": 3, "maxLength": 10, "pattern": "^[A-Za-z]+$", }, "age": { "type": "integer", "exclusiveMinimum": 17, "minimum": 18, "exclusiveMaximum": 100, "maximum": 99, }, "year": { "type": "string", }, "tags": { "type": "array", "uniqueItems": True, "items": { "type": "string", }, "maxItems": 50, }, }, "required": ["name", "age", "year", "tags"], } check_schemas(actual, expected) scrapy-itemadapter-78cc818/tests/test_adapter_scrapy.py000066400000000000000000000062111527060762600234460ustar00rootroot00000000000000from types import MappingProxyType import pytest from itemadapter.utils import get_field_meta_from_class from tests import ( AttrsItem, DataClassItem, PydanticModel, ScrapyItem, ScrapySubclassedItem, clear_itemadapter_imports, make_mock_import, requires_scrapy, ) class TestScrapyItem: def test_false(self): from itemadapter.adapter import ScrapyItemAdapter assert not ScrapyItemAdapter.is_item(int) assert not ScrapyItemAdapter.is_item(sum) assert not ScrapyItemAdapter.is_item(1234) assert not ScrapyItemAdapter.is_item(object()) assert not ScrapyItemAdapter.is_item(DataClassItem()) assert not ScrapyItemAdapter.is_item("a string") assert not ScrapyItemAdapter.is_item(b"some bytes") assert not ScrapyItemAdapter.is_item({"a": "dict"}) assert not ScrapyItemAdapter.is_item(["a", "list"]) assert not ScrapyItemAdapter.is_item(("a", "tuple")) assert not ScrapyItemAdapter.is_item({"a", "set"}) assert not ScrapyItemAdapter.is_item(ScrapySubclassedItem) try: import attrs # noqa: F401 # pylint: disable=unused-import except ImportError: pass else: assert not ScrapyItemAdapter.is_item(AttrsItem()) if PydanticModel is not None: assert not ScrapyItemAdapter.is_item(PydanticModel()) @requires_scrapy def test_module_import_error(self, monkeypatch: pytest.MonkeyPatch) -> None: with monkeypatch.context() as m, clear_itemadapter_imports(): m.setattr("builtins.__import__", make_mock_import("scrapy")) from itemadapter import utils from itemadapter.adapter import ScrapyItemAdapter assert not ScrapyItemAdapter.is_item(ScrapySubclassedItem(name="asdf", value=1234)) with pytest.raises( TypeError, match=r"tests.ScrapySubclassedItem'\> is not a valid item class" ): utils.get_field_meta_from_class(ScrapySubclassedItem, "name") @requires_scrapy def test_module_not_available(self, monkeypatch: pytest.MonkeyPatch) -> None: from itemadapter.adapter import ScrapyItemAdapter monkeypatch.setattr("itemadapter.adapter._scrapy_item_classes", ()) assert not ScrapyItemAdapter.is_item(ScrapySubclassedItem(name="asdf", value=1234)) with pytest.raises( TypeError, match=r"tests.ScrapySubclassedItem'\> is not a valid item class" ): get_field_meta_from_class(ScrapySubclassedItem, "name") @requires_scrapy def test_true(self): from itemadapter.adapter import ScrapyItemAdapter assert ScrapyItemAdapter.is_item(ScrapyItem()) assert ScrapyItemAdapter.is_item(ScrapySubclassedItem()) assert ScrapyItemAdapter.is_item(ScrapySubclassedItem(name="asdf", value=1234)) # field metadata assert get_field_meta_from_class(ScrapySubclassedItem, "name") == MappingProxyType( {"serializer": str} ) assert get_field_meta_from_class(ScrapySubclassedItem, "value") == MappingProxyType( {"serializer": int} ) scrapy-itemadapter-78cc818/tests/test_adapter_typed_dict.py000066400000000000000000000307231527060762600243020ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING, Annotated, NotRequired, Required, TypedDict import pytest from itemadapter.adapter import DictAdapter, ItemAdapter from itemadapter.utils import get_field_meta_from_class from tests import clear_itemadapter_imports, make_mock_import from tests.test_json_schema import check_schemas try: import typing_extensions except ImportError: typing_extensions = None # type: ignore[assignment] if TYPE_CHECKING: from decimal import Decimal class TypedDictItem(TypedDict): name: str """Display name""" value: Annotated[int, {"json_schema_extra": {"minimum": 0}}] class TypedDictItemNested(TypedDict): nested: TypedDictItem tags: list[str] brand: str | None class TypedDictItemRecursive(TypedDict): child: TypedDictItemRecursive class TypedDictItemOptionalNested(TypedDict): nested: TypedDictItem | None class TypedDictItemBase(TypedDict): base: int class TypedDictItemSubclass(TypedDictItemBase, total=False): __json_schema_extra__ = {"title": "Subclass"} optional: str class TestTypedDict: def test_is_item_class(self): assert ItemAdapter.is_item_class(TypedDictItem) assert DictAdapter.is_item_class(TypedDictItem) assert ItemAdapter._get_adapter_class(TypedDictItem) is DictAdapter def test_instances_are_dicts(self): """TypedDict instances are plain dicts at run time, with no reference to the TypedDict subclass they were declared as, so they are handled like any other dict.""" item: TypedDictItem = {"name": "asdf", "value": 1234} adapter = ItemAdapter(item) assert isinstance(adapter.adapter, DictAdapter) assert adapter["name"] == "asdf" adapter["undeclared"] = True assert adapter.item == {"name": "asdf", "value": 1234, "undeclared": True} def test_get_field_names_from_class(self): assert ItemAdapter.get_field_names_from_class(TypedDictItem) == ["name", "value"] assert ItemAdapter.get_field_names_from_class(TypedDictItemSubclass) == [ "base", "optional", ] def test_get_field_names_from_class_dict(self): assert ItemAdapter.get_field_names_from_class(dict) is None def test_get_field_meta_from_class(self): assert get_field_meta_from_class(TypedDictItem, "value") == { "json_schema_extra": {"minimum": 0} } assert get_field_meta_from_class(TypedDictItem, "name") == {} with pytest.raises(KeyError, match="TypedDictItem does not support field: non_existent"): get_field_meta_from_class(TypedDictItem, "non_existent") def test_get_field_meta_from_class_dict(self): assert get_field_meta_from_class(dict, "any") == {} def test_get_field_meta_from_class_non_mapping_annotation(self): class TypedDictItemStringAnnotation(TypedDict): name: Annotated[str, "not metadata"] assert get_field_meta_from_class(TypedDictItemStringAnnotation, "name") == {} def test_json_schema(self): actual = ItemAdapter.get_json_schema(TypedDictItem) expected = { "type": "object", "additionalProperties": False, "properties": { "name": {"type": "string", "description": "Display name"}, "value": {"minimum": 0, "type": "integer"}, }, "required": ["name", "value"], } check_schemas(actual, expected) def test_json_schema_nested(self): actual = ItemAdapter.get_json_schema(TypedDictItemNested) expected = { "type": "object", "additionalProperties": False, "properties": { "nested": { "type": "object", "additionalProperties": False, "properties": { "name": {"type": "string", "description": "Display name"}, "value": {"minimum": 0, "type": "integer"}, }, "required": ["name", "value"], }, "tags": {"type": "array", "items": {"type": "string"}}, "brand": {"type": ["null", "string"]}, }, "required": ["nested", "tags", "brand"], } check_schemas(actual, expected) def test_json_schema_total_false(self): actual = ItemAdapter.get_json_schema(TypedDictItemSubclass) expected = { "title": "Subclass", "type": "object", "additionalProperties": False, "properties": { "base": {"type": "integer"}, "optional": {"type": "string"}, }, "required": ["base"], } check_schemas(actual, expected) def test_json_schema_no_required_fields(self): class TypedDictItemAllOptional(TypedDict, total=False): optional: str actual = ItemAdapter.get_json_schema(TypedDictItemAllOptional) expected = { "type": "object", "additionalProperties": False, "properties": {"optional": {"type": "string"}}, } check_schemas(actual, expected) def test_json_schema_no_fields(self): class TypedDictItemEmpty(TypedDict): pass actual = ItemAdapter.get_json_schema(TypedDictItemEmpty) expected = {"type": "object", "additionalProperties": False} check_schemas(actual, expected) def test_json_schema_dict(self): check_schemas(ItemAdapter.get_json_schema(dict), {"type": "object"}) def test_json_schema_recursion(self): actual = ItemAdapter.get_json_schema(TypedDictItemRecursive) expected = { "type": "object", "additionalProperties": False, "properties": {"child": {"type": "object"}}, "required": ["child"], } check_schemas(actual, expected) def test_json_schema_not_required(self): class TypedDictItemNotRequired(TypedDict): required: int optional: NotRequired[Annotated[str, {"json_schema_extra": {"minLength": 2}}]] unannotated: NotRequired[float] assert get_field_meta_from_class(TypedDictItemNotRequired, "optional") == { "json_schema_extra": {"minLength": 2} } actual = ItemAdapter.get_json_schema(TypedDictItemNotRequired) expected = { "type": "object", "additionalProperties": False, "properties": { "required": {"type": "integer"}, "optional": {"minLength": 2, "type": "string"}, "unannotated": {"type": "number"}, }, "required": ["required"], } check_schemas(actual, expected) def test_json_schema_required(self): """Required and NotRequired are missing from __required_keys__ and __optional_keys__ when annotations are postponed, so requiredness is read from the field type hints as well.""" class TypedDictItemRequired(TypedDict, total=False): required: Required[int] optional: str actual = ItemAdapter.get_json_schema(TypedDictItemRequired) expected = { "type": "object", "additionalProperties": False, "properties": { "required": {"type": "integer"}, "optional": {"type": "string"}, }, "required": ["required"], } check_schemas(actual, expected) class TestTypedDictTypeHint: def test_qualifiers_in_any_order(self): class TypedDictItemQualifiers(TypedDict, total=False): required: Annotated[Required[int], {"json_schema_extra": {"minimum": 0}}] optional: Annotated[NotRequired[str], {"json_schema_extra": {"minLength": 2}}] assert get_field_meta_from_class(TypedDictItemQualifiers, "required") == { "json_schema_extra": {"minimum": 0} } actual = ItemAdapter.get_json_schema(TypedDictItemQualifiers) expected = { "type": "object", "additionalProperties": False, "properties": { "required": {"minimum": 0, "type": "integer"}, "optional": {"minLength": 2, "type": "string"}, }, "required": ["required"], } check_schemas(actual, expected) def test_nested_annotated(self): class TypedDictItemNestedAnnotated(TypedDict): values: list[Annotated[int, {"json_schema_extra": {"minimum": 0}}]] assert get_field_meta_from_class(TypedDictItemNestedAnnotated, "values") == {} actual = ItemAdapter.get_json_schema(TypedDictItemNestedAnnotated) expected = { "type": "object", "additionalProperties": False, "properties": {"values": {"type": "array", "items": {"type": "integer"}}}, "required": ["values"], } check_schemas(actual, expected) def test_get_field_names_from_class_unresolvable(self): class TypedDictItemUnresolvable(TypedDict): price: Decimal assert ItemAdapter.get_field_names_from_class(TypedDictItemUnresolvable) == ["price"] @pytest.mark.skipif(not typing_extensions, reason="typing_extensions module is not available") def test_typing_extensions(self): class TypedDictItemTypingExtensions(typing_extensions.TypedDict): name: typing_extensions.ReadOnly[Annotated[str, {"json_schema_extra": {"x": 1}}]] value: NotRequired[typing_extensions.ReadOnly[int]] assert ItemAdapter.is_item_class(TypedDictItemTypingExtensions) assert ItemAdapter.get_field_names_from_class(TypedDictItemTypingExtensions) == [ "name", "value", ] assert get_field_meta_from_class(TypedDictItemTypingExtensions, "name") == { "json_schema_extra": {"x": 1} } actual = ItemAdapter.get_json_schema(TypedDictItemTypingExtensions) expected = { "type": "object", "additionalProperties": False, "properties": { "name": {"x": 1, "type": "string"}, "value": {"type": "integer"}, }, "required": ["name"], } check_schemas(actual, expected) def test_typing_extensions_import_error(self, monkeypatch: pytest.MonkeyPatch) -> None: with monkeypatch.context() as m, clear_itemadapter_imports(): m.setattr("builtins.__import__", make_mock_import("typing_extensions")) import itemadapter.adapter names = itemadapter.adapter.ItemAdapter.get_field_names_from_class(TypedDictItem) assert names == ["name", "value"] class TestTypedDictCrossNesting: def test_typed_dict_in_dataclass(self): from dataclasses import dataclass @dataclass class DataClassItemTypedDictNested: nested: TypedDictItem actual = ItemAdapter.get_json_schema(DataClassItemTypedDictNested) expected = { "type": "object", "additionalProperties": False, "properties": { "nested": { "type": "object", "additionalProperties": False, "properties": { "name": {"type": "string", "description": "Display name"}, "value": {"minimum": 0, "type": "integer"}, }, "required": ["name", "value"], } }, "required": ["nested"], } check_schemas(actual, expected) def test_optional_typed_dict(self): actual = ItemAdapter.get_json_schema(TypedDictItemOptionalNested) expected = { "type": "object", "additionalProperties": False, "properties": { "nested": { "anyOf": [ {"type": "null"}, { "type": "object", "additionalProperties": False, "properties": { "name": {"type": "string", "description": "Display name"}, "value": {"minimum": 0, "type": "integer"}, }, "required": ["name", "value"], }, ] } }, "required": ["nested"], } check_schemas(actual, expected) scrapy-itemadapter-78cc818/tests/test_interface.py000066400000000000000000000167031527060762600224140ustar00rootroot00000000000000from __future__ import annotations from collections.abc import Iterator, KeysView from types import MappingProxyType from typing import Any import pytest from itemadapter.adapter import AdapterInterface, ItemAdapter def test_interface_class_methods() -> None: with pytest.raises(NotImplementedError): AdapterInterface.is_item(object()) with pytest.raises(NotImplementedError): AdapterInterface.is_item_class(object) class FakeItemClass: _fields = { "name": {"serializer": str}, "value": {"serializer": int}, } def __init__(self, **kwargs) -> None: self._values = {**kwargs} class BaseFakeItemAdapter(AdapterInterface): """An adapter that only implements the required methods.""" @classmethod def is_item_class(cls, item_class: type) -> bool: return issubclass(item_class, FakeItemClass) def __getitem__(self, field_name: str) -> Any: if field_name in self.item._fields: return self.item._values[field_name] raise KeyError(field_name) def __setitem__(self, field_name: str, value: Any) -> None: if field_name in self.item._fields: self.item._values[field_name] = value else: raise KeyError(field_name) def __delitem__(self, field_name: str) -> None: if field_name in self.item._fields and field_name in self.item._values: del self.item._values[field_name] else: raise KeyError(field_name) def __iter__(self) -> Iterator: return iter(self.item._values) def __len__(self) -> int: return len(self.item._values) class FieldNamesFakeItemAdapter(BaseFakeItemAdapter): """An adapter that also implements the field_names method.""" def field_names(self) -> KeysView: return KeysView({key.upper(): value for key, value in self.item._fields.items()}) class MetadataFakeItemAdapter(BaseFakeItemAdapter): """An adapter that also implements metadata-related methods.""" @classmethod def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: return MappingProxyType(item_class._fields.get(field_name) or {}) class TestBaseFakeItemAdapter: item_class = FakeItemClass adapter_class = BaseFakeItemAdapter @pytest.fixture(autouse=True) def _prepend_adapter_class(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( ItemAdapter, "ADAPTER_CLASSES", (self.adapter_class, *ItemAdapter.ADAPTER_CLASSES) ) def test_repr(self): item = self.item_class() adapter = ItemAdapter(item) assert repr(adapter) == "" adapter["name"] = "asdf" adapter["value"] = 1234 assert repr(adapter) == "" def test_get_field_names_from_class(self): assert ItemAdapter.get_field_names_from_class(self.item_class) is None def test_get_set_value(self): item = self.item_class() adapter = ItemAdapter(item) assert adapter.get("name") is None assert adapter.get("value") is None adapter["name"] = "asdf" adapter["value"] = 1234 assert adapter.get("name") == "asdf" assert adapter.get("value") == 1234 assert adapter["name"] == "asdf" assert adapter["value"] == 1234 def test_get_set_value_init(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) assert adapter.get("name") == "asdf" assert adapter.get("value") == 1234 assert adapter["name"] == "asdf" assert adapter["value"] == 1234 def test_get_value_keyerror(self): item = self.item_class() adapter = ItemAdapter(item) with pytest.raises(KeyError): adapter["_undefined_"] def test_as_dict(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) assert dict(adapter) == {"name": "asdf", "value": 1234} def test_set_value_keyerror(self): item = self.item_class() adapter = ItemAdapter(item) with pytest.raises(KeyError): adapter["_undefined_"] = "some value" def test_delitem_len_iter(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) assert len(adapter) == 2 assert sorted(iter(adapter)) == ["name", "value"] del adapter["name"] assert len(adapter) == 1 assert sorted(iter(adapter)) == ["value"] del adapter["value"] assert len(adapter) == 0 assert sorted(iter(adapter)) == [] with pytest.raises(KeyError): del adapter["name"] with pytest.raises(KeyError): del adapter["value"] with pytest.raises(KeyError): del adapter["_undefined_"] def test_get_value_keyerror_item_dict(self): """Instantiate without default values.""" adapter = ItemAdapter(self.item_class()) with pytest.raises(KeyError): adapter["name"] def test_get_field_meta(self): """Metadata is always empty for the default implementation.""" adapter = ItemAdapter(self.item_class()) assert adapter.get_field_meta("_undefined_") == MappingProxyType({}) assert adapter.get_field_meta("name") == MappingProxyType({}) assert adapter.get_field_meta("value") == MappingProxyType({}) def test_get_field_meta_from_class(self): """Metadata is always empty for the default implementation.""" assert ItemAdapter.get_field_meta_from_class( self.item_class, "_undefined_" ) == MappingProxyType({}) assert ItemAdapter.get_field_meta_from_class(self.item_class, "name") == MappingProxyType( {} ) assert ItemAdapter.get_field_meta_from_class(self.item_class, "value") == MappingProxyType( {} ) def test_field_names(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) assert isinstance(adapter.field_names(), KeysView) assert sorted(adapter.field_names()) == ["name", "value"] class TestMetadataFakeItemAdapter(TestBaseFakeItemAdapter): item_class = FakeItemClass adapter_class = MetadataFakeItemAdapter def test_get_field_meta(self): adapter = ItemAdapter(self.item_class()) assert adapter.get_field_meta("_undefined_") == MappingProxyType({}) assert adapter.get_field_meta("name") == MappingProxyType({"serializer": str}) assert adapter.get_field_meta("value") == MappingProxyType({"serializer": int}) def test_get_field_meta_from_class(self): assert ItemAdapter.get_field_meta_from_class( self.item_class, "_undefined_" ) == MappingProxyType({}) assert ItemAdapter.get_field_meta_from_class(self.item_class, "name") == MappingProxyType( {"serializer": str} ) assert ItemAdapter.get_field_meta_from_class(self.item_class, "value") == MappingProxyType( {"serializer": int} ) class TestFieldNamesFakeItemAdapter(TestBaseFakeItemAdapter): item_class = FakeItemClass adapter_class = FieldNamesFakeItemAdapter def test_field_names(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) assert isinstance(adapter.field_names(), KeysView) assert sorted(adapter.field_names()) == ["NAME", "VALUE"] scrapy-itemadapter-78cc818/tests/test_itemadapter.py000066400000000000000000000031721527060762600227470ustar00rootroot00000000000000from typing import Any import pytest from itemadapter.adapter import DictAdapter, ItemAdapter from tests import DataClassItem class DictOnlyItemAdapter(ItemAdapter): ADAPTER_CLASSES = [DictAdapter] class EmptyDictAdapter(DictAdapter): """An adapter that only handles empty dicts, which it can only tell on an item basis.""" @classmethod def is_item(cls, item: Any) -> bool: return isinstance(item, dict) and not item class TestItemAdapter: def test_repr(self): adapter = ItemAdapter({"foo": "bar"}) assert repr(adapter) == "" def test_adapter_classes_change(self, monkeypatch: pytest.MonkeyPatch) -> None: item = DataClassItem() assert ItemAdapter.is_item(item) with monkeypatch.context() as m: m.setattr(ItemAdapter, "ADAPTER_CLASSES", (DictAdapter,)) assert not ItemAdapter.is_item(item) assert ItemAdapter.is_item(item) def test_repr_subclass(self): adapter = DictOnlyItemAdapter({"foo": "bar"}) assert repr(adapter) == "" def test_is_item_override(self): class EmptyDictItemAdapter(ItemAdapter): ADAPTER_CLASSES = (EmptyDictAdapter,) with pytest.warns(DeprecationWarning, match="EmptyDictAdapter"): assert EmptyDictItemAdapter.is_item({}) assert not EmptyDictItemAdapter.is_item({"foo": "bar"}) assert isinstance(EmptyDictItemAdapter({}).adapter, EmptyDictAdapter) with pytest.raises(TypeError, match="No adapter found"): EmptyDictItemAdapter({"foo": "bar"}) scrapy-itemadapter-78cc818/tests/test_json_schema.py000066400000000000000000000711201527060762600227370ustar00rootroot00000000000000from __future__ import annotations import inspect import json import re import typing from collections.abc import Mapping, Sequence # noqa: TC003 from dataclasses import dataclass, field, make_dataclass from enum import Enum from typing import Any, Generic, TypeVar, Union import pytest from itemadapter._imports import pydantic from itemadapter._json_schema import json_schema_pattern from itemadapter.adapter import AttrsAdapter, ItemAdapter, PydanticAdapter, ScrapyItemAdapter from tests import ( AttrsItem, AttrsItemJsonSchemaNested, DataClassItemJsonSchemaNested, PydanticModel, PydanticModelJsonSchemaNested, ScrapySubclassedItem, ScrapySubclassedItemJsonSchemaNested, requires_attrs, requires_pydantic, requires_scrapy, ) _T = TypeVar("_T") if ScrapySubclassedItem and AttrsItem: from scrapy import Field as ScrapyField from scrapy import Item as ScrapyItem class ScrapySubclassedItemCrossNested(ScrapyItem): nested: AttrsItemJsonSchemaNested = ScrapyField() @dataclass class Brand: name: str @dataclass class OptionalItemListNestedItem: is_nested: bool = True @dataclass class OptionalItemListItem: foo: list[OptionalItemListNestedItem] | None = None T = typing.TypeVar("T") @dataclass class UnhandledTypeItem: """Type hints that map to no JSON Schema type.""" mapping: typing.Mapping """A mapping with no key or value types.""" meta: type[int] """A generic alias whose origin is neither a container nor a union.""" var: T """A type variable, which is not a type.""" @dataclass class ExplicitRequiredItem: __json_schema_extra__ = {"required": ["bar"]} foo: str bar: str = "asdf" @dataclass class MultiItemUnionItem: either: Brand | OptionalItemListNestedItem or_none: Brand | OptionalItemListNestedItem | None containers: list[int] | dict @dataclass class RecursionItem: child: RecursionNestedItem sibling: RecursionItem @dataclass class RecursionNestedItem: parent: RecursionItem sibling: RecursionNestedItem @dataclass class SimpleItem: foo: str class CustomMapping: # noqa: PLW1641 def __init__(self, data): self._data = dict(data) def __getitem__(self, key): return self._data[key] def __iter__(self): return iter(self._data) def __len__(self): return len(self._data) def __contains__(self, key): return key in self._data def keys(self): return self._data.keys() def items(self): return self._data.items() def values(self): return self._data.values() def get(self, key, default=None): return self._data.get(key, default) def __eq__(self, other): if isinstance(other, CustomMapping): return self._data == other._data if isinstance(other, dict): return self._data == other return NotImplemented def __ne__(self, other): eq = self.__eq__(other) if eq is NotImplemented: return NotImplemented return not eq class SimpleEnum(Enum): foo = "foo" if PydanticModel: class PydanticEnumModel(pydantic.BaseModel): enum: SimpleEnum def check_schemas(actual, expected): """Helper function to check if the actual JSON schema matches the expected one. It uses json.dumps() WITHOUT sorting the keys, to ensure key sorting matches in both schemas as well. Maintaining the source order of fields can be important The indentation is set for better readability or mismatch output. """ assert json.dumps(actual, indent=2) == json.dumps(expected, indent=2) @pytest.mark.parametrize( ("field_type", "expected"), [ (int | float, {"type": ["number"]}), (float | int, {"type": ["number"]}), (int | float | None, {"type": ["null", "number"]}), (bool | int | float, {"type": ["boolean", "number"]}), (int | str, {"type": ["integer", "string"]}), (int | None, {"type": ["null", "integer"]}), (list[int | float], {"type": "array", "items": {"type": ["number"]}}), (tuple[int, float], {"type": "array", "items": {"type": ["number"]}}), ], ) def test_numeric_union(field_type, expected): item_class = make_dataclass("NumericItem", [("value", field_type)]) actual = ItemAdapter.get_json_schema(item_class)["properties"]["value"] check_schemas(actual, expected) class TestJsonSchema: @requires_attrs @requires_pydantic def test_attrs_pydantic_enum(self): """This test exists to ensure that we do not let the JSON Schema generation of Pydantic item classes generate nested $defs (which we don’t since we do not run Pydantic’s JSON Schema generation but our own).""" import attrs @attrs.define class TestAttrsItem: pydantic: PydanticEnumModel actual = ItemAdapter.get_json_schema(TestAttrsItem) expected = { "type": "object", "additionalProperties": False, "properties": { "pydantic": { "type": "object", "properties": { "enum": {"type": "string", "enum": ["foo"]}, }, "required": ["enum"], } }, "required": ["pydantic"], } check_schemas(actual, expected) @requires_scrapy def test_unreachable_source(self, monkeypatch: pytest.MonkeyPatch) -> None: """Reading the item class source to find attribute docstrings is not always a possibility, e.g. for classes defined in a REPL. In those cases, only the extraction of those docstrings should fail.""" class ScrapySubclassedItemUnreachable(ScrapyItem): name: str = ScrapyField(json_schema_extra={"example": "Foo"}) """Display name""" expected = { "type": "object", "additionalProperties": False, "properties": { "name": { "example": "Foo", "type": "string", } }, } for exception in (OSError, TypeError): def getsource(_obj: Any, _exception: type[Exception] = exception) -> str: raise _exception with monkeypatch.context() as m: m.setattr(inspect, "getsource", getsource) actual = ItemAdapter.get_json_schema(ScrapySubclassedItemUnreachable) check_schemas(actual, expected) def test_recursion(self): actual = ItemAdapter.get_json_schema(RecursionItem) expected = { "type": "object", "additionalProperties": False, "properties": { "child": { "type": "object", "additionalProperties": False, "properties": { "parent": { "type": "object", }, "sibling": { "type": "object", }, }, "required": ["parent", "sibling"], }, "sibling": { "type": "object", }, }, "required": ["child", "sibling"], } check_schemas(actual, expected) def test_nested_dict(self): @dataclass class TestItem: foo: dict actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": { "type": "object", }, }, "required": ["foo"], } check_schemas(actual, expected) def test_optional_item_list(self): actual = ItemAdapter.get_json_schema(OptionalItemListItem) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": { "anyOf": [ { "type": "null", }, { "type": "array", "items": { "type": "object", "additionalProperties": False, "properties": { "is_nested": { "type": "boolean", "default": True, }, }, }, }, ], "default": None, }, }, } check_schemas(actual, expected) def test_sequence_untyped(self): @dataclass class TestItem: foo: Sequence actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": { "type": "array", }, }, "required": ["foo"], } check_schemas(actual, expected) def test_tuple_ellipsis(self): @dataclass class TestItem: foo: tuple[Any, ...] actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": { "type": "array", }, }, "required": ["foo"], } check_schemas(actual, expected) def test_tuple_multiple_types(self): @dataclass class TestItem: foo: tuple[str, int, int] actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": { "type": "array", "items": { "type": ["integer", "string"], }, }, }, "required": ["foo"], } check_schemas(actual, expected) def test_type_var(self): # this tests the update_prop_from_type() branch where prop_type is not a type @dataclass class TestItem(Generic[_T]): foo: _T actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": {}, }, "required": ["foo"], } check_schemas(actual, expected) def test_union_single(self): @dataclass class TestItem: foo: Union[str] # noqa: UP007 actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": {"type": "string"}, }, "required": ["foo"], } check_schemas(actual, expected) def test_custom_any_of(self): @dataclass class TestItem: foo: str | SimpleItem = field( metadata={"json_schema_extra": {"anyOf": []}}, ) actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": {"anyOf": []}, }, "required": ["foo"], } check_schemas(actual, expected) def test_set_untyped(self): @dataclass class TestItem: foo: set actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": {"type": "array", "uniqueItems": True}, }, "required": ["foo"], } check_schemas(actual, expected) def test_mapping_untyped(self): @dataclass class TestItem: foo: Mapping actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": {"type": "object"}, }, "required": ["foo"], } check_schemas(actual, expected) def test_custom_mapping(self): @dataclass class TestItem: foo: CustomMapping actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": {"type": "object"}, }, "required": ["foo"], } check_schemas(actual, expected) def test_item_without_attributes(self): @dataclass class TestItem: pass actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, } check_schemas(actual, expected) def test_typing_sequence_untyped(self): @dataclass class TestItem: foo: typing.Sequence actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": { "type": "array", }, }, "required": ["foo"], } check_schemas(actual, expected) def test_custom_items(self): @dataclass class TestItem: foo: typing.Sequence = field(metadata={"json_schema_extra": {"items": {}}}) actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": { "items": {}, "type": "array", }, }, "required": ["foo"], } check_schemas(actual, expected) @requires_attrs def test_modern_optional_annotations(self): import attrs @attrs.define class Product: name: str """Product name""" brand: Brand | None in_stock: bool = True actual = ItemAdapter.get_json_schema(Product) expected = { "type": "object", "additionalProperties": False, "properties": { "name": {"type": "string", "description": "Product name"}, "brand": { "anyOf": [ {"type": "null"}, { "type": "object", "additionalProperties": False, "properties": {"name": {"type": "string"}}, "required": ["name"], }, ] }, "in_stock": {"type": "boolean", "default": True}, }, "required": ["name", "brand"], } check_schemas(actual, expected) def test_field_docstring_inheritance(self): """Test that field docstrings are inherited from parent classes.""" @dataclass class ParentItem: foo: str """Parent item foo""" @dataclass class ChildItem(ParentItem): pass actual = ItemAdapter.get_json_schema(ChildItem) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": {"type": "string", "description": "Parent item foo"}, }, "required": ["foo"], } check_schemas(actual, expected) def test_field_docstring_source_reads(self, monkeypatch: pytest.MonkeyPatch) -> None: """The source of each class in the MRO of an item class is read once, not once per field, and not again on later calls.""" @dataclass class ParentItem: foo: str """Parent item foo""" @dataclass class ChildItem(ParentItem): bar: str baz: str """Child item baz""" expected = { "type": "object", "additionalProperties": False, "properties": { "foo": {"type": "string", "description": "Parent item foo"}, "bar": {"type": "string"}, "baz": {"type": "string", "description": "Child item baz"}, }, "required": ["foo", "bar", "baz"], } original_getsource = inspect.getsource calls = [] def getsource(obj: Any) -> str: calls.append(obj) return original_getsource(obj) with monkeypatch.context() as m: m.setattr(inspect, "getsource", getsource) check_schemas(ItemAdapter.get_json_schema(ChildItem), expected) check_schemas(ItemAdapter.get_json_schema(ChildItem), expected) assert calls == list(ChildItem.__mro__) def test_field_docstring_cleandoc(self): """Test that field docstrings are properly dedented and trimmed.""" @dataclass class TestItem: foo: str """ This is a multiline docstring. It has multiple paragraphs and indentation that should be cleaned up. """ actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, "properties": { "foo": { "type": "string", "description": ( "This is a multiline docstring.\n\n" "It has multiple paragraphs and indentation\n" "that should be cleaned up." ), }, }, "required": ["foo"], } check_schemas(actual, expected) class TestCrossNesting: """Test item nesting across different item types, with all supported types acting as parent or child in one test.""" @requires_pydantic @requires_attrs def test_attrs_pydantic(self): import attrs @attrs.define class TestItem: nested: PydanticModelJsonSchemaNested actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, "properties": { "nested": { "type": "object", "properties": { "is_nested": {"type": "boolean", "default": True}, }, } }, "required": ["nested"], } check_schemas(actual, expected) actual = AttrsAdapter.get_json_schema(TestItem) expected = { "type": "object", "additionalProperties": False, "properties": {"nested": {}}, "required": ["nested"], } check_schemas(actual, expected) @requires_scrapy @requires_attrs def test_scrapy_attrs(self): actual = ItemAdapter.get_json_schema(ScrapySubclassedItemCrossNested) # pylint: disable=possibly-used-before-assignment expected = { "type": "object", "additionalProperties": False, "properties": { "nested": { "type": "object", "additionalProperties": False, "properties": { "is_nested": {"type": "boolean", "default": True}, }, } }, } check_schemas(actual, expected) actual = ScrapyItemAdapter.get_json_schema(ScrapySubclassedItemCrossNested) expected = { "type": "object", "additionalProperties": False, "properties": {"nested": {}}, } check_schemas(actual, expected) @requires_pydantic def test_pydantic_dataclass(self): class TestItem(pydantic.BaseModel): nested: DataClassItemJsonSchemaNested actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "properties": { "nested": { "type": "object", "additionalProperties": False, "properties": { "is_nested": {"type": "boolean", "default": True}, }, }, }, "required": ["nested"], } check_schemas(actual, expected) actual = PydanticAdapter.get_json_schema(TestItem) expected = { "type": "object", "properties": { "nested": {}, }, "required": ["nested"], } check_schemas(actual, expected) @requires_pydantic @requires_scrapy def test_pydantic_scrapy(self): class TestItem(pydantic.BaseModel): nested: ScrapySubclassedItemJsonSchemaNested model_config = { "arbitrary_types_allowed": True, } actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "properties": { "nested": { "type": "object", "additionalProperties": False, "properties": { "is_nested": {"default": True, "type": "boolean"}, }, }, }, "required": ["nested"], } check_schemas(actual, expected) actual = PydanticAdapter.get_json_schema(TestItem) expected = { "type": "object", "properties": { "nested": {"type": "object"}, }, "required": ["nested"], } check_schemas(actual, expected) class TestJsonSchemaEdgeCase: """Inputs that map to no JSON Schema output, or that the JSON Schema generation cannot take at face value.""" def test_unhandled_types(self): actual = ItemAdapter.get_json_schema(UnhandledTypeItem) expected = { "type": "object", "additionalProperties": False, "properties": { "mapping": { "type": "object", "description": "A mapping with no key or value types.", }, "meta": { "description": ( "A generic alias whose origin is neither a container nor a union." ) }, "var": {"description": "A type variable, which is not a type."}, }, "required": ["mapping", "meta", "var"], } check_schemas(actual, expected) def test_multi_item_union(self): """Unions keep the declaration order of their members, and only get a leading entry for their simple types if they have any.""" actual = ItemAdapter.get_json_schema(MultiItemUnionItem) brand_schema = { "type": "object", "additionalProperties": False, "properties": {"name": {"type": "string"}}, "required": ["name"], } nested_schema = { "type": "object", "additionalProperties": False, "properties": {"is_nested": {"type": "boolean", "default": True}}, } expected = { "type": "object", "additionalProperties": False, "properties": { "either": {"anyOf": [brand_schema, nested_schema]}, "or_none": {"anyOf": [{"type": "null"}, brand_schema, nested_schema]}, "containers": { "anyOf": [ {"type": "array", "items": {"type": "integer"}}, {"type": "object"}, ] }, }, "required": ["either", "or_none", "containers"], } check_schemas(actual, expected) def test_explicit_required(self): """An explicit required list in the class metadata is not overridden.""" actual = ItemAdapter.get_json_schema(ExplicitRequiredItem) expected = { "required": ["bar"], "type": "object", "additionalProperties": False, "properties": { "foo": {"type": "string"}, "bar": {"type": "string", "default": "asdf"}, }, } check_schemas(actual, expected) def test_source_of_a_different_class(self, monkeypatch: pytest.MonkeyPatch) -> None: """Reading the item class source can find the source of a different class, e.g. for classes that are renamed after their definition. Only the extraction of attribute docstrings should be affected.""" @dataclass class RenamedItem: foo: str """Foo""" source = "@dataclass\nclass SomethingElse:\n foo: str\n '''Bar'''\n" with monkeypatch.context() as m: m.setattr(inspect, "getsource", lambda _obj: source) actual = ItemAdapter.get_json_schema(RenamedItem) expected = { "type": "object", "additionalProperties": False, "properties": {"foo": {"type": "string"}}, "required": ["foo"], } check_schemas(actual, expected) @requires_attrs def test_attrs_unhandled_validator(self): """Validators that map to no JSON Schema keyword are ignored.""" import attrs @attrs.define class TestAttrsItem: foo: str = attrs.field(validator=attrs.validators.instance_of(str)) actual = ItemAdapter.get_json_schema(TestAttrsItem) expected = { "type": "object", "additionalProperties": False, "properties": {"foo": {"type": "string"}}, "required": ["foo"], } check_schemas(actual, expected) @requires_pydantic def test_pydantic_general_metadata_without_pattern(self): """General metadata that carries no pattern maps to no JSON Schema keyword.""" class TestItem(pydantic.BaseModel): foo: float = pydantic.Field(allow_inf_nan=False) actual = ItemAdapter.get_json_schema(TestItem) expected = { "type": "object", "properties": {"foo": {"type": "number"}}, "required": ["foo"], } check_schemas(actual, expected) @pytest.mark.parametrize( ("pattern", "expected"), [ # Shared syntax. (r"^[A-Za-z]+$", r"^[A-Za-z]+$"), (r"\bD\d{2}\b", r"\bD\d{2}\b"), (r"a(b|c)?(?:d)(?=e)(?!f)\.\\", r"a(b|c)?(?:d)(?=e)(?!f)\.\\"), (r"\x41\u00e9\t\n", r"\x41\u00e9\t\n"), (r"(a)\1", r"(a)\1"), (r"[a\]b^-]", r"[a\]b^-]"), (r"[a.b]", r"[a.b]"), (r"(?<=a)b", r"(?<=a)b"), (r"(?\d{4})", r"(\d{4})"), (r"a(?#comment)b", r"ab"), (r"(?s)a.b", r"a[\s\S]b"), (re.compile(r"a.b", re.DOTALL), r"a[\s\S]b"), ("(?x)a # comment\n \\d+", r"a\d+"), (re.compile("a # comment\n b", re.VERBOSE), r"ab"), (r"(?ax)a b", r"ab"), ("(?x)a # unterminated comment", "a"), # Python-only syntax. (r"\a", None), (r"[\a]", None), (r"\N{BULLET}", None), (r"\U0001F600", None), (r"(?P=name)", None), (r"(?i)a", None), (re.compile(r"a", re.IGNORECASE), None), (r"(?m)a", None), (r"(?i:a)", None), (r"(?>a)", None), (r"a*+", None), (r"a{2,3}+", None), # Syntax that both dialects accept with a different meaning. (r"[]]", None), (r"[^]]", None), (r"\0", None), (r"\01", None), (r"(?x)a{2, 3}", None), # Malformed patterns. ("\\", None), (r"[abc", None), (r"\x4", None), (r"(?#unterminated", None), (r"(?P None: assert json_schema_pattern(pattern) == expected scrapy-itemadapter-78cc818/tests/test_utils.py000066400000000000000000000053041527060762600216070ustar00rootroot00000000000000from types import MappingProxyType import pytest from itemadapter import ItemAdapter from itemadapter.utils import get_field_meta_from_class, is_item from tests import ( AttrsClassicItem, AttrsItem, DataClassItem, PydanticModel, ScrapyItem, ScrapySubclassedItem, requires_attrs, requires_pydantic, requires_scrapy, ) class TestFieldMetaFromClass: def test_invalid_item_class(self): with pytest.raises(TypeError, match=r"issubclass\(\) arg 1 must be a class"): get_field_meta_from_class(1, "field") with pytest.raises(TypeError, match=r"'list'\> is not a valid item class"): get_field_meta_from_class(list, "field") def test_empty_meta_for_dict(self): class DictSubclass(dict): pass assert get_field_meta_from_class(DictSubclass, "name") == MappingProxyType({}) assert get_field_meta_from_class(dict, "name") == MappingProxyType({}) class TestItemLike: def test_false(self): assert not is_item(int) assert not is_item(sum) assert not is_item(1234) assert not is_item(object()) assert not is_item("a string") assert not is_item(b"some bytes") assert not is_item(["a", "list"]) assert not is_item(("a", "tuple")) assert not is_item({"a", "set"}) assert not is_item(dict) assert not is_item(ScrapyItem) assert not is_item(DataClassItem) assert not is_item(ScrapySubclassedItem) assert not is_item(AttrsItem) assert not is_item(PydanticModel) assert not ItemAdapter.is_item_class(list) assert not ItemAdapter.is_item_class(int) assert not ItemAdapter.is_item_class(tuple) def test_true_dict(self): assert is_item({"a": "dict"}) assert ItemAdapter.is_item_class(dict) @requires_scrapy def test_true_scrapy(self): assert is_item(ScrapyItem()) assert is_item(ScrapySubclassedItem(name="asdf", value=1234)) assert ItemAdapter.is_item_class(ScrapyItem) assert ItemAdapter.is_item_class(ScrapySubclassedItem) def test_true_dataclass(self): assert is_item(DataClassItem(name="asdf", value=1234)) assert ItemAdapter.is_item_class(DataClassItem) @requires_attrs def test_true_attrs(self): assert is_item(AttrsItem(name="asdf", value=1234)) assert ItemAdapter.is_item_class(AttrsItem) assert is_item(AttrsClassicItem(name="asdf", value=1234)) assert ItemAdapter.is_item_class(AttrsClassicItem) @requires_pydantic def test_true_pydantic(self): assert is_item(PydanticModel(name="asdf", value=1234)) assert ItemAdapter.is_item_class(PydanticModel) scrapy-itemadapter-78cc818/tox.ini000066400000000000000000000036021527060762600172060ustar00rootroot00000000000000[tox] envlist = min-{attrs,pydantic,scrapy,extra},py{311-315},attrs,pydantic,scrapy,extra,benchmark,pre-commit,mypy,docs,twinecheck,pylint minversion = 4.25.0 [testenv] basepython = min-attrs,min-pydantic,min-scrapy,min-extra: python3.11 deps = pytest>=9.0 pytest-cov>=7.0.0 packaging min-attrs,min-extra: attrs==22.1.0 min-pydantic,min-extra: pydantic==2.7 min-scrapy,min-extra: scrapy==2.7 # newer w3lib breaks Scrapy < 2.16 min-scrapy,min-extra: w3lib < 2.5.0 extras = min-attrs,attrs,min-extra,extra: attrs min-pydantic,pydantic,min-extra,extra: pydantic min-scrapy,scrapy,min-extra,extra: scrapy commands = pytest -vv --cov=itemadapter --cov-report=term-missing --cov-report=html --cov-report=xml {posargs: itemadapter tests} [testenv:docs] deps = {[testenv]deps} zyte-common-items extras = attrs pydantic scrapy commands = pytest -vv --cov=itemadapter --cov-report=term-missing --cov-report=html --cov-report=xml --doctest-glob=README.md {posargs:README.md} # CPU benchmarks, tracked on CodSpeed. [testenv:benchmark] basepython = python3.14 deps = pytest>=9.0 pytest-codspeed extras = attrs pydantic scrapy passenv = *codspeed* *ci* commands = pytest {posargs:tests/benchmarks} --codspeed --codspeed-mode=simulation [testenv:mypy] basepython = python3 deps = mypy==2.4.0 attrs==26.1.0 pydantic==2.13.5 Scrapy==2.19.0 commands = # not checking tests for now mypy {posargs:itemadapter} [testenv:pylint] deps = pylint==4.1.2 pylint-per-file-ignores==4.0.0 commands = pylint {posargs:itemadapter tests} [testenv:twinecheck] basepython = python3 deps = twine==7.0.0 build==1.6.1 commands = python -m build --sdist twine check dist/* [testenv:pre-commit] deps = pre-commit commands = pre-commit run --all-files --show-diff-on-failure skip_install = true