pax_global_header 0000666 0000000 0000000 00000000064 15270054500 0014510 g ustar 00root root 0000000 0000000 52 comment=73b3a909cbbb60f5e083bad35d3b84744aa62520
scrapy-sphinx-scrapy-e29b0a7/ 0000775 0000000 0000000 00000000000 15270054500 0016232 5 ustar 00root root 0000000 0000000 scrapy-sphinx-scrapy-e29b0a7/.github/ 0000775 0000000 0000000 00000000000 15270054500 0017572 5 ustar 00root root 0000000 0000000 scrapy-sphinx-scrapy-e29b0a7/.github/actionlint.yaml 0000664 0000000 0000000 00000000317 15270054500 0022623 0 ustar 00root root 0000000 0000000 paths:
.github/workflows/publish.yml:
ignore:
# The job is disabled on purpose while the package cannot be published to
# PyPI, see the README.
- 'constant expression "false" in condition'
scrapy-sphinx-scrapy-e29b0a7/.github/dependabot.yml 0000664 0000000 0000000 00000000333 15270054500 0022421 0 ustar 00root root 0000000 0000000 version: 2
updates:
- package-ecosystem: github-actions
directory: "/"
schedule:
interval: monthly
groups:
github-actions:
patterns:
- "*"
cooldown:
default-days: 7
scrapy-sphinx-scrapy-e29b0a7/.github/workflows/ 0000775 0000000 0000000 00000000000 15270054500 0021627 5 ustar 00root root 0000000 0000000 scrapy-sphinx-scrapy-e29b0a7/.github/workflows/ci.yml 0000664 0000000 0000000 00000001074 15270054500 0022747 0 ustar 00root root 0000000 0000000 name: CI
permissions:
contents: read
on:
push:
branches:
- main
pull_request:
concurrency:
group: ${{github.workflow}}-${{ github.ref }}
cancel-in-progress: true
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: 3.14
- uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
scrapy-sphinx-scrapy-e29b0a7/.github/workflows/publish.yml 0000664 0000000 0000000 00000001543 15270054500 0024023 0 ustar 00root root 0000000 0000000 name: Publish
permissions:
contents: read
on:
push:
tags:
- '[0-9]+.[0-9]+.[0-9]+'
concurrency:
group: ${{github.workflow}}-${{ github.ref }}
cancel-in-progress: true
jobs:
publish:
runs-on: ubuntu-latest
environment:
name: pypi
url: https://pypi.org/p/${{ github.event.repository.name }}
permissions:
id-token: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: 3.14
- run: |
python -m pip install --upgrade build
python -m build
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
with:
verbose: true
scrapy-sphinx-scrapy-e29b0a7/.pre-commit-config.yaml 0000664 0000000 0000000 00000001575 15270054500 0022523 0 ustar 00root root 0000000 0000000 repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.11.13
hooks:
- id: ruff-check
args: [ --fix ]
- id: ruff-format
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.16.0
hooks:
- id: mypy
additional_dependencies:
- sphinx
- tox
- types-docutils
- repo: https://github.com/adamchainz/blacken-docs
rev: 1.19.1
hooks:
- id: blacken-docs
additional_dependencies:
- black==25.1.0
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v6.0.0
hooks:
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/sphinx-contrib/sphinx-lint
rev: v1.0.2
hooks:
- id: sphinx-lint
- repo: https://github.com/rhysd/actionlint
rev: v1.7.12
hooks:
- id: actionlint
- repo: https://github.com/zizmorcore/zizmor-pre-commit
rev: v1.29.0
hooks:
- id: zizmor
args: [--no-progress, --fix]
scrapy-sphinx-scrapy-e29b0a7/.pre-commit-hooks.yaml 0000664 0000000 0000000 00000000176 15270054500 0022375 0 ustar 00root root 0000000 0000000 - id: sphinx-scrapy
name: sphinx-scrapy
entry: sphinx-scrapy update-rtd-config
language: python
pass_filenames: false
scrapy-sphinx-scrapy-e29b0a7/CHANGES.rst 0000664 0000000 0000000 00000024510 15270054500 0020036 0 ustar 00root root 0000000 0000000 =============
Release notes
=============
0.15.0 (2026-10-27)
===================
- The ``docs`` tox environment now installs sphinx-scrapy at the version of
the tox plugin, plus the ``docs`` `dependency group
`_
of :file:`pyproject.toml`, if defined, instead of
:file:`docs/requirements.txt`. Documentation dependencies are thus no
longer pinned, and sphinx-scrapy is pinned only in :file:`tox.ini` and
:file:`.pre-commit-config.yaml`.
To update, move documentation dependencies other than sphinx-scrapy to
that group, e.g. ``docs = ["sphinx-rtd-theme"]`` under
``[dependency-groups]``, and remove :file:`docs/requirements.in` and
:file:`docs/requirements.txt`. ``sphinx-scrapy build`` and the
``sphinx-scrapy`` pre-commit hook fail while either file exists.
0.14.0 (2026-10-14)
===================
- | Extended easy Intersphinx configuration to
| `websockets `_
- Intersphinx mappings now point to the documentation of the latest release
(``/en/stable/``) instead of the development version (``/en/latest/``)
for projects that publish it.
- The ``sphinx-scrapy`` pre-commit hook now also updates Git-based
sphinx-scrapy pins, e.g. ``sphinx-scrapy @
git+https://github.com/scrapy/sphinx-scrapy.git@0.9.0``, turning them into
PyPI pins.
0.13.0 (2026-09-30)
===================
- sphinx-llm-friendly 0.5.1 or higher is now required.
- Enabled `sphinx_design `_
and `sphinxcontrib.youtube `_,
so their directives, e.g. ``.. tab-set::`` or ``.. youtube::``, work
without adding them to ``extensions``, and YouTube videos render as links
in the Markdown output.
- | Extended easy Intersphinx configuration to
| `requests `_
0.12.0 (2026-09-24)
===================
- The ``sphinx-scrapy`` pre-commit hook now also updates the sphinx-scrapy
pins in :file:`tox.ini` and :file:`docs/requirements.in` to match its
``rev``, so upgrading sphinx-scrapy only takes changing that ``rev``.
After :file:`docs/requirements.in` changes, recompile
:file:`docs/requirements.txt` with ``uv pip compile requirements.in -o
requirements.txt`` from the :file:`docs` directory.
0.11.0 (2026-09-24)
===================
- Documentation builds now fail on Sphinx warnings.
While you fix existing warnings, you can `suppress them by type
`_
in :file:`docs/conf.py`, or allow all of them in :file:`pyproject.toml`
with ``fail-on-warning = false`` under ``[tool.sphinx-scrapy]``.
- Documentation is now built with a regular parallel ``sphinx-build -b
html``, which writes the Markdown pages, ``llms.txt`` and
``llms-full.txt`` next to the HTML pages, in :file:`docs/_build/html`
instead of :file:`docs/_build/all`.
``sphinx-scrapy build`` and ``tox -e docs`` accept an optional output
directory, e.g. ``tox -e docs -- $READTHEDOCS_OUTPUT/html``.
This is backward-incompatible. To update:
- Run ``sphinx-scrapy update-rtd-config`` to regenerate
:file:`.readthedocs.yml`.
- In ``app.add_node()`` calls, drop ``llm_singlemarkdown=`` handlers;
``llm_markdown=`` handlers cover all Markdown output.
- Sphinx 8.0.0 or higher is now required, and sphinx-llm-friendly 0.3.0 or
higher.
0.10.0 (2026-09-24)
===================
- Replaced the Git-based sphinx-llms-txt and sphinx-markdown-builder
dependencies with `sphinx-llm-friendly
`_, so sphinx-scrapy can be
installed from PyPI again, e.g. ``sphinx-scrapy==0.10.0`` in
:file:`docs/requirements.txt` and ``sphinx-scrapy[tox]==0.10.0`` in the
``requires`` of :file:`tox.ini`.
This is backward-incompatible. To update:
- Rename ``llms_txt_*`` settings to their ``llm_friendly_*``
counterparts, or drop them where the new defaults cover them, and drop
``markdown_flavor``.
- Remove ``"sphinx_llms_txt"`` from ``extensions``.
- In ``app.add_node()`` calls, rename the ``markdown=`` and
``singlemarkdown=`` handlers to ``llm_markdown=`` and
``llm_singlemarkdown=``.
- Replace ``exclude_patterns`` workarounds that check the builder name
with ``llm_friendly_exclude`` or
``llm_friendly_llms_full_txt_exclude``.
0.9.0 (2026-08-26)
==================
- HTML pages now include an ````
tag pointing to their ``.md`` counterpart.
0.8.13 (2026-08-25)
===================
- ``llms-full.txt`` now includes a ``Source:`` line before each page's
content.
0.8.12 (2026-08-14)
===================
- | Extended easy Intersphinx configuration to
| `platformdirs `_
- Added an explicit dependency on ``docutils``.
0.8.11 (2026-08-10)
===================
- Added the ``:gh:`` role, to link to GitHub issues and pull requests, of the
current repository or of a different one.
- Fixed project data from ``pyproject.toml``, like the automatic
``html_baseurl`` value, being read relative to the current working
directory instead of relative to ``conf.py``.
- Fixed the ``docs`` tox environment getting an invalid Python interpreter in
projects without a ``pyproject.toml``.
0.8.10 (2026-07-31)
===================
- Fixed the ``docs`` tox environment failing to find its Python interpreter
when `tox-uv `_ is installed.
0.8.9 (2026-07-29)
==================
- Added the ``stat`` cross-reference type, for the ``.. stat::`` directive
and the ``:stat:`` role, to document and link to Scrapy stats.
0.8.8 (2026-06-15)
==================
- | Extended easy Intersphinx configuration to
| `scrapy-lint `_
0.8.7 (2026-06-11)
==================
- | Extended easy Intersphinx configuration to
| `pypug `_
0.8.6 (2026-05-12)
==================
- Add index templates for extension-defined cross-reference types so related
objects appear in generated Sphinx indexes.
0.8.5 (2026-04-29)
==================
- Normalize project IDs loaded from :file:`pyproject.toml` and inferred from
Sphinx metadata.
0.8.4 (2026-04-28)
==================
- Fix links in :file:`llms.txt`.
0.8.3 (2026-04-22)
==================
- Rewrite generated Markdown links for supported Intersphinx targets from
``.html`` to ``.md``.
- | Extended easy Intersphinx configuration to
| `aiohttp `_
| `curl-cffi `_
| `niquests `_
- Document installation from Git for ``sphinx-scrapy`` and
``sphinx-scrapy[tox]``.
- Upgrade sphinx-markdown-builder.
0.8.2 (2026-04-10)
==================
- Upgrade sphinx-markdown-builder.
- Make the copy-as-Markdown button smaller and place it to the right of the
page title.
0.8.1 (2026-04-10)
==================
- Addressed warnings affecting sitemap generation due to missing ``/`` at the
end of Intersphinx URLs.
0.8.0 (2026-04-09)
==================
- Now generates a sitemap.
0.7.2 (2026-04-06)
==================
- Added a ``tox`` extra, to be used in the ``requires`` section of
``tox.ini`` to ensure a supported version of ``tox`` is used.
0.7.1 (2026-04-06)
==================
- Fixed the generated Read the Docs configuration, which was causing builds
to fail.
- When the pre-commit hook fails, it now reports that it updated
:file:`.readthedocs.yml`.
0.7.0 (2026-04-02)
==================
- Dropped support for Python 3.8 and 3.9, added official support for Python
3.14.
- Added a tox plugin that provides a ``docs`` environment.
- Added a pre-commit hook to create or update ``.readthedocs.yml``.
- Enabling the ``sphinx_scrapy`` extension now automatically enables
``sphinx.ext.autodoc``, ``sphinx.ext.viewcode``, ``sphinx_copybutton``, and
``sphinx_llms_txt``.
- Added a "Copy as Markdown" button on HTML pages that copies each page's
corresponding ``.md`` counterpart.
- | Extended easy Intersphinx configuration to
| `coverage `_
| `cryptography `_
| `multidict `_
| `sphinx `_
| `tox `_
0.6.1 (2025-09-16)
==================
Fixed Python 3.8 support.
0.6.0 (2025-09-16)
==================
| Extended easy Intersphinx configuration to:
| `pydantic `_
0.5.0 (2025-09-16)
==================
Change the minimum version of Python from 3.9 to 3.8.
0.4.0 (2025-09-16)
==================
| Extended easy Intersphinx configuration to:
| `packaging `_
| `pytest `_
0.3.0 (2025-09-02)
==================
| Extended easy Intersphinx configuration to:
| `jinja `_
| `lxml `_
0.2.0 (2025-06-16)
==================
| Extended easy Intersphinx configuration to:
| `dateparser `_
| `form2request `_
| `formasaurus `_
| `python-scrapinghub `_
| `scrapy-spider-metadata `_
| `scrapy-zyte-api `_
| `scrapy-zyte-smartproxy `_
| `scrapyd `_
| `shub `_
| `shub-image `_
| `spidermon `_
| `url-matcher `_
| `zyte-parsers `_
0.1.0 (2025-06-14)
==================
Initial release.
scrapy-sphinx-scrapy-e29b0a7/LICENSE 0000664 0000000 0000000 00000002755 15270054500 0017250 0 ustar 00root root 0000000 0000000 Copyright (c) Scrapy developers.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions, and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions, and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of Scrapy nor the names of its contributors may be used
to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
scrapy-sphinx-scrapy-e29b0a7/README.rst 0000664 0000000 0000000 00000020112 15270054500 0017715 0 ustar 00root root 0000000 0000000 =============
sphinx-scrapy
=============
|version| |python_version|
.. |version| image:: https://img.shields.io/pypi/v/sphinx-scrapy.svg
:target: https://pypi.org/pypi/sphinx-scrapy
:alt: PyPI version
.. |python_version| image:: https://img.shields.io/pypi/pyversions/sphinx-scrapy.svg
:target: https://pypi.org/pypi/sphinx-scrapy
:alt: Supported Python versions
`Sphinx `_ extension for documentation in the
`Scrapy `_ ecosystem.
Features
========
- Provides a ``docs`` `tox `_
environment.
- Generates a `Read the Docs `_ configuration.
- Keeps the sphinx-scrapy pin of ``tox.ini`` in sync with the pre-commit
``rev``.
- Makes documentation builds fail on Sphinx warnings. While you fix existing
warnings, you can either `suppress them by type
`_
in ``docs/conf.py``, or allow all of them in ``pyproject.toml``:
.. code-block:: toml
[tool.sphinx-scrapy]
fail-on-warning = false
- Enables `commonly-used Sphinx extensions <#sphinx-extensions>`_.
- Configures `sphinx.ext.intersphinx`_ for `Python
`_ and Scrapy_, and streamlines configuration for
`additional packages <#intersphinx-packages>`_.
- Allows you to easily link to Scrapy settings, request metadata keys,
signals, commands and stats:
.. code-block:: rst
:setting:`BOT_NAME`
:setting:`LOG_LEVEL `
:reqmeta:`download_slot`
:signal:`spider_opened`
:command:`crawl`
:stat:`item_scraped_count`
- Allows you to easily link to GitHub issues and pull requests:
.. code-block:: rst
:gh:`1908`
:gh:`parsel#12`
:gh:`sphinx-doc/sphinx#13000`
They render as ``#1908``, ``parsel#12`` and ``sphinx#13000``. Use the number
alone for the current repository, which comes from the GitHub URLs in
``pyproject.toml``. For a different repository, its name alone is enough if
it is a known repository of the Scrapy ecosystem or it belongs to the same
owner as the current repository.
Setup
=====
#. Configure in ``pyproject.toml`` the Python version for documentation
builds, e.g.:
.. code-block:: toml
[tool.sphinx-scrapy]
python-version = "3.14"
It must be `supported by Read the Docs
`_.
#. Add any other documentation dependencies, e.g. a Sphinx theme, to a
``docs`` dependency group in ``pyproject.toml``:
.. code-block:: toml
[dependency-groups]
docs = ["sphinx-rtd-theme"]
#. Add to ``docs/conf.py``:
.. code-block:: python
extensions = [
"sphinx_scrapy",
]
To automatically configure `sphinx.ext.intersphinx`_ for installed
`supported packages <#intersphinx-packages>`_, set:
.. code-block:: python
scrapy_intersphinx_enable_installed = True
You can also enable or disable the automatic `sphinx.ext.intersphinx`_
configuration of packages manually:
.. code-block:: python
scrapy_intersphinx_enable = [
"parsel",
"w3lib",
]
scrapy_intersphinx_disable = [
"scrapy",
]
The ``html_baseurl`` option for `sitemap generation
`_
is generated automatically based on the project name (``pyproject.toml`` or
``conf.py``) and known documentation URLs (the same used for easy
intersphinx configuration) with a fallback to
https://.readthedocs.io/en/latest/. You can alternatively define
the setting yourself in ``conf.py``.
#. Add to ``docs/.gitignore``:
.. code-block::
/_build/
#. Add to ``.pre-commit-config.yaml``:
.. code-block:: yaml
repos:
- repo: https://github.com/scrapy/sphinx-scrapy
rev: 0.15.0
hooks:
- id: sphinx-scrapy
#. Add to ``tox.ini``:
.. code-block:: ini
[tox]
requires =
sphinx-scrapy[tox]==0.15.0
envlist = …,docs
.. note:: ``docs`` in ``envlist`` is required.
You can now build the docs with:
.. code-block:: bash
tox -e docs
.. _sphinx-extensions:
Sphinx extensions
=================
The following Sphinx extensions are automatically enabled when you enable
``sphinx_scrapy``:
- `sphinx.ext.autodoc
`_
- `sphinx.ext.intersphinx`_
- `sphinx.ext.viewcode
`_
- `sphinx_copybutton `_
- `sphinx_design `_
- `sphinx-llm-friendly `_,
which documents the ``llm_friendly_*`` settings
- `sphinxcontrib.youtube `_
.. _intersphinx-packages:
Intersphinx packages
====================
``scrapy_intersphinx_enable`` supports the following packages:
| `aiohttp `_
| `attrs `_
| `coverage `_
| `cryptography `_
| `cssselect `_
| `curl-cffi `_
| `dateparser `_
| `form2request `_
| `formasaurus `_
| `itemloaders `_
| `jinja `_
| `lxml `_
| `multidict `_
| `niquests `_
| `packaging `_
| `parsel `_
| `platformdirs `_
| `pydantic `_
| `pypug `_
| `pytest `_
| `python-scrapinghub `_
| `python-zyte-api `_
| `requests `_
| `scrapy-poet `_
| `scrapy-spider-metadata `_
| `scrapy-zyte-api `_
| `scrapy-zyte-smartproxy `_
| `scrapyd `_
| `shub `_
| `shub-image `_
| `sphinx `_
| `spidermon `_
| `tenacity `_
| `tox `_
| `twisted `_ (and `twistedapi `_)
| `url-matcher `_
| `w3lib `_
| `web-poet `_
| `websockets `_
| `zyte `_
| `zyte-common-items `_
| `zyte-parsers `_
| `zyte-spider-templates `_
Release notes
=============
See the `release notes
`_ for a list of
releases and their changes.
.. _sphinx.ext.intersphinx: https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html
scrapy-sphinx-scrapy-e29b0a7/pyproject.toml 0000664 0000000 0000000 00000006370 15270054500 0021154 0 ustar 00root root 0000000 0000000 [build-system]
requires = ["hatchling>=1.27.0"]
build-backend = "hatchling.build"
[project]
name = "sphinx-scrapy"
version = "0.15.0"
description = "Sphinx extension for documentation in the Scrapy ecosystem"
dependencies = [
"docutils",
"packaging",
"sphinx>=8.0.0",
"sphinx-copybutton",
"sphinx-design",
"sphinx-llm-friendly>=0.5.1",
"sphinx-sitemap",
"sphinxcontrib-youtube",
"tomli; python_version < '3.11'",
]
classifiers = [
"Development Status :: 4 - Beta",
"Framework :: Scrapy",
"Intended Audience :: Developers",
"Operating System :: OS Independent",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
"Topic :: Internet :: WWW/HTTP",
"Topic :: Software Development :: Libraries :: Application Frameworks",
"Topic :: Software Development :: Libraries :: Python Modules",
]
license = "BSD-3-Clause"
license-files = ["LICENSE"]
readme = "README.rst"
requires-python = ">=3.10"
authors = [{ name = "Scrapy developers", email = "opensource@zyte.com" }]
maintainers = [{ name = "Scrapy developers", email = "opensource@zyte.com" }]
[project.scripts]
sphinx-scrapy = "sphinx_scrapy.cli:main"
[project.entry-points.tox]
sphinx_scrapy = "sphinx_scrapy.tox_plugin"
[project.urls]
Source = "https://github.com/scrapy/sphinx-scrapy"
Tracker = "https://github.com/scrapy/sphinx-scrapy/issues"
"Release notes" = "https://github.com/scrapy/sphinx-scrapy/blob/main/CHANGES.rst"
[project.optional-dependencies]
tox = ["tox>=4.29.0"]
[tool.mypy]
strict = true
[tool.bumpversion]
current_version = "0.15.0"
commit = true
tag = true
tag_name = "{new_version}"
[[tool.bumpversion.files]]
filename = "CHANGES.rst"
search = "\\(unreleased\\)$"
replace = "({now:%Y-%m-%d})"
regex = true
[[tool.bumpversion.files]]
filename = "README.rst"
search = "rev: {current_version}"
replace = "rev: {new_version}"
[[tool.bumpversion.files]]
filename = "README.rst"
search = "sphinx-scrapy[tox]=={current_version}"
replace = "sphinx-scrapy[tox]=={new_version}"
[tool.ruff.lint]
extend-select = [
# flake8-bugbear
"B",
# flake8-comprehensions
"C4",
# pydocstyle
"D",
# flake8-future-annotations
"FA",
# flynt
"FLY",
# refurb
"FURB",
# isort
"I",
# flake8-implicit-str-concat
"ISC",
# flake8-logging
"LOG",
# Perflint
"PERF",
# pygrep-hooks
"PGH",
# flake8-pie
"PIE",
# pylint
"PL",
# flake8-pytest-style
"PT",
# flake8-use-pathlib
"PTH",
# flake8-pyi
"PYI",
# flake8-quotes
"Q",
# flake8-return
"RET",
# flake8-raise
"RSE",
# Ruff-specific rules
"RUF",
# flake8-bandit
"S",
# flake8-simplify
"SIM",
# flake8-slots
"SLOT",
# flake8-debugger
"T10",
# flake8-type-checking
"TC",
# flake8-tidy-imports
"TID",
# pyupgrade
"UP",
# pycodestyle warnings
"W",
# flake8-2020
"YTT",
]
ignore = [
# No public API to document
"D",
]
[tool.ruff.lint.pydocstyle]
convention = "pep257"
scrapy-sphinx-scrapy-e29b0a7/sphinx_scrapy/ 0000775 0000000 0000000 00000000000 15270054500 0021124 5 ustar 00root root 0000000 0000000 scrapy-sphinx-scrapy-e29b0a7/sphinx_scrapy/__init__.py 0000664 0000000 0000000 00000026304 15270054500 0023242 0 ustar 00root root 0000000 0000000 from __future__ import annotations
import re
from functools import cache
from logging import getLogger
from pathlib import Path
from typing import TYPE_CHECKING
from docutils import nodes
from sphinx.util.docutils import SphinxRole
from .config import load_project_config, normalize_project_id
if TYPE_CHECKING:
from collections.abc import Generator
from sphinx.application import Sphinx
from sphinx.config import Config
from sphinx.util.typing import ExtensionMetadata
from .config import ProjectConfig
logger = getLogger(__name__)
INTERSPHINX_MAPPING = {
"aiohttp": ("https://docs.aiohttp.org/en/stable/", None),
"attrs": ("https://www.attrs.org/en/stable/", None),
"coverage": ("https://coverage.readthedocs.io/en/latest/", None),
"cryptography": ("https://cryptography.io/en/latest/", None),
"cssselect": ("https://cssselect.readthedocs.io/en/stable/", None),
"curl-cffi": ("https://curl-cffi.readthedocs.io/en/latest/", None),
"dateparser": ("https://dateparser.readthedocs.io/en/stable/", None),
"form2request": ("https://form2request.readthedocs.io/en/stable/", None),
"formasaurus": ("https://formasaurus.readthedocs.io/en/stable/", None),
"itemloaders": ("https://itemloaders.readthedocs.io/en/stable/", None),
"jinja": ("https://jinja.palletsprojects.com/en/stable/", None),
"lxml": ("https://lxml.de/apidoc/", None),
"multidict": ("https://multidict.aio-libs.org/en/stable/", None),
"niquests": ("https://niquests.readthedocs.io/en/latest/", None),
"packaging": ("https://packaging.pypa.io/en/stable/", None),
"parsel": ("https://parsel.readthedocs.io/en/stable/", None),
"platformdirs": ("https://platformdirs.readthedocs.io/en/latest/", None),
"pydantic": ("https://pydantic.dev/docs/validation/latest/", None),
"pypug": ("https://packaging.python.org/en/latest/", None),
"pytest": ("https://docs.pytest.org/en/stable/", None),
"python": ("https://docs.python.org/3/", None),
"python-scrapinghub": (
"https://python-scrapinghub.readthedocs.io/en/stable/",
None,
),
"python-zyte-api": ("https://python-zyte-api.readthedocs.io/en/stable/", None),
"requests": ("https://docs.python-requests.org/en/stable/", None),
"scrapy": ("https://docs.scrapy.org/en/latest/", None),
"scrapy-lint": ("https://scrapy-lint.readthedocs.io/en/stable/", None),
"scrapy-poet": ("https://scrapy-poet.readthedocs.io/en/stable/", None),
"scrapy-spider-metadata": (
"https://scrapy-spider-metadata.readthedocs.io/en/stable/",
None,
),
"scrapy-zyte-api": ("https://scrapy-zyte-api.readthedocs.io/en/stable/", None),
"scrapy-zyte-smartproxy": (
"https://scrapy-zyte-smartproxy.readthedocs.io/en/latest/",
None,
),
"scrapyd": ("https://scrapyd.readthedocs.io/en/stable/", None),
"shub": ("https://shub.readthedocs.io/en/stable/", None),
"shub-image": ("https://shub-image.readthedocs.io/en/stable/", None),
"sphinx": ("https://www.sphinx-doc.org/en/master/", None),
"spidermon": ("https://spidermon.readthedocs.io/en/stable/", None),
"tenacity": ("https://tenacity.readthedocs.io/en/latest/", None),
"tox": ("https://tox.wiki/en/latest/", None),
"twisted": ("https://docs.twisted.org/en/stable/", None),
"twistedapi": ("https://docs.twisted.org/en/stable/api/", None),
"url-matcher": ("https://url-matcher.readthedocs.io/en/stable/", None),
"w3lib": ("https://w3lib.readthedocs.io/en/stable/", None),
"web-poet": ("https://web-poet.readthedocs.io/en/stable/", None),
"websockets": ("https://websockets.readthedocs.io/en/stable/", None),
"zyte": ("https://docs.zyte.com/", None),
"zyte-common-items": ("https://zyte-common-items.readthedocs.io/en/stable/", None),
"zyte-parsers": ("https://zyte-parsers.readthedocs.io/en/stable/", None),
"zyte-spider-templates": (
"https://zyte-spider-templates.readthedocs.io/en/stable/",
None,
),
}
# By default, interphinx entries are configured if a same-name module is
# installed. Here you can set True to always configure the entry unless
# explicitly disabled, False to never configure the entry unless explicitly
# requested, or a string to use as the package name to check for.
PACKAGE_OVERRIDES = {
"python": True,
"python-zyte-api": "zyte-api",
"scrapy": True,
"twistedapi": "twisted",
"zyte": False,
}
# Repositories that :gh: can target by name alone, e.g. :gh:`parsel#12`.
_GITHUB_OWNERS = {
"scrapinghub": (
"andi",
"dateparser",
"extruct",
"price-parser",
"python-scrapinghub",
"scrapy-poet",
"scrapyrt",
"shub",
"spidermon",
"web-poet",
),
"scrapy": (
"cssselect",
"form2request",
"itemadapter",
"itemloaders",
"parsel",
"protego",
"queuelib",
"scrapy",
"scrapy-lint",
"scrapyd",
"scrapyd-client",
"sphinx-scrapy",
"w3lib",
),
"scrapy-plugins": (
"scrapy-playwright",
"scrapy-splash",
"scrapy-zyte-api",
"scrapy-zyte-smartproxy",
),
"zytedata": (
"python-zyte-api",
"scrapy-spider-metadata",
"url-matcher",
"zyte-common-items",
"zyte-parsers",
"zyte-spider-templates",
),
}
_GITHUB_REPO_OWNERS = {
repo: owner for owner, repos in _GITHUB_OWNERS.items() for repo in repos
}
_GITHUB_REFERENCE = re.compile(
r"(?:(?:(?P[\w.-]+)/)?(?P[\w.-]+)#)?(?P\d+)"
)
def setup(app: Sphinx) -> ExtensionMetadata:
app.add_config_value(
"scrapy_intersphinx_enable", [], "env", types=frozenset({list})
)
app.add_config_value(
"scrapy_intersphinx_enable_installed", False, "env", types=frozenset({bool})
)
app.add_config_value(
"scrapy_intersphinx_disable", [], "env", types=frozenset({list})
)
for extension in (
"sphinx.ext.autodoc",
"sphinx.ext.intersphinx",
"sphinx.ext.viewcode",
"sphinx_copybutton",
"sphinx_design",
"sphinx_llm_friendly",
"sphinx_sitemap",
"sphinxcontrib.youtube",
):
app.setup_extension(extension)
app.add_role("gh", _GitHubRole())
app.connect("builder-inited", set_better_defaults)
app.connect("config-inited", update_config)
# https://github.com/scrapy/scrapy/blob/dba37674e6eaa6c2030c8eb35ebf8127cd488062/docs/_ext/scrapydocs.py#L90C16-L110C6
for crossref_type in ("setting", "signal", "command", "reqmeta", "stat"):
app.add_crossref_type(
directivename=crossref_type,
rolename=crossref_type,
indextemplate=f"pair: %s; {crossref_type}",
)
return {"parallel_read_safe": True}
class _GitHubRole(SphinxRole):
"""Link to a GitHub issue or pull request, given ``123`` for the current
repository, or ``repo#123`` or ``owner/repo#123`` for a different one.
"""
def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]:
current_repo = _project_config(self.env.app.confdir).github_repo or ""
match = _GITHUB_REFERENCE.fullmatch(self.text.strip())
error = owner = repo = number = ""
if not match:
error = f"Unsupported GitHub reference: {self.text!r}"
else:
number = match["number"]
repo = match["repo"] or current_repo.partition("/")[2]
owner = (
match["owner"]
or _GITHUB_REPO_OWNERS.get(repo)
or current_repo.partition("/")[0]
)
if not (owner and repo):
error = (
f"Could not determine the repository of {self.text!r}; use "
f"the owner/repo#number syntax"
)
elif match["repo"] and f"{owner}/{repo}" == current_repo:
error = (
f"{self.text!r} targets the current repository; use the "
f"number alone"
)
if error:
message = self.inliner.reporter.error(error, line=self.lineno)
problem = self.inliner.problematic(self.rawtext, self.rawtext, message)
return [problem], [message]
label = (
f"#{number}" if f"{owner}/{repo}" == current_repo else f"{repo}#{number}"
)
# GitHub redirects the issue URL of a pull request to the pull request.
url = f"https://github.com/{owner}/{repo}/issues/{number}"
return [nodes.reference(self.rawtext, label, refuri=url)], []
@cache
def _project_config(confdir: str | Path) -> ProjectConfig:
return load_project_config(Path(confdir))
def update_config(app: Sphinx, config: Config) -> None:
configure_intersphinx(config)
configure_sitemap(config, _project_config(app.confdir))
def set_better_defaults(app: Sphinx) -> None:
manual_conf = getattr(app, "_raw_config", {})
better_defaults = {
"sitemap_excludes": ["genindex.html", "search.html"],
"sitemap_url_scheme": "{link}",
}
for key, value in better_defaults.items():
if key in manual_conf:
continue
setattr(app.config, key, value)
def configure_intersphinx(config: Config) -> None:
known = set(INTERSPHINX_MAPPING)
default = {k for k in INTERSPHINX_MAPPING if PACKAGE_OVERRIDES.get(k) is True}
disabled = set(config.scrapy_intersphinx_disable)
non_disabled = known - disabled
requested = set(config.scrapy_intersphinx_enable)
to_configure = (
requested
| (default - disabled)
| (
set(installed(non_disabled - requested - default))
if config.scrapy_intersphinx_enable_installed
else set()
)
)
for k in to_configure:
config.intersphinx_mapping[k] = INTERSPHINX_MAPPING[k]
def configure_sitemap(config: Config, project_config: ProjectConfig) -> None:
if not config.html_baseurl:
package: str | None = None
if project_config.project_id:
package = normalize_project_id(project_config.project_id)
elif hasattr(config, "project"):
package = normalize_project_id(re.sub(r"\s+", "-", str(config.project)))
if not package:
return
if package in INTERSPHINX_MAPPING:
base_url = INTERSPHINX_MAPPING[package][0]
else:
base_url = f"https://{package}.readthedocs.io/en/latest/"
config.html_baseurl = base_url
if not config.html_baseurl.endswith("/"):
config.html_baseurl = config.html_baseurl + "/"
logger.warning(
"html_baseurl should end with a slash; automatically fixed to %r",
config.html_baseurl,
)
def installed(names: set[str]) -> Generator[str, None, None]:
checked: dict[str, bool] = {}
for name in names:
if name in PACKAGE_OVERRIDES:
package = PACKAGE_OVERRIDES[name]
if package is False:
continue
if name in checked:
if checked[name]:
yield name
continue
module_name = name.replace("-", "_")
try:
__import__(module_name)
except ImportError:
checked[name] = False
continue
checked[name] = True
yield name
scrapy-sphinx-scrapy-e29b0a7/sphinx_scrapy/cli.py 0000664 0000000 0000000 00000007531 15270054500 0022253 0 ustar 00root root 0000000 0000000 from __future__ import annotations
import argparse
import re
import sys
from importlib.metadata import version
from pathlib import Path
from packaging.version import InvalidVersion, Version
from sphinx.cmd.build import main as sphinx_build
from .config import LATEST_RTD_PYTHON_VERSION, ProjectConfig, load_project_config
def _check_docs_requirements(config: ProjectConfig) -> bool:
found = [
path
for path in ("docs/requirements.in", "docs/requirements.txt")
if (config.root / path).is_file()
]
if found:
print(
f"Found {' and '.join(found)}. Move documentation dependencies "
f"other than sphinx-scrapy to a docs dependency group in "
f"pyproject.toml, and remove {' and '.join(found)}.",
file=sys.stderr,
)
return False
return True
def build_docs(output_dir: Path | None) -> int:
config = load_project_config()
if not _check_docs_requirements(config):
return 1
docs_dir = config.root / "docs"
if not docs_dir.is_dir():
print("docs directory not found", file=sys.stderr)
return 1
build_dir = docs_dir / "_build"
args = [
"-b",
"html",
"-d",
str(build_dir / "doctrees"),
"-j",
"auto",
str(docs_dir),
str(output_dir or build_dir / "html"),
]
if config.fail_on_warning:
args += ["-W", "--keep-going"]
return sphinx_build(args)
def update_rtd_config() -> int:
config = load_project_config()
if not _check_docs_requirements(config):
return 1
try:
config_python_version = Version(str(config.python_version))
except InvalidVersion:
print(
f"Invalid Python version in pyproject.toml: {config.python_version}",
file=sys.stderr,
)
return 1
if config_python_version > Version(LATEST_RTD_PYTHON_VERSION):
print(
f"Configured Python ({config.python_version}) is newer than the "
f"latest Read the Docs supported Python version known to "
f"sphinx-scrapy ({LATEST_RTD_PYTHON_VERSION}).",
file=sys.stderr,
)
return 1
output = config.root / ".readthedocs.yml"
output.write_text(
"\n".join(
[
"version: 2",
"build:",
" os: ubuntu-24.04",
" tools:",
f' python: "{config.python_version}"',
" commands:",
" - pip install tox",
" - tox -e docs -- $READTHEDOCS_OUTPUT/html",
"",
]
),
encoding="utf-8",
)
print("Updated .readthedocs.yml")
tox_ini = config.root / "tox.ini"
if tox_ini.is_file():
text = tox_ini.read_text(encoding="utf-8")
new_text = re.sub(
r"(sphinx-scrapy(?:\[tox\])?)(?:==\S+|\s*@\s*git\+\S+)",
rf"\g<1>=={version('sphinx-scrapy')}",
text,
)
if new_text != text:
tox_ini.write_text(new_text, encoding="utf-8")
print("Updated tox.ini")
return 0
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="sphinx-scrapy")
subparsers = parser.add_subparsers(dest="command", required=True)
build_parser = subparsers.add_parser("build")
build_parser.add_argument("output_dir", nargs="?", type=Path)
subparsers.add_parser("update-rtd-config")
return parser
def main(argv: list[str] | None = None) -> int:
parser = build_parser()
args = parser.parse_args(argv)
if args.command == "build":
return build_docs(args.output_dir)
if args.command == "update-rtd-config":
return update_rtd_config()
parser.error(f"Unknown command: {args.command}")
return 2
if __name__ == "__main__":
raise SystemExit(main())
scrapy-sphinx-scrapy-e29b0a7/sphinx_scrapy/config.py 0000664 0000000 0000000 00000007210 15270054500 0022743 0 ustar 00root root 0000000 0000000 from __future__ import annotations
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from packaging.utils import canonicalize_name
if sys.version_info >= (3, 11):
import tomllib
else:
import tomli as tomllib
LATEST_RTD_PYTHON_VERSION = "3.14"
_GITHUB_URL = re.compile(r"https?://github\.com/(?P[\w.-]+)/(?P[\w.-]+)")
@dataclass(frozen=True)
class ProjectConfig:
root: Path
python_version: str | None = None
extras: set[str] = field(default_factory=set)
project_id: str | None = None
github_repo: str | None = None
fail_on_warning: bool = True
docs_dependency_group: bool = False
def find_project_root(start: Path | None = None) -> Path:
path = (start or Path.cwd()).resolve()
for candidate in (path, *path.parents):
if (candidate / "pyproject.toml").is_file():
return candidate
msg = "Could not find pyproject.toml in the current directory or parent directories"
raise FileNotFoundError(msg)
def get_extras(pyproject_data: dict[str, Any]) -> set[str]:
project_data = pyproject_data.get("project", {})
optional_dependencies = project_data.get("optional-dependencies", {})
return {str(key) for key in optional_dependencies}
def _get_github_repo(pyproject_data: dict[str, Any]) -> str | None:
urls: dict[str, Any] = pyproject_data.get("project", {}).get("urls", {})
# Repository URLs first, since other entries may point to a different
# repository, e.g. a documentation page hosted elsewhere.
keys = sorted(urls, key=lambda key: key.lower() not in {"repository", "source"})
for key in keys:
match = _GITHUB_URL.match(str(urls[key]))
if match:
owner, repo = match.group("owner", "repo")
return f"{owner}/{repo.removesuffix('.git')}"
return None
def normalize_project_id(project_id: str) -> str:
return canonicalize_name(project_id.strip())
def load_project_config(start: Path | None = None) -> ProjectConfig:
"""Load project configuration from a :file:`pyproject.toml` if available.
The file is searched for in *start* (CWD by default) and its parents. If
there is none, return a minimal ``ProjectConfig`` containing only the
resolved project root and leaving other fields empty/None.
"""
try:
project_root = find_project_root(start)
except FileNotFoundError:
resolved_root = (start or Path.cwd()).resolve()
return ProjectConfig(root=resolved_root)
pyproject_path = project_root / "pyproject.toml"
with pyproject_path.open("rb") as fp:
pyproject_data = tomllib.load(fp)
tool_data = pyproject_data.get("tool", {})
scrapy_data = tool_data.get("sphinx-scrapy", {})
python_version = scrapy_data.get("python-version", LATEST_RTD_PYTHON_VERSION)
fail_on_warning = bool(scrapy_data.get("fail-on-warning", True))
extras = get_extras(pyproject_data)
dependency_groups = {
canonicalize_name(name) for name in pyproject_data.get("dependency-groups", {})
}
raw_project_id = pyproject_data.get("project", {}).get("name")
project_id = None
if raw_project_id is not None:
normalized_project_id = normalize_project_id(str(raw_project_id))
if normalized_project_id:
project_id = normalized_project_id
return ProjectConfig(
root=project_root,
python_version=str(python_version) if python_version is not None else None,
extras=extras,
project_id=project_id,
github_repo=_get_github_repo(pyproject_data),
fail_on_warning=fail_on_warning,
docs_dependency_group="docs" in dependency_groups,
)
scrapy-sphinx-scrapy-e29b0a7/sphinx_scrapy/tox_plugin.py 0000664 0000000 0000000 00000002423 15270054500 0023667 0 ustar 00root root 0000000 0000000 from __future__ import annotations
from importlib.metadata import version
from typing import TYPE_CHECKING
from tox.config.loader.memory import MemoryLoader
from tox.config.types import Command
from tox.plugin import impl
from .config import LATEST_RTD_PYTHON_VERSION, load_project_config
if TYPE_CHECKING:
from tox.config.sets import ConfigSet
from tox.session.state import State
def _python_executable(version: str) -> str:
return f"python{version}"
@impl
def tox_extend_envs() -> tuple[str, ...]:
return ("docs",)
@impl
def tox_add_core_config(core_conf: ConfigSet, state: State) -> None:
project_config = load_project_config()
state.conf.memory_seed_loaders["docs"].append(
MemoryLoader(
description="build documentation",
base_python=[
_python_executable(
project_config.python_version or LATEST_RTD_PYTHON_VERSION
)
],
deps=[f"sphinx-scrapy=={version('sphinx-scrapy')}"],
dependency_groups=["docs"] if project_config.docs_dependency_group else [],
extras=tuple(project_config.extras),
commands=[
Command(["sphinx-scrapy", "build", *(state.conf.pos_args(None) or ())])
],
)
)