pax_global_header 0000666 0000000 0000000 00000000064 15237605002 0014512 g ustar 00root root 0000000 0000000 52 comment=3581b4148e62f34f0a835cdd06eee5a6e0f5f843
scrapy-sphinx-scrapy-9357640/ 0000775 0000000 0000000 00000000000 15237605002 0016024 5 ustar 00root root 0000000 0000000 scrapy-sphinx-scrapy-9357640/.github/ 0000775 0000000 0000000 00000000000 15237605002 0017364 5 ustar 00root root 0000000 0000000 scrapy-sphinx-scrapy-9357640/.github/actionlint.yaml 0000664 0000000 0000000 00000000317 15237605002 0022415 0 ustar 00root root 0000000 0000000 paths:
.github/workflows/publish.yml:
ignore:
# The job is disabled on purpose while the package cannot be published to
# PyPI, see the README.
- 'constant expression "false" in condition'
scrapy-sphinx-scrapy-9357640/.github/dependabot.yml 0000664 0000000 0000000 00000000333 15237605002 0022213 0 ustar 00root root 0000000 0000000 version: 2
updates:
- package-ecosystem: github-actions
directory: "/"
schedule:
interval: monthly
groups:
github-actions:
patterns:
- "*"
cooldown:
default-days: 7
scrapy-sphinx-scrapy-9357640/.github/workflows/ 0000775 0000000 0000000 00000000000 15237605002 0021421 5 ustar 00root root 0000000 0000000 scrapy-sphinx-scrapy-9357640/.github/workflows/ci.yml 0000664 0000000 0000000 00000001074 15237605002 0022541 0 ustar 00root root 0000000 0000000 name: CI
permissions:
contents: read
on:
push:
branches:
- main
pull_request:
concurrency:
group: ${{github.workflow}}-${{ github.ref }}
cancel-in-progress: true
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: 3.14
- uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
scrapy-sphinx-scrapy-9357640/.github/workflows/publish.yml 0000664 0000000 0000000 00000001657 15237605002 0023623 0 ustar 00root root 0000000 0000000 name: Publish
permissions:
contents: read
on:
push:
tags:
- '[0-9]+.[0-9]+.[0-9]+'
concurrency:
group: ${{github.workflow}}-${{ github.ref }}
cancel-in-progress: true
jobs:
publish:
if: false # Cannot be published to PyPI at the moment, see the README.
runs-on: ubuntu-latest
environment:
name: pypi
url: https://pypi.org/p/${{ github.event.repository.name }}
permissions:
id-token: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: 3.14
- run: |
python -m pip install --upgrade build
python -m build
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
with:
verbose: true
scrapy-sphinx-scrapy-9357640/.pre-commit-config.yaml 0000664 0000000 0000000 00000001575 15237605002 0022315 0 ustar 00root root 0000000 0000000 repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.11.13
hooks:
- id: ruff-check
args: [ --fix ]
- id: ruff-format
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.16.0
hooks:
- id: mypy
additional_dependencies:
- sphinx
- tox
- types-docutils
- repo: https://github.com/adamchainz/blacken-docs
rev: 1.19.1
hooks:
- id: blacken-docs
additional_dependencies:
- black==25.1.0
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v6.0.0
hooks:
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/sphinx-contrib/sphinx-lint
rev: v1.0.2
hooks:
- id: sphinx-lint
- repo: https://github.com/rhysd/actionlint
rev: v1.7.12
hooks:
- id: actionlint
- repo: https://github.com/zizmorcore/zizmor-pre-commit
rev: v1.29.0
hooks:
- id: zizmor
args: [--no-progress, --fix]
scrapy-sphinx-scrapy-9357640/.pre-commit-hooks.yaml 0000664 0000000 0000000 00000000176 15237605002 0022167 0 ustar 00root root 0000000 0000000 - id: sphinx-scrapy
name: sphinx-scrapy
entry: sphinx-scrapy update-rtd-config
language: python
pass_filenames: false
scrapy-sphinx-scrapy-9357640/CHANGES.rst 0000664 0000000 0000000 00000013055 15237605002 0017632 0 ustar 00root root 0000000 0000000 =============
Release notes
=============
0.8.12 (2026-08-14)
===================
- | Extended easy Intersphinx configuration to
| `platformdirs `_
- Added an explicit dependency on ``docutils``.
0.8.11 (2026-08-10)
===================
- Added the ``:gh:`` role, to link to GitHub issues and pull requests, of the
current repository or of a different one.
- Fixed project data from ``pyproject.toml``, like the automatic
``html_baseurl`` value, being read relative to the current working
directory instead of relative to ``conf.py``.
- Fixed the ``docs`` tox environment getting an invalid Python interpreter in
projects without a ``pyproject.toml``.
0.8.10 (2026-07-31)
===================
- Fixed the ``docs`` tox environment failing to find its Python interpreter
when `tox-uv `_ is installed.
0.8.9 (2026-07-29)
==================
- Added the ``stat`` cross-reference type, for the ``.. stat::`` directive
and the ``:stat:`` role, to document and link to Scrapy stats.
0.8.8 (2026-06-15)
==================
- | Extended easy Intersphinx configuration to
| `scrapy-lint `_
0.8.7 (2026-06-11)
==================
- | Extended easy Intersphinx configuration to
| `pypug `_
0.8.6 (2026-05-12)
==================
- Add index templates for extension-defined cross-reference types so related
objects appear in generated Sphinx indexes.
0.8.5 (2026-04-29)
==================
- Normalize project IDs loaded from :file:`pyproject.toml` and inferred from
Sphinx metadata.
0.8.4 (2026-04-28)
==================
- Fix links in :file:`llms.txt`.
0.8.3 (2026-04-22)
==================
- Rewrite generated Markdown links for supported Intersphinx targets from
``.html`` to ``.md``.
- | Extended easy Intersphinx configuration to
| `aiohttp `_
| `curl-cffi `_
| `niquests `_
- Document installation from Git for ``sphinx-scrapy`` and
``sphinx-scrapy[tox]``.
- Upgrade sphinx-markdown-builder.
0.8.2 (2026-04-10)
==================
- Upgrade sphinx-markdown-builder.
- Make the copy-as-Markdown button smaller and place it to the right of the
page title.
0.8.1 (2026-04-10)
==================
- Addressed warnings affecting sitemap generation due to missing ``/`` at the
end of Intersphinx URLs.
0.8.0 (2026-04-09)
==================
- Now generates a sitemap.
0.7.2 (2026-04-06)
==================
- Added a ``tox`` extra, to be used in the ``requires`` section of
``tox.ini`` to ensure a supported version of ``tox`` is used.
0.7.1 (2026-04-06)
==================
- Fixed the generated Read the Docs configuration, which was causing builds
to fail.
- When the pre-commit hook fails, it now reports that it updated
:file:`.readthedocs.yml`.
0.7.0 (2026-04-02)
==================
- Dropped support for Python 3.8 and 3.9, added official support for Python
3.14.
- Added a tox plugin that provides a ``docs`` environment.
- Added a pre-commit hook to create or update ``.readthedocs.yml``.
- Enabling the ``sphinx_scrapy`` extension now automatically enables
``sphinx.ext.autodoc``, ``sphinx.ext.viewcode``, ``sphinx_copybutton``, and
``sphinx_llms_txt``.
- Added a "Copy as Markdown" button on HTML pages that copies each page's
corresponding ``.md`` counterpart.
- | Extended easy Intersphinx configuration to
| `coverage `_
| `cryptography `_
| `multidict `_
| `sphinx `_
| `tox `_
0.6.1 (2025-09-16)
==================
Fixed Python 3.8 support.
0.6.0 (2025-09-16)
==================
| Extended easy Intersphinx configuration to:
| `pydantic `_
0.5.0 (2025-09-16)
==================
Change the minimum version of Python from 3.9 to 3.8.
0.4.0 (2025-09-16)
==================
| Extended easy Intersphinx configuration to:
| `packaging `_
| `pytest `_
0.3.0 (2025-09-02)
==================
| Extended easy Intersphinx configuration to:
| `jinja `_
| `lxml `_
0.2.0 (2025-06-16)
==================
| Extended easy Intersphinx configuration to:
| `dateparser `_
| `form2request `_
| `formasaurus `_
| `python-scrapinghub `_
| `scrapy-spider-metadata `_
| `scrapy-zyte-api `_
| `scrapy-zyte-smartproxy `_
| `scrapyd `_
| `shub `_
| `shub-image `_
| `spidermon `_
| `url-matcher `_
| `zyte-parsers `_
0.1.0 (2025-06-14)
==================
Initial release.
scrapy-sphinx-scrapy-9357640/LICENSE 0000664 0000000 0000000 00000002755 15237605002 0017042 0 ustar 00root root 0000000 0000000 Copyright (c) Scrapy developers.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions, and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions, and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of Scrapy nor the names of its contributors may be used
to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
scrapy-sphinx-scrapy-9357640/README.rst 0000664 0000000 0000000 00000016615 15237605002 0017524 0 ustar 00root root 0000000 0000000 =============
sphinx-scrapy
=============
|version| |python_version|
.. |version| image:: https://img.shields.io/pypi/v/sphinx-scrapy.svg
:target: https://pypi.org/pypi/sphinx-scrapy
:alt: PyPI version
.. |python_version| image:: https://img.shields.io/pypi/pyversions/sphinx-scrapy.svg
:target: https://pypi.org/pypi/sphinx-scrapy
:alt: Supported Python versions
`Sphinx `_ extension for documentation in the
`Scrapy `_ ecosystem.
Features
========
- Provides a ``docs`` `tox `_
environment.
- Generates a `Read the Docs `_ configuration.
- Enables `commonly-used Sphinx extensions <#sphinx-extensions>`_.
- Configures `sphinx.ext.intersphinx`_ for `Python
`_ and Scrapy_, and streamlines configuration for
`additional packages <#intersphinx-packages>`_.
- Allows you to easily link to Scrapy settings, request metadata keys,
signals, commands and stats:
.. code-block:: rst
:setting:`BOT_NAME`
:setting:`LOG_LEVEL `
:reqmeta:`download_slot`
:signal:`spider_opened`
:command:`crawl`
:stat:`item_scraped_count`
- Allows you to easily link to GitHub issues and pull requests:
.. code-block:: rst
:gh:`1908`
:gh:`parsel#12`
:gh:`sphinx-doc/sphinx#13000`
They render as ``#1908``, ``parsel#12`` and ``sphinx#13000``. Use the number
alone for the current repository, which comes from the GitHub URLs in
``pyproject.toml``. For a different repository, its name alone is enough if
it is a known repository of the Scrapy ecosystem or it belongs to the same
owner as the current repository.
Setup
=====
#. Configure in ``pyproject.toml`` the Python version for documentation
builds, e.g.:
.. code-block:: toml
[tool.sphinx-scrapy]
python-version = "3.14"
It must be `supported by Read the Docs
`_.
#. Add to ``docs/requirements.txt``:
Install from Git, because this project currently depends on Git-based
packages and cannot be published to PyPI:
.. code-block:: text
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.12
#. Add to ``docs/conf.py``:
.. code-block:: python
extensions = [
"sphinx_scrapy",
]
To automatically configure `sphinx.ext.intersphinx`_ for installed
`supported packages <#intersphinx-packages>`_, set:
.. code-block:: python
scrapy_intersphinx_enable_installed = True
You can also enable or disable the automatic `sphinx.ext.intersphinx`_
configuration of packages manually:
.. code-block:: python
scrapy_intersphinx_enable = [
"parsel",
"w3lib",
]
scrapy_intersphinx_disable = [
"scrapy",
]
The ``html_baseurl`` option for `sitemap generation
`_
is generated automatically based on the project name (``pyproject.toml`` or
``conf.py``) and known documentation URLs (the same used for easy
intersphinx configuration) with a fallback to
https://.readthedocs.io/en/latest/. You can alternatively define
the setting yourself in ``conf.py``.
#. Add to ``docs/.gitignore``:
.. code-block::
/_build/
#. Add to ``.pre-commit-config.yaml``:
.. code-block:: yaml
repos:
- repo: https://github.com/scrapy/sphinx-scrapy
rev: 0.8.12
hooks:
- id: sphinx-scrapy
#. Add to ``tox.ini``:
.. code-block:: ini
[tox]
requires =
sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.12
envlist = …,docs
.. note:: ``docs`` in ``envlist`` is required.
You can now build the docs with:
.. code-block:: bash
tox -e docs
.. _sphinx-extensions:
Sphinx extensions
=================
The following Sphinx extensions are automatically enabled when you enable
``sphinx_scrapy``:
- `sphinx.ext.autodoc
`_
- `sphinx.ext.intersphinx`_
- `sphinx.ext.viewcode
`_
- `sphinx_copybutton `_
- `sphinx_llms_txt `_
.. _intersphinx-packages:
Intersphinx packages
====================
``scrapy_intersphinx_enable`` supports the following packages:
| `aiohttp `_
| `attrs `_
| `coverage `_
| `cryptography `_
| `cssselect `_
| `curl-cffi `_
| `dateparser `_
| `form2request `_
| `formasaurus `_
| `itemloaders `_
| `jinja `_
| `lxml `_
| `multidict `_
| `niquests `_
| `packaging `_
| `parsel `_
| `platformdirs `_
| `pydantic `_
| `pypug `_
| `pytest `_
| `python-scrapinghub `_
| `python-zyte-api `_
| `scrapy-poet `_
| `scrapy-spider-metadata `_
| `scrapy-zyte-api `_
| `scrapy-zyte-smartproxy `_
| `scrapyd `_
| `shub `_
| `shub-image `_
| `sphinx `_
| `spidermon `_
| `tenacity `_
| `tox `_
| `twisted `_ (and `twistedapi `_)
| `url-matcher `_
| `w3lib `_
| `web-poet `_
| `zyte `_
| `zyte-common-items `_
| `zyte-parsers `_
| `zyte-spider-templates `_
Release notes
=============
See the `release notes
`_ for a list of
releases and their changes.
.. _sphinx.ext.intersphinx: https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html
scrapy-sphinx-scrapy-9357640/pyproject.toml 0000664 0000000 0000000 00000007434 15237605002 0020750 0 ustar 00root root 0000000 0000000 [build-system]
requires = ["hatchling>=1.27.0"]
build-backend = "hatchling.build"
[project]
name = "sphinx-scrapy"
version = "0.8.12"
description = "Sphinx extension for documentation in the Scrapy ecosystem"
dependencies = [
"docutils",
"packaging",
"sphinx",
"sphinx-copybutton",
"sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69",
"sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@cfe4c0bfd7b4542f7e6b65a58cdf9ec765829940",
"sphinx-sitemap",
"tomli; python_version < '3.11'",
]
classifiers = [
"Development Status :: 4 - Beta",
"Framework :: Scrapy",
"Intended Audience :: Developers",
"Operating System :: OS Independent",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
"Topic :: Internet :: WWW/HTTP",
"Topic :: Software Development :: Libraries :: Application Frameworks",
"Topic :: Software Development :: Libraries :: Python Modules",
]
license = "BSD-3-Clause"
license-files = ["LICENSE"]
readme = "README.rst"
requires-python = ">=3.10"
authors = [{ name = "Scrapy developers", email = "opensource@zyte.com" }]
maintainers = [{ name = "Scrapy developers", email = "opensource@zyte.com" }]
[project.scripts]
sphinx-scrapy = "sphinx_scrapy.cli:main"
[project.entry-points.tox]
sphinx_scrapy = "sphinx_scrapy.tox_plugin"
[project.urls]
Source = "https://github.com/scrapy/sphinx-scrapy"
Tracker = "https://github.com/scrapy/sphinx-scrapy/issues"
"Release notes" = "https://github.com/scrapy/sphinx-scrapy/blob/main/CHANGES.rst"
[project.optional-dependencies]
tox = ["tox>=4.29.0"]
[tool.hatch.metadata]
allow-direct-references = true
[tool.mypy]
strict = true
[tool.bumpversion]
current_version = "0.8.12"
commit = true
tag = true
tag_name = "{new_version}"
[[tool.bumpversion.files]]
filename = "CHANGES.rst"
search = "\\(unreleased\\)$"
replace = "({now:%Y-%m-%d})"
regex = true
[[tool.bumpversion.files]]
filename = "README.rst"
search = "rev: {current_version}"
replace = "rev: {new_version}"
[[tool.bumpversion.files]]
filename = "README.rst"
search = "sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@{current_version}"
replace = "sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@{new_version}"
[[tool.bumpversion.files]]
filename = "README.rst"
search = "sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@{current_version}"
replace = "sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@{new_version}"
[tool.ruff.lint]
extend-select = [
# flake8-bugbear
"B",
# flake8-comprehensions
"C4",
# pydocstyle
"D",
# flake8-future-annotations
"FA",
# flynt
"FLY",
# refurb
"FURB",
# isort
"I",
# flake8-implicit-str-concat
"ISC",
# flake8-logging
"LOG",
# Perflint
"PERF",
# pygrep-hooks
"PGH",
# flake8-pie
"PIE",
# pylint
"PL",
# flake8-pytest-style
"PT",
# flake8-use-pathlib
"PTH",
# flake8-pyi
"PYI",
# flake8-quotes
"Q",
# flake8-return
"RET",
# flake8-raise
"RSE",
# Ruff-specific rules
"RUF",
# flake8-bandit
"S",
# flake8-simplify
"SIM",
# flake8-slots
"SLOT",
# flake8-debugger
"T10",
# flake8-type-checking
"TC",
# flake8-tidy-imports
"TID",
# pyupgrade
"UP",
# pycodestyle warnings
"W",
# flake8-2020
"YTT",
]
ignore = [
# No public API to document
"D",
]
[tool.ruff.lint.pydocstyle]
convention = "pep257"
scrapy-sphinx-scrapy-9357640/sphinx_scrapy/ 0000775 0000000 0000000 00000000000 15237605002 0020716 5 ustar 00root root 0000000 0000000 scrapy-sphinx-scrapy-9357640/sphinx_scrapy/__init__.py 0000664 0000000 0000000 00000037667 15237605002 0023052 0 ustar 00root root 0000000 0000000 from __future__ import annotations
import re
from functools import cache
from logging import getLogger
from pathlib import Path
from typing import TYPE_CHECKING
from docutils import nodes
from sphinx.util.docutils import SphinxRole
from .config import load_project_config, normalize_project_id
if TYPE_CHECKING:
from collections.abc import Generator
from sphinx.application import Sphinx
from sphinx.config import Config
from .config import ProjectConfig
logger = getLogger(__name__)
INTERSPHINX_MAPPING = {
"aiohttp": ("https://docs.aiohttp.org/en/stable/", None),
"attrs": ("https://www.attrs.org/en/stable/", None),
"coverage": ("https://coverage.readthedocs.io/en/latest/", None),
"cryptography": ("https://cryptography.io/en/latest/", None),
"cssselect": ("https://cssselect.readthedocs.io/en/latest/", None),
"curl-cffi": ("https://curl-cffi.readthedocs.io/en/latest/", None),
"dateparser": ("https://dateparser.readthedocs.io/en/latest/", None),
"form2request": ("https://form2request.readthedocs.io/en/latest/", None),
"formasaurus": ("https://formasaurus.readthedocs.io/en/latest/", None),
"itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None),
"jinja": ("https://jinja.palletsprojects.com/en/latest/", None),
"lxml": ("https://lxml.de/apidoc/", None),
"multidict": ("https://multidict.aio-libs.org/en/latest/", None),
"niquests": ("https://niquests.readthedocs.io/en/latest/", None),
"packaging": ("https://packaging.pypa.io/en/stable/", None),
"parsel": ("https://parsel.readthedocs.io/en/latest/", None),
"platformdirs": ("https://platformdirs.readthedocs.io/en/latest/", None),
"pydantic": ("https://pydantic.dev/docs/validation/latest/", None),
"pypug": ("https://packaging.python.org/en/latest/", None),
"pytest": ("https://docs.pytest.org/en/stable/", None),
"python": ("https://docs.python.org/3/", None),
"python-scrapinghub": (
"https://python-scrapinghub.readthedocs.io/en/latest/",
None,
),
"python-zyte-api": ("https://python-zyte-api.readthedocs.io/en/stable/", None),
"scrapy": ("https://docs.scrapy.org/en/latest/", None),
"scrapy-lint": ("https://scrapy-lint.readthedocs.io/en/latest/", None),
"scrapy-poet": ("https://scrapy-poet.readthedocs.io/en/stable/", None),
"scrapy-spider-metadata": (
"https://scrapy-spider-metadata.readthedocs.io/en/latest/",
None,
),
"scrapy-zyte-api": ("https://scrapy-zyte-api.readthedocs.io/en/latest/", None),
"scrapy-zyte-smartproxy": (
"https://scrapy-zyte-smartproxy.readthedocs.io/en/latest/",
None,
),
"scrapyd": ("https://scrapyd.readthedocs.io/en/latest/", None),
"shub": ("https://shub.readthedocs.io/en/latest/", None),
"shub-image": ("https://shub-image.readthedocs.io/en/latest/", None),
"sphinx": ("https://www.sphinx-doc.org/en/master/", None),
"spidermon": ("https://spidermon.readthedocs.io/en/latest/", None),
"tenacity": ("https://tenacity.readthedocs.io/en/latest/", None),
"tox": ("https://tox.wiki/en/latest/", None),
"twisted": ("https://docs.twisted.org/en/stable/", None),
"twistedapi": ("https://docs.twisted.org/en/stable/api/", None),
"url-matcher": ("https://url-matcher.readthedocs.io/en/latest/", None),
"w3lib": ("https://w3lib.readthedocs.io/en/latest/", None),
"web-poet": ("https://web-poet.readthedocs.io/en/stable/", None),
"zyte": ("https://docs.zyte.com/", None),
"zyte-common-items": ("https://zyte-common-items.readthedocs.io/en/latest/", None),
"zyte-parsers": ("https://zyte-parsers.readthedocs.io/en/latest/", None),
"zyte-spider-templates": (
"https://zyte-spider-templates.readthedocs.io/en/latest/",
None,
),
}
# By default, interphinx entries are configured if a same-name module is
# installed. Here you can set True to always configure the entry unless
# explicitly disabled, False to never configure the entry unless explicitly
# requested, or a string to use as the package name to check for.
PACKAGE_OVERRIDES = {
"python": True,
"python-zyte-api": "zyte-api",
"scrapy": True,
"twistedapi": "twisted",
"zyte": False,
}
# Repositories that :gh: can target by name alone, e.g. :gh:`parsel#12`.
_GITHUB_OWNERS = {
"scrapinghub": (
"andi",
"dateparser",
"extruct",
"price-parser",
"python-scrapinghub",
"scrapy-poet",
"scrapyrt",
"shub",
"spidermon",
"web-poet",
),
"scrapy": (
"cssselect",
"form2request",
"itemadapter",
"itemloaders",
"parsel",
"protego",
"queuelib",
"scrapy",
"scrapy-lint",
"scrapyd",
"scrapyd-client",
"sphinx-scrapy",
"w3lib",
),
"scrapy-plugins": (
"scrapy-playwright",
"scrapy-splash",
"scrapy-zyte-api",
"scrapy-zyte-smartproxy",
),
"zytedata": (
"python-zyte-api",
"scrapy-spider-metadata",
"url-matcher",
"zyte-common-items",
"zyte-parsers",
"zyte-spider-templates",
),
}
_GITHUB_REPO_OWNERS = {
repo: owner for owner, repos in _GITHUB_OWNERS.items() for repo in repos
}
_GITHUB_REFERENCE = re.compile(
r"(?:(?:(?P[\w.-]+)/)?(?P[\w.-]+)#)?(?P\d+)"
)
COPY_AS_MARKDOWN_BUTTON_JS = """
(function () {
var DEFAULT_LABEL = 'M\u2193';
var SUCCESS_LABEL = 'Copied';
var ERROR_LABEL = 'Error';
function markdownPathFromCurrentPage(pathname) {
if (pathname.endsWith('.html')) {
return pathname.slice(0, -5) + '.md';
}
if (pathname.endsWith('/')) {
return pathname + 'index.md';
}
var lastPart = pathname.split('/').pop() || '';
if (!lastPart.includes('.')) {
return pathname + '.md';
}
return pathname + '.md';
}
async function copyToClipboard(text) {
if (navigator.clipboard && window.isSecureContext) {
await navigator.clipboard.writeText(text);
return;
}
var textarea = document.createElement('textarea');
textarea.value = text;
textarea.setAttribute('readonly', 'readonly');
textarea.style.position = 'fixed';
textarea.style.left = '-9999px';
document.body.appendChild(textarea);
textarea.select();
document.execCommand('copy');
document.body.removeChild(textarea);
}
function setTemporaryLabel(button, label, previousLabel) {
var prev = typeof previousLabel !== 'undefined' ? previousLabel : button.textContent;
button.textContent = label;
window.setTimeout(function () {
button.textContent = prev;
button.disabled = false;
}, 1000);
}
async function onButtonClick(button) {
var previousLabel = button.textContent;
button.disabled = true;
button.textContent = '...';
try {
var mdPath = markdownPathFromCurrentPage(window.location.pathname);
var response = await fetch(mdPath, { credentials: 'same-origin' });
if (!response.ok) {
throw new Error('Unable to fetch markdown source');
}
var markdown = await response.text();
await copyToClipboard(markdown);
setTemporaryLabel(button, SUCCESS_LABEL, previousLabel);
} catch (_error) {
setTemporaryLabel(button, ERROR_LABEL, previousLabel);
}
}
function addStyles() {
var style = document.createElement('style');
style.textContent = [
'.scrapy-copy-as-markdown {',
' display: inline-block;',
' margin-left: 0.25rem;',
' border: 1px solid #c9d4de;',
' border-radius: 0.45rem;',
' background: #ffffff;',
' color: #233a50;',
' font: inherit;',
' font-size: 0.875rem;',
' line-height: 1;',
' padding: 0.25rem 0.25rem;',
' box-shadow: 0 2px 10px rgba(0, 0, 0, 0.08);',
' cursor: pointer;',
'}',
'.scrapy-copy-as-markdown:hover {',
' background: #f3f7fb;',
'}',
'.scrapy-copy-as-markdown:disabled {',
' opacity: 0.75;',
' cursor: default;',
'}',
'.scrapy-copy-as-markdown-title-wrapper {',
' display: flex;',
' align-items: center;',
' justify-content: space-between;',
' gap: 1rem;',
' width: 100%;',
'}',
'.scrapy-copy-as-markdown-title-wrapper h1 {',
' margin: 0;',
'}',
].join('\\n');
document.head.appendChild(style);
}
function addButton() {
if (!document.body || document.querySelector('.scrapy-copy-as-markdown')) {
return;
}
addStyles();
var button = document.createElement('button');
button.type = 'button';
button.className = 'scrapy-copy-as-markdown';
button.title = 'Copy this page as Markdown';
button.setAttribute('aria-label', 'Copy this page as Markdown');
button.textContent = DEFAULT_LABEL;
button.addEventListener('click', function () {
onButtonClick(button);
});
var h1 = document.querySelector('#main h1') || document.querySelector('h1');
if (h1 && h1.parentNode) {
var parent = h1.parentNode;
var wrapper = document.createElement('div');
wrapper.className = 'scrapy-copy-as-markdown-title-wrapper';
parent.replaceChild(wrapper, h1);
wrapper.appendChild(h1);
wrapper.appendChild(button);
} else {
// fallback: insert at top of body but keep within the first container
var container = document.body.firstElementChild || document.body;
container.insertBefore(button, container.firstChild);
}
}
if (document.readyState === 'loading') {
document.addEventListener('DOMContentLoaded', addButton);
} else {
addButton();
}
})();
"""
def setup(app: Sphinx) -> None:
app.add_config_value(
"scrapy_intersphinx_enable", [], "env", types=frozenset({list})
)
app.add_config_value(
"scrapy_intersphinx_enable_installed", False, "env", types=frozenset({bool})
)
app.add_config_value(
"scrapy_intersphinx_disable", [], "env", types=frozenset({list})
)
for extension in (
"sphinx.ext.autodoc",
"sphinx.ext.intersphinx",
"sphinx.ext.viewcode",
"sphinx_copybutton",
"sphinx_llms_txt",
"sphinx_sitemap",
):
app.setup_extension(extension)
app.add_role("gh", _GitHubRole())
app.connect("builder-inited", add_copy_as_markdown_button)
app.connect("builder-inited", set_better_defaults)
app.connect("config-inited", update_config)
# https://github.com/scrapy/scrapy/blob/dba37674e6eaa6c2030c8eb35ebf8127cd488062/docs/_ext/scrapydocs.py#L90C16-L110C6
for crossref_type in ("setting", "signal", "command", "reqmeta", "stat"):
app.add_crossref_type(
directivename=crossref_type,
rolename=crossref_type,
indextemplate=f"pair: %s; {crossref_type}",
)
def add_copy_as_markdown_button(app: Sphinx) -> None:
if app.builder.format != "html":
return
app.add_js_file(None, body=COPY_AS_MARKDOWN_BUTTON_JS)
class _GitHubRole(SphinxRole):
"""Link to a GitHub issue or pull request, given ``123`` for the current
repository, or ``repo#123`` or ``owner/repo#123`` for a different one.
"""
def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]:
current_repo = _project_config(self.env.app.confdir).github_repo or ""
match = _GITHUB_REFERENCE.fullmatch(self.text.strip())
error = owner = repo = number = ""
if not match:
error = f"Unsupported GitHub reference: {self.text!r}"
else:
number = match["number"]
repo = match["repo"] or current_repo.partition("/")[2]
owner = (
match["owner"]
or _GITHUB_REPO_OWNERS.get(repo)
or current_repo.partition("/")[0]
)
if not (owner and repo):
error = (
f"Could not determine the repository of {self.text!r}; use "
f"the owner/repo#number syntax"
)
elif match["repo"] and f"{owner}/{repo}" == current_repo:
error = (
f"{self.text!r} targets the current repository; use the "
f"number alone"
)
if error:
message = self.inliner.reporter.error(error, line=self.lineno)
problem = self.inliner.problematic(self.rawtext, self.rawtext, message)
return [problem], [message]
label = (
f"#{number}" if f"{owner}/{repo}" == current_repo else f"{repo}#{number}"
)
# GitHub redirects the issue URL of a pull request to the pull request.
url = f"https://github.com/{owner}/{repo}/issues/{number}"
return [nodes.reference(self.rawtext, label, refuri=url)], []
@cache
def _project_config(confdir: str | Path) -> ProjectConfig:
return load_project_config(Path(confdir))
def update_config(app: Sphinx, config: Config) -> None:
configure_intersphinx(config)
configure_sitemap(config, _project_config(app.confdir))
def set_better_defaults(app: Sphinx) -> None:
manual_conf = getattr(app, "_raw_config", {})
better_defaults = {
"sitemap_excludes": ["genindex.html", "search.html"],
"sitemap_url_scheme": "{link}",
}
for key, value in better_defaults.items():
if key in manual_conf:
continue
setattr(app.config, key, value)
def configure_intersphinx(config: Config) -> None:
known = set(INTERSPHINX_MAPPING)
default = {k for k in INTERSPHINX_MAPPING if PACKAGE_OVERRIDES.get(k) is True}
disabled = set(config.scrapy_intersphinx_disable)
non_disabled = known - disabled
requested = set(config.scrapy_intersphinx_enable)
to_configure = (
requested
| (default - disabled)
| (
set(installed(non_disabled - requested - default))
if config.scrapy_intersphinx_enable_installed
else set()
)
)
for k in to_configure:
config.intersphinx_mapping[k] = INTERSPHINX_MAPPING[k]
def configure_sitemap(config: Config, project_config: ProjectConfig) -> None:
if not config.html_baseurl:
package: str | None = None
if project_config.project_id:
package = normalize_project_id(project_config.project_id)
elif hasattr(config, "project"):
package = normalize_project_id(re.sub(r"\s+", "-", str(config.project)))
if not package:
return
if package in INTERSPHINX_MAPPING:
base_url = INTERSPHINX_MAPPING[package][0]
else:
base_url = f"https://{package}.readthedocs.io/en/latest/"
config.html_baseurl = base_url
if not config.html_baseurl.endswith("/"):
config.html_baseurl = config.html_baseurl + "/"
logger.warning(
"html_baseurl should end with a slash; automatically fixed to %r",
config.html_baseurl,
)
def installed(names: set[str]) -> Generator[str, None, None]:
checked: dict[str, bool] = {}
for name in names:
if name in PACKAGE_OVERRIDES:
package = PACKAGE_OVERRIDES[name]
if package is False:
continue
if name in checked:
if checked[name]:
yield name
continue
module_name = name.replace("-", "_")
try:
__import__(module_name)
except ImportError:
checked[name] = False
continue
checked[name] = True
yield name
scrapy-sphinx-scrapy-9357640/sphinx_scrapy/cli.py 0000664 0000000 0000000 00000027054 15237605002 0022047 0 ustar 00root root 0000000 0000000 from __future__ import annotations
import argparse
import concurrent.futures
import re
import shutil
import sys
from http import HTTPStatus
from typing import TYPE_CHECKING
from urllib.error import HTTPError, URLError
from urllib.parse import urlsplit, urlunsplit
from urllib.request import Request, urlopen
from packaging.version import InvalidVersion, Version
from sphinx.cmd.build import main as sphinx_build_main
from . import INTERSPHINX_MAPPING
from .config import LATEST_RTD_PYTHON_VERSION, load_project_config, normalize_project_id
if TYPE_CHECKING:
from collections.abc import Callable
from pathlib import Path
URL_PATTERN = re.compile(r"https?://[^\s<>()\[\]{}\"']+")
MARKDOWN_LINK_PATTERN = re.compile(
r"(?P!?\[[^\]]*\]\()"
r"(?P[^)\s]+)"
r"(?P\s+\"[^\"]*\")?"
r"(?P<suffix>\))"
)
def _intersphinx_base_urls() -> tuple[str, ...]:
bases = {url for url, _inventory in INTERSPHINX_MAPPING.values()}
return tuple(sorted(bases, key=len, reverse=True))
def _project_docs_base_url(project_id: str | None) -> str | None:
if not project_id:
return None
normalized_project_id = normalize_project_id(project_id)
if not normalized_project_id:
return None
if normalized_project_id in INTERSPHINX_MAPPING:
return INTERSPHINX_MAPPING[normalized_project_id][0]
return f"https://{normalized_project_id}.readthedocs.io/en/latest/"
def _split_wrapped_markdown_target(target: str) -> tuple[str, str, str]:
if target.startswith("<") and target.endswith(">"):
return "<", target[1:-1], ">"
return "", target, ""
def _rewrite_markdown_links(
content: str,
rewrite_target: Callable[[str], str],
) -> str:
in_fence = False
fence_char = ""
rewritten_lines: list[str] = []
def replacement(match: re.Match[str]) -> str:
target = match.group("target")
rewritten_target = rewrite_target(target)
if rewritten_target == target:
return match.group(0)
title = match.group("title") or ""
return (
f"{match.group('prefix')}{rewritten_target}{title}{match.group('suffix')}"
)
for line in content.splitlines(keepends=True):
stripped = line.lstrip()
if stripped.startswith(("```", "~~~")):
marker = stripped[0]
if not in_fence:
in_fence = True
fence_char = marker
elif marker == fence_char:
in_fence = False
fence_char = ""
rewritten_lines.append(line)
continue
if in_fence:
rewritten_lines.append(line)
continue
rewritten_lines.append(MARKDOWN_LINK_PATTERN.sub(replacement, line))
return "".join(rewritten_lines)
def _rewrite_llms_link_target(
target: str,
docs_host: str,
docs_path_prefix: str,
) -> str:
wrapper_prefix, wrapped_target, wrapper_suffix = _split_wrapped_markdown_target(
target
)
parts = urlsplit(wrapped_target)
if parts.scheme or parts.netloc:
if parts.scheme not in {"http", "https"}:
return target
if parts.netloc.lower() != docs_host:
return target
if not parts.path.endswith(".md"):
return target
clean_path = parts.path.lstrip("/")
while clean_path.startswith("./"):
clean_path = clean_path[2:]
if clean_path.startswith("../"):
return target
if docs_path_prefix and not clean_path.startswith(docs_path_prefix):
clean_path = f"{docs_path_prefix}{clean_path}"
rewritten_target = urlunsplit(("", "", clean_path, parts.query, parts.fragment))
return f"{wrapper_prefix}{rewritten_target}{wrapper_suffix}"
def _rewrite_llms_links_to_docs_path(
output_dir: Path,
docs_base_url: str | None,
) -> None:
llms_path = output_dir / "llms.txt"
if docs_base_url is None or not llms_path.is_file():
return
docs_parts = urlsplit(docs_base_url)
docs_host = docs_parts.netloc.lower()
docs_path = docs_parts.path.strip("/")
docs_path_prefix = f"{docs_path}/" if docs_path else ""
content = llms_path.read_text(encoding="utf-8")
rewritten = _rewrite_markdown_links(
content,
lambda target: _rewrite_llms_link_target(target, docs_host, docs_path_prefix),
)
if rewritten != content:
llms_path.write_text(rewritten, encoding="utf-8")
def _iter_markdown_outputs(output_dir: Path) -> list[Path]:
files = sorted(output_dir.rglob("*.md"))
for filename in ("llms.txt", "llms-full.txt"):
path = output_dir / filename
if path.is_file():
files.append(path)
return files
def _matching_base_url(url: str, base_urls: tuple[str, ...]) -> str | None:
for base_url in base_urls:
if url.startswith(base_url):
return base_url
return None
def _supports_markdown(base_url: str) -> bool:
probe_url = f"{base_url.rstrip('/')}/index.md"
headers = {"User-Agent": "sphinx-scrapy"}
for method in ("HEAD", "GET"):
request = Request(probe_url, headers=headers, method=method) # noqa: S310
try:
with urlopen(request, timeout=5) as response: # noqa: S310
status = int(response.status)
return HTTPStatus.OK <= status < HTTPStatus.BAD_REQUEST
except HTTPError as error:
if method == "HEAD" and error.code in {
HTTPStatus.METHOD_NOT_ALLOWED,
HTTPStatus.NOT_IMPLEMENTED,
}:
continue
return False
except URLError:
return False
return False
def _rewrite_url_to_markdown(
url: str, enabled_base_urls: set[str], base_urls: tuple[str, ...]
) -> str:
base_url = _matching_base_url(url, base_urls)
if not base_url or base_url not in enabled_base_urls:
return url
parts = urlsplit(url)
path = parts.path
if not path.endswith(".html"):
return url
new_path = f"{path[:-5]}.md"
return urlunsplit(
(parts.scheme, parts.netloc, new_path, parts.query, parts.fragment)
)
def _rewrite_intersphinx_links_to_markdown(output_dir: Path) -> None:
target_files = _iter_markdown_outputs(output_dir)
if not target_files:
return
base_urls = _intersphinx_base_urls()
file_contents: dict[Path, str] = {}
candidate_base_urls: set[str] = set()
for file in target_files:
content = file.read_text(encoding="utf-8")
file_contents[file] = content
for match in URL_PATTERN.finditer(content):
url = match.group(0)
if ".html" not in url:
continue
base_url = _matching_base_url(url, base_urls)
if base_url is not None:
candidate_base_urls.add(base_url)
if not candidate_base_urls:
return
with concurrent.futures.ThreadPoolExecutor(
max_workers=min(8, len(candidate_base_urls))
) as executor:
availability = {
base_url: executor.submit(_supports_markdown, base_url)
for base_url in candidate_base_urls
}
enabled_base_urls = {
base_url for base_url, future in availability.items() if future.result()
}
if not enabled_base_urls:
return
for file, content in file_contents.items():
rewritten = URL_PATTERN.sub(
lambda match: _rewrite_url_to_markdown(
match.group(0),
enabled_base_urls,
base_urls,
),
content,
)
if rewritten != content:
file.write_text(rewritten, encoding="utf-8")
def _builder_settings(builder: str) -> list[str]:
if builder == "html":
return ["-D", "llms_txt_uri_template={docname}.md"]
if builder == "singlemarkdown":
return [
"-D",
"llms_txt_uri_template={docname}.md",
"-D",
"singlemarkdown_flavor=llm",
]
return []
def _run_builder(builder: str, source_dir: Path, build_dir: Path) -> None:
args = [
"-b",
builder,
*_builder_settings(builder),
str(source_dir),
str(build_dir / builder),
]
exit_code = sphinx_build_main(args)
if exit_code != 0:
msg = f"sphinx builder failed for '{builder}' with exit code {exit_code}"
raise RuntimeError(msg)
def build_docs() -> int:
config = load_project_config()
docs_base_url = _project_docs_base_url(config.project_id)
docs_dir = config.root / "docs"
if not docs_dir.is_dir():
print("docs directory not found", file=sys.stderr)
return 1
sphinx_build_dir = docs_dir / "_build"
sphinx_build_dir.mkdir(parents=True, exist_ok=True)
builders = ["html", "markdown", "singlemarkdown"]
with concurrent.futures.ProcessPoolExecutor(max_workers=len(builders)) as executor:
futures = [
executor.submit(
_run_builder,
builder,
docs_dir,
sphinx_build_dir,
)
for builder in builders
]
for future in concurrent.futures.as_completed(futures):
future.result()
all_dir = sphinx_build_dir / "all"
all_dir.mkdir(parents=True, exist_ok=True)
shutil.copytree(sphinx_build_dir / "html", all_dir, dirs_exist_ok=True)
shutil.copytree(sphinx_build_dir / "markdown", all_dir, dirs_exist_ok=True)
shutil.copy2(
sphinx_build_dir / "singlemarkdown" / "index.md", all_dir / "llms-full.txt"
)
_rewrite_intersphinx_links_to_markdown(all_dir)
_rewrite_llms_links_to_docs_path(all_dir, docs_base_url)
print("\nDocumentation generated in docs/_build/all.")
return 0
def update_rtd_config() -> int:
config = load_project_config()
try:
config_python_version = Version(str(config.python_version))
except InvalidVersion:
print(
f"Invalid Python version in pyproject.toml: {config.python_version}",
file=sys.stderr,
)
return 1
if config_python_version > Version(LATEST_RTD_PYTHON_VERSION):
print(
f"Configured Python ({config.python_version}) is newer than the "
f"latest Read the Docs supported Python version known to "
f"sphinx-scrapy ({LATEST_RTD_PYTHON_VERSION}).",
file=sys.stderr,
)
return 1
output = config.root / ".readthedocs.yml"
output.write_text(
"\n".join(
[
"version: 2",
"build:",
" os: ubuntu-24.04",
" tools:",
f' python: "{config.python_version}"',
" commands:",
" - pip install tox",
" - tox -e docs",
" - mkdir -p $READTHEDOCS_OUTPUT/html",
" - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/",
"",
]
),
encoding="utf-8",
)
print("Updated .readthedocs.yml")
return 0
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="sphinx-scrapy")
subparsers = parser.add_subparsers(dest="command", required=True)
subparsers.add_parser("build")
subparsers.add_parser("update-rtd-config")
return parser
def main(argv: list[str] | None = None) -> int:
parser = build_parser()
args = parser.parse_args(argv)
if args.command == "build":
return build_docs()
if args.command == "update-rtd-config":
return update_rtd_config()
parser.error(f"Unknown command: {args.command}")
return 2
if __name__ == "__main__":
raise SystemExit(main())
scrapy-sphinx-scrapy-9357640/sphinx_scrapy/config.py 0000664 0000000 0000000 00000006436 15237605002 0022546 0 ustar 00root root 0000000 0000000 from __future__ import annotations
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from packaging.utils import canonicalize_name
if sys.version_info >= (3, 11):
import tomllib
else:
import tomli as tomllib
LATEST_RTD_PYTHON_VERSION = "3.14"
_GITHUB_URL = re.compile(r"https?://github\.com/(?P<owner>[\w.-]+)/(?P<repo>[\w.-]+)")
@dataclass(frozen=True)
class ProjectConfig:
root: Path
python_version: str | None = None
extras: set[str] = field(default_factory=set)
project_id: str | None = None
github_repo: str | None = None
def find_project_root(start: Path | None = None) -> Path:
path = (start or Path.cwd()).resolve()
for candidate in (path, *path.parents):
if (candidate / "pyproject.toml").is_file():
return candidate
msg = "Could not find pyproject.toml in the current directory or parent directories"
raise FileNotFoundError(msg)
def get_extras(pyproject_data: dict[str, Any]) -> set[str]:
project_data = pyproject_data.get("project", {})
optional_dependencies = project_data.get("optional-dependencies", {})
return {str(key) for key in optional_dependencies}
def _get_github_repo(pyproject_data: dict[str, Any]) -> str | None:
urls: dict[str, Any] = pyproject_data.get("project", {}).get("urls", {})
# Repository URLs first, since other entries may point to a different
# repository, e.g. a documentation page hosted elsewhere.
keys = sorted(urls, key=lambda key: key.lower() not in {"repository", "source"})
for key in keys:
match = _GITHUB_URL.match(str(urls[key]))
if match:
owner, repo = match.group("owner", "repo")
return f"{owner}/{repo.removesuffix('.git')}"
return None
def normalize_project_id(project_id: str) -> str:
return canonicalize_name(project_id.strip())
def load_project_config(start: Path | None = None) -> ProjectConfig:
"""Load project configuration from a :file:`pyproject.toml` if available.
The file is searched for in *start* (CWD by default) and its parents. If
there is none, return a minimal ``ProjectConfig`` containing only the
resolved project root and leaving other fields empty/None.
"""
try:
project_root = find_project_root(start)
except FileNotFoundError:
resolved_root = (start or Path.cwd()).resolve()
return ProjectConfig(root=resolved_root)
pyproject_path = project_root / "pyproject.toml"
with pyproject_path.open("rb") as fp:
pyproject_data = tomllib.load(fp)
tool_data = pyproject_data.get("tool", {})
scrapy_data = tool_data.get("sphinx-scrapy", {})
python_version = scrapy_data.get("python-version", LATEST_RTD_PYTHON_VERSION)
extras = get_extras(pyproject_data)
raw_project_id = pyproject_data.get("project", {}).get("name")
project_id = None
if raw_project_id is not None:
normalized_project_id = normalize_project_id(str(raw_project_id))
if normalized_project_id:
project_id = normalized_project_id
return ProjectConfig(
root=project_root,
python_version=str(python_version) if python_version is not None else None,
extras=extras,
project_id=project_id,
github_repo=_get_github_repo(pyproject_data),
)
scrapy-sphinx-scrapy-9357640/sphinx_scrapy/tox_plugin.py 0000664 0000000 0000000 00000002014 15237605002 0023455 0 ustar 00root root 0000000 0000000 from __future__ import annotations
from typing import TYPE_CHECKING
from tox.config.loader.memory import MemoryLoader
from tox.plugin import impl
from .config import LATEST_RTD_PYTHON_VERSION, load_project_config
if TYPE_CHECKING:
from tox.config.sets import ConfigSet
from tox.session.state import State
def _python_executable(version: str) -> str:
return f"python{version}"
@impl
def tox_extend_envs() -> tuple[str, ...]:
return ("docs",)
@impl
def tox_add_core_config(core_conf: ConfigSet, state: State) -> None:
project_config = load_project_config()
state.conf.memory_seed_loaders["docs"].append(
MemoryLoader(
description="build documentation",
base_python=[
_python_executable(
project_config.python_version or LATEST_RTD_PYTHON_VERSION
)
],
deps=["-rdocs/requirements.txt"],
extras=tuple(project_config.extras),
commands=["sphinx-scrapy build"],
)
)