pax_global_header 0000666 0000000 0000000 00000000064 15257157677 0014540 g ustar 00root root 0000000 0000000 52 comment=537c5d46455ae8b2c67b53fc03b36ef1da8c4837
scrapy-w3lib-d6b4a87/ 0000775 0000000 0000000 00000000000 15257157677 0014500 5 ustar 00root root 0000000 0000000 scrapy-w3lib-d6b4a87/.git-blame-ignore-revs 0000664 0000000 0000000 00000000207 15257157677 0020577 0 ustar 00root root 0000000 0000000 # This is a list of commits that git blame should ignore
# running pre-commit on all the code
47be6dd35faf78f06b6183ea9dad6b5f78f5c756
scrapy-w3lib-d6b4a87/.github/ 0000775 0000000 0000000 00000000000 15257157677 0016040 5 ustar 00root root 0000000 0000000 scrapy-w3lib-d6b4a87/.github/dependabot.yml 0000664 0000000 0000000 00000000401 15257157677 0020663 0 ustar 00root root 0000000 0000000 ---
version: 2
updates:
- package-ecosystem: github-actions
directory: /
schedule:
interval: monthly
groups:
github-actions:
patterns:
- '*'
cooldown:
default-days: 7
scrapy-w3lib-d6b4a87/.github/workflows/ 0000775 0000000 0000000 00000000000 15257157677 0020075 5 ustar 00root root 0000000 0000000 scrapy-w3lib-d6b4a87/.github/workflows/checks.yml 0000664 0000000 0000000 00000003000 15257157677 0022051 0 ustar 00root root 0000000 0000000 ---
name: Checks
permissions:
contents: read
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
build:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
# Keep in sync with .readthedocs.yml
- python-version: '3.14'
env:
TOXENV: docs
- python-version: '3.14'
env:
TOXENV: pre-commit
- python-version: '3.14'
env:
TOXENV: pylint
- python-version: '3.14'
env:
TOXENV: typing
- python-version: '3.14'
env:
TOXENV: twinecheck
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
- name: Run check
env: ${{ matrix.env }}
run: |
pip install --upgrade pip
pip install --upgrade tox
tox
scrapy-w3lib-d6b4a87/.github/workflows/codspeed.yml 0000664 0000000 0000000 00000002315 15257157677 0022407 0 ustar 00root root 0000000 0000000 ---
name: codspeed
on:
push:
branches:
- master
pull_request:
paths:
- w3lib/**
- tests/benchmarks/**
- .github/workflows/codspeed.yml
- pyproject.toml
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
jobs:
benchmark:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python 3.13
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.13'
- name: Install dependencies
run: |
pip install --upgrade pip
pip install --upgrade tox
tox -n -e benchmark
- name: Run benchmarks
uses: CodSpeedHQ/action@373d6868929f444bc08d901fd0eb0ad52a8875ea # v5.2.1
with:
mode: simulation
run: tox -e benchmark
scrapy-w3lib-d6b4a87/.github/workflows/publish.yml 0000664 0000000 0000000 00000003152 15257157677 0022267 0 ustar 00root root 0000000 0000000 ---
name: Publish
permissions:
contents: read
on:
push:
tags:
- v[0-9]+.[0-9]+.[0-9]+
jobs:
build:
name: Build distribution
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.14'
- name: Install pypa/build
run: python3 -m pip install build --user
- name: Build a binary wheel and a source tarball
run: python3 -m build
- name: Store the distribution packages
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: python-package-distributions
path: dist/
publish:
name: Publish to PyPI
needs:
- build
runs-on: ubuntu-latest
environment:
name: pypi
url: https://pypi.org/p/w3lib
permissions:
id-token: write
steps:
- name: Download all the dists
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: python-package-distributions
path: dist/
- name: Publish distribution to PyPI
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
scrapy-w3lib-d6b4a87/.github/workflows/test.yml 0000664 0000000 0000000 00000002520 15257157677 0021576 0 ustar 00root root 0000000 0000000 ---
name: Test
permissions:
contents: read
on:
push:
branches:
- master
pull_request:
branches:
- master
env:
FORCE_COLOR: 1
jobs:
tests:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
python-version:
- '3.10'
- '3.11'
- '3.12'
- '3.13'
- '3.14'
- '3.15'
- pypy3.11
os: [windows-latest, macos-latest, ubuntu-latest]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
allow-prereleases: true
- name: Run tests
run: |
pip install --upgrade pip
pip install --upgrade tox
tox -e py
- name: Upload coverage report
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
scrapy-w3lib-d6b4a87/.gitignore 0000664 0000000 0000000 00000000256 15257157677 0016473 0 ustar 00root root 0000000 0000000 *.pyc
*egg-info
MANIFEST
.tox
venv
build
dist
docs/_build
_trial_temp
.coverage
coverage.xml
.cache
.mypy_cache/
/index.txt
.dmypy.json
.hypothesis/
.idea/
.codspeed
uv.lock
scrapy-w3lib-d6b4a87/.pre-commit-config.yaml 0000664 0000000 0000000 00000002366 15257157677 0020770 0 ustar 00root root 0000000 0000000 ---
exclude: ^tests/benchmarks/pages/
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.12
hooks:
- id: ruff-check
args: [--fix]
- id: ruff-format
- repo: https://github.com/adamchainz/blacken-docs
rev: 1.20.0
hooks:
- id: blacken-docs
additional_dependencies:
- black==26.1.0
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v6.0.0
hooks:
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/sphinx-contrib/sphinx-lint
rev: v1.0.2
hooks:
- id: sphinx-lint
- repo: https://github.com/adrienverge/yamllint.git
rev: v1.38.0
hooks:
- id: yamllint
args: [--format, parsable, --strict, -d, '{rules: {truthy: {check-keys: false}}}']
- repo: https://github.com/jumanjihouse/pre-commit-hook-yamlfmt
rev: 0.2.3
hooks:
- id: yamlfmt
- repo: https://github.com/rhysd/actionlint
rev: v1.7.12
hooks:
- id: actionlint
- repo: https://github.com/zizmorcore/zizmor-pre-commit
rev: v1.29.0
hooks:
- id: zizmor
args: [--no-progress, --fix]
scrapy-w3lib-d6b4a87/.readthedocs.yml 0000664 0000000 0000000 00000000641 15257157677 0017567 0 ustar 00root root 0000000 0000000 ---
version: 2
formats: all
sphinx:
configuration: docs/conf.py
fail_on_warning: true
build:
os: ubuntu-24.04
tools:
# For available versions, see:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
python: '3.14' # Keep in sync with .github/workflows/checks.yml
python:
install:
- requirements: docs/requirements.txt
- path: .
scrapy-w3lib-d6b4a87/CODE_OF_CONDUCT.md 0000664 0000000 0000000 00000012611 15257157677 0017300 0 ustar 00root root 0000000 0000000 # Contributor Covenant Code of Conduct
## Our Pledge
We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, caste, color, religion, or sexual
identity and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to a positive environment for our
community include:
* Demonstrating empathy and kindness toward other people
* Being respectful of differing opinions, viewpoints, and experiences
* Giving and gracefully accepting constructive feedback
* Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
* Focusing on what is best not just for us as individuals, but for the overall
community
Examples of unacceptable behavior include:
* The use of sexualized language or imagery, and sexual attention or advances of
any kind
* Trolling, insulting or derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or email address,
without their explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Enforcement Responsibilities
Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.
Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.
## Scope
This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
[opensource@zyte.com](mailto:opensource@zyte.com)
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series of
actions.
**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or permanent
ban.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within the
community.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 2.1, available at
[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
Community Impact Guidelines were inspired by
[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
For answers to common questions about this code of conduct, see the FAQ at
[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
[https://www.contributor-covenant.org/translations][translations].
[homepage]: https://www.contributor-covenant.org
[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
[Mozilla CoC]: https://github.com/mozilla/diversity
[FAQ]: https://www.contributor-covenant.org/faq
[translations]: https://www.contributor-covenant.org/translations
scrapy-w3lib-d6b4a87/LICENSE 0000664 0000000 0000000 00000002765 15257157677 0015517 0 ustar 00root root 0000000 0000000 Copyright (c) w3lib and Scrapy developers.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of Scrapy nor the names of its contributors may be used
to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
scrapy-w3lib-d6b4a87/NEWS 0000664 0000000 0000000 00000050523 15257157677 0015204 0 ustar 00root root 0000000 0000000 w3lib release notes
===================
2.5.0 (2026-09-30)
------------------
New features:
- Added support for Python 3.15 (#295).
- Added :func:`~w3lib.url.add_http_if_no_scheme`, ported from Scrapy, which
adds ``http`` as the default scheme to a URL that has none (#309).
- ``w3lib.url.parse_qsl_to_bytes``, :func:`~w3lib.url.url_query_parameter`,
:func:`~w3lib.url.add_or_replace_parameter`,
:func:`~w3lib.url.add_or_replace_parameters` and
:func:`~w3lib.url.canonicalize_url` now accept a *separator* (or
*query_separator* for :func:`~w3lib.url.canonicalize_url`) keyword argument,
to support query strings that use a separator other than ``&`` (#167).
- :func:`~w3lib.html.get_base_url` and :func:`~w3lib.html.get_meta_refresh`
now accept a *max_scan* keyword argument, an upper bound on how much of the
document they look at (#336).
- Improved the performance of most :mod:`w3lib.url` functions,
:func:`~w3lib.url.safe_url_string` in particular (#257), and of some
:mod:`w3lib.html` functions (#256).
- :func:`~w3lib.html.get_base_url` and :func:`~w3lib.html.get_meta_refresh`
are now several times faster on typical pages, which have no ```` tag
and no ```` refresh tag (#331, #332, #334, #338).
Deprecations and removals:
- The ``w3lib.util`` module is deprecated, and now emits a
:exc:`DeprecationWarning` on import (#322).
- The undocumented ``w3lib_replace`` codec error handler is no longer
registered (#318).
Security and correctness fixes:
- :func:`~w3lib.url.safe_url_string`, :func:`~w3lib.url.canonicalize_url` and
``w3lib.url.parse_url`` no longer disagree with how browsers and
:mod:`urllib.parse` read a URL in the following cases, which could let a URL
resolve to a different host or path than the one these functions reported:
- ``\`` is now treated like ``/`` in the authority and path of
special-scheme URLs (#285).
- An NFKC-normalized backslash in the host is now rejected, like other
normalized authority delimiters already were (#280).
- ASCII tab, carriage return and line feed characters are now stripped
from the whole URL, including the host (#301).
Additionally, :func:`~w3lib.url.safe_url_string` now adds a ``/`` path to a
URL that has a query but no path, since some HTTP clients otherwise send
the query alone as the request target (#297), and no longer raises
:exc:`UnicodeDecodeError` for a host that cannot be IDNA-encoded, such as
one with an empty label, when *encoding* is not UTF-8 (#319).
- :func:`~w3lib.url.canonicalize_url` now resolves dot segments (``.`` and
``..``) in the path and normalizes IPv6 addresses in the host, so that
equivalent URLs canonicalize the same (#308), and keeps percent-encoded
characters whose decoding would change their meaning, such as a semicolon
(``%3B``, a path parameter delimiter, #323) or a percent sign (``%25``,
which would otherwise decode to a bare ``%``, #306), in the path encoded.
- :func:`~w3lib.url.canonicalize_url` no longer IDNA-encodes the userinfo of a
URL together with its host, which changed both, e.g.
``http://user@éxample.com`` became ``http://xn--user@xample-fbb.com`` (#279).
- :func:`~w3lib.url.safe_download_url` now resolves percent-encoded dot
segments, such as ``%2e`` or ``%2e%2e``, like ``.`` and ``..`` (#320), and
decides whether the path keeps its trailing ``/`` based on the path alone,
rather than on whether the whole URL ends with ``/`` (#311).
- :func:`~w3lib.url.url_query_parameter` now accepts a ``bytes`` URL, instead
of parsing its ``b'…'`` representation (#325), and
:func:`~w3lib.url.url_query_cleaner` accepts one as well, instead of
raising :exc:`TypeError`, while the type hint of its *parameterlist*
argument no longer claims to accept ``bytes`` items, which never matched
anything (#257, #326).
- :func:`~w3lib.url.path_to_file_uri` and :func:`~w3lib.url.file_uri_to_path`
now round-trip Windows UNC paths, ``\\server\share\file`` becoming
``file://server/share/file`` (#324), and :func:`~w3lib.url.file_uri_to_path`
no longer raises for a path that starts with an extra pair of slashes, such
as ``////foo/bar`` (#327).
- Fixed excessive backtracking on malformed input, which could be used for
denial of service, in :func:`~w3lib.html.get_base_url` (#264),
:func:`~w3lib.html.get_meta_refresh` (#266, #288),
:func:`~w3lib.html.remove_tags_with_content` (#271),
:func:`~w3lib.html.replace_tags` and :func:`~w3lib.html.remove_tags` (#268,
#286), and :func:`~w3lib.html.unquote_markup` on unclosed CDATA sections
(#289).
- :func:`~w3lib.html.get_base_url` no longer picks up a ```` tag
written as text inside ````, now also
recognizing a closing tag followed by whitespace, ``/`` or another token
(#302).
- :func:`~w3lib.url.parse_data_uri` no longer raises on an empty quoted
parameter value, such as ``charset=""`` (#310).
- :func:`~w3lib.html.replace_entities` and :func:`~w3lib.html.get_meta_refresh`
now only treat ASCII digits as numeric character references and refresh
intervals, matching browsers (#292).
- :func:`~w3lib.html.replace_entities` now resolves a null or surrogate
numeric character reference, such as ``�`` or ``�``, to U+FFFD,
as browsers do, instead of producing a NUL or a lone surrogate that cannot
be encoded (#321).
- :func:`~w3lib.encoding.resolve_encoding`, and with it every function that
detects the encoding of a response, no longer resolves UTF-7, which
browsers do not support and which lets ASCII-only bytes smuggle markup
past filters (#307).
- :func:`~w3lib.encoding.http_content_type_encoding` now recognizes a quoted
``charset`` parameter value, and no longer stops matching a valid
``charset`` because of what follows it or of a quoted-string parameter
value that itself contains ``charset`` (#291, #293).
- :func:`~w3lib.encoding.html_body_declared_encoding` now skips HTML
comments, like :func:`~w3lib.html.get_base_url` and
:func:`~w3lib.html.get_meta_refresh` already did (#305), and finds a
charset declaration regardless of the order of the attributes in the tag,
of how the ``http-equiv`` pragma is spelled, e.g.
``httpequiv="ContentType"``, and of what surrounds ``charset=`` within a
``content`` attribute value (#299, #337).
- :func:`~w3lib.encoding.html_to_unicode` now decodes a lead ``0x80`` byte as
the euro sign when the encoding is GB18030, matching the WHATWG Encoding
Standard (#300), and decodes a BOM-less ``utf-16`` or ``utf-32`` encoding
declared in the body or auto-detected as big-endian, as it already did for
one declared in the ``Content-Type`` header (#317).
- :func:`~w3lib.html.replace_tags`, :func:`~w3lib.html.remove_tags` and
:func:`~w3lib.html.remove_tags_with_content` no longer end a tag at an
angle bracket inside a quoted attribute value, such as the ``>`` of
```` (#343).
- :func:`~w3lib.html.replace_entities` and :func:`~w3lib.html.unquote_markup`
no longer consume *keep* when it is an iterator, such as a generator, which
left some entities unkept (#342, #345), and :func:`~w3lib.html.remove_tags`
no longer raises :exc:`ValueError` when *which_ones* or *keep* is an empty
iterator and the other one is not (#345).
- :func:`~w3lib.html.get_meta_refresh` no longer reports a redirect for a
refresh ``content`` value that browsers do not follow because it uses
non-ASCII whitespace, such as U+00A0 (#353).
- :func:`~w3lib.html.get_meta_refresh` no longer prints the HTML content it
was given to stdout when a :exc:`UnicodeDecodeError` is raised (#256).
- Tests, benchmarking and CI improvements (#247, #259, #262, #281, #296,
#304, #313, #314, #329, #335, #340, #344, #350).
2.4.1 (2026-03-20)
------------------
- :func:`~w3lib.url.safe_url_string` now preserves IPv6 brackets in the URL
netloc (#253).
2.4.0 (2026-01-29)
------------------
- Dropped support for Python 3.9 and PyPy 3.10 (#250).
- Added support for Python 3.14 and PyPy 3.11 (#241, #245).
- Improved performance of :func:`~w3lib.http.headers_raw_to_dict` and
:func:`~w3lib.http.headers_dict_to_raw` (#246).
- Switched the build system to ``hatchling`` (#243).
- The obsolete ``sphinx-hoverxref`` extension is no longer used to build the
docs (#244).
- Tests and CI improvements (#237, #238, #240, #242, #248, #251).
2.3.1 (2025-01-27)
------------------
- Fix a merge error, no code changes.
2.3.0 (2025-01-27)
------------------
- Dropped Python 3.8 support (#232).
- Removed the following functions, deprecated in 2.0.0:
- ``w3lib.util.str_to_unicode``
- ``w3lib.util.to_native_str``
- ``w3lib.util.unicode_to_str``
(#235).
- Added Python 3.13 support (#232).
- Fixed running tests with newer point releases of Python 3.10 and 3.11 (#233).
- Cleanup and CI improvements (#232, #234).
2.2.1 (2024-06-12)
------------------
- :func:`~w3lib.url.canonicalize_url` no longer applies lowercase to the
userinfo URL component. (#229, #230)
2.2.0 (2024-06-05)
------------------
- Dropped Python 3.7 support (#214).
- Added Python 3.12 and PyPy 3.10 support (#218).
- Added the description to the package metadata (#227).
- Improved type hints (#226).
- Added ``.readthedocs.yml`` (#219).
- Updated the intersphinx URLs (#224).
- Added the ``pre-commit`` configuration, code reformatted with ``black``
(#220).
- Updated CI configuration (#217, #227).
2.1.2 (2023-08-03)
------------------
- Fix test failures on Python 3.11.4+ (#212, #213).
- Fix an incorrect type hint (#211).
- Add project URLs to setup.py (#215).
2.1.1 (2022-12-09)
------------------
- :func:`~w3lib.url.safe_url_string`, :func:`~w3lib.url.safe_download_url`
and :func:`~w3lib.url.canonicalize_url` now strip whitespace and control
characters urls according to the URL living standard.
2.1.0 (2022-11-28)
------------------
- Dropped Python 3.6 support, and made Python 3.11 support official. (#195,
#200)
- :func:`~w3lib.url.safe_url_string` now generates safer URLs.
To make URLs safer for the `URL living standard`_:
.. _URL living standard: https://url.spec.whatwg.org/
- ``;=`` are percent-encoded in the URL username.
- ``;:=`` are percent-encoded in the URL password.
- ``'`` is percent-encoded in the URL query if the URL scheme is `special
`__.
To make URLs safer for `RFC 2396`_ and `RFC 3986`_, ``|[]`` are
percent-encoded in URL paths, queries, and fragments.
.. _RFC 2396: https://www.ietf.org/rfc/rfc2396.txt
.. _RFC 3986: https://www.ietf.org/rfc/rfc3986.txt
(#80, #203)
- :func:`~w3lib.encoding.html_to_unicode` now checks for the `byte order
mark`_ before inspecting the ``Content-Type`` header when determining the
content encoding, in line with the `URL living standard`_. (#189, #191)
.. _byte order mark: https://en.wikipedia.org/wiki/Byte_order_mark
- :func:`~w3lib.url.canonicalize_url` now strips spaces from the input URL,
to be more in line with the `URL living standard`_. (#132, #136)
- :func:`~w3lib.html.get_base_url` now ignores HTML comments. (#70, #77)
- Fixed :func:`~w3lib.url.safe_url_string` re-encoding percent signs on
the URL username and password even when they were being used as part of an
escape sequence. (#187, #196)
- Fixed :func:`~w3lib.http.basic_auth_header` using the wrong flavor of
base64 encoding, which could prevent authentication in rare cases. (#181,
#192)
- Fixed :func:`~w3lib.html.replace_entities` raising :exc:`OverflowError` in
some cases due to `a bug in CPython
`__. (#199, #202)
- Improved typing and fixed typing issues. (#190, #206)
- Made CI and test improvements. (#197, #198)
- Adopted a Code of Conduct. (#194)
2.0.1 (2022-08-11)
------------------
Minor documentation fix (release date is set in the changelog).
2.0.0 (2022-08-11)
------------------
Backwards incompatible changes:
- Python 2 is no longer supported; Python 3.6+ is required now (#168, #175).
- :func:`w3lib.url.safe_url_string` and :func:`w3lib.url.canonicalize_url`
no longer convert "%23" to "#" when it appears in the URL path. This is a bug
fix. It's listed as a backward-incomatible change because in some cases the
output of :func:`w3lib.url.canonicalize_url` is going to change, and so, if
this output is used to generate URL fingerprints, new fingerprints might be
incompatible with those created with the previous w3lib versions
(#141).
Deprecation removals (#169):
- The ``w3lib.form`` module is removed.
- The ``w3lib.html.remove_entities`` function is removed.
- The ``w3lib.url.urljoin_rfc`` function is removed.
The following functions are deprecated, and will be removed in future releases
(#170):
- ``w3lib.util.str_to_unicode``
- ``w3lib.util.unicode_to_str``
- ``w3lib.util.to_native_str``
Other improvements and bug fixes:
- Type annotations are added (#172, #184).
- Added support for Python 3.9 and 3.10 (#168, #176).
- Fixed :func:`w3lib.html.get_meta_refresh` for ```` tags where
``http-equiv`` is written after ``content`` (#179).
- Fixed :func:`w3lib.url.safe_url_string` for IDNA domains with ports (#174).
- :func:`w3lib.url.url_query_cleaner` no longer adds an unneeded ``#`` when
``keep_fragments=True`` is passed, and the URL doesn't have a fragment
(#159).
- Removed a workaround for an ancient pathname2url bug (#142)
- CI is migrated to GitHub Actions (#166, #177); other CI improvements (#160,
#182).
- The code is formatted using black (#173).
1.22.0 (2020-05-13)
-------------------
- Python 3.4 is no longer supported (issue #156)
- :func:`w3lib.url.safe_url_string` now supports an optional ``quote_path``
parameter to disable the percent-encoding of the URL path (issue #119)
- :func:`w3lib.url.add_or_replace_parameter` and
:func:`w3lib.url.add_or_replace_parameters` no longer remove duplicate
parameters from the original query string that are not being added or
replaced (issue #126)
- :func:`w3lib.html.remove_tags` now raises a :exc:`ValueError` exception
instead of :exc:`AssertionError` when using both the ``which_ones`` and the
``keep`` parameters (issue #154)
- Test improvements (issues #143, #146, #148, #149)
- Documentation improvements (issues #140, #144, #145, #151, #152, #153)
- Code cleanup (issue #139)
1.21.0 (2019-08-09)
-------------------
- Add the ``encoding`` and ``path_encoding`` parameters to
:func:`w3lib.url.safe_download_url` (issue #118)
- :func:`w3lib.url.safe_url_string` now also removes tabs and new lines
(issue #133)
- :func:`w3lib.html.remove_comments` now also removes truncated comments
(issue #129)
- :func:`w3lib.html.remove_tags_with_content` no longer removes tags which
start with the same text as one of the specified tags (issue #114)
- Recommend pytest instead of nose to run tests (issue #124)
1.20.0 (2019-01-11)
-------------------
- Fix url_query_cleaner to do not append "?" to urls without a query string (issue #109)
- Add support for Python 3.7 and drop Python 3.3 (issue #113)
- Add `w3lib.url.add_or_replace_parameters` helper (issue #117)
- Documentation fixes (issue #115)
1.19.0 (2018-01-25)
-------------------
- Add a workaround for CPython segfault (https://bugs.python.org/issue32583)
which affect w3lib.encoding functions. This is technically **backwards
incompatible** because it changes the way non-decodable bytes are replaced
(in some cases instead of two ``\ufffd`` chars you can get one).
As a side effect, the fix speeds up decoding in Python 3.4+.
- Add 'encoding' parameter for w3lib.http.basic_auth_header.
- Fix pypy testing setup, add pypy3 to CI.
1.18.0 (2017-08-03)
-------------------
- Include additional assets used for distribution packages in the source tarball
- Consider ``[`` and ``]`` as safe characters in path and query components
of URLs, i.e. they are not escaped anymore
- Disable codecov project coverage check
1.17.0 (2017-02-08)
-------------------
- Add Python 3.5 and 3.6 support
- Add ``w3lib.url.parse_data_uri`` helper for parsing "data:" URIs
- Add ``w3lib.html.strip_html5_whitespace`` function to strip leading and
trailing whitespace as per W3C recommendations, e.g. for cleaning
"href" attribute values
- Fix ``w3lib.http.headers_raw_to_dict`` for multiple headers with same name
- Do not distribute tests/test_*.pyc artifacts
1.16.0 (2016-11-10)
-------------------
- ``canonicalize_url()`` and ``safe_url_string()``:
strip ":" when no port is specified (as per `RFC 3986`_;
see also https://github.com/scrapy/scrapy/issues/2377)
- ``url_query_cleaner()``: support new ``keep_fragments`` argument
(defaulting to ``False``)
1.15.0 (2016-07-29)
-------------------
- Add ``canonicalize_url()`` to ``w3lib.url``
1.14.3 (2016-07-14)
-------------------
Bugfix release:
- Handle IDNA encoding failures in ``safe_url_string()`` (issue #62)
1.14.2 (2016-04-11)
-------------------
Bugfix release:
- fix function import for (deprecated) ``urljoin_rfc`` (issue #51)
- only expose wanted functions from ``w3lib.url``, via ``__all__``
(see issue #54, https://github.com/scrapy/scrapy/issues/1917)
1.14.1 (2016-04-07)
-------------------
Bugfix release:
- For bytes URLs, when supplied encoding (or default UTF8) is wrong,
``safe_url_string`` falls back to percent-encoding offending bytes.
1.14.0 (2016-04-06)
-------------------
Changes to safe_url_string:
- proper handling of non-ASCII characters in Python2 and Python3
- support IDNs
- new `path_encoding` to override default UTF-8 when serializing non-ASCII
characters before percent-encoding
html_body_declared_encoding also detects encoding when not sole attribute
in ````.
Package is now properly marked as ``zip_safe``.
1.13.0 (2015-11-05)
-------------------
- remove_tags removes uppercase tags as well;
- ignore meta-redirects inside script or noscript tags by default,
but add an option to not ignore them;
- replace_entities now handles entities without trailing semicolon;
- fixed uncaught UnicodeDecodeError when decoding entities.
1.12.0 (2015-06-29)
-------------------
- meta_refresh regex now handles leading newlines and whitespaces in the url;
- include tests folder in source distribution.
1.11.0 (2015-01-13)
-------------------
- url_query_cleaner now supports str or list parameters;
- add support for resolving base URLs in tags with attributes
before href.
1.10.0 (2014-08-20)
-------------------
- reverted all 1.9.0 changes.
1.9.0 (2014-08-16)
------------------
- all url-related functions accept bytes and unicode and now return bytes.
1.8.1 (2014-08-14)
------------------
- w3lib.http.basic_auth_header now returns bytes
1.8.0 (2014-07-31)
------------------
- add support for big5-hkscs encoding.
1.7.1 (2014-07-26)
------------------
- PY3 fixed headers_raw_to_dict and headers_dict_to_raw;
- documentation improvements;
- provide wheels.
1.6 (2014-06-03)
----------------
- w3lib.form.encode_multipart is deprecated;
- docstrings and docs are improved;
- w3lib.url.add_or_replace_parameter is re-implemented on top of
stdlib functions;
- remove_entities is renamed to replace_entities.
1.5 (2013-11-09)
----------------
- Python 2.6 support is dropped.
1.4 (2013-10-18)
----------------
- Python 3 support;
- get_meta_refresh encoding handling is fixed;
- check for '?' in add_or_replace_parameter;
- ISO-8859-1 is used for HTTP Basic Auth;
- fixed unicode handling in replace_escape_chars;
1.3 (2012-05-13)
----------------
- support non-standard gb_2312_80 encoding;
- drop Python 2.5 support.
1.2 (2012-05-02)
----------------
- Detect encoding for content attr before http-equiv in meta tag.
1.1 (2012-04-18)
----------------
- w3lib.html.remove_comments handles multiline comments;
- Added w3lib.encoding module, containing functions for working with character
encoding, like encoding autodetection from HTML pages.
- w3lib.url.urljoin_rfc is deprecated.
1.0 (2011-04-17)
----------------
First release of w3lib.
scrapy-w3lib-d6b4a87/README.rst 0000664 0000000 0000000 00000001647 15257157677 0016177 0 ustar 00root root 0000000 0000000 =====
w3lib
=====
.. image:: https://github.com/scrapy/w3lib/actions/workflows/test.yml/badge.svg
:target: https://github.com/scrapy/w3lib/actions/workflows/test.yml
.. image:: https://img.shields.io/codecov/c/github/scrapy/w3lib/master.svg
:target: https://codecov.io/github/scrapy/w3lib?branch=master
:alt: Coverage report
Overview
========
This is a Python library of web-related functions, such as:
* remove comments, or tags from HTML snippets
* extract base url from HTML snippets
* translate entities on HTML strings
* convert raw HTTP headers to dicts and vice-versa
* construct HTTP auth header
* converting HTML pages to unicode
* sanitize urls (like browsers do)
* extract arguments from urls
Requirements
============
Python 3.10+
Install
=======
``pip install w3lib``
Documentation
=============
See https://w3lib.readthedocs.io/
License
=======
The w3lib library is licensed under the BSD license.
scrapy-w3lib-d6b4a87/codecov.yml 0000664 0000000 0000000 00000000233 15257157677 0016643 0 ustar 00root root 0000000 0000000 ---
comment:
layout: header, diff, tree
coverage:
status:
patch: false
project:
default:
target: 100%
scrapy-w3lib-d6b4a87/conftest.py 0000664 0000000 0000000 00000000102 15257157677 0016670 0 ustar 00root root 0000000 0000000 collect_ignore = [
# warning on import
"w3lib/util.py",
]
scrapy-w3lib-d6b4a87/docs/ 0000775 0000000 0000000 00000000000 15257157677 0015430 5 ustar 00root root 0000000 0000000 scrapy-w3lib-d6b4a87/docs/Makefile 0000664 0000000 0000000 00000012670 15257157677 0017076 0 ustar 00root root 0000000 0000000 # Makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
PAPER =
BUILDDIR = _build
# Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
# the i18n builder cannot share the environment and doctrees with the others
I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
help:
@echo "Please use \`make ' where is one of"
@echo " html to make standalone HTML files"
@echo " dirhtml to make HTML files named index.html in directories"
@echo " singlehtml to make a single large HTML file"
@echo " pickle to make pickle files"
@echo " json to make JSON files"
@echo " htmlhelp to make HTML files and a HTML help project"
@echo " qthelp to make HTML files and a qthelp project"
@echo " devhelp to make HTML files and a Devhelp project"
@echo " epub to make an epub"
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
@echo " latexpdf to make LaTeX files and run them through pdflatex"
@echo " text to make text files"
@echo " man to make manual pages"
@echo " texinfo to make Texinfo files"
@echo " info to make Texinfo files and run them through makeinfo"
@echo " gettext to make PO message catalogs"
@echo " changes to make an overview of all changed/added/deprecated items"
@echo " linkcheck to check all external links for integrity"
@echo " doctest to run all doctests embedded in the documentation (if enabled)"
clean:
-rm -rf $(BUILDDIR)/*
html:
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
dirhtml:
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
singlehtml:
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
@echo
@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
pickle:
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
@echo
@echo "Build finished; now you can process the pickle files."
json:
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
@echo
@echo "Build finished; now you can process the JSON files."
htmlhelp:
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
@echo
@echo "Build finished; now you can run HTML Help Workshop with the" \
".hhp project file in $(BUILDDIR)/htmlhelp."
qthelp:
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
@echo
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
".qhcp project file in $(BUILDDIR)/qthelp, like this:"
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/w3lib.qhcp"
@echo "To view the help file:"
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/w3lib.qhc"
devhelp:
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
@echo
@echo "Build finished."
@echo "To view the help file:"
@echo "# mkdir -p $$HOME/.local/share/devhelp/w3lib"
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/w3lib"
@echo "# devhelp"
epub:
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
@echo
@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
latex:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
@echo "Run \`make' in that directory to run these through (pdf)latex" \
"(use \`make latexpdf' here to do that automatically)."
latexpdf:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through pdflatex..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
text:
$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
@echo
@echo "Build finished. The text files are in $(BUILDDIR)/text."
man:
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
@echo
@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
texinfo:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo
@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
@echo "Run \`make' in that directory to run these through makeinfo" \
"(use \`make info' here to do that automatically)."
info:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo "Running Texinfo files through makeinfo..."
make -C $(BUILDDIR)/texinfo info
@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
gettext:
$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
@echo
@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
changes:
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
@echo
@echo "The overview file is in $(BUILDDIR)/changes."
linkcheck:
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
@echo
@echo "Link check complete; look for any errors in the above output " \
"or in $(BUILDDIR)/linkcheck/output.txt."
doctest:
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
@echo "Testing of doctests in the sources finished, look at the " \
"results in $(BUILDDIR)/doctest/output.txt."
scrapy-w3lib-d6b4a87/docs/conf.py 0000664 0000000 0000000 00000020376 15257157677 0016737 0 ustar 00root root 0000000 0000000 # w3lib documentation build configuration file, created by
# sphinx-quickstart on Sun Jan 26 22:19:38 2014.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
import sys
from pathlib import Path
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, str(Path(__file__).parent.parent))
# -- General configuration -----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
"notfound.extension",
"sphinx.ext.autodoc",
"sphinx.ext.doctest",
"sphinx.ext.intersphinx",
"sphinx.ext.viewcode",
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]
# The suffix of source filenames.
source_suffix = {".rst": "restructuredtext"}
# The encoding of source files.
# source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = "index"
# General information about the project.
project = "w3lib"
project_copyright = "2014, w3lib developers"
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The full version, including alpha/beta/rc tags.
release = "2.5.0"
# The short X.Y version.
version = ".".join(release.split(".")[:2])
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
# language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ["_build"]
# The reST default role (used for this markup: `text`) to use for all documents.
# default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
# add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = "sphinx"
# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []
# -- Options for HTML output ---------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = "sphinx_rtd_theme"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []
# The name for this set of Sphinx documents. If None, it defaults to
# " v documentation".
# html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
# html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = []
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}
# If false, no module index is generated.
# html_domain_indices = True
# If false, no index is generated.
# html_use_index = True
# If true, the index is split into individual pages for each letter.
# html_split_index = False
# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = "w3libdoc"
# -- Options for LaTeX output --------------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
# 'preamble': '',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
("index", "w3lib.tex", "w3lib Documentation", "w3lib developers", "manual"),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False
# If true, show page references after internal links.
# latex_show_pagerefs = False
# If true, show URL addresses after external links.
# latex_show_urls = False
# Documents to append as an appendix to all manuals.
# latex_appendices = []
# If false, no module index is generated.
# latex_domain_indices = True
# -- Options for manual page output --------------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [("index", "w3lib", "w3lib Documentation", ["w3lib developers"], 1)]
# If true, show URL addresses after external links.
# man_show_urls = False
# -- Options for Texinfo output ------------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(
"index",
"w3lib",
"w3lib Documentation",
"w3lib developers",
"w3lib",
"One line description of project.",
"Miscellaneous",
),
]
# Documents to append as an appendix to all manuals.
# texinfo_appendices = []
# If false, no module index is generated.
# texinfo_domain_indices = True
# How to display URL addresses: 'footnote', 'no', or 'inline'.
# texinfo_show_urls = 'footnote'
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {
"pytest": ("https://docs.pytest.org/en/latest", None),
"python": ("https://docs.python.org/3", None),
"scrapy": ("https://docs.scrapy.org/en/latest", None),
"tox": ("https://tox.wiki/en/latest", None),
}
# -- Nitpicking options -------------------------------------------------------
nitpicky = True
scrapy-w3lib-d6b4a87/docs/index.rst 0000664 0000000 0000000 00000002671 15257157677 0017277 0 ustar 00root root 0000000 0000000 Welcome to w3lib's documentation!
=================================
Overview
========
This is a Python library of web-related functions, such as:
* remove comments, or tags from HTML snippets
* extract base url from HTML snippets
* translate entities on HTML strings
* convert raw HTTP headers to dicts and vice-versa
* construct HTTP auth header
* converting HTML pages to unicode
* sanitize urls (like browsers do)
* extract arguments from urls
The w3lib library is licensed under the BSD license.
Modules
=======
.. toctree::
:maxdepth: 4
w3lib
Requirements
============
Python 3.10+
Install
=======
``pip install w3lib``
Tests
=====
:doc:`pytest ` is the preferred way to run tests. Just run:
``pytest`` from the root directory to execute tests using the default Python
interpreter.
:doc:`tox ` could be used to run tests for all supported Python
versions. Install it (using 'pip install tox') and then run ``tox`` from
the root directory - tests will be executed for all available
Python interpreters.
Changelog
=========
.. include:: ../NEWS
:start-line: 3
History
-------
The code of w3lib was originally part of the :doc:`Scrapy framework
` but was later stripped out of Scrapy, with the aim of make it
more reusable and to provide a useful library of web functions without
depending on Scrapy.
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
scrapy-w3lib-d6b4a87/docs/make.bat 0000664 0000000 0000000 00000011746 15257157677 0017046 0 ustar 00root root 0000000 0000000 @ECHO OFF
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set BUILDDIR=_build
set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
set I18NSPHINXOPTS=%SPHINXOPTS% .
if NOT "%PAPER%" == "" (
set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
)
if "%1" == "" goto help
if "%1" == "help" (
:help
echo.Please use `make ^` where ^ is one of
echo. html to make standalone HTML files
echo. dirhtml to make HTML files named index.html in directories
echo. singlehtml to make a single large HTML file
echo. pickle to make pickle files
echo. json to make JSON files
echo. htmlhelp to make HTML files and a HTML help project
echo. qthelp to make HTML files and a qthelp project
echo. devhelp to make HTML files and a Devhelp project
echo. epub to make an epub
echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
echo. text to make text files
echo. man to make manual pages
echo. texinfo to make Texinfo files
echo. gettext to make PO message catalogs
echo. changes to make an overview over all changed/added/deprecated items
echo. linkcheck to check all external links for integrity
echo. doctest to run all doctests embedded in the documentation if enabled
goto end
)
if "%1" == "clean" (
for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
del /q /s %BUILDDIR%\*
goto end
)
if "%1" == "html" (
%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The HTML pages are in %BUILDDIR%/html.
goto end
)
if "%1" == "dirhtml" (
%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
goto end
)
if "%1" == "singlehtml" (
%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
goto end
)
if "%1" == "pickle" (
%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can process the pickle files.
goto end
)
if "%1" == "json" (
%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can process the JSON files.
goto end
)
if "%1" == "htmlhelp" (
%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can run HTML Help Workshop with the ^
.hhp project file in %BUILDDIR%/htmlhelp.
goto end
)
if "%1" == "qthelp" (
%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can run "qcollectiongenerator" with the ^
.qhcp project file in %BUILDDIR%/qthelp, like this:
echo.^> qcollectiongenerator %BUILDDIR%\qthelp\w3lib.qhcp
echo.To view the help file:
echo.^> assistant -collectionFile %BUILDDIR%\qthelp\w3lib.ghc
goto end
)
if "%1" == "devhelp" (
%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
if errorlevel 1 exit /b 1
echo.
echo.Build finished.
goto end
)
if "%1" == "epub" (
%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The epub file is in %BUILDDIR%/epub.
goto end
)
if "%1" == "latex" (
%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
if errorlevel 1 exit /b 1
echo.
echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
goto end
)
if "%1" == "text" (
%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The text files are in %BUILDDIR%/text.
goto end
)
if "%1" == "man" (
%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The manual pages are in %BUILDDIR%/man.
goto end
)
if "%1" == "texinfo" (
%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
goto end
)
if "%1" == "gettext" (
%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
goto end
)
if "%1" == "changes" (
%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
if errorlevel 1 exit /b 1
echo.
echo.The overview file is in %BUILDDIR%/changes.
goto end
)
if "%1" == "linkcheck" (
%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
if errorlevel 1 exit /b 1
echo.
echo.Link check complete; look for any errors in the above output ^
or in %BUILDDIR%/linkcheck/output.txt.
goto end
)
if "%1" == "doctest" (
%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
if errorlevel 1 exit /b 1
echo.
echo.Testing of doctests in the sources finished, look at the ^
results in %BUILDDIR%/doctest/output.txt.
goto end
)
:end
scrapy-w3lib-d6b4a87/docs/requirements.txt 0000664 0000000 0000000 00000000055 15257157677 0020714 0 ustar 00root root 0000000 0000000 sphinx
sphinx-notfound-page
sphinx_rtd_theme
scrapy-w3lib-d6b4a87/docs/w3lib.rst 0000664 0000000 0000000 00000000733 15257157677 0017205 0 ustar 00root root 0000000 0000000 w3lib Package
=============
:mod:`~w3lib.encoding` Module
-----------------------------
.. automodule:: w3lib.encoding
:members:
:mod:`~w3lib.html` Module
-------------------------
.. automodule:: w3lib.html
:members:
:mod:`~w3lib.http` Module
-------------------------
.. automodule:: w3lib.http
:members:
:mod:`~w3lib.url` Module
------------------------
.. automodule:: w3lib.url
:members:
.. autoclass:: ParseDataURIResult
:members:
scrapy-w3lib-d6b4a87/pyproject.toml 0000664 0000000 0000000 00000013300 15257157677 0017411 0 ustar 00root root 0000000 0000000 [build-system]
requires = ["hatchling>=1.27.0"]
build-backend = "hatchling.build"
[project]
name = "w3lib"
description = "Library of web-related functions"
readme = "README.rst"
license = "BSD-3-Clause"
license-files = ["LICENSE"]
authors = [
{ name = "Scrapy project", email = "info@scrapy.org" },
]
classifiers = [
"Development Status :: 5 - Production/Stable",
"Intended Audience :: Developers",
"Operating System :: OS Independent",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
"Programming Language :: Python :: 3.15",
"Programming Language :: Python :: Implementation :: CPython",
"Programming Language :: Python :: Implementation :: PyPy",
"Topic :: Internet :: WWW/HTTP",
"Topic :: Software Development :: Libraries :: Python Modules",
]
requires-python = ">=3.10"
dynamic = ["version"]
[project.urls]
Homepage = "https://github.com/scrapy/w3lib"
Documentation = "https://w3lib.readthedocs.io/en/latest/"
Source = "https://github.com/scrapy/w3lib"
Tracker = "https://github.com/scrapy/w3lib/issues"
"Release notes" = "https://w3lib.readthedocs.io/en/latest/#changelog"
[tool.hatch.version]
path = "w3lib/__init__.py"
[tool.hatch.build.targets.sdist]
include = [
"/docs",
"/w3lib",
"/tests",
"/NEWS",
"/codecov.yml",
"/tox.ini",
]
[tool.bumpversion]
current_version = "2.5.0"
commit = true
tag = true
tag_name = "v{new_version}"
[[tool.bumpversion.files]]
filename = "w3lib/__init__.py"
[[tool.bumpversion.files]]
filename = "docs/conf.py"
[[tool.bumpversion.files]]
filename = "NEWS"
search = "\\(unreleased\\)$"
replace = "({now:%Y-%m-%d})"
regex = true
[tool.coverage.run]
branch = true
[tool.mypy]
strict = true
[[tool.mypy.overrides]]
# Allow test functions to be untyped
module = "tests.*"
allow_untyped_defs = true
[tool.pylint.MASTER]
persistent = "no"
[tool.pylint."MESSAGES CONTROL"]
enable = [
"useless-suppression",
]
disable = [
"fixme",
"import-error",
"import-outside-toplevel",
"invalid-name",
"line-too-long",
"missing-class-docstring",
"missing-function-docstring",
"missing-module-docstring",
"raise-missing-from",
"redefined-builtin",
"redefined-outer-name",
"too-few-public-methods",
"too-many-arguments",
"too-many-branches",
"too-many-lines",
"too-many-positional-arguments",
"too-many-public-methods",
"unused-argument",
"unused-import",
"unused-variable",
"useless-import-alias",
"possibly-used-before-assignment",
]
[tool.ruff.lint]
extend-select = [
# flake8-builtins
"A",
# flake8-async
"ASYNC",
# flake8-bugbear
"B",
# flake8-comprehensions
"C4",
# flake8-commas
"COM",
# pydocstyle
"D",
# flake8-future-annotations
"FA",
# flynt
"FLY",
# refurb
"FURB",
# isort
"I",
# flake8-implicit-str-concat
"ISC",
# flake8-logging
"LOG",
# Perflint
"PERF",
# pygrep-hooks
"PGH",
# flake8-pie
"PIE",
# pylint
"PL",
# flake8-pytest-style
"PT",
# flake8-use-pathlib
"PTH",
# flake8-pyi
"PYI",
# flake8-quotes
"Q",
# flake8-return
"RET",
# flake8-raise
"RSE",
# Ruff-specific rules
"RUF",
# flake8-bandit
"S",
# flake8-simplify
"SIM",
# flake8-slots
"SLOT",
# flake8-debugger
"T10",
# flake8-type-checking
"TC",
# pyupgrade
"UP",
# pycodestyle warnings
"W",
# flake8-2020
"YTT",
]
ignore = [
# Within an `except` clause, raise exceptions with `raise ... from`
"B904",
# Trailing comma missing
"COM812",
# Missing docstring in public module
"D100",
# Missing docstring in public class
"D101",
# Missing docstring in public method
"D102",
# Missing docstring in public function
"D103",
# Missing docstring in public package
"D104",
# Missing docstring in magic method
"D105",
# Missing docstring in public nested class
"D106",
# Missing docstring in __init__
"D107",
# One-line docstring should fit on one line with quotes
"D200",
# No blank lines allowed after function docstring
"D202",
# 1 blank line required between summary line and description
"D205",
# Multi-line docstring closing quotes should be on a separate line
"D209",
# First line should end with a period
"D400",
# First line should be in imperative mood; try rephrasing
"D401",
# First line should not be the function's "signature"
"D402",
# First word of the first line should be properly capitalized
"D403",
# No blank lines allowed between a section header and its content
"D412",
# Too many return statements
"PLR0911",
# Too many branches
"PLR0912",
# Too many arguments in function definition
"PLR0913",
# Too many statements
"PLR0915",
# Magic value used in comparison
"PLR2004",
# String contains ambiguous {}.
"RUF001",
# Docstring contains ambiguous {}.
"RUF002",
# Comment contains ambiguous {}.
"RUF003",
# Mutable class attributes should be annotated with `typing.ClassVar`
"RUF012",
# Use of `assert` detected
"S101",
# Import alias does not rename original package
"PLC0414",
# Too many positional arguments
"PLR0917",
# Private name import
"PLC2701",
]
[tool.ruff.lint.isort]
split-on-trailing-comma = false
combine-as-imports = true
[tool.ruff.lint.pydocstyle]
convention = "pep257"
scrapy-w3lib-d6b4a87/tests/ 0000775 0000000 0000000 00000000000 15257157677 0015642 5 ustar 00root root 0000000 0000000 scrapy-w3lib-d6b4a87/tests/__init__.py 0000664 0000000 0000000 00000000000 15257157677 0017741 0 ustar 00root root 0000000 0000000 scrapy-w3lib-d6b4a87/tests/benchmarks/ 0000775 0000000 0000000 00000000000 15257157677 0017757 5 ustar 00root root 0000000 0000000 scrapy-w3lib-d6b4a87/tests/benchmarks/__init__.py 0000664 0000000 0000000 00000000621 15257157677 0022067 0 ustar 00root root 0000000 0000000 import sys
from collections.abc import Callable, Iterable
from typing import Any, TypeAlias
import pytest
pytest.importorskip("pytest_codspeed", reason="Benchmark tests require pytest-codspeed")
BENCHMARK_MARKS = [
pytest.mark.benchmark,
pytest.mark.skipif("PyPy" in sys.version, reason="CodSpeed doesn't support PyPy"),
]
CasesMapType: TypeAlias = dict[Callable[..., Any], Iterable[Any]]
scrapy-w3lib-d6b4a87/tests/benchmarks/pages/ 0000775 0000000 0000000 00000000000 15257157677 0021056 5 ustar 00root root 0000000 0000000 scrapy-w3lib-d6b4a87/tests/benchmarks/pages/base-href-in-head.html 0000664 0000000 0000000 00000034210 15257157677 0025103 0 ustar 00root root 0000000 0000000
scrapy-w3lib-d6b4a87/tests/benchmarks/pages/base-without-href.html 0000664 0000000 0000000 00006173560 15257157677 0025323 0 ustar 00root root 0000000 0000000
Distinctio quo Amet in Dolores-Deleniti Veniam, UT - 796 Eveniet in Dolores-Deleniti Veniam, UT | Itaque
Distinctio quo Amet in Dolores-Deleniti Veniam, UT
Sed Odio do Dolores quidem debitis harum sit sapiente assumenda in l soluta doloribus pariatur in Dolores, Nostrud. Et l occaecat, sed'id lorem sint dolore ad harum voluptas, non similique, sit...
Distinctio quo Amet in Dolores-Deleniti Veniam, UT
Laboris quis ad Sed Dolorum do Dolores. Tempor sunt qui 1- sit 2-ducimus distinctio in Dolores, Nostrud, sit quos et cum. Sunt l proident facere, nobis-labore atque, sit blanditiis occaecat sit...
Distinctio quo Amet in Dolores-Deleniti Veniam, UT
Laboris ad Sed Labore, debitis at 1020 Anim Nam in Dolores, UT. Qui cum-officiis cupidatat quidem proident 1, 2, 3, sit 4-ducimus distinctio laboris sunt 820 ad 1,466 id.ad., dolorum quo l dolorum do...
Sed esse officiis ad nisi labore quo fuga at odio non sint omnis in fuga sunt sed Mollit Consectetur Laboris do Nostrud NAM. Nisi labore expedita esse do similique animi culpa quas Itaque sit tempor quod sed Mollit Consectetur quod sit corrupti dignissimos dolor sunt sapiente sed quod do sed ullamco laborum. Dignissimos tempor deleniti nam sed cupiditate. Similique 2026 Nostrud NAM. Cum maxime corrupti.
Cum corrupti officia pariatur sit dignissimos sit expedita do Maxime Corrupti, QUI | NMLS #3030; www.NMLSConsumerAccess.org. Voluptas in 50 soluta. Odio sint ex sed reiciendis do sed Cum Fuga Culpa Blanditiis do Accusamus Delectus quo corrupti exercitation do quas exercitation reiciendis eiusmod ad reiciendis debitis in sed Culpa do Cum Fuga.
Itaque Praesentium ex ad distinctio sapiente do Maxime Placeat Dignissimos. Fuga ducimus, sit atque voluptatibus, sit deserunt ipsum mollitia occaecat sit aliquip eiusmod sit est cupiditate sit asperiores voluptatum.
Do sed sit alias l aliqua cumque, do fugiat debitis tempore odio facilis, veniam quas Itaque Expedita Ducimus quo vero at 1-555-012-1949.
Aute in 2006, assumenda est nam incididunt d’delectus qui similique l’harum v ad voluptatibus ad eveniet duis, recusandae ad voluptas dolore et fuga do ullamco at velit ut maiores ad quas.
assumenda l in s’dolorum in deleniti eos…
Non Tempore Tenetur Recusandae - Vero (B/F/M) (M/F/M)
Facere d'quidem
Doloribus : Facere d'quidem.
Earum 75010 10z Necessitatibus
Itaque v Earum 75010 10z Necessitatibus.
24/08/2026
Aute ad omnis : 24/08/2026.
maiores: release-2026-09-15.21120069
scrapy-w3lib-d6b4a87/tests/benchmarks/test_benchmark_encoding.py 0000664 0000000 0000000 00000015321 15257157677 0025172 0 ustar 00root root 0000000 0000000 from __future__ import annotations
from typing import TYPE_CHECKING, Any
import pytest
from tests.benchmarks import BENCHMARK_MARKS, CasesMapType
from w3lib.encoding import (
html_body_declared_encoding,
html_to_unicode,
http_content_type_encoding,
read_bom,
resolve_encoding,
to_unicode,
)
if TYPE_CHECKING:
from collections.abc import Callable
from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]
pytestmark = BENCHMARK_MARKS
BENCHMARK_CASES: CasesMapType = {
read_bom: [
((b"\xfe\xff\x6c\x34",), {}),
((b"\xff\xfe\x34\x6c",), {}),
((b"\x00\x00\xfe\xff\x00\x00\x6c\x34",), {}),
((b"\xff\xfe\x00\x00\x34\x6c\x00\x00",), {}),
((b"foo",), {}),
((b"",), {}),
],
http_content_type_encoding: [
(("Content-Type: text/html; charset=ISO-8859-4",), {}),
(("something else",), {}),
],
html_body_declared_encoding: [
(
(
b"""""",
),
{},
),
(
(
b"""\n""",
),
{},
),
(
(
b"""""",
),
{},
),
(
(
b"""""",
),
{},
),
(
(
b"""""",
),
{},
),
(
(
b"""""",
),
{},
),
(
(
b""" bad html still supported < meta http-equiv='Content-Type'\n content="text/html; charset=utf-8">""",
),
{},
),
((b"""""",), {}),
((b"""""",), {}),
((b"""""",), {}),
((b"something else",), {}),
(
(
b"""
this isn't searched
""",
),
{},
),
(
(
b"""""",
),
{},
),
(("something else",), {}),
(("""""",), {}),
(
(
"""
this isn't searched
""",
),
{},
),
(
(
"""""",
),
{},
),
],
resolve_encoding: [
(("latin1",), {}),
((" Latin-1",), {}),
(("gb_2312-80",), {}),
(("unknown encoding",), {}),
],
to_unicode: [
((b"\xc2\xa3", "utf-8"), {}),
((b"\xc2\xc2\xa3", "utf-8"), {}),
],
html_to_unicode: [
(
(
"Content-Type: text/html; charset=cp1251",
b"\xea\xe8\xf0\xe8\xeb\xeb\xe8\xf7\xe5\xf1\xea\xe8\xe9 \xf2\xe5\xea\xf1\xf2",
),
{},
),
(("Content-Type: text/html; charset=utf-8", b"\xc2\xa3"), {}),
(("Content-Type: text/html; charset=iso-8859-1", b"\xa3"), {}),
(("Content-Type: text/html; charset=", b"\xc2\xa3"), {}),
(("Content-Type: text/html; charset=none", b"\xc2\xa3"), {}),
(("Content-Type: text/html; charset=gb2312", b"\xa8D"), {}),
(("Content-Type: text/html; charset=gbk", b"\xa8D"), {}),
(("Content-Type: text/html; charset=big5", b"\xf9\xda"), {}),
(
(
"Content-Type: text/html; charset=utf-8",
b"\xef\xbb\xbfWORD\xe3\xabWORD2",
),
{},
),
((None, b"\xef\xbb\xbfWORD\xe3\xabWORD2"), {}),
(("Content-Type: text/html; charset=utf-8", b"\xef\xbb\xbfWORD\xe3\xab"), {}),
((None, b"\xef\xbb\xbfWORD\xe3\xab"), {}),
(("Content-Type: text/html; charset=utf-8", b"PREFIX\xe3\xabSUFFIX"), {}),
(("Content-Type: text/html; charset=utf-8", b"\xf0value"), {}),
(("Content-Type: text/html; charset=utf-8", b"\xef\xbb\xbfhi"), {}),
((None, b"\xef\xbb\xbfhi"), {}),
(("Content-Type: text/html; charset=utf-16", b"\xfe\xff\x00h\x00i"), {}),
((None, b"\xfe\xff\x00h\x00i"), {}),
(("Content-Type: text/html; charset=utf-16", b"\xff\xfeh\x00i\x00"), {}),
((None, b"\xff\xfeh\x00i\x00"), {}),
(
(
"Content-Type: text/html; charset=utf-32",
b"\x00\x00\xfe\xff\x00\x00\x00h\x00\x00\x00i",
),
{},
),
((None, b"\x00\x00\xfe\xff\x00\x00\x00h\x00\x00\x00i"), {}),
(
(
None,
b"""blah blah < meta http-equiv="Content-Type"
content="text/html; charset=iso-8859-1"> other stuff""",
),
{},
),
(
(
"Content-Type: text/html; charset=utf-8",
b"""blah blah < meta http-equiv="Content-Type"
content="text/html; charset=iso-8859-1"> other stuff""",
),
{},
),
((None, b"\xef\xbb\xbfblah blah"), {}),
((None, b""""""), {"auto_detect_fun": lambda x: "ascii"}),
((None, b"no encoding info"), {"auto_detect_fun": lambda x: "ascii"}),
((None, b"no encoding info"), {}),
((None, b"no encoding info"), {"default_encoding": "ascii"}),
((None, b""), {}),
],
}
@pytest.mark.parametrize("func", BENCHMARK_CASES)
def test_benchmark_encoding(
benchmark: BenchmarkFixture,
func: Callable[..., Any],
) -> None:
@benchmark
def factory():
for args, kwargs in BENCHMARK_CASES[func]:
func(*args, **kwargs)
@pytest.mark.parametrize("func", BENCHMARK_CASES)
def test_benchmark_encoding_cold(
benchmark: BenchmarkFixture,
func: Callable[..., Any],
) -> None:
"""Same cases as test_benchmark_encoding but with the resolve_encoding LRU
cache cleared before every round."""
def factory():
for args, kwargs in BENCHMARK_CASES[func]:
func(*args, **kwargs)
benchmark.pedantic(factory, setup=resolve_encoding.cache_clear, rounds=100)
scrapy-w3lib-d6b4a87/tests/benchmarks/test_benchmark_html.py 0000664 0000000 0000000 00000023137 15257157677 0024354 0 ustar 00root root 0000000 0000000 from __future__ import annotations
from pathlib import Path
from typing import TYPE_CHECKING, Any
import pytest
from tests.benchmarks import BENCHMARK_MARKS, CasesMapType
from w3lib.html import (
get_base_url,
get_meta_refresh,
has_entities,
remove_comments,
remove_tags,
remove_tags_with_content,
replace_entities,
replace_escape_chars,
replace_tags,
unquote_markup,
)
if TYPE_CHECKING:
from collections.abc import Callable
from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]
pytestmark = BENCHMARK_MARKS
# Real pages, one per way a document can shape the cost of a lookup. In
# a sample of 246 pages from 96 popular sites (September 2026), 235 had no
# at all, 3 had a without href, and 1 had a in the
# head, so the no-base page is the representative workload and the others are
# the edge cases worth watching.
PAGES = {
path.stem: path.read_bytes()
for path in sorted((Path(__file__).parent / "pages").glob("*.html"))
}
BENCHMARK_CASES: CasesMapType = {
replace_entities: [
((b"no entities",), {}),
((b"Price: £100!",), {}),
(("no entities",), {}),
(("Price: £100!",), {}),
(("As low as £100!",), {}),
((b"As low as £100!",), {}),
(
(
"redirectTo=search&searchtext=MR0221Y&aff=buyat&affsrc=d_data&cm_mmc=buyat-_-ELECTRICAL & SEASONAL-_-MR0221Y-_-9-carat gold ½oz solid crucifix pendant",
),
{},
),
((b"Low < High & Medium £ six",), {"keep": ["lt", "amp"]}),
(("Low < High & Medium £ six",), {"keep": ["lt", "amp"]}),
(("a < b &illegal; c � six",), {"remove_illegal": False}),
(("a < b &illegal; c � six",), {"remove_illegal": True}),
(("x≤y",), {}),
(("xy",), {}),
(("xy",), {"remove_illegal": False}),
(("�",), {}),
(("�",), {"remove_illegal": False}),
(("x™y",), {"encoding": "cp1252"}),
(("x™y",), {"encoding": "cp1252"}),
(("x\x99™™y",), {"encoding": "cp1252"}),
(("<<!",), {"encoding": "cp1252"}),
(("<!",), {"encoding": "cp1252"}),
(("A ",), {"encoding": "cp1252"}),
(("A!",), {"encoding": "cp1252"}),
(("Ah",), {"encoding": "cp1252"}),
(("A!",), {"encoding": "cp1252"}),
(("Ax",), {"encoding": "cp1252"}),
(("³!",), {"encoding": "cp1252"}),
(("Á!",), {"encoding": "cp1252"}),
(("☃!",), {"encoding": "cp1252"}),
(("™",), {"encoding": "cp1252"}),
(("™",), {"encoding": "cp1252"}),
],
replace_tags: [
((b"no entities",), {}),
(("no entities",), {}),
(("This text contains some tag",), {}),
((b"This text is very important", " "), {}),
((b'Click here',), {}),
],
remove_comments: [
((b"without comments",), {}),
((b"",), {}),
(("without comments",), {}),
(("",), {}),
(("text without comments",), {}),
(("",), {}),
(("Hello",), {}),
(("Hello",), {}),
((b"test whatever",), {}),
((b"test whatever",), {}),
((b"test ') is None
assert (
html_body_declared_encoding(b'')
== "utf-8"
)
@pytest.mark.parametrize(
("body", "expected"),
[
# a "charset=" inside an unrelated attribute value is not a
# declaration, and neither is the tail of another attribute's name
(
b'',
None,
),
(b'', None),
(
b'',
"utf-8",
),
(
b'',
"utf-8",
),
# a charset inside a content attribute needs the http-equiv
# pragma, however the pragma is spelled, but with a value
(b'', None),
(b'', None),
(
b'',
"gb18030",
),
# another pragma is not the content-type one, however its own
# content value spells "charset="
(b'', None),
(
b'',
None,
),
(
b'',
None,
),
# only the first content attribute of a tag is read, and an empty
# charset attribute does not hide it
(
b'',
"gb18030",
),
(
b'',
"gb18030",
),
# a charset attribute wins over a content one, wherever it is
(
b'',
"utf-8",
),
# a quoted ">" does not end the tag, and a quoted tag written
# inside one neither ends nor stops the scan
(b'', "utf-8"),
(b'', "utf-8"),
# an empty charset attribute is not a declaration
(b'', None),
(b'', "utf-8"),
# a quoted "a charset="big5">', None),
(b'>', "big5hkscs"),
# an unterminated comment hides everything after it
(b'"), str)
assert isinstance(remove_comments("without comments"), str)
assert isinstance(remove_comments(""), str)
def test_no_comments(self):
# text without comments
assert remove_comments("text without comments") == "text without comments"
def test_remove_comments(self):
# text with comments
assert remove_comments("") == ""
assert remove_comments("Hello") == "Hello"
assert remove_comments("Hello") == "Hello"
assert remove_comments(b"test whatever") == "test whatever"
assert (
remove_comments(b"test whatever") == "test whatever"
)
assert remove_comments(b"test """) == ""
assert get_base_url(""" """
)
== "http://example_2.com/"
)
assert (
get_base_url(
""" """
)
== "http://example_3.com/"
)
def test_base_url_in_script(self):
baseurl = "https://example.org"
# A browser does not parse tags inside """,
baseurl,
)
== "https://example.org"
)
assert (
get_base_url(
"""""",
baseurl,
)
== "https://example.org"
)
# a real after the ignored one is still picked up
assert (
get_base_url(
""""""
"""""",
baseurl,
)
== "http://example.org/found/"
)
# an unterminated "
)
assert get_meta_refresh(body, baseurl) == (None, None)
def test_unterminated_ignored_tag(self):
# an unterminated
"""
assert get_meta_refresh(body, baseurl) == (None, None)
assert get_meta_refresh(body, baseurl, ignore_tags=()) == (
0.0,
"http://example.org/foobar_required",
)
def test_redirections_in_different_ordering__in_meta_tag(self):
baseurl = "http://localhost:8000"
url1 = ''
url2 = ''
assert get_meta_refresh(url1, baseurl) == (
0.0,
"http://localhost:8000/dummy.html",
)
assert get_meta_refresh(url2, baseurl) == (
0.0,
"http://localhost:8000/dummy.html",
)
@pytest.mark.parametrize(
"body",
[
'',
'',
'',
'',
'',
"",
],
)
def test_attribute_containing_the_substring_content(self, body: str) -> None:
assert get_meta_refresh(body, "http://example.org") == (
3.0,
"http://example.org/next",
)
def test_first_content_attribute_with_a_payload_wins(self) -> None:
baseurl = "http://example.org"
body = ''
assert get_meta_refresh(body, baseurl) == (3.0, "http://example.org/next")
def test_http_equiv_without_refresh(self) -> None:
# "refresh" elsewhere in the tag gets it scanned, but the pragma is
# only the http-equiv attribute with refresh in its value
baseurl = "http://example.org"
body = ''
assert get_meta_refresh(body, baseurl) == (None, None)
body = ''
assert get_meta_refresh(body, baseurl) == (3.0, "http://example.org/next")
def test_unquoted_whitespace(self) -> None:
body = ""
assert get_meta_refresh(body, "http://example.org") == (None, None)
def test_no_catastrophic_backtracking(self) -> None:
# a long run of whitespace inside a refresh tag must be skipped in
# one step, not retried from every position
evil = '", "http://example.org") == (None, None)
assert get_meta_refresh(
evil + 'content="3; url=/next">', "http://example.org"
) == (3.0, "http://example.org/next")
assert time.process_time() - start < 2
def test_non_refresh_meta_is_skipped(self):
baseurl = "http://example.org"
body = """
"""
assert get_meta_refresh(body, baseurl) == (
2.0,
"http://example.org/next",
)
@pytest.mark.parametrize(
("encoding", "char"),
[
("utf-8", "\u793e"),
("latin-1", "\u00e9"),
("shift_jis", "\u793e"),
("gb18030", "\u793e"),
("iso2022_jp", "\u793e"),
],
)
def test_get_meta_refresh_bytes(self, encoding: str, char: str) -> None:
baseurl = "http://example.org"
with_refresh = (
f"{char}"
""
)
without_refresh = f"{char}"
assert (
get_meta_refresh(with_refresh.encode(encoding), baseurl, encoding)
== get_meta_refresh(with_refresh, baseurl, encoding)
== (3.0, "http://example.org/next")
)
assert (
get_meta_refresh(without_refresh.encode(encoding), baseurl, encoding)
== get_meta_refresh(without_refresh, baseurl, encoding)
== (None, None)
)
def test_get_meta_refresh_spelled_by_characters(self) -> None:
# These kanji encode to bytes that spell "refresh", which neither a
# browser nor the scan of the decoded document sees.
text = "
\u9c5a\u80d9\u7e89\u8515
"
assert text.encode("iso2022_jp") == b"
\x1b$Brefreshx\x1b(B
"
assert get_meta_refresh(
text.encode("iso2022_jp"), "http://example.org", "iso2022_jp"
) == (None, None)
def test_get_meta_refresh_non_ascii_compatible(self) -> None:
# UTF-16 writes ASCII characters as something else than their ASCII
# bytes, so the document is decoded before it is scanned.
body = ""
assert get_meta_refresh(
body.encode("utf-16"), "http://example.org", "utf-16"
) == get_meta_refresh(body, "http://example.org", "utf-16")
def test_get_meta_refresh_split_by_a_dropped_escape(self) -> None:
# ESC ( B redesignates ASCII where ASCII is already in use, so it
# divides the tag in the bytes but not in the document.
raw = b''
assert (
raw.decode("iso2022_jp")
== ''
)
assert get_meta_refresh(raw, "http://example.org", "iso2022_jp") == (
3.0,
"http://example.org/next",
)
def test_get_meta_refresh_non_ascii_whitespace(self) -> None:
# U+3000 does not separate a tag name from its attributes.
body = ""
assert get_meta_refresh(body, "http://example.org") == (None, None)
assert get_meta_refresh(body.encode(), "http://example.org") == (None, None)
def test_get_meta_refresh_content_non_ascii_whitespace(self) -> None:
# The refresh content is parsed with ASCII whitespace only, so U+00A0
# around the interval or the ";" leaves it unparsed and no redirect is
# reported, as a browser reloads the same document instead.
for content in (" 0;url=/next", "0 ;url=/next", "0; url=/next"):
body = f""
assert get_meta_refresh(body, "http://example.org") == (None, None)
# An ASCII-whitespace variant is still parsed.
assert get_meta_refresh(
"", "http://example.org"
) == (0.0, "http://example.org/next")
def test_get_meta_refresh_non_ascii_case_folding(self) -> None:
# U+017F uppercases to "S", but it is no "s" in a tag name, so this is
# no ",
'
',
'
',
"a < b",
"a > b",
"<",
">",
"x",
]
def base(text: str | bytes, **kwargs: object) -> object:
return get_base_url(text, "https://example.org", **kwargs) # type: ignore[arg-type]
def meta(text: str | bytes, **kwargs: object) -> object:
return get_meta_refresh(text, "https://example.org", **kwargs) # type: ignore[arg-type]
@pytest.mark.parametrize(
("func", "tag", "found", "missing"),
[
(base, BASE_TAG, "http://example.org/found", "https://example.org"),
(meta, META_TAG, (5, "http://example.org/found"), (None, None)),
],
ids=["base", "meta"],
)
class TestMaxScan:
def test_straddling_tag(self, func, tag, found, missing):
# A tag that starts before the limit and ends after it is read whole,
# wherever in it the limit falls, inside a quoted value included.
for offset in range(1, len(tag)):
text = "a" * (LIMIT - offset) + tag
assert func(text, max_scan=LIMIT) == found
assert func(text.encode(), max_scan=LIMIT) == found
def test_before_limit(self, func, tag, found, missing):
assert func(tag + "a" * LIMIT, max_scan=LIMIT) == found
def test_unterminated_tag_within_limit(self, func, tag, found, missing):
assert func(tag[:-1], max_scan=LIMIT) == found
def test_stray_lt_before_limit(self, func, tag, found, missing):
# A "<" that starts no tag must not extend the scan past the limit.
assert func("1 < 2 " + "a" * LIMIT + tag, max_scan=LIMIT) == missing
def test_bytes_limit_counts_bytes(self, func, tag, found, missing):
# The limit reaches the tag in characters, but not in bytes.
text = "\u00e9" * 20 + tag
assert func(text, max_scan=30) == found
assert func(text.encode(), max_scan=30) == missing
def test_bytes_split_character(self, func, tag, found, missing):
# The cut splits the last character, which decoding replaces.
assert func(("\u00e9" * 5).encode() + tag.encode(), max_scan=9) == missing
@given(
markup=st.lists(st.sampled_from(FRAGMENTS), max_size=12).map("".join),
max_scan=st.integers(min_value=0, max_value=400),
)
def test_cut_never_invents(self, func, tag, found, missing, markup, max_scan):
# A cut can misread an angle bracket in a comment or an attribute value
# only into looking at less, never into a result of its own.
assert func(markup, max_scan=max_scan) in (func(markup), missing)
assert func(markup.encode(), max_scan=max_scan) in (func(markup), missing)
def test_reading_on_stops_at_the_next_tag(self, func, tag, found, missing):
# Reading a split tag to its end stops where the next tag begins.
assert func("
Fugiat Soluta
Cillum
![Occaecat at Nisi Est Cum Vero - 926 Debitis Nihil Ex, Dolores, UT, 30907, 1-2 hic, 1-2 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/ad422r69-ad68-936m-5id2-738337r4ea25/pariatur/maxIme.0_7.webp)
Nisi Est Cum Vero - 926 Debitis Nihil Ex, Dolores, UT, 30907![Occaecat at Libero Dolores - 3232 Mollitia Eos, Dolores, UT, 30909, 1-2 hic, 1-2 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/l3z35440-6279-6r4z-5ea2-ea859935m783/pariatur/maxIme.0_1.webp)
Libero Dolores - 3232 Mollitia Eos, Dolores, UT, 30909![Occaecat at Pariatur at Voluptas Minim Distinctio - 1075 Placeat Ex, Dolores, UT, 30909, 1-3 hic, 1-2 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/harum533-19ad-26r8-t944-r138in5at17t/pariatur/maxIme.0_3.webp)
Pariatur at Voluptas Minim Distinctio - 1075 Placeat Ex, Dolores, UT, 30909![Occaecat at Nostrud - 249 Hic Iusto Ex, Dolores, UT, 30909, 1-2 hic, 1 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/80id81ad-f221-021s-m3b0-68g5b0f3667f/pariatur/maxIme.0_4.webp)
Nostrud - 249 Hic Iusto Ex, Dolores, UT, 30909![Occaecat at Debitis Nihil Nostrud - 100 Est Est Id, Dolores, UT, 30907, 1-2 hic, 1-2 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/r0860157-1643-919c-d381-2et402192m49/pariatur/maxIme.0_5.webp)
Debitis Nihil Nostrud - 100 Est Est Id, Dolores, UT, 30907![Occaecat at Sed Nobis - 9000 Provident Eos, Dolores, UT, 30909, 1-3 hic, 1-2 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/4ex05t97-ad0m-4d06-b64c-t46r863c4531/pariatur/maxIme.0_3.webp)
Sed Nobis - 9000 Provident Eos, Dolores, UT, 30909![Occaecat at Mollit Saepe Dolores - 1400 Mollit Saepe Id, Dolores, UT, 30909, 2 aute, 2 ipsum, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/est2m793-11c2-58f8-c70t-3429887do6z8/pariatur/maxIme.0_5.webp)
Mollit Saepe Dolores - 1400 Mollit Saepe Id, Dolores, UT, 30909![Occaecat at Cillum Minim Debitis Elit - 934 Accusamus Id, Dolores, UT, 30909, 0-2 aute, 1-1.5 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/m5m09f9r-7353-7l41-2at5-ut0149non2m5/pariatur/maxIme.0_7.webp)
Cillum Minim Debitis Elit - 934 Accusamus Id, Dolores, UT, 30909![Occaecat at Eligendi Minim - 1100 Laboris ID, Consequat, UT, 30813, 1-2 hic, 1-2 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/l6499212-m36f-857g-0est-f46m1quo5c9r/pariatur/maxIme.0_2.webp)
Eligendi Minim - 1100 Laboris ID, Consequat, UT, 30813![Occaecat at Molestias Maxime Quibusdam - 1903 Maxime Ex, Dolores, UT, 30909, 3 aute, 3 ipsum, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/425t5t4f-5m20-8327-d994-9451est37l72/pariatur/maxIme.0_2.webp)
Molestias Maxime Quibusdam - 1903 Maxime Ex, Dolores, UT, 30909![Occaecat at Occaecat Distinctio - 2905 Voluptate Id, Dolores, UT, 30909, 2 aute, 2 ipsum, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/t7m7b19b-32m1-12d3-9l2d-b646901r466g/pariatur/maxIme.0_7.webp)
Occaecat Distinctio - 2905 Voluptate Id, Dolores, UT, 30909![Occaecat at Nobis at sed Optio - 3122 Sapiente Ex, Dolores, UT, 30909, 1-2 hic, 1-2 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/in906s7t-in67-171l-at39-0ex448f18536/pariatur/maxIme.0_1.webp)
Nobis at sed Optio - 3122 Sapiente Ex, Dolores, UT, 30909![Occaecat at Cupidatat - 1840 Reprehenderit Ex, Dolores, UT, 30904, 1 hic, 1 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/8d508d37-71do-7323-l304-ea1qui5do042/pariatur/maxIme.0_5.webp)
Cupidatat - 1840 Reprehenderit Ex, Dolores, UT, 30904![Occaecat at Laboris Quibusdam - 220 Hic Iusto Ex, Dolores, UT, 30909, 2 aute, 1.5 ipsum, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/in1m1m77-t2d4-1id8-6d77-0m0d91m4in25/pariatur/maxIme.0_7.webp)
Laboris Quibusdam - 220 Hic Iusto Ex, Dolores, UT, 30909![Occaecat at Commodo Earum Distinctio - 4011 Commodo Earum Ex, Dolores, UT, 30909, 2 aute, 1 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/6d96aute-in30-3r15-ex9d-82341f280d2d/pariatur/maxIme.0_8.webp)
Commodo Earum Distinctio - 4011 Commodo Earum Ex, Dolores, UT, 30909![Occaecat at Consectetur Dolore Distinctio - Molestias Excepteur Tempore Dolores - 2476 DeSerunt Ut, Dolores, UT, 30904, 1-2 hic, 1-1.5 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/z2at9121-9l50-2z24-8g0c-ut0est8ut964/pariatur/maxIme.0_5.webp)
Consectetur Dolore Distinctio - Molestias Excepteur Tempore Dolores - 2476 DeSerunt Ut, Dolores, UT, 30904![Occaecat at Sed Officiis at Veniam - 1247 Dolores Vero Sint, Dolores, UT, 30909, 3 aute, 2 ipsum, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/22071ea7-ea61-66r1-d978-39et4s8279m4/pariatur/maxIme.0_7.webp)
Sed Officiis at Veniam - 1247 Dolores Vero Sint, Dolores, UT, 30909![Occaecat at Molestias Distinctio & Cumque - 1414 Possimus Id, Dolores, UT, 30906, 2-3 aute, 2 ipsum, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/s89m72et-041z-4360-l200-r244z5595581/pariatur/maxIme.0_6.webp)
Molestias Distinctio & Cumque - 1414 Possimus Id, Dolores, UT, 30906![Occaecat at Vero at Debitis Dolor Assumenda Harum - 1150 Reiciendis Sint, Dolores, UT, 30909, 1-3 hic, 1-2 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/eos75f65-m48f-196d-s5r1-3ea4z420136r/pariatur/maxIme.0_5.webp)
Vero at Debitis Dolor Assumenda Harum - 1150 Reiciendis Sint, Dolores, UT, 30909![Occaecat at Cupiditate at Dolores Assumenda Harum - 3722 Libero Nam Nam, Dolores, UT, 30907, 1-3 hic, 1-2 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/in032m2m-0854-6m01-95ad-7hic25g1s347/pariatur/maxIme.0_2.webp)
Cupiditate at Dolores Assumenda Harum - 3722 Libero Nam Nam, Dolores, UT, 30907![Occaecat at Nihil Nihil Distinctio - 950 Debitis Nihil Ex, Dolores, UT, 30907, 2 aute, 2 ipsum, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/3in0g64r-23et-2ea3-6do0-442952c16d57/pariatur/maxIme.0_3.webp)
Nihil Nihil Distinctio - 950 Debitis Nihil Ex, Dolores, UT, 30907![Occaecat at Laboris Nihil Distinctio - 1924 B Non Ex, Dolores, UT, 30909, 1-2 hic, 1 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/69s197l9-3est-099l-30s4-453247facere/pariatur/maxIme.0_6.webp)
Laboris Nihil Distinctio - 1924 B Non Ex, Dolores, UT, 30909![Occaecat at Impedit at 2663 - 2663 Fugiat Ex, Dolores, UT, 30906, 1-3 hic, 1-2.5 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/514t879b-ex3r-50b0-f41g-458velit00r4/pariatur/maxIme.0_2.webp)
Impedit at 2663 - 2663 Fugiat Ex, Dolores, UT, 30906![Occaecat at Sed Eos - 1812 Dolore Ut, Dolores, UT, 30904, 1-3 hic, 1-2 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/m5222412-8g18-95do-m2z6-m30519r074l9/pariatur/maxIme.0_6.webp)
Sed Eos - 1812 Dolore Ut, Dolores, UT, 30904![Occaecat at Laborum Occaecat - 601 Corrupti Ad, Dolores, UT, 30907, 1-3 hic, 1-2 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/d16l35m1-d964-257m-7922-795030z1f16f/pariatur/maxIme.0_5.webp)
Laborum Occaecat - 601 Corrupti Ad, Dolores, UT, 30907![Occaecat at Sed Amet - 1035 Consequat Id, Dolores, UT, 30909, 1-3 hic, 1-2 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/8hic3ea8-6900-81r1-m8s7-7do3ad39in9f/pariatur/maxIme.0_1.webp)
Sed Amet - 1035 Consequat Id, Dolores, UT, 30909![Occaecat at Tempore at Blanditiis Distinctio - 420 Debitis Nam, Dolores, UT, 30909, 1-3 hic, 1-2 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/amet0262-7845-1d69-5m70-407ad44l5r41/pariatur/maxIme.0_1.webp)
Tempore at Blanditiis Distinctio - 420 Debitis Nam, Dolores, UT, 30909![Occaecat at Occaecati Distinctio - 2512 Ipsum Est, Dolores, UT, 30909, 2 aute, 2 ipsum, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/m8186592-f5r2-77at-5sed-m510937c0non/pariatur/maxIme.0_11.webp)
Occaecati Distinctio - 2512 Ipsum Est, Dolores, UT, 30909![Occaecat at Mollitia at Veniam - 4210 Deserunt Ex, Dolores, UT, 30906, 4 aute, 2.5 ipsum, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/1f3m676l-9649-36b4-d3ad-z05est088cum/pariatur/maxIme.0_2.webp)
Mollitia at Veniam - 4210 Deserunt Ex, Dolores, UT, 30906![Occaecat at Sunt Nihil - 1315 Magna Tempor Ex, Dolores, UT, 30909, 2 aute, 2 ipsum, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/8829t0z8-5m2z-0eos-7b65-0ad9m3ut7ut7/pariatur/maxIme.0_7.webp)
Sunt Nihil - 1315 Magna Tempor Ex, Dolores, UT, 30909![Occaecat at Irure Facilis Anim - 3630 Irure Facilis Ex, Dolores, UT, 30906, 2 aute, 2 ipsum, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/z34nam2t-m5c3-2c4z-at11-ea73d9515244/pariatur/maxIme.0_5.webp)
Irure Facilis Anim - 3630 Irure Facilis Ex, Dolores, UT, 30906![Occaecat at Laborum Tenetur - 1311 Magna Tempor Ex, Dolores, UT, 30909, 0 aute, 1 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/0b771sed-340r-4591-995l-2890m7l819do/pariatur/maxIme.0_5.webp)
Laborum Tenetur - 1311 Magna Tempor Ex, Dolores, UT, 30909![Occaecat at Sed Mollit at Dolores - 2205 Doloribus Id, Dolores, UT, 30906, 2 aute, 1-1.5 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/t3ex4b85-03l8-50ea-f1m4-7r1nisi5ex3t/pariatur/maxIme.0_5.webp)
Sed Mollit at Dolores - 2205 Doloribus Id, Dolores, UT, 30906![Occaecat at Dolores Commodo Distinctio 55+ - 1730 Facere Ex, Dolores, UT, 30909, 1-2 hic, 1 sunt, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/d3mollit-0r73-9742-z3id-3ad63831t862/pariatur/maxIme.0_4.webp)
Dolores Commodo Distinctio 55+ - 1730 Facere Ex, Dolores, UT, 30909![Occaecat at Sed Ducimus at Similique - 50 Earum Eveniet Id, Dolores, UT, 30909, 2-3 aute, 2 ipsum, [veniam Veniam]](https://ssl.cdn-ho.example/dolor/amet/35at1403-647r-5027-z2m7-74m25l5in42b/pariatur/maxIme.0_4.webp)
Sed Ducimus at Similique - 50 Earum Eveniet Id, Dolores, UT, 30909
Sed esse officiis ad nisi labore quo fuga at odio non sint omnis in fuga sunt sed Mollit Consectetur Laboris do Nostrud NAM. Nisi labore expedita esse do similique animi culpa quas Itaque sit tempor quod sed Mollit Consectetur quod sit corrupti dignissimos dolor sunt sapiente sed quod do sed ullamco laborum. Dignissimos tempor deleniti nam sed cupiditate. Similique 2026 Nostrud NAM. Cum maxime corrupti.
ITAQUE EX MOLESTIAS AD SIT TEMPOR DO SED VERO TEMPORE QUI SIT ALIAS CONSECTETUR QUI. ELIT ITAQUE'S VERO TEMPORE MOLLIT SIT SED CUM FUGA CULPA VERO TEMPORE CUMQUE.
