pax_global_header00006660000000000000000000000064152353043170014515gustar00rootroot0000000000000052 comment=49c381d5a039aee6655cff5149c9177aec19663c python-nameparser-2.1.0/000077500000000000000000000000001523530431700151715ustar00rootroot00000000000000python-nameparser-2.1.0/.editorconfig000066400000000000000000000004431523530431700176470ustar00rootroot00000000000000# http://editorconfig.org root = true [*] charset = utf-8 end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true [*.{py,rst,ini}] indent_style = space indent_size = 4 [*.{html,json,yml}] indent_style = space indent_size = 2 [*.md] trim_trailing_whitespace = false python-nameparser-2.1.0/.github/000077500000000000000000000000001523530431700165315ustar00rootroot00000000000000python-nameparser-2.1.0/.github/dependabot.yml000066400000000000000000000011521523530431700213600ustar00rootroot00000000000000version: 2 updates: - package-ecosystem: "github-actions" directory: "/" schedule: interval: "monthly" # nameparser has no runtime dependencies; this watches the one optional-extra # dependency, namedivider-python (the [ja] segmenter). pyproject floats >=0.4 so # fresh installs always get the newest release, but CI's ja-extra job installs # from uv.lock — without these bumps, CI keeps proving an old version while # users install a newer one. - package-ecosystem: "uv" directory: "/" schedule: interval: "monthly" allow: - dependency-name: "namedivider-python" python-nameparser-2.1.0/.github/workflows/000077500000000000000000000000001523530431700205665ustar00rootroot00000000000000python-nameparser-2.1.0/.github/workflows/codeql.yml000066400000000000000000000010131523530431700225530ustar00rootroot00000000000000name: CodeQL on: push: branches: [ master ] pull_request: branches: [ master ] schedule: - cron: '30 4 * * 1' permissions: contents: read jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: security-events: write steps: - uses: actions/checkout@v7 - name: Initialize CodeQL uses: github/codeql-action/init@v4.37.3 with: languages: python, actions - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4.37.3 python-nameparser-2.1.0/.github/workflows/python-package.yml000066400000000000000000000066741523530431700242400ustar00rootroot00000000000000# This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Test the Python package on: workflow_dispatch: push: branches: [ master ] pull_request: branches: [ master ] permissions: contents: read jobs: build: runs-on: ubuntu-latest strategy: fail-fast: false matrix: python-version: ["3.11", "3.12", "3.13", "3.14", "3.15"] # 3.15 is pre-release until 2026-10-01 (#259): surface breakage # without blocking the branch; flip to blocking when it goes final. continue-on-error: ${{ matrix.python-version == '3.15' }} env: # Without this, uv sync honors .python-version (3.11) over the # matrix interpreter and every job silently tests 3.11. UV_PYTHON: ${{ matrix.python-version }} steps: - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} allow-prereleases: true - name: Install uv uses: astral-sh/setup-uv@v7 with: enable-cache: true - name: Install dependencies run: uv sync --group dev - name: Run linter run: uv run ruff check - name: Check types run: uv run mypy - name: Run Tests run: | uv run pytest --cov=nameparser --cov-report=xml uv run --with build python -m build --sdist uv run --with twine twine check dist/* uv run sphinx-build -b html docs dist/docs uv run sphinx-build -b doctest docs dist/doctest uv run python -m doctest README.rst - name: Upload coverage to Codecov if: matrix.python-version == '3.13' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml fail_ci_if_error: false # The optional nameparser[ja] segmenter (#272). Its own job, not a # matrix entry: the matrix above must keep proving that the package # installs and passes with NO third-party dependency at all, which is # the promise the extra is carved out of. Everything the extra adds # is under tests/v2/ (the ja pack's integration tests, which skip # without it). ja-extra: runs-on: ubuntu-latest # 3.14, not the matrix's newest: namedivider-python pulls lightgbm, # which pulls scipy/numpy, and those have no 3.15 wheels yet (the # sdist build fails). 3.15 is pre-release and this matrix is ahead # of the ecosystem; revisit when it goes final. env: UV_PYTHON: "3.14" steps: - uses: actions/checkout@v7 - name: Set up Python 3.14 uses: actions/setup-python@v7 with: python-version: "3.14" - name: Install uv uses: astral-sh/setup-uv@v7 with: enable-cache: true - name: Install dependencies run: uv sync --group dev --extra ja - name: Run Tests # The import check first: every test the extra adds is guarded by # a find_spec("namedivider") skipif, so a job where the extra # silently failed to install is a job where all of them SKIP and # the step still passes green. This line is what makes that # failure loud -- without it the job proves nothing on the day it # matters. run: | uv run python -c "import namedivider" uv run pytest tests/v2/ -q python-nameparser-2.1.0/.github/workflows/python-publish.yml000066400000000000000000000011021523530431700242700ustar00rootroot00000000000000name: Upload Published Python Package on: release: types: [published] jobs: deploy: runs-on: ubuntu-latest environment: pypi permissions: id-token: write steps: - uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v7 with: python-version: '3.x' - name: Install dependencies run: | python -m pip install --upgrade pip pip install build - name: Build package run: python -m build - name: Publish package uses: pypa/gh-action-pypi-publish@release/v1 python-nameparser-2.1.0/.gitignore000066400000000000000000000004321523530431700171600ustar00rootroot00000000000000*.hgrc *.DS_Store __pycache__/ *.py[cod] .python2/ MANIFEST nameparser.egg-info/ build *.egg .coverage dist .idea Pipfile Pipfile.lock .tm_properties # virtual environments .env/ .venv/ venv/ env/ # tools .claude/ .codex/ .gemini/ .pytest_cache/ # docs docs/_* docs/superpowers/ python-nameparser-2.1.0/.python-version000066400000000000000000000000051523530431700201710ustar00rootroot000000000000003.11 python-nameparser-2.1.0/.readthedocs.yaml000066400000000000000000000003071523530431700204200ustar00rootroot00000000000000version: 2 build: os: ubuntu-24.04 tools: python: "3.12" sphinx: configuration: docs/conf.py python: install: - method: pip path: . - requirements: docs/requirements.txt python-nameparser-2.1.0/AGENTS.md000066400000000000000000002110361523530431700164770ustar00rootroot00000000000000# CLAUDE.md This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. ## Workflow For non-trivial changes, use a feature branch and open a PR. Branch naming: `fix/issue-NNN-short-description` or `feat/short-description`. Before opening the PR, if the change alters parser behavior or internals, *read* the Architecture, Extension Patterns, and Gotchas sections of this file against the change — don't grep for it: AGENTS.md paraphrases behavior in its own words, so text made stale by a code change rarely matches the code's phrasing (a doc-staleness sweep driven by grep terms from the diff will miss it every time). The same applies when scoping a doc-review pass or a subagent prompt: include AGENTS.md in the list of docs to check, or it won't be checked. ## Commands ```bash # Preferred: use uv run (works without activating the venv) # Alternative: .venv/bin/ if the venv is already active # Run all tests (includes --doctest-modules, so doctests in nameparser/ are also run; # the dual-parametrize fixture doubles the count, so ~630 methods → ~1250 results) uv run pytest # --doctest-modules is set in pyproject.toml, so doctests run automatically # Run a single test file / class / method uv run pytest tests/test_python_api.py uv run pytest tests/test_python_api.py::HumanNamePythonTests::test_utf8 # Type check (covers nameparser/ and tests/, per pyproject.toml's [tool.mypy] packages) uv run mypy # Lint uv run ruff check nameparser/ # Debug how a specific name string is parsed: 2.0 core parse() -- # prints the ParsedName repr raw and capitalized, then initials; # --json emits the component dict uv run python -m nameparser "Dr. Juan Q. Xavier de la Vega III" # Build docs uv run sphinx-build -b html docs dist/docs # Maintain docs/release_log.rst as changes land: # - Keep an "Unreleased" entry at the top: `* X.Y.Z - Unreleased` # - Add one bullet per notable change; prefix with Add/Fix/Remove/Change # - Reference the issue or PR in parentheses: (#123) or (#123, #124) # Use "closes #N" when the change directly resolves the issue # - Version is decided at release time (patch/minor/major per semver) # - Format matches existing entries — see 1.3.0 block for a current example # Release checklist (PyPI publish is triggered automatically by GitHub Actions on release creation) # 0. Sweep for stale prose. Grepping for the changed SYMBOL finds almost none of # it: prose describes behavior in words, not identifiers. Walk these sites # explicitly — every one of them has gone stale at least once. # - docs/*.rst: usage.rst examples and API surface, plus any file listing # config constants or HumanName kwargs # - docstrings that autodoc renders (nameparser/_policy.py, _types.py, ...). # These ARE the API reference in modules.html, so a docs/*.rst-only sweep # structurally misses half the rendered page — that is how the guide and # the reference ended up teaching different Policy spellings in 2.1 # - user-facing MESSAGES: warnings and raise text often hand the reader code # to paste, and it has to be code that still works (and type-checks) # - pipeline stage header blocks: each _pipeline/*.py docstring declares # "Reads:". That is checkable, so check it rather than reading it: # compare it against grep -oE '\b(policy|lexicon)\.[a-z_]+' on the module # - tests/v2/cases.py notes, which explain why a row lands where it does # - AGENTS.md itself, for stale commands, architecture notes, or gotchas # And check for open Dependabot PRs on uv.lock (namedivider-python) and merge them # first — pyproject floats >=0.4 so fresh installs get the newest namedivider, but # CI's ja-extra job installs from uv.lock and only tests what the lock pins # 1. Bump VERSION in nameparser/_version.py (and the `version:` field in CITATION.cff to match) # For a pre-release, set PRE_RELEASE = 'rc1' (etc.) — __version__ joins it without a dot ('2.0.0rc1'); '' means final # 2. Stamp "Unreleased" → "X.Y.Z - Month DD, YYYY" in docs/release_log.rst # 3. git commit + git tag -a vX.Y.Z -m "Release X.Y.Z" # 4. git push origin master && git push origin vX.Y.Z ← tag must be pushed separately before gh release create # 5. gh release create vX.Y.Z --title "vX.Y.Z" --notes "..." # 6. Close the vX.Y.Z milestone and create a new "Next Release" one: # MILESTONE=$(gh api repos/derek73/python-nameparser/milestones --jq '.[] | select(.title=="vX.Y.Z") | .number') # gh api -X PATCH repos/derek73/python-nameparser/milestones/$MILESTONE -f state=closed # gh api -X POST repos/derek73/python-nameparser/milestones -f title="Next Release" ``` Enable debug logging to see the parser's internal decisions: ```python import logging logging.getLogger('HumanName').setLevel(logging.DEBUG) ``` ## Architecture The library has two layers: `nameparser/config/` (data) and `nameparser/parser.py` (logic). **Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271/#272): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Five defaults fall out of it, plus a sixth that applies the same not-a-guess standard to specific WORDS rather than to a script (#308's honorific peel, below). Wholly-Han, wholly-Hangul and kana-licensed names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Hiragana licenses Japanese (#272) — a name whose characters stay inside Han∪kana while carrying at least one kana cannot be Chinese (the kana rules it out) and is not a transcription (foreign names are transcribed in katakana ALONE, マイケル has no kanji), so 高橋みなみ and 山田 エミ read family-first too; mechanically they resolve to the HIRAGANA entry, the license's carrier key. PURE katakana is excluded and keeps the positional default: マイケル・ジャクソン is a transcribed foreign name in its source order. And the 间隔号 U+00B7 (#298) is the transcription marker for scripts that HAVE no transcription script: a name it divides (威廉·莎士比亚 — flanked by classified characters on both sides, so Catalan's Gal·la is untouched) keeps its source order and never segments — the orthography names the convention, exactly as pure katakana does, with the divider carrying the signal instead of the script. And a listed CJK honorific glued to the END of a name token is split off it (#308) — 田中さん is 田中 plus さん — on the same orthography-settles-it test, narrowed for the glued position: an entry peels only where it can never end a name, so 씨/님/さん/様/先生 peel while 양/군/氏/博士/殿 stay spaced-only (김지양 and 田中博士 are names, and ~90 Japanese surnames end in 殿) and 君 is in NEITHER set (王君 is a complete Chinese name), though its kana spelling くん peels. Like the nakaguro's tokenize-level separation described next, it is reached by neither policy opt-out — but for its own reason: the vocabulary carries the license itself rather than borrowing the script's, so `segment_scripts` has nothing to say about it. Since #312 it also crosses the FAMILY comma and the 间隔号, which is the more surprising fact, since both of those still stop the surname split standing right beside it: each answers where a name DIVIDES into surname and given, and the peel never asks that question — so `김, 민준씨` reads exactly as the spaced `김 민준씨` does (family 김, given 민준, suffix 씨) while the split stands down as before. Its site is accordingly the name-bearing segment runs — `segments[:2]` under a family comma, and `segments[0]` as before otherwise, the family comma being the one structure that splits the name itself across two runs, with the honorific as often glued to the given side as to the family. That is the whole reach and nothing past it (`김, 민준 지훈씨` peels; `김, 민준, 지훈씨` and `김,, 민준씨` do not, both landing in a third run), and whether `segments[1]` is name text at all is now ASKED rather than inferred from the structure — `segment` does not guarantee it, since a one-word part before the comma reads as FAMILY_COMMA even when the part after it is entirely suffix-shaped, and the peel walking into such a run took `V.` for its site, found no listed tail and abandoned (#319). The question is `segment`'s own suffix-comma predicate, lifted into `_vocab.is_wholly_suffix` so the two stages cannot drift: a wholly suffix-shaped second run is declined and the scan stays in `segments[0]`, so `田中さん, V.` and `田中さん, Ph. D.` give さん up as `田中さん, PhD` always did. The test is necessary but NOT sufficient, and the second condition is not decoration: every honorific tail is also a suffix word, so a glued honorific is itself part of what makes its run read as suffix-shaped, and declining a run that holds the ONLY site loses the peel outright. `segments[0]` must therefore offer a peel site of its own before the second run is declined — `이, J.씨` and `선생님, J.씨` pass the suffix test and are scanned anyway, keeping the pre-#319 reading, while `김민준씨, J.씨` has a site on both sides and peels the person's own 씨 rather than the junk one behind the comma. Uniform in the PEEL, that is — where the credential itself lands is `assign`'s question and still differs by spelling (`PhD` → `title`, `V.` → `given`, `Ph. D.` → `suffix`). Not `_is_post_nominal` pluralized: the run predicate says yes both to what the token predicate vetoes (`V.`, `V`, `I` — the class the defect was reported as) and to what the token predicate never sees at all, since `period_joined_vocab` and the delimiter routes are the run predicate's alone (`Msc.Ed.` and `J.씨` reach it that way, and `田中さん, Msc.Ed.` moves with the rest). `Policy(lenient_comma_suffixes=False)` drops this call to the strict token test too — so those three read as name text again and keep the pre-#319 answer, while `Ph. D.` peels under the knob regardless, its merged `phd` passing the strict test. `田中さん, 太郎` is unchanged, and not because of its comma — the honorific there is not at the end of the name, 太郎 is. The nakaguro belongs to the same doctrine but is decided a layer down: U+30FB and its halfwidth twin U+FF65 separate tokens like whitespace, unconditionally and in tokenize, so neither policy opt-out (`script_orders={}`, `segment_scripts=()`) reaches it — the codepoints are CJK-only and appear in no other script's names, which is what licenses a tokenize-level rule where U+00B7 (also the Catalan punt volat, interior to Gal·la) needs the flanked-by-classified-script guard `_tokenize_region` gives it (#298). Han segmentation stays OPT-IN (`locales.ZH` for Chinese, `locales.JA` for Japanese) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎; no surname list divides a kanji name at all, so `locales.JA` activates the stage and a pluggable `Parser(segmenter=...)` does the dividing. Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. ### Configuration layer (`nameparser/config/`) Most modules define a plain Python set of known name pieces; `capitalization.py` and `regexes.py` define dicts: - `titles.py` — `TITLES` (prenominals) and `FIRST_NAME_TITLES` (e.g. "Sir", which treat the following name as first, not last) - `suffixes.py` — `SUFFIX_ACRONYMS` (with periods, e.g. "M.D.") and `SUFFIX_NOT_ACRONYMS` (e.g. "Jr."), plus `GLUED_HONORIFICS` (#308), the subset of `SUFFIX_NOT_ACRONYMS` the peel may split off the END of a name token — a separate, harsher set, since the glued position has no writer-drawn boundary to lean on - `prefixes.py` — `PREFIXES` (lastname particles, e.g. "de", "van") - `bound_first_names.py` — `BOUND_FIRST_NAMES` (bound given-name prefixes, e.g. "abdul", "abu"); `_join_bound_first_name` joins the first non-title piece to its following piece before the main assignment loop - `conjunctions.py` — `CONJUNCTIONS` (e.g. "and", "of") used to chain multi-word titles - `maiden_markers.py` — `MAIDEN_MARKERS` (e.g. "née", "geb.") routing the following name to `maiden` - `surnames.py` — `KOREAN_SURNAMES`, the census list the 2.0 API splits unspaced hangul on (#271). With `maiden_markers.py` it is one of the two data modules `Constants` has **no** attribute for: both reach the parse only through `Constants._snapshot()` → `Lexicon`, so the v1 surface stays frozen and there is no v1 knob to turn either off (the opt-out is the 2.0 `Policy`) - `capitalization.py` — `CAPITALIZATION_EXCEPTIONS` mapping (e.g. `{'phd': 'Ph.D.'}`) - `regexes.py` — dict of compiled regular expressions (wrapped in `RegexTupleManager` by `Constants`) `config/__init__.py` wraps everything into `SetManager` and `TupleManager` instances inside a `Constants` class. A module-level singleton `CONSTANTS` is shared across all `HumanName` instances by default. **Two-tier config pattern**: `CONSTANTS` is global; passing `None` as the second arg to `HumanName` creates a fresh per-instance `Constants()`. After modifying per-instance config you must call `hn.parse_full_name()` again. `SetManager` normalizes elements (lowercase, leading/trailing periods stripped) at every entry point — constructor, `add()`/`remove()`, set operators, and `in` membership checks (`__contains__`, #244) — and raises `TypeError` for bare strings/bytes and non-str elements, so callers don't need to worry about case. `TupleManager`/`RegexTupleManager` apply the equivalent bare-string/bytes guard to their own constructor, plus a check that no element of an iterable-of-pairs argument is itself a `str`/`bytes` — a 2-character string is a valid dict-update-sequence element, so without the check it silently shreds into a key/value pair (#242). **`_SetManagerAttribute`/`_CachedUnionMember` descriptors**: all nine `SetManager`-backed `Constants` attributes (`prefixes`, `suffix_acronyms`, `suffix_not_acronyms`, `titles`, `first_name_titles`, `conjunctions`, `bound_first_names`, `non_first_name_prefixes`, `suffix_acronyms_ambiguous`) are managed by a descriptor so a non-`SetManager` assignment (e.g. `c.conjunctions = 'and'`) raises `TypeError` immediately instead of silently degrading later `in` checks into substring tests (#241). The four PST-contributing attrs (`prefixes`, `suffix_acronyms`, `suffix_not_acronyms`, `titles`) additionally use `_CachedUnionMember`, a subclass that also wires the `_pst` cache-invalidation callback on assignment; the other five use the plain `_SetManagerAttribute` base with no cache involved. Both store their values under the *private* name (`_prefixes`, `_conjunctions`, etc.) in the instance `__dict__` so the descriptor's `__set__` owns every assignment. Any code that inspects `__dict__` directly (e.g. `__getstate__`) must map `_xxx` → `xxx` for descriptor-managed attrs rather than filtering on `not k.startswith('_')`. **`_EmptyAttributeDefaultAttribute` descriptor** backs `empty_attribute_default` (1.4, #255): unlike `_SetManagerAttribute`, `__get__(obj=None, ...)` returns `''`, not `self` — deliberate, so `Constants.__repr__`'s `getattr(type(self), name)` default comparison still works and the pre-existing `str`-only mypy inference on the attribute (never widened to `str | None`, matching the descriptor-free days) is preserved. Because of that, `__getstate__`/`__setstate__` use `inspect.getattr_static(type(self), name)`, not plain `getattr`, to detect this descriptor (or the `property` used by the legacy-pickle migration shim below) without triggering `__get__` — a plain `getattr` there would silently miss both. `__setstate__` also bypasses the descriptor's own `__set__` (writes `_empty_attribute_default` directly) so restoring pickled/copied state doesn't itself emit the deprecation warning meant for user assignment. ### Parser (`nameparser/parser.py`) `HumanName` is the single public class. Assigning to `full_name` (or instantiating with a string) triggers `parse_full_name()`. Parse flow: 1. `pre_process()` — strips nicknames/maiden names (parenthesis/quotes, routed to `nickname_list`/`maiden_list` per `Constants.nickname_delimiters`/`maiden_delimiters`) and emoji, fixes "Ph.D." variant spellings 2. Split on commas → 1 part (no comma), 2 parts (suffix-comma or lastname-comma), 3+ parts 3. `parse_pieces()` — splits on spaces, detects dotted abbreviations like "Lt.Gov." and registers them as per-parse derived titles/suffixes (the `_derived_*` sets — never the config; see Gotchas) 4. `join_on_conjunctions()` — merges pieces adjacent to conjunctions into single tokens (e.g. `['Secretary', 'of', 'State']` → `['Secretary of State']`); also joins prefix particles to the following lastname token — a piece merged with a title *or prefix* neighbor is re-registered into the per-parse derived title/prefix set so later steps still recognize it, e.g. `von`+`und`+`zu` → registered as a derived prefix so the whole phrase joins to the last name (German "von und zu"; PR #191) 4a. `_join_bound_first_name()` — called immediately after step 4 in all three paths that build a first-name-bearing token sequence: no-comma, lastname-comma (post-comma pieces), and suffix-comma (`parts[0]`); merges bound given-name prefixes (e.g. "abdul") with the next piece before the assignment loop runs; suffixes are still in `pieces` at this point, so the `reserve_last` guard must count non-suffix pieces only. Not called for the lastname portion itself (`lastname_pieces` in the lastname-comma path) — that token sequence is a surname, not first-name text, so the join must not apply there. The join lives at each of these three call sites individually rather than inside `parse_pieces()` itself. That's because `parse_pieces()` is also called for the lastname portion with the same `additional_parts_count` value used for the (joinable) post-comma given-names portion, so `additional_parts_count` alone can't disambiguate which callers want the join. 5. Iterates pieces, assigning to `title_list`, `first_list`, `middle_list`, `last_list`, `suffix_list` 6. `post_process()` — `handle_firstnames()` swaps first/last when only a title + one name; then, gated on `patronymic_name_order`, `handle_east_slavic_patronymic_name_order()` and `handle_turkic_patronymic_name_order()` reorder Russian-formal-order and reversed Turkic patronymics; then, gated on `middle_name_as_last`, `handle_middle_name_as_last()` folds `middle_list` into `last_list`; finally `handle_capitalization()` applies optional auto-cap. Any new `self._attr` used by `post_process()` helpers must be initialized in `__init__` (with its default value) — the direct-kwargs path bypasses `parse_full_name()`, so the attribute won't exist otherwise. Each named attribute (`title`, `first`, etc.) is a `@property` that joins its corresponding `_list`. Setters call `_set_list()` which runs the value through `parse_pieces()`, so assigning `hn.last = "de la Vega"` correctly re-parses prefix tokens. ## 2.0 API modules (`nameparser/_*.py`, `tests/v2/`) The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These conventions apply to all new-API code and are stricter than the v1 sections above. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. - **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_pipeline/`, `_parser.py`, plus the facade layer: `_facade.py`, `_config_shim.py`). The public import surface is exactly `nameparser` and `nameparser.locales`; `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Since the M11 swap, the old paths are import-path-preserving re-exports: `nameparser.parser` re-exports the `_facade` `HumanName`, `nameparser.config` re-exports the `_config_shim` names (`Constants`, `CONSTANTS`, `SetManager`, `TupleManager`, `RegexTupleManager`); the `config/` DATA modules stay the vocabulary source through 2.x. The whole facade layer is deleted in 3.0. - **Facade layer** (`_facade.py`, `_config_shim.py`): the v1-compat `HumanName`/`Constants` over the core. Key mechanisms: `Constants._generation` dirty-tracking (every mutation bumps; facades resolve their `Parser` lazily via `_cached_parser(lexicon, policy)`); `Constants._snapshot()` mirrors `_lexicon._default_lexicon()` (equality-pinned); the facade pickles v1-SHAPED state (component lists, one `__setstate__` path for 1.4 and 2.x blobs; components rebuild via `replace()`, never a re-parse); `_V1_HOOKS` overrides warn once per subclass (#280). The compat contract is the migration spec's promise: warning-free 1.4 code behaves identically except release-log-classified fixes — `tools/differential/` (dev-only, not shipped) verifies this against 1.4-on-PyPI, and `--baseline` points the same gate at any released version — run it at 1.4.0 for the compat contract, at the previous minor for a release's blast radius. `tools/differential/README.md` owns the corpus roster and what each file is blind to; don't restate it here. `parser.py:NNNN` citations throughout the 2.0 code refer to the PRE-swap v1 file, deleted at the M11 swap; resolve them with `git show 2d5d8c2:nameparser/parser.py`. - **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`; the facade layer (`_facade`, `_config_shim`, `parser`, `config/__init__`, `__main__`) may import anything public plus `_render`; locale pack modules (`locales/*.py`) import `_locale`/`_lexicon`/`_policy`/`_types` only (`_types` joined the list in #272 — `locales/ja.py`'s segmenter factory constructs a `Segmentation`), and the `locales/__init__` additionally lazy-imports its packs (PEP 562). Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. `Role` is a `StrEnum`: members compare as their field-name strings, and `tokens_for()` coerces strings via `_coerce_enum`. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. Sanctioned deviation, facade layer only: `HumanName` and the shim `Constants` organize by v1 concern groups (`# -- render defaults --`, `# -- config / parsing --`, `# -- fields --`, ..., dunders and pickle last) — the classes mirror v1's own surface and die in 3.0; the canonical order still binds every core type. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). - **Guard, hint, and emit for the WHOLE family, and parametrize the test over it**: a check added to one member of a set belongs on all of it, and the test must sweep the family, not one example. This session shipped `_reject_str_and_mapping` on `Policy` but not `PolicyPatch`, the bytes decode hint on three of five config entry points, and a regex-sync roster missing four of its copies — each a separate follow-up bug that a `{class} × {field} × {bad-value}` parametrization would have caught and a per-example test hid. When you find you're guarding member N, grep for the other members first. - **Ambiguities are emitted at the DECISION site**: an `Ambiguity` records a fork the parse had to call, not a token that sits in an ambiguous vocabulary. Emit where the branch is taken — the trailing-suffix peel in `_assign`, the delimiter escape's follow-up in `classify` — never by scanning for a `vocab:*-ambiguous` tag. The same tagged token is a genuine fork in one position and unremarkable in another (`do` mid-name in "Joao da Silva do Amaral de Souza" chooses nothing). **A branch that runs but changes nothing is not a decision either** -- the prefix chain's `merge(k, j)` executes even when `j == k + 1`, folding a piece into itself, and keying on "the code got here" reported a fork for all 39 ambiguous particles on "Dr. Van Jr.", where the particle stayed the GIVEN name and `_assign` reported the same token again. Check that the branch actually claimed something (`j > k + 1`) before recording. Structure also structure often settles the question before it arises, which is why `PARTICLE_OR_GIVEN` is deliberately not emitted on the `FAMILY_COMMA` path and `SUFFIX_OR_NAME` is not emitted for "Ma, Jack". The decision site also has the token index and the detail text in hand, which the tag scan would have to reconstruct. **If a fork's two branches are taken in DIFFERENT stages, every one of them needs the emitter** -- `PARTICLE_OR_GIVEN` is decided in `_assign` when the ambiguous particle stays a lone leading piece and in `_group` when a title shifts it off index 0 and the prefix chain claims it, so both report; for two years only the first did. The stage-ownership map in `tests/v2/pipeline/test_state.py` must list `ambiguities` for each such stage, and it passes vacuously until a case row exercises the path, so add the row too. Report BOTH directions of a two-way fork — "John Smith MA" (read as a suffix) and "Jack MA" (read as the family name) are equally guesses. Every kind needs a trigger in `tests/v2/test_contracts.py::_AMBIGUITY_TRIGGERS` (an explicit `None`, strict-xfail, while reserved), and case-table rows pin expected kinds exactly, so a new emitter shows up in both immediately. **Pin the decision, not the vocabulary**: the only titled-particle test used an UNAMBIGUOUS particle, so it walked the right code path and proved nothing about the branch under test -- two criticals passed 1539 tests. A row contrasting the two readings ("John Smith V" against "John Smith B") is what makes an emitter's absence meaningful. - **A kind is worth adding only if a reader would hesitate too**: the test is not "does the code take a branch" but whether a person reading that input would genuinely be unsure. "Smith, John V" reads as a middle initial to anyone -- the comma settles it -- so reporting it would be noise that teaches callers to ignore the field, which costs more than the missing report. Reachability of the second branch is necessary, not sufficient. Prefer leaving a fork silent and documenting the omission (see the comma paths in concepts.rst) over emitting on input nobody finds ambiguous. - **Parser owns config-dependent conveniences**: `Parser.matches`/`Parser.capitalized`/`Parser.revise` exist because the `ParsedName` equivalents fall back to DEFAULT config for str/omitted arguments (documented loudly in both docstrings). `revise` harvests tokens from a full sub-parse of each replacement value (tags kept minus `FOLDED_TAG`, roles forced, ambiguities discarded); the merge tail is shared with `replace()` via `ParsedName._with_field_tokens`. `Parser.capitalized` delegates through `name.capitalized(self.lexicon)` specifically so `_parser` never imports `_render` — keep it that way. - **Per-word vocabulary fields warn on multi-word entries** (`_normset`/`_normpairs` via `_warn_dead_entry`, UserWarning, never a raise — see the given_name_titles Gotcha for why raising is wrong). `given_name_titles` is the one multi-word-matched field and is exempt; `_edit` passes `warn=False` (add() warns once via the new instance's `__post_init__`; remove() stores nothing). The default vocabulary and every locale pack must stay warning-free (`test_default_lexicon_builds_warning_free`, `test_pack_vocabulary_entries_are_single_words`). - **Invariants guard harm, not no-ops**: add a constructor check when violating it produces a *wrong parse*, not when it produces *nothing*. A false positive costs a working configuration; a true positive on an inert condition costs the user nothing, so that trade is never worth taking. `suffix_acronyms_ambiguous ∩ suffix_words` is guarded because the overlap loses a family name; `given_name_titles` is not, because an unreachable entry is simply never consulted (see Gotchas). Before adding one, construct the config it forbids and check what actually breaks. - **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Six exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`), `honorific_tails = GLUED_HONORIFICS ∩ suffix_words` (#308 behavior with no v1 manager of its own, so the one v1 knob that reaches it is deleting the suffix word — which turns the peel off, `test_snapshot_removing_a_honorific_word_turns_the_peel_off`), and `maiden_delimiters − nickname_delimiters` on the POLICY half of the same method (v1 precedence: a pair in both v1 buckets parses as a nickname, while `Policy` resolves the overlap the other way, so the subtraction is what keeps the facade at v1 behavior, `test_snapshot_overlap_keeps_v1_nickname_precedence`). Note that last one is on the `Policy`, not the `Lexicon` — the roster is per-`_snapshot()`, not per-vocabulary-field, so a sweep that only reads the `Lexicon(...)` call misses it. When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). `PolicyPatch`'s repr shows only set (non-UNSET) fields; `_order_repr` must never raise even on an unvalidated patch's garbage `name_order` (PolicyPatch defers validation to apply time); the sweep test in `tests/v2/test_reprs.py` pins that no config repr leaks the UNSET sentinel. - **A claim about WHICH STAGE does something is checkable — check it before writing it.** The pipeline is eight stages with a written ownership map (`ParseState`'s docstring, pinned by `tests/v2/pipeline/test_state.py`), and `parse(s).tokens` prints every token's role and tags, so "extract assigns this", "classify never sees that", "group consumes it" each have a one-command answer. #329's prose claimed delimited maiden content is *"claimed whole before classify has tagged anything inside it"*; measured, `classify` tags the marker fine and only the CONSUMING is missing, because `_group`'s rule walks `pieces` and a token that already carries a role is not in `pieces`. Two different mechanisms, one plausible sentence covering both. **And when a mechanism claim turns out wrong, sweep for where else you wrote it**: that one shipped in three places — the release-log entry, `config/maiden_markers.py`'s docstring, and a case-row note — because the correction reached the issue it was found on and nowhere else. Prose density here means one idea routinely lives in a docstring, a case note, a release-log entry and this file; a claim worth writing is worth grepping for when it changes. It then shipped a fourth and fifth time — the fix's own comment in `_group.py` and its test docstring — written by the commit that closed the bug, after this bullet existed: a wrong mechanism gets reused most readily by the person implementing against it, so the sweep belongs at the END of the change too, over the words the change itself just added. The correction pass then made a FRESH stage error in the very sentence fixing the old one — "extract assigned the whole clause `Role.MAIDEN`", when `extract_delimited` produces no tokens at all (it records `extracted`/`masked` spans; `tokenize` sets the role) — which is this bullet turned on itself: rewriting a mechanism claim is writing one, and earns the same one-command check. - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. **Document the positive direction of a partial property**: "a non-empty `ambiguities` is a signal to act on" is checkable, while "an empty one means no fork occurred" is a universal negative needing exhaustive verification -- that claim was written twice and falsified twice, at sites the author had not audited. - **The segmenter contract**: the optional `Parser(segmenter=...)` hook is parse-totality's ONE exception (locales spec section 4). Everything inside that exception is a bug in USER CODE, never a fact about the name, so it is surfaced rather than absorbed: the segmenter's own exceptions propagate, and the two protocol violations the stage can detect for itself — an answer of the wrong type, and one cutting at or past the end of the token it was handed — raise `TypeError`/`ValueError` from `_script_segment` for the same reason. The line to hold when adding a check there: a protocol violation by the segmenter's AUTHOR raises, while an adapter's defense against its own third-party library (`locales/ja.py`'s repertoire, length, reconstruction and score guards) declines with `None`, because what those catch is a fact about the content. - **Pickling**: v2 types must round-trip (`Parser` is picklable by construction, and it holds a `Lexicon`; the one qualifier is that a `Parser` pickles iff its segmenter does — see the segmenter bullet above). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. - **One sanctioned global**: the (future) cached default `Parser`. Lazily cached FROZEN singletons (`Lexicon.default()`'s `functools.cache`, the future default parser) are constants, not state; any second piece of module-level MUTABLE state requires amending the conventions doc, on purpose, in review. Sanctioned exceptions, facade layer only: `_config_shim.CONSTANTS` (the v1 shared singleton, mutable by design) and `_facade._WARNED_SUBCLASSES` (the once-per-subclass hook-warning dedup set) — both deleted with the layer in 3.0. - **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module plus the cross-cutting ones (`test_reprs.py`, `test_layering.py`, `test_contracts.py`, `test_properties.py`, `test_benchmark.py`, the `cases.py`/`test_cases.py` table, and `test_regex_sync.py`, which pins every hand-copied pattern or codepoint table against its source wherever the copy lives — including copies outside the package), names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. ## Extension Patterns **Adding a scalar `Constants` attribute + `HumanName` kwarg** (e.g. `initials_separator`, `suffix_delimiter`): 1. Add class attr to `Constants` in `config/__init__.py` with docstring 2. Add `x: str | None = None` to `HumanName.__init__` signature after related kwargs 3. Add `self.x = x if x is not None else self.C.x` in body — use `is not None`, not `or`, to allow falsy values like `""` 4. conftest auto-restores scalar CONSTANTS between tests, but tests that *set* CONSTANTS mid-run still need their own try/finally 5. Update `docs/customize.rst`'s constants list (and `docs/usage.rst` if it affects a documented example) — don't wait for the release checklist, these `.rst` docs aren't covered by CI so a stale one can ship silently. Check `AGENTS.md` itself too (Extension Patterns, Gotchas, the Architecture section) for now-stale attribute/test names or descriptions — it has the same blind spot as the `.rst` docs and the release checklist only catches it at release time. **Adding a new mutable/collection `Constants` attribute** (a `SetManager`/`TupleManager`-backed group, e.g. `nickname_delimiters`/`maiden_delimiters`): add it to `_COLLECTION_CONFIG_ATTRS` in `tests/conftest.py`, or tests that mutate the global `CONSTANTS` copy will leak state into later tests. Contents must be deep-copyable (the snapshot uses `copy.deepcopy`) — already true for the existing manager types. (The parse-no-mutation tests in `tests/test_constants.py` need no such registration — they discover collections structurally via `Constants.__getstate__()`; conftest's list is the only manual one.) Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_deepcopy_roundtrip`/`test_nickname_delimiters_deepcopy_roundtrip` in `tests/test_constants.py`), not just reliance on conftest's autouse snapshot/restore exercising it incidentally. `TupleManager`/`RegexTupleManager.__getattr__` answer *any* unknown attribute lookup — dunder probes like `__deepcopy__` raise `AttributeError` (never answered as config keys), everything else still returns `self.get(attr)`/`EMPTY_REGEX` — so a new manager subtype or a `__getattr__` tweak can silently break `copy.deepcopy` (this bit `RegexTupleManager` before the dunder-lookup guard was added). A direct test on the new attribute's own manager instance catches that where the conftest fixture, which never asserts on the copy, would not. As with the scalar-attribute pattern above, also check `AGENTS.md` itself for now-stale references when you touch this. **Unknown-key attribute access on `TupleManager`/`RegexTupleManager` warns (1.4, #256) — 2.0 note: the warning became a hard `AttributeError` naming the known keys (shim `TupleManager`); the paragraph below describes the deleted v1 machinery and applies only when reading v1 history.** — a key not currently in the dict emits `DeprecationWarning` naming the miss and the known keys (`_warn_unknown_key`), before falling back to the same `None`/`EMPTY_REGEX` default as before; `.get()` stays silent. Dunder probes (`__deepcopy__`) still raise `AttributeError` outright, and single-underscore probes (`_repr_html_`, IPython's `_ipython_canary_method_should_not_exist_`, etc.) are excluded from the warning too — no real config key starts with `_`, so both guards just keep protocol/introspection probes from misfiring as "typo" warnings. This means internal parser code that reads `self.C.regexes.` unconditionally (e.g. `squash_bidi`'s `bidi`) now warns if a caller's custom `regexes` dict omits that key — a previously-silent partial-override pattern is on the same deprecation path as an actual typo. **Adding a word to a config set** — first check the *other* sets for the same word (grep `nameparser/config/` or intersect the sets in a `python3 -c`). Real overlaps exist: `do`/`st`/`mc` ∈ `PREFIXES` ∩ `TITLES`/`SUFFIX_ACRONYMS`; `abd` = "ABD" ∈ `SUFFIX_ACRONYMS`; `abu` ∈ `PREFIXES` ∩ `bound_first_names` (position-dependent: leading token → first-name join, mid-name → last-name join). Usually position-dependent and harmless, but can force a guard or an exclusion (the `last_base` all-particles guard; dropping `abd` from `bound_first_names`). **Before adding a short/common word to `PREFIXES` globally**, test it mid-string against realistic 3-token names, not just check for English-word collisions: Korean/Vietnamese given names put a short syllable in the middle slot (`Park In Hwan`, `Nguyen To Nga`), and Western names put a bare initial there (`John V. Smith`). A word that looks safe ("nobody is named 'to'") can still swallow a real middle name/initial into the last name once it's a global prefix — confirmed regressions for `to`/`in`/`an`/`ten`/`then` and bare `v` this way (PR #191). **Adding a curated sub-set of an existing config set** (must stay ⊆ its parent, e.g. `FIRST_NAME_TITLES` ⊂ `TITLES`) — define the parent as a *static* union in the config module: `TITLES = FIRST_NAME_TITLES | set([...])`. The sub-set is a `_SetManagerAttribute` on `Constants` (like `first_name_titles`), **not** a `_CachedUnionMember` — only `prefixes`/`suffix_acronyms`/`suffix_not_acronyms`/`titles` feed the `_pst` hot-path cache, so a sub-set costs nothing at runtime and stays out of `is_rootname`. The union is import-time only: a runtime `.add()` to the sub-set does **not** propagate to the parent's `SetManager` (same as `first_name_titles`→`titles`), so a caller adding a brand-new word adds it to both. Pin the relationships with import-time `assert`s in the config module itself (see the bottom of `prefixes.py`): `subset ⊆ parent`, and `∩ == ∅` with any set it logically can't overlap (a sub-set member that's also in `TITLES` is silently inert — title handling consumes it first). A violated assert fails at import — before any test runs — so don't also duplicate them as tests. Do **not** assert `titles ∩ prefixes == ∅` — that overlap is intentional (`st`, `do`). **Adding a flag-gated post-parse transform** (reorder/adjust) — add a `Constants` boolean (default `False`), implement a `handle_*()` method, and call it in `post_process()` after `handle_firstnames()` and before `handle_capitalization()`, gated on the flag. Default-off keeps existing parses byte-for-byte unchanged. Two shipped examples: `patronymic_name_order` gates both `handle_east_slavic_patronymic_name_order()` (#85) and `handle_turkic_patronymic_name_order()` (#185) — one flag driving two independent handlers, added in the same `post_process()` slot; `middle_name_as_last` gates `handle_middle_name_as_last()` (#133), which folds `middle_list` into `last_list`. **Validating a new parsing rule** — before implementing, simulate it in a throwaway script against `TEST_NAMES` (plus a few target-language examples) to catch regressions/false-positives early. E.g. this surfaced the `last_base` `do`/`st`/`mc` empties and the patronymic `"David Michael Abramovich"` false-reorder. ## Gotchas **Parsing must never write into `Constants`** — parse-time derived recognition (dotted abbreviations like "Lt.Gov.", conjunction-joined titles/prefixes like "Mr. and Mrs." / "von und zu") lives in per-instance `HumanName._derived_titles` / `_derived_suffixes` / `_derived_conjunctions` / `_derived_prefixes` sets, consulted by `is_title`/`is_suffix`/`is_conjunction`/`is_prefix`/`is_rootname`, reset at the start of `parse_full_name()`, and backfilled in `__setstate__` for pre-existing pickles. Never `self.C..add()` during parsing — `self.C` is usually the shared `CONSTANTS` singleton, so a write makes parse results order-dependent and thread-unsafe (this was a real bug through 1.2.x). `ParsingDoesNotMutateConfigTests` (`tests/test_constants.py`) enforces the invariant by snapshotting the whole config around a parse; it discovers collections structurally via `Constants.__getstate__()`, so new `Constants` collections are watched automatically — nothing to register. If a new derived category is ever needed: add a `_derived_*` set in `__init__`, reset it in `parse_full_name()`, backfill it in `__setstate__`, consult it in the matching `is_*` predicate (store `lc()`-normalized values, mirroring `SetManager`), and add a leak test with a name that triggers it. **`HumanName.C` is a property backed by `_C`, but pickles under the public key `'C'`** — `__init__`/direct assignment route through the `C` setter, which calls the shared `_validate_constants` staticmethod (also used by `__init__`) so an invalid value raises `TypeError` immediately instead of surfacing later as an unrelated `AttributeError` deep in parsing (#239). `__getstate__`/`__setstate__` deliberately translate `self._C` ↔ a `'C'` key in the pickled dict (with the usual `CONSTANTS`-singleton-becomes-`None` sentinel) rather than pickling `_C` directly, so the on-disk pickle format hasn't changed across this fix — don't "simplify" that translation away or old pickles/tests that hand-build a state dict with a `'C'` key will break. **Titles permanently shadow first names — be conservative** — any word in `TITLES` is always consumed as a title and can never be parsed as a first name. `"Dean"` is the canonical example: it's a common academic title *and* a common given name, so it is intentionally absent from the default titles (see `docs/customize.rst` — users who need it add it via opt-in `Constants`). Before adding a word to `TITLES`, ask: "Could this plausibly be someone's given name in any culture?" If yes, don't add it globally; it belongs in caller-supplied `Constants` instead. This same caution applies to international honorifics — `Prince`, `Sheikh`, `Frau` are all first names in some contexts. It also applies to any prefix sub-set gated on "never a first name": obscure-looking foreign particles are surprisingly often real given names — `Von` (Von Miller), `Vander` (Brazilian, also the Arcane character). When unsure, exclude — a missing member just means that name isn't auto-handled, whereas a wrong member misparses a real person. **The period-abbreviation title inference runs at the head of the GIVEN-NAME part, not the head of the name** — an unrecognized multi-letter word ending in a single trailing period (`_assign._PERIOD_ABBREV`, a hand copy of the `period_abbreviation` regex, `{2,}` letters) is treated as a title in the leading title run, e.g. `"Insp. Jane Morse"` → `title='Insp.'`. "Leading" is per SEGMENT: `_peel_leading_titles` is called for NO_COMMA segment 0, SUFFIX_COMMA segment 0, and FAMILY_COMMA **segment 1**, so `"Morse, Det. Insp. Jane"` → `title='Det. Insp.'` and a lone `"Smith, Xyz."` → `title='Xyz.'` — long-standing, verified against 1.4.0, and the mechanism behind #296 (`"Smith, Jr."` → title, which the shape rule claims even once `jr` leaves `TITLES`). The docs said "leading word" until 2026-08-01 and were wrong for every comma path. It does not mutate `C.titles`, so the periodless form (`"Insp"`) is unaffected elsewhere. The `{2,}` length requirement — not a separate initials check — is what excludes single-letter initials like `"J."`; the same word after the given name is left as a middle name. **The inference OUTRANKS vocabulary where it runs**: `"Esq. Smith"` → `title='Esq.'` even though `esq` is suffix-only vocabulary, because the shape rule fires before anything consults the suffix sets. **And it runs in one direction only**: a trailing abbreviation has no structural counterpart and is matched against the suffix vocabulary alone, so a trailing TITLE word is not a title (`"John Smith Prof."` → `family='Prof.'`, and the comma path disagrees — `"Smith, Prof."` → `title='Prof.'`). Meanwhile `period_joined_vocab` resolves INTERIOR-period tokens (`Lt.Gov.`, `Msc.Ed.`) to title-or-suffix by vocabulary, and `_extract._suffix_shaped` treats any period-final delimited content as not-a-nickname. Four trailing-period behaviors, four different resolutions; unifying them is open design work, not settled. (#109; see `docs/usage.rst` "Titles you didn't configure") **Cyrillic suffix regexes need `re.I` even when the pattern is suffix-only** — a Latin title-cased word (`Ivanovich`) keeps its suffix lowercase, so `re.I` seemed skippable; but an irregular Cyrillic suffix can be nearly the whole word (`ильич`), so title-casing capitalizes into the suffix itself (`Ильич`). `east_slavic_patronymic_cyrillic` shipped without `re.I` on the Latin reasoning and silently failed on capitalized irregular forms — don't assume Latin's title-case safety transfers to Cyrillic. (#185) **`suffix_not_acronyms` vs `is_an_initial` tension** — single-letter roman numeral suffixes (`i`, `v`) are in `suffix_not_acronyms` but also match the `is_an_initial` regex (single uppercase letter), so `is_suffix()` rejects them. The lenient test lives in `is_suffix_lenient()`, which accepts `suffix_not_acronyms` members unconditionally and is only safe in unambiguous positions: (1) suffix-comma detection uses it via `are_suffixes_after_comma()`; (2) lastname-comma post-comma parsing uses it inline, only when `nxt is None` and `len(parts)==2` (no `parts[2]` suffix segment). See issues #136, #144. **The tension is Latin-only since #320** — the veto is now scoped to scripts that HAVE initials, and v1's single `is_an_initial` is now two predicates, of which the tension is about `_vocab.is_initial` (the one `is_suffix()`/`is_suffix_strict` vetoes with; `is_initial_shaped` is the bare v1 shape test and vetoes nothing) — `_vocab.is_initial` ANDs that shape test with "not in `_policy._NO_INITIALS`", that constant listing the four scripts which do NOT (Han, Hangul, Hiragana, Katakana) — so the seven single-character CJK members of `suffix_not_acronyms` (씨, 様, 氏, 군, 양, 님, 殿) no longer collide with it when written with a period: `"김민준 씨."` gives suffix `씨.` where the veto had made it the family name, and `"김민준, 씨."` where it had made it the given name. Latin is untouched — `V.`/`I.` still lose to the veto and still need `is_suffix_lenient()`. **Comparing against v1 needs `PYTHONSAFEPATH=1` AND a directory outside the worktree** — `uv run --isolated --no-project --with 'nameparser==1.4.0'` still puts the checkout's `nameparser/` ahead of the pinned wheel on `sys.path`, so the "v1" side silently imports the branch and every comparison reports parity. Run it as `cd && PYTHONSAFEPATH=1 uv run --isolated --no-project --with 'nameparser==1.4.0' python -c "..."`. This produces false confidence rather than an error, so it invalidates results without ever looking wrong. **`esq` is in both `SUFFIX_ACRONYMS` and `SUFFIX_NOT_ACRONYMS` on purpose — do not "deduplicate" it** — the two branches normalize differently (see the asymmetry gotcha above): the word test strips only edge periods, the acronym test strips all of them. `Esq` matches only through the word set, `E.S.Q.` only through the acronym set. Removing either membership drops a spelling and, for the acronym one, loses the family name (`"John Smith E.S.Q."` → `family='E.S.Q.'`). Pinned by the `suffix_acronym_multidot_spelling` case-table row. The disjointness that IS asserted is `SUFFIX_ACRONYMS_AMBIGUOUS ∩ SUFFIX_NOT_ACRONYMS`, which is a different claim: `suffix_as_written` ORs the branches, so a word membership bypasses the period gate the ambiguous set exists to impose. **`Lexicon.given_name_titles` is deliberately unvalidated against `titles` — do not add a check** — the lookup key is the space-joined run of `Role.TITLE` tokens, built by the parse, and a conjunction inside a run is itself tagged `Role.TITLE`, so `"sir and dame"` is a matchable key whose middle word lives in `conjunctions`. A whole-entry check rejected multi-word entries; a per-word check rejected that one. No static relation over the vocabulary sets decides reachability. An unreachable entry is inert — nothing consults it and nothing misparses — so the condition being "guarded" costs the user nothing while each guard cost a working configuration. If a diagnostic is wanted, it must be non-blocking — the multi-word UserWarning in `_normset` is the shipped example; a raise remains wrong. Note the SHIPPED data cannot carry a multi-word given-name title without a spurious warning: `FIRST_NAME_TITLES ⊆ TITLES` puts the entry in `titles` too, which is per-word warned; user-supplied v2 Lexicons are unaffected (`add(given_name_titles=...)` alone is silent). **`_normalize` must reach a fixed point** — storage and match-time share the one fold, and `Lexicon.__setstate__` re-validates, so a value that changes on re-normalization changes under its owner. `strip().strip(".")` alone is not idempotent (`'. a .'` → `' a '` → `'a'`). The loop is the fix; keep any new stripping inside it. **Anything built on `_normalize` must converge too** — `_title_key` joins per-word `_normalize` and DROPS words that fold away; keeping the empty slot stored `'lt .'` as `'lt '`, a key match-time can never rebuild (so the entry is silently inert) and `__setstate__` rejects on the next round-trip as "not written by this version". **Perf regressions are caught by the scaling test, not the absolute-time ones** — `tests/v2/test_benchmark.py::test_parse_cost_grows_no_worse_than_linearly` times a repeated unit at n vs 4n over ten shapes (one per pipeline inner loop) and bounds the ratio; the `_thousand_names` tests use constant-size, delimiter-free input and are structurally blind to a complexity regression. Two rules when touching it: calibrate `_MAX_RATIO` against the WEAKEST quadratic's signal (a mixed quadratic surfaces far below the textbook 16×, so the operating point `_BASE` matters more than the bound), and confirm a planted regression fails it across REPEATED runs — one failure is a coin-flip on a timing test. The ten shapes cover different dimensions (segment count only via `commas`, intra-piece accumulation only via `particles`/`conjunctions`, non-ASCII input only via `honorifics` — the other nine are pure ASCII, so `script_segment` returns at its bail and the CJK stages go unmeasured); measure before pruning one. A stage gated on an opt-in `Policy` field needs a `_POLICY_SHAPES` entry instead, since bare `parse()` never enters it — and that table's rows carry a **reachability probe** run before the measurement, because a precedence change can quietly stop the shape reaching the stage and leave a green test measuring a no-op (`_POLICY_SHAPES` is also asserted non-empty: an empty `parametrize` is a skip, not a failure, so deleting its last row would retire the guard silently). **Expected-failure tests use `@pytest.mark.xfail`** — the conftest parametrized fixture breaks `@unittest.expectedFailure`; always use `@pytest.mark.xfail` instead. **`lc()` strips leading and trailing periods** — `'M.D.'` → `'m.d'`, not `'md'` (interior periods are preserved). Exception keys in `capitalization_exceptions` are dot-free, so lookups must also try `.replace('.', '')`. **`is_suffix()`'s period-stripping is asymmetric** — it does `lc(piece).replace('.', '')` (strips *all* periods) before checking `suffix_acronyms`, but only `lc(piece)` (leading/trailing only) before checking `suffix_not_acronyms`. Code that reimplements this check instead of calling `is_suffix()` must mirror both branches or it will misclassify acronym suffixes with internal-only periods (e.g. `"M.D"` with no trailing dot) — bit `parse_nicknames()`'s `handle_match()` in PR #189. **`nickname_delimiters`/`maiden_delimiters` built-ins are string sentinels, not compiled patterns** — the three default keys (`quoted_word`, `double_quotes`, `parenthesis`) store the *name* of a `Constants.regexes` entry (a plain `str`), resolved via `getattr(self.C.regexes, name)` at parse time in `parse_nicknames()` — not the compiled `re.Pattern` itself. This is what lets `CONSTANTS.regexes.parenthesis = ...` keep affecting nickname/maiden parsing after construction, same as before this mechanism existed. Routing a built-in between buckets must be a `pop()` + assign (`maiden_delimiters['parenthesis'] = nickname_delimiters.pop('parenthesis')`) to carry that string sentinel over — copying `CONSTANTS.regexes['parenthesis']` directly into the new bucket instead would freeze it as a snapshot and silently stop tracking further `regexes` overrides. A key added by a caller for a *custom* delimiter is a real compiled pattern, distinguished at parse time via `isinstance(raw_pattern, re.Pattern)`. (#22) **`TupleManager.__setattr__`/`__delattr__` guard dunder names too, not just `__getattr__`** — constructing a subscripted generic, e.g. `TupleManager[re.Pattern[str] | str]({...})` (needed so mypy sees the right value type instead of inferring one from the dict literal), makes `typing`'s `GenericAlias.__call__` set `__orig_class__` on the new instance right after `__init__` returns. Before this guard existed, `__setattr__` was a bare `dict.__setitem__` alias, so that assignment silently inserted a bogus `'__orig_class__'` entry into the dict itself, corrupting `.values()`/iteration for *every* `TupleManager`/`RegexTupleManager` instance, not just the one being constructed — this bit `nickname_delimiters`'s construction (#22) before the guard was added. Same fix shape as the `__getattr__` dunder guard above: fall back to `object.__setattr__`/`object.__delattr__` for dunder names, dict-backed storage for everything else. **`_join_bound_first_name` guard must exclude trailing suffixes** — suffix tokens are still in `pieces` when the helper runs (suffix detection happens in the assignment loop, later). The `reserve_last` guard must count `if not self.is_suffix(p)` to avoid treating a trailing suffix like "Jr." as a last-name slot; otherwise `"abdul salam jr"` → `last='jr'`. **Doctests** — docstring examples in `nameparser/*.py` run under `uv run pytest` (`--doctest-modules`; `testpaths` is `tests` + `nameparser` only). The `.rst` doctests in `docs/` (`usage.rst`, `customize.rst`) and `README.rst` are also run in CI: the `python-package.yml` workflow runs `sphinx-build -b doctest docs dist/doctest` (covers every `.rst` under `docs/`) and `python -m doctest README.rst` as separate steps after the existing `-b html` build. Both should pass with zero failures — expect a clean run, not baked-in noise. Every `.. doctest::` block in one `.rst` file shares a single namespace — the per-document default group — so **both** local variables and global state carry forward, in document order. Two consequences, and the first is the one that bites. A variable bound in one block clobbers that name in every later block, so inserting a section that binds something common breaks examples further down the file that were passing and that you never touched: writing the #271 usage.rst section with `name = parse("김민준")` in it made `name.initials()` six blocks later return `'민. 김.'` instead of `'J. Q. X. V.'`. Pick a distinctive variable name (`minjun`), or reuse whatever the surrounding blocks already bind. Second, examples that mutate the shared `CONSTANTS` singleton (e.g. `capitalize_name`, `titles.clear()`) must reset it at the end of the same block, because nothing else will — an unreset mutation leaks into every later example in the same build (this caused several real failures before it was fixed). If an example isn't actually demonstrating shared-singleton behavior, prefer a local `Constants()` instance instead of `CONSTANTS` — no reset needed, and no ordering dependency on other examples. `SetManager.__repr__` sorts its elements for this reason too: plain `set()` iteration order depends on per-process string hash randomization, so an unsorted repr would make ellipsis-style doc examples (`SetManager({'10th', ..., 'zoologist'})`) unreproducible across runs. Don't use the bare `python3 -m doctest .rst` CLI (no `optionflags`) to check files under `docs/` — it ignores each block's `:options:` directive (e.g. `+NORMALIZE_WHITESPACE`, `+ELLIPSIS`) and reports false-positive whitespace failures that look like real regressions. `uv run sphinx-build -b doctest docs ` is the faithful check for those. `README.rst` has no `:options:` directives (plain `::` blocks, not `.. doctest::`), so the bare `python -m doctest README.rst` CLI is fine there — that's what CI actually runs. `Constants` class attributes (e.g. `patronymic_name_order`, `middle_name_as_last`) document behavior with a bare string literal placed right after the assignment — Sphinx's attribute-docstring convention. That string never becomes a real `__doc__`, so `--doctest-modules` (which walks `__doc__` attributes) never sees any `.. doctest::` examples inside it — this let a stale example slip through CI once (`middle_name_as_last`, #133). `tests/test_config_attribute_docstrings.py` (#195) closes that gap: it parses `nameparser/config/__init__.py` with `ast` to recover those literals and runs any doctest examples through `doctest.DocTestParser`/`DocTestRunner` explicitly, so `pytest -q` now exercises them too. When adding or editing a `.. doctest::` example in a `Constants` attribute's bare-string docstring, this is the mechanism that actually runs it — don't assume `--doctest-modules` covers it. **`uv run mypy` covers `tests/` too** (`pyproject.toml`'s `[tool.mypy] packages = ["nameparser", "tests"]`, PR #250) — a test that deliberately passes an off-contract value to assert a `TypeError`/`ValueError`, or exercises a documented falsy-disables-the-feature toggle (e.g. `regexes.emoji = False`), needs a `# type: ignore[code]` comment on that line rather than a signature change. Before reaching for an ignore, check whether the "error" is really a source annotation that hasn't caught up with already-supported runtime behavior (e.g. `is_prefix`/`is_conjunction`/`is_suffix` accepting `list[str]`, `add_with_encoding` accepting `bytes`) — widen the annotation instead in that case. **`initials_separator` is intra-group only** — it controls the joiner between consecutive initials *within* a name group (e.g. two middle names in `middle_list`). Spaces *between* groups come from `initials_format`. To fully concatenate initials you need both `initials_separator=""` and `initials_format="{first}{middle}{last}"`. **`pr/NNN` local branches** track upstream PRs — don't commit to them by accident. Check `git branch --show-current` before starting work. **Prefix-join uses value-based `list.index()`** in `join_on_conjunctions` — fragile when a token value repeats (e.g. a trailing title that's also a suffix acronym, or two `van`s); constrain such lookups to start at `i + 1`. See #100. **Title vs suffix is positional for BARE words, and the leading period-abbreviation rule overrides even that** — a word matching `TITLES` at the front of a name becomes `title`; the same word matching `SUFFIX_ACRONYMS`/`SUFFIX_NOT_ACRONYMS` at the end becomes `suffix` (never both, regardless of the word's real-world meaning). External test sources (old issue gists, etc.) sometimes assert `suffix` for a leading professional abbreviation like `RA`/`PD`/`Dipl.-Ing.` — that's the source data being wrong, not a parser bug. Verify position before "fixing" it. Two qualifications the older "purely positional" wording papered over, both measured 2026-08-01: a PERIOD-marked leading word is claimed by the shape rule before any vocabulary is read (`"Esq. Smith"` → `title`, though `esq` is suffix-only), and trailing position has no such rule at all, so a title word there is neither title nor suffix but a NAME part (`"John Smith Prof."` → `family='Prof.'`) — which is what the comma path already disagrees with. Why it is not simply inverted to "vocabulary decides": `TITLES` holds 692 words that are in no suffix set, and many are ordinary surnames (`king`, `bishop`, `prince`, `pope`, `judge`, `sheriff`, `baron`, `master`, ...), so a vocabulary-first trailing rule would read `"Mary Jane King"` as `title='King'`, `family='Jane'`. The period is what separates the safe case from that one — `King` is a surname, `King.` is not. ### Tests (`tests/`) **Prefer behavior tests over constant-content tests** — don't assert on the literal contents/structure of config constants (e.g. `SUFFIX_ACRONYMS == SET_A | SET_B`, `'x' in SOME_SET`). Test observable parsing behavior instead (`HumanName(...)` output). Constant-content assertions just create a second place to update whenever the lists change. A parse-behavior test can still be a disguised constant-content test if it only exercises one or two hardcoded entries from a large list (e.g. guarding a specific missing-comma typo in `SUFFIX_ACRONYMS`) — it documents that one instance without catching the same bug class elsewhere in the list. Prefer a test on the structural property itself, or skip a dedicated test for narrow, unrepeatable typo fixes. **When adding a new aggregate property** (like `given_names` or `surnames`), always include a test for the empty path — a name that produces no value for that property — so the `or self.C.empty_attribute_default` guard is covered. The conftest dual-fixture then automatically exercises both `""` and `None` variants. Example: `HumanName("Williams")` for a given-names property (last-name-only string has no first or middle). **`python_classes = ["*Tests", "*TestCase"]`** in `pyproject.toml` — suffix style (`FooTests`), NOT prefix (`TestFoo`); wrong style silently skips discovery. Tests run under **pytest** (via `uv run pytest`) and are split one file per concern (`tests/test_titles.py`, `tests/test_suffixes.py`, etc.). `tests/base.py` holds `HumanNameTestBase` — a plain (non-`unittest`) base whose `m()` helper is a custom assert that prints the original name string on failure (plus thin `assert*` shims so the moved test bodies are unchanged). `tests/conftest.py` defines an autouse fixture that runs **every test twice** — once with `empty_attribute_default = ''` and once with `None` — so reported counts are doubled (e.g. 11 methods → 22 results); it also snapshots/restores the scalar `CONSTANTS` config around each test to keep tests order-independent. `TEST_NAMES` (in `tests/test_variations.py`) is a list of name strings permuted into comma-separated variants as a regression check. Tests that should fail use `@pytest.mark.xfail`. When adding a parsing case, add it to the relevant `tests/test_*.py` file and consider adding the base form to `TEST_NAMES`. Test classes that parse with a dedicated flagged `Constants` (e.g. `patronymic_name_order=True`) subclass `FlaggedConstantsTestBase` (`tests/base.py`) and set `constants_kwargs = {...}` — don't hand-write per-class `setup_method`/`hn()` fixture pairs. python-nameparser-2.1.0/AUTHORS000066400000000000000000000000451523530431700162400ustar00rootroot00000000000000Derek Gulbranson python-nameparser-2.1.0/CITATION.cff000066400000000000000000000006631523530431700170700ustar00rootroot00000000000000cff-version: 1.2.0 message: "If you use this software, please cite it as below." title: python-nameparser abstract: "A simple Python module for parsing human names into their individual components." type: software authors: - family-names: Gulbranson given-names: Derek repository-code: "https://github.com/derek73/python-nameparser" url: "https://github.com/derek73/python-nameparser" license: LGPL-2.1-or-later version: "2.1.0" python-nameparser-2.1.0/CONTRIBUTING.md000066400000000000000000000033721523530431700174270ustar00rootroot00000000000000Contributing ============== Development Environment Setup -------------------------------- Install dev dependencies: uv sync Running Tests --------------- uv run pytest # --doctest-modules is set in pyproject.toml, so doctests run automatically Run a single test file or test: uv run pytest tests/test_titles.py uv run pytest tests/test_titles.py::TitleTestCase You can also pass a name string to see how it will be parsed: $ python -m nameparser "secretary of state hillary rodham-clinton" Initials: H. R. CI runs tests against Python 3.10–3.14 via GitHub Actions on every push and pull request. Writing Tests ---------------- If you make changes, please include tests with example names that should parse correctly. It's a good idea to include tests of alternate comma placement formats of the name to ensure that the 3 code paths for the 3 formats work in the same way. Unless you add better coverage someplace else, add a few examples of your names to `TEST_NAMES`. A test attempts to try the 3 different comma variations of these names automatically and make sure things don't blow up, so it can be a helpful regression indicator. New Releases ------------ Releases are published automatically to PyPI via GitHub Actions. To cut a release, create and publish a new GitHub Release — the workflow will build and upload the package using trusted publishing (no API token or twine needed). python-nameparser-2.1.0/LICENSE000066400000000000000000000011751523530431700162020ustar00rootroot00000000000000Copyright Derek Gulbranson . http://derekgulbranson.com/ ----- LGPL-2.1+ http://www.opensource.org/licenses/lgpl-license.html This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. python-nameparser-2.1.0/MANIFEST.in000066400000000000000000000001201523530431700167200ustar00rootroot00000000000000include AUTHORS include LICENSE include README.rst recursive-include tests *.py python-nameparser-2.1.0/README.rst000066400000000000000000000116741523530431700166710ustar00rootroot00000000000000Name Parser =========== |Build Status| |PyPI| |PyPI version| |Documentation| |License| |Downloads| |Codecov| nameparser parses human names into seven fields — title, given, middle, family, suffix, nickname, maiden. Results are immutable, configuration is composable, and locale packs are opt-in. 📣 **nameparser 2.0 is out.** Existing ``HumanName`` code keeps working through 2.x, and most 1.x code needs no changes. The `migration guide `__ has the field-by-field map. Please `open an issue `__ for anything that parses wrong. **2.1 adds East Asian name support.** Chinese, Japanese and Korean names written in their own scripts are read family-first, unspaced Korean names are split against the census surname list, and CJK honorifics are recognized. See `East Asian names `__. Installation ------------ :: pip install nameparser Requires Python 3.11+. Quick Start Example -------------------- .. code-block:: python >>> from nameparser import parse >>> name = parse("Dr. Juan Q. Xavier de la Vega III (Doc Vega)") >>> name >>> name.family_base, name.family_particles ('Vega', 'de la') >>> name.render("{family}, {given}") 'de la Vega, Juan' >>> parse("김민준").family # Korean: unspaced, split on the census list '김' >>> parse("高橋 みなみ").family # Japanese: kanji with kana, family first '高橋' >>> parse("김민준씨").suffix # an honorific written against the name '씨' >>> parse("г-н Иван Петров").title # Cyrillic title 'г-н' >>> parse("محمد بن سلمان").family # Arabic: بن chains onto the family name 'بن سلمان' >>> from nameparser import locales, parser_for >>> chinese = parser_for(locales.ZH) # Han text does not say which language >>> chinese.parse("毛泽东").family # so splitting it is opt-in '毛' >>> russian = parser_for(locales.RU) >>> russian.parse("Сидоров Иван Петрович").family 'Сидоров' >>> locales.available() ('ja', 'ru', 'tr_az', 'zh') Learn more ---------- * `Using the parser `__ — the full tour: input shapes, aggregates, rendering, comparison, ambiguities, tokens * `Customizing the parser `__ — vocabulary, behavior, and presentation * `Locale packs `__ — opt-in bundles for East Slavic patronymics, Turkic markers, and more * There's also a CLI: ``python -m nameparser --json "Doe, John"`` Coming from 1.x ---------------- ``HumanName`` and ``CONSTANTS`` keep working in 2.0 — same imports, same attributes, same mutation API. What 2.0 removes is the batch of deprecations 1.3 and 1.4 announced, so if your test suite runs clean on 1.4 under ``python -W error::DeprecationWarning``, you are nearly done. Two things that check will not catch: four removals 1.4 never warned about (three raise on contact, the fourth only warns), and one that changes results silently — ``name == "John Smith"`` is now ``False``. `Migrating from HumanName `__ covers both, and translates a v1 customization into the new API whenever that's convenient for you. See the `release log `__ for the full list of changes in the 2.0 series. License ------- LGPL licensed. See `LICENSE `__ for details. .. |Build Status| image:: https://github.com/derek73/python-nameparser/actions/workflows/python-package.yml/badge.svg :target: https://github.com/derek73/python-nameparser/actions/workflows/python-package.yml .. |PyPI| image:: https://img.shields.io/pypi/v/nameparser.svg :target: https://pypi.org/project/nameparser/ .. |Documentation| image:: https://readthedocs.org/projects/nameparser/badge/?version=latest :target: http://nameparser.readthedocs.io/en/latest/?badge=latest .. |PyPI version| image:: https://img.shields.io/pypi/pyversions/nameparser.svg :target: https://pypi.org/project/nameparser/ .. |License| image:: https://img.shields.io/pypi/l/nameparser.svg :target: https://pypi.org/project/nameparser/ .. |Downloads| image:: https://static.pepy.tech/badge/nameparser :target: https://pepy.tech/project/nameparser .. |Codecov| image:: https://codecov.io/gh/derek73/python-nameparser/branch/master/graph/badge.svg :target: https://codecov.io/gh/derek73/python-nameparser python-nameparser-2.1.0/docs/000077500000000000000000000000001523530431700161215ustar00rootroot00000000000000python-nameparser-2.1.0/docs/Makefile000066400000000000000000000151721523530431700175670ustar00rootroot00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) endif # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Nameparser.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Nameparser.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/Nameparser" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Nameparser" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." python-nameparser-2.1.0/docs/concepts.rst000066400000000000000000000241241523530431700204740ustar00rootroot00000000000000How the parser works ==================== ``parse()`` turns a name string into a :class:`~nameparser.ParsedName`. This page explains the model behind that call: how a string becomes tokens and tokens become fields, how those tokens get their roles, where configuration lives and why it is split the way it is, why parsers are plain values, and what happens when a name is genuinely ambiguous. The task pages all build on these five ideas. From string to name -------------------- Every parse follows the same path: the input string is split into :class:`tokens `, each token is assigned one of the seven roles — ``title``, ``given``, ``middle``, ``family``, ``suffix``, ``nickname``, ``maiden`` — and every string you read off the result is computed from those tokens at read time. Parsing ``"Dr. Juan Q. Xavier de la Vega III"`` produces eight tokens. The first is ``Dr.`` with the ``title`` role; ``de``, ``la``, and ``Vega`` each carry the ``family`` role, which is why ``name.family`` returns ``"de la Vega"`` — the field is a view that joins the family-role tokens in order, not a stored string. Each token also records where it came from. A :class:`~nameparser.Span` is a pair of character positions bounding the token in the original string: ``Dr.`` has span ``(0, 3)``, and ``name.original[0:3]`` is exactly ``"Dr."``. Internally, spans let the pipeline refer to a token by position instead of by text. v1 re-found name pieces by searching for matching text, so a name with a repeated word could make the parser rewrite the wrong occurrence (`issue #100 `_ and its relatives); a position cannot be confused with a look-alike. :class:`~nameparser.ParsedName` is frozen: there is no attribute assignment, ever. If a parse is almost right and you want to fix one field, you call ``.replace()``, which returns a new :class:`~nameparser.ParsedName` with that field changed and everything else — tokens, spans, the rest of the roles — carried over unchanged (``replace()`` tokens carry no vocabulary tags; :meth:`Parser.revise ` is the tag-preserving form). ``str()`` renders the default view; nothing about calling it mutates the value you called it on. Two layers decide the roles ---------------------------- Roles are not assigned in one pass. A vocabulary layer runs first and claims words for what they *are*, wherever they sit: titles, particles, conjunctions, recognized suffixes, and anything set off by nickname or maiden delimiters. Titles chain, so ``"Asst. Vice Chancellor"`` is one title; particles join forward, so ``de la`` attaches to ``Vega``. Whatever the vocabulary layer has not claimed is left to a positional layer, which assigns purely by where a word sits: the first unclaimed word is the given name, the last is the family name, and anything between them is the middle name. ``name_order``, an explicit comma, and — for a name written wholly in one East Asian script — ``script_orders`` change what "first" and "last" mean here; the Chinese interpunct ``·`` dividing such a name walks the last of those back, marking a transcription that keeps its source order; nothing else does. This is the whole parser in two sentences, and it explains its character. A word nameparser has never seen still gets a sensible role, because the positional layer does not need to recognize anything. The same word can play different parts in different places — ``Dr.`` is a title before a name and a suffix after it, which is why the field names ``title`` and ``suffix`` are really "pre-nominal" and "post-nominal". And nothing is statistical: there is no model and no training data, so the same input always parses the same way, and a parse that is wrong is wrong reproducibly, which is what makes it fixable by configuration. The split also tells you which container a setting belongs in, before you look anything up: if you are teaching the parser a *word*, it goes in the :class:`~nameparser.Lexicon`; if you are changing how unclaimed words are *arranged*, it goes in the :class:`~nameparser.Policy`. Configuration lives in three containers ---------------------------------------- Every piece of nameparser configuration falls into exactly one of three places, and which one is decided by a single question: what does this setting vary with? * :class:`~nameparser.Lexicon` holds everything that varies by **language**: the vocabulary — titles, particles, suffixes, conjunctions, and the rest of the word lists the parser matches against. * :class:`~nameparser.Policy` holds everything that varies by **data source or application**: the behavior switches — name order, patronymic rules, delimiters, strip flags — anything that changes how the pipeline runs, not what words it recognizes. * :ref:`Rendering arguments ` cover everything that varies by **output destination**: the ``spec`` you pass to ``render(spec)``, or a keyword to ``initials()``/``capitalized()``. "Dean" is a common given name, so it is not in the default titles vocabulary. But in some data it is more common as a title. The right reading is a fact about the domain the names come from: that makes it a :class:`~nameparser.Lexicon` entry. A CRM that wraps nicknames in square brackets — ``John [Johnny] Smith`` — instead of quotes is a fact about that one data source's export format, not about the language of the names in it — that's a :class:`~nameparser.Policy` (``nickname_delimiters=frozenset({('[', ']')})``). One particular report wanting names formatted as "Family, Given" while every other consumer of the same parsed data wants "Given Family" is a fact about where the string is going next, decided at the moment you render it — that's a rendering argument, not something baked into how the name was parsed. This replaces v1's single ``Constants`` object, which mixed all three concerns — vocabulary, behavior, and output formatting — into one mutable bag plus a ``string_format`` template string. Sorting a setting into the right container is largely mechanical once you ask the "varies by what?" question above; see :doc:`migrate` for the attribute-by-attribute mapping from the old ``Constants`` to the new containers. Parsers are values -------------------- ``parse()`` is a convenience function over a module-level default :class:`~nameparser.Parser`. You only need to build your own :class:`~nameparser.Parser` (directly, or via :func:`~nameparser.parser_for`) when you want non-default vocabulary or behavior — and when you do, build it once and reuse it. Constructing a :class:`~nameparser.Parser` validates its configuration up front, so it's cheap but not free; parsing individual names is the hot path, and a :class:`~nameparser.Parser` is designed to be called many times without reconstruction. :class:`~nameparser.Lexicon`, :class:`~nameparser.Policy`, :class:`~nameparser.Parser`, and :class:`~nameparser.ParsedName` are all frozen and hashable. That means they're safe to share across threads without locking, safe to use as dict keys or cache keys, and equality means exactly what it says — two :class:`~nameparser.Parser` instances built from equal configuration are equal values, not merely two objects that happen to behave the same. Every piece of configuration in the 2.0 API is a frozen value — including the module-level default parser itself. Honest ambiguity ------------------ Parsing never raises. Pass in a string that doesn't look like a name at all, and you get back a :class:`~nameparser.ParsedName` with empty fields, not an exception. The parser's job is to make a reasonable call on real-world text, not to reject it. The single exception is code you supplied yourself: a :data:`~nameparser.Segmenter` passed to ``Parser(segmenter=...)`` runs inside the parse, and its own exceptions propagate rather than being swallowed — a failure there is a bug in your callable, not a fact about the name. Some calls are irreducibly ambiguous — both readings are legitimate, and no amount of rule-tuning resolves them without breaking some other name. Those surface as entries on ``ParsedName.ambiguities`` instead of being silently guessed away. The canonical example: a leading "Van" reads as a given name — the right call for the actor Van Johnson, the wrong one for a bare "Van Buren", and nothing in the two-word shape distinguishes them — so the parse records a ``particle-or-given`` ambiguity alongside its answer. You can inspect ``ambiguities`` to decide, case by case, whether your data needs a second look. An ambiguity records a *decision*, not a word. The same token in a different position may present no fork at all: ``do`` is in the ambiguous post-nominal vocabulary, but in ``"Joao da Silva do Amaral de Souza"`` it sits mid-name, where nothing has to choose between readings — so nothing is recorded. A comma can settle the question before it arises, too: ``"Ma, Jack"`` fixes the family name, so the credential reading never comes up, while ``"John Smith MA"`` has to call it and says so. An empty ``ambiguities`` is therefore not a certificate of certainty. Reporting is deliberately partial: :class:`~nameparser.AmbiguityKind` lists the forks worth flagging, and even those are not reported everywhere they occur — the comma paths stay quiet on purpose, since a comma usually settles the structure before the question arises. Coverage grows over releases. Treat a non-empty ``ambiguities`` as a signal to act on; do not read an empty one as a guarantee. :class:`Tokens ` also carry tags — a second, independent label alongside their role, recording how a token was classified rather than what part of the name it belongs to — but only a handful of them are part of the stable API, collected in :data:`~nameparser.STABLE_TAGS`: ``particle``, ``conjunction``, ``initial``, and ``joined``. Any tag written with a namespace prefix, like ``vocab:...``, is provenance information for debugging how a token got classified — it can change shape between releases and isn't something to match against in your own code. If you need to branch on how a token was classified, branch on role or on one of the four stable tags above, not on a namespaced one. python-nameparser-2.1.0/docs/conf.py000066400000000000000000000215361523530431700174270ustar00rootroot00000000000000# ruff: noqa: E402 # # Nameparser documentation build configuration file, created by # sphinx-quickstart on Fri May 16 01:29:58 2014. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys import os from datetime import date # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('..')) import nameparser # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.viewcode', ] # Declaration order is canonical in this codebase (Role order drives the # seven-field order everywhere), so the reference lists members in # source order, not alphabetically -- Lexicon's vocabulary grouping and # Policy's field ordering match the customize.rst table. autodoc_member_order = 'bysource' # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = 'Nameparser' copyright = f'{date.today():%Y}, Derek Gulbranson' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = nameparser.__version__ # The full version, including alpha/beta/rc tags. release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all # documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. #keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'furo' html_theme_options = { 'source_repository': 'https://github.com/derek73/python-nameparser/', 'source_branch': 'master', 'source_directory': 'docs/', } # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". # (empty: docs/_static does not exist; a non-existent entry warns on # every build and blocks -W) html_static_path = [] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. #html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'Nameparserdoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ ('index', 'Nameparser.tex', 'Nameparser Documentation', 'Derek Gulbranson', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'nameparser', 'Nameparser Documentation', ['Derek Gulbranson'], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'Nameparser', 'Nameparser Documentation', 'Derek Gulbranson', 'Nameparser', 'A simple python modules for parsing human names into components.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False doctest_global_setup = """from nameparser import HumanName from nameparser.config import CONSTANTS, Constants CONSTANTS = Constants() """ python-nameparser-2.1.0/docs/contributing.rst000066400000000000000000000004411523530431700213610ustar00rootroot00000000000000Contributing ============ The project is hosted on GitHub: https://github.com/derek73/python-nameparser Find more information about running tests and contributing the project at the projects contribution guide. https://github.com/derek73/python-nameparser/blob/master/CONTRIBUTING.md python-nameparser-2.1.0/docs/customize.rst000066400000000000000000000524641523530431700207100ustar00rootroot00000000000000Customizing the parser ======================= Every piece of nameparser configuration sorts into one of three places by asking what it varies with: vocabulary varies by **language** (:class:`~nameparser.Lexicon`), behavior varies by **data source or application** (:class:`~nameparser.Policy`), and presentation varies by **output destination** (a rendering argument). See :doc:`concepts` for why the split is drawn there. Vocabulary: Lexicon -------------------- Adding and removing words ~~~~~~~~~~~~~~~~~~~~~~~~~~ .. doctest:: >>> from nameparser import Lexicon, Parser >>> lex = Lexicon.default().add(titles={"dean"}) >>> Parser(lexicon=lex).parse("Dean Robert Johns").title 'Dean' :meth:`~nameparser.Lexicon.add` and :meth:`~nameparser.Lexicon.remove` both return a new :class:`~nameparser.Lexicon` — the one you started from (here, ``Lexicon.default()``) is never mutated. Every field accepts a plain set of lowercase words, keyword by field name (``titles`` above; ``particles``, ``suffix_words``, and the rest work the same way) — see :doc:`modules` for the full field list. Vocabulary entries are matched one word at a time (``given_name_titles`` excepted), so a multi-word entry like ``titles={"grand moff"}`` can never match; the constructor warns when it sees one (``capitalization_exceptions`` keys included — they are looked up per word too). Removing works the same way, and drops the word from recognition: .. doctest:: >>> lean = Lexicon.default().remove(titles={"professor"}) >>> Parser(lexicon=lean).parse("Professor Robert Johns").title '' A few fields mark a subset of another — ``given_name_titles`` over ``titles``, ``particles_ambiguous`` over ``particles``, ``suffix_acronyms_ambiguous`` over ``suffix_acronyms``, and ``honorific_tails`` over ``suffix_words``. Entries belong in the base field too, so add to both and remove from the marker first. The last three enforce that: anything else raises ``ValueError`` naming the orphans rather than leaving a marker entry that no rule will ever consult. ``given_name_titles`` is deliberately unchecked — a title run is matched as one space-joined string, so a legitimate entry like ``"sir and dame"`` is no single word in ``titles`` — and an orphan there is inert rather than harmful. Turning title detection off ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The subset rule matters most when clearing a field wholesale. Emptying ``titles`` alone orphans every ``given_name_titles`` entry, so the two go together: .. doctest:: >>> d = Lexicon.default() >>> lean = d.remove(titles=set(d.titles), ... given_name_titles=set(d.given_name_titles)) >>> Parser(lexicon=lean).parse("Hon Solo").given 'Hon' Emptying the vocabulary does not switch titles off entirely, though. A word ending in a period, standing at the front of the part that carries the given name, is read as a title structurally, without consulting ``titles`` at all — that is what lets unfamiliar ranks and abbreviations work (see :ref:`abbreviated-titles`): .. doctest:: >>> bare = Parser(lexicon=Lexicon.empty()) >>> bare.parse("Professor John Smith").title # vocabulary gone '' >>> bare.parse("Dr. John Smith").title # structural, stays 'Dr.' Combining two lexicons ~~~~~~~~~~~~~~~~~~~~~~~ Whole lexicons compose with ``|``, which unions field by field — handy for keeping a shared house vocabulary separate from a per-source one and combining them at parser construction: .. doctest:: >>> house = Lexicon.empty().add(titles={"dean"}) >>> per_source = Lexicon.empty().add(titles={"provost"}) >>> sorted((house | per_source).titles) ['dean', 'provost'] Fixing the case of a particular word ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``capitalization_exceptions`` is the one pair-valued field — each entry maps a lowercase key to its exact-cased replacement (``"phd"`` → ``"PhD"``), so it isn't a fit for ``add()``/``remove()``. Change it with ``dataclasses.replace()`` instead, and pass the result to ``capitalized()``: .. doctest:: >>> import dataclasses >>> from nameparser import parse >>> str(parse("jane smith dds").capitalized()) 'Jane Smith Dds' >>> default = Lexicon.default() >>> lex = dataclasses.replace( ... default, ... capitalization_exceptions=tuple(default.capitalization_exceptions) ... + (("dds", "DDS"),)) >>> str(parse("jane smith dds").capitalized(lex)) 'Jane Smith DDS' Note the ``tuple(...) + ...``: assigning a bare ``(("dds", "DDS"),)`` would *replace* the default exceptions rather than extend them, so ``"phd"`` and the rest would stop being fixed. The key is matched against the token with punctuation normalized away, not against the raw text, so one ``"phd"`` entry covers ``"phd"``, ``"Phd"``, and ``"Ph.D."`` alike — you don't need a separate key for each way a source might punctuate it. Words that are also ordinary names ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Two fields — ``suffix_acronyms_ambiguous`` and ``particles_ambiguous`` — mark entries from ``suffix_acronyms`` and ``particles`` that are also plausible as ordinary name words on their own (an acronym suffix that doubles as a nickname, a particle that doubles as a given name). They don't add new vocabulary by themselves; they narrow how an existing entry is read when it appears alone. If you're not sure whether a word you're adding is one of these ambiguous cases, leave it out — an unrecognized word usually still parses reasonably, while a wrongly disambiguated one silently picks the less likely reading. (That conservatism is why ``dean`` above isn't in the default vocabulary in the first place: "Dean" is also a common given name, and a default that swallowed it as a title would misparse "Dean Martin" for everyone.) ``ma`` is a shipped example. It is both a credential and a common surname, so it is listed in ``suffix_acronyms_ambiguous`` and counts as a suffix only when written with periods: .. doctest:: >>> parse("Jack Ma").family 'Ma' >>> parse("Jack M.A.").suffix 'M.A.' ``particles_ambiguous`` is the same idea for surname particles. A particle listed there may also be a given name, so a name that starts with one keeps its given name; a particle *not* listed there is never a given name, so a name starting with it has no given name at all — the whole thing is the surname: .. doctest:: >>> parse("van Gogh").given # 'van' may be a given name 'van' >>> parse("de Mesnil").given # 'de' may not '' >>> parse("de Mesnil").family 'de Mesnil' If your data never uses ``Van`` as a given name, take it out of the ambiguous set and leading ``van`` becomes part of the surname: .. doctest:: >>> lex = Lexicon.default().remove(particles_ambiguous={"van"}) >>> Parser(lexicon=lex).parse("van Gogh").family 'van Gogh' Bound given names ~~~~~~~~~~~~~~~~~~ ``bound_given_names`` holds given-name prefixes that attach to the following word to form one given name — ``abdul``, ``abu``, ``umm`` and their Arabic-script spellings (``عبد``, ``أبو``, ``أم``) among them: .. doctest:: >>> parse("abdul salam ahmed salem").given 'abdul salam' Add your own, or empty the set to switch the behavior off entirely: .. doctest:: >>> lex = Lexicon.default().add(bound_given_names={"mohamad"}) >>> Parser(lexicon=lex).parse("mohamad salam ahmed salem").given 'mohamad salam' >>> d = Lexicon.default() >>> off = d.remove(bound_given_names=set(d.bound_given_names)) >>> Parser(lexicon=off).parse("abdul salam ahmed salem").given 'abdul' Behavior: Policy ----------------- When your data source or application needs different parsing behavior — a different name order, stricter suffix rules, extra delimiters — set it on :class:`~nameparser.Policy`, a small, closed set of fields, listed below. .. list-table:: :header-rows: 1 :widths: 22 28 50 * - Field - Type - Effect * - ``name_order`` - one of the three exported order constants - Assigns positional (no-comma) input to given/middle/family in this order. Use the exported ``GIVEN_FIRST`` (default), ``FAMILY_FIRST``, or ``FAMILY_FIRST_GIVEN_LAST`` constants. Ignored when a comma separates family from given ("Thomas, John" puts the family name first); a comma that only sets off suffixes ("John Smith, Jr.") leaves it governing the name part. * - ``patronymic_rules`` - ``frozenset[PatronymicRule]`` - Reorders patronymic-shaped names via opt-in detectors — East Slavic formal order (``EAST_SLAVIC``) or Turkic reversed order (``TURKIC``). Defaults to empty. * - ``middle_as_family`` - ``bool`` - Folds ``middle`` into ``family`` instead of splitting them — for naming systems with no middle-name concept. Defaults to ``False``. * - ``nickname_delimiters`` - ``frozenset[tuple[str, str]]`` - Routes content enclosed by these delimiter pairs to ``nickname``. Defaults to :data:`~nameparser.DEFAULT_NICKNAME_DELIMITERS` — straight quotes and parentheses plus the typographic conventions (smart quotes, guillemets, CJK brackets, ...). * - ``maiden_delimiters`` - ``frozenset[tuple[str, str]]`` - Routes content enclosed by these delimiter pairs to ``maiden`` instead, and drops them from the effective nickname set. A marker word opening the enclosed content is dropped from the value, so ``"Jane Smith (née Jones)"`` gives maiden ``Jones`` — but only where that content holds more than one *token*, since a lone ``"(Nee)"`` is a maiden name rather than a marker. Tokens, not words: a marker written against the name it marks is one token with them, so ``"山田花子（旧姓佐藤）"`` keeps its ``旧姓``. Defaults to empty — see the routing example below. * - ``extra_suffix_delimiters`` - ``frozenset[str]`` - Adds separators that split suffix groups, e.g. ``" - "`` for ``"Jane Smith, RN - CRNA"``. Additions only — the comma always splits suffix groups and cannot be replaced. * - ``lenient_comma_suffixes`` - ``bool`` - Reads an initial-shaped suffix word after a comma as a suffix: ``"John Smith, V"`` is John Smith the fifth when ``True`` (default); ``False`` reads ``V`` as a given-name initial instead. Multi-letter suffixes (``III``, ``MD``) are unaffected. The same test is also one of the two the glued-honorific peel asks before crossing a family comma (#319), so the setting reaches CJK names too: ``"田中さん, V."`` gives family ``田中``, suffix ``さん`` when ``True``, and family ``田中さん``, given ``V.`` when ``False``. * - ``strip_emoji`` - ``bool`` - Excludes emoji from tokenization — they appear in no field or rendered view, though ``original`` keeps them. Defaults to ``True``. * - ``strip_bidi`` - ``bool`` - Excludes bidirectional control characters the same way. Defaults to ``True``. To apply a :class:`PolicyPatch ` directly -- without going through a locale pack -- call :meth:`Policy.patched() `: .. doctest:: >>> from nameparser import Policy, PolicyPatch >>> Policy().patched(PolicyPatch(middle_as_family=True)) Policy(middle_as_family=True) Family-first name order ~~~~~~~~~~~~~~~~~~~~~~~~ ``name_order`` is the one most likely to matter for non-Western data. Positional input is assigned in the order you declare, so a family-first name parses as written instead of needing to be rearranged afterwards: .. doctest:: >>> from nameparser import Parser, Policy, FAMILY_FIRST, parse >>> parse("Nguyen Van Minh").family # default GIVEN_FIRST 'Van Minh' >>> family_first = Parser(policy=Policy(name_order=FAMILY_FIRST)) >>> name = family_first.parse("Nguyen Van Minh") >>> name.family, name.given ('Nguyen', 'Van Minh') An explicit comma still wins, on the reasoning that someone who wrote one meant it — so the same parser reads ``"Thomas, John"`` as family-then-given regardless of the configured order: .. doctest:: >>> family_first.parse("Thomas, John").family 'Thomas' East Asian defaults, and turning them off ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Two defaults key on the *script* a name is written in rather than on anything you set: a name written wholly in Han or Hangul — or one mixing kanji with kana — is assigned family-first (``script_orders``), and an unspaced hangul name is split into surname and given name against the shipped Korean census list (``segment_scripts``). :ref:`east-asian-names` explains the naming conventions both rest on — this section is how to switch them off, which you can do separately: .. doctest:: >>> parse("김민준").family # both defaults on '김' >>> positional = Parser(policy=Policy(script_orders=())) >>> positional.parse("김민준").family # still split '민준' >>> unsplit = Parser(policy=Policy(segment_scripts=frozenset())) >>> unsplit.parse("김민준").family # one token, not split '김민준' The two switches interact, and clearing only ``script_orders`` produces a third behavior rather than the old one: the split still runs, so ``김민준`` still becomes two tokens, and the positional default then assigns them given-first — the surname lands in ``given``. To restore nameparser 2.0's reading exactly, clear both fields. To teach the splitter a surname it doesn't ship with, add it to the ``surnames`` vocabulary like any other word: .. doctest:: >>> lex = Lexicon.default().add(surnames={"김민"}) >>> Parser(lexicon=lex).parse("김민준").family '김민' Chinese surnames are deliberately absent from that default set, because splitting Han text requires knowing Chinese from Japanese; :doc:`locales` covers the opt-in ``zh`` pack that supplies them. The Japanese behaviors ride these same two fields, so they need no switches of their own: ``script_orders=()`` clears the kana-licensed entry along with the Han and Hangul ones, and ``segment_scripts=frozenset()`` deactivates every script at once, which also stops a parser consulting whatever segmenter it was given. The segmenter has an off-switch as well — ``Parser(segmenter=None)``, which is the default; see :ref:`segmenter-contract` for what one is expected to do with text it does not handle. Two behaviors are not policy fields at all, and apply however these two fields are set. The katakana middle dot ・ separates tokens the way a space does, decided in tokenization. And a listed CJK honorific glued to the end of a name token is split off it — ``田中さん`` reads family ``田中`` with ``さん`` in ``suffix`` — because the tail vocabulary carries its own license rather than borrowing a script's: every entry is a word that can never end a name, so there is no per-script trust question for ``segment_scripts`` to answer. That vocabulary is also the peel's off-switch — ``Lexicon.default().remove(honorific_tails={"さん"})`` leaves ``田中さん`` unsplit while the spaced ``田中 さん`` still reads ``さん`` as a suffix. Dropping a word from a marker field alone orphans nothing, so that one needs no matching ``suffix_words`` edit. Emptying the field is also the way to opt out of what the peel *costs* a non-ASCII parse — an empty ``honorific_tails`` stops it at its first guard, whereas ``segment_scripts`` never gated it and so cannot turn it off. .. note:: Every field here is annotated with its canonical *storage* type rather than with everything the constructor accepts — the same as ``capitalization_exceptions``, and for the same reason: the annotation is what you get back when you READ the attribute, which is the commoner operation. The constructor is deliberately wider. It takes any mapping for ``script_orders``, any iterable of ``Script`` for ``segment_scripts``, and plain strings wherever a ``Role`` is wanted (``Role`` is a ``StrEnum`` precisely so that works). A dataclass cannot express those two types separately, so the examples in this guide use the spellings that check clean under mypy — ``()`` and ``frozenset(...)`` rather than ``{}`` and a bare set literal. The wider spellings parse identically; they just need a ``# type: ignore[arg-type]`` if you run a type checker. Nicknames, maiden names, and brackets ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A delimiter pair routes to exactly one field, and ``maiden_delimiters`` states the more specific intent — so listing a pair there drops it from the effective ``nickname_delimiters`` set automatically, and the one-liner is the whole recipe: .. doctest:: >>> policy = Policy(maiden_delimiters=frozenset({("(", ")")})) >>> Parser(policy=policy).parse("Jane (Jones) Smith").maiden 'Jones' To *add* a delimiter pair rather than reroute one, build on the exported default — assigning a bare set replaces the built-in pairs instead of extending them, the same trap as ``capitalization_exceptions``: .. doctest:: >>> from nameparser import DEFAULT_NICKNAME_DELIMITERS >>> parse("Benjamin {Ben} Franklin").middle # not a pair by default '{Ben}' >>> policy = Policy( ... nickname_delimiters=DEFAULT_NICKNAME_DELIMITERS | {("{", "}")}) >>> Parser(policy=policy).parse("Benjamin {Ben} Franklin").nickname 'Ben' Suffixes not separated by commas ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``extra_suffix_delimiters`` handles sources that separate post-nominals with something other than a comma. The default reading of such a name is bad enough to be the reason you'd go looking: .. doctest:: >>> name = parse("Jane Smith, RN - CRNA") >>> name.given, name.family, name.suffix ('RN', 'Jane Smith', 'CRNA') >>> policy = Policy(extra_suffix_delimiters={" - "}) >>> name = Parser(policy=policy).parse("Jane Smith, RN - CRNA") >>> name.given, name.family, name.suffix ('Jane', 'Smith', 'RN, CRNA') Keeping emoji and control characters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The strip flags keep characters the parser removes by default. Note what happens to an emoji you keep — it becomes a token like any other, and lands in the middle name: .. doctest:: >>> str(parse("Sam 😊 Smith")) # stripped by default 'Sam Smith' >>> kept = Parser(policy=Policy(strip_emoji=False)).parse("Sam 😊 Smith") >>> str(kept), kept.middle ('Sam 😊 Smith', '😊') ``strip_bidi=False`` does the same for invisible bidirectional control characters, which is occasionally what you want when round-tripping right-to-left text verbatim. .. _rendering-arguments: Presentation: rendering arguments ---------------------------------- Once a name is parsed, how it's displayed is a separate decision made at the point of output, not baked into the parse. Three methods on :class:`~nameparser.ParsedName` cover it — see :doc:`modules` for full signatures: * :meth:`~nameparser.ParsedName.render` fills a format spec from the seven role fields. * :meth:`~nameparser.ParsedName.initials` is the same idea narrowed to first letters, with its own ``delimiter``/``separator`` arguments. * :meth:`~nameparser.ParsedName.capitalized` returns a new, case-fixed :class:`~nameparser.ParsedName` instead of a string. It only touches input that's already single-case (all lower, all upper) unless you pass ``force=True`` — mixed case is left alone by default on the assumption that someone already capitalized it on purpose. .. doctest:: >>> from nameparser import parse >>> name = parse("Dr. Juan Q. Xavier de la Vega III") >>> name.render("{family}, {given} {middle}") 'de la Vega, Juan Q. Xavier' >>> name.initials(spec="{given}{middle}{family}", delimiter="", separator="") 'JQXV' >>> str(parse("DR. JUAN DE LA VEGA").capitalized()) 'Dr. Juan de la Vega' >>> str(parse("JuAn DE LA vEGA").capitalized()) 'JuAn DE LA vEGA' >>> str(parse("JuAn DE LA vEGA").capitalized(force=True)) 'Juan de la Vega' Looking for v1's ``string_format``? It's the ``render(spec)`` argument now — pass your own format string per call instead of setting it once on a shared config object. A spec chooses what the output is *for*. The default is written for display and does not survive a reparse — it parenthesizes the maiden name, which reads back as a nickname. When the rendered string will be parsed again, spell the marker out (``née {maiden}``) so the field round-trips; see :ref:`the round-trip note in the tour `. Sharing a configured parser ---------------------------- A :class:`~nameparser.Parser` is a frozen value, so the way to share one configuration across a codebase is the same way you'd share any other constant: build it once at module level and import it wherever you parse. .. code-block:: python # myapp/names.py from nameparser import Lexicon, Parser, Policy lex = Lexicon.default().add(titles={"dean"}) policy = Policy(strip_emoji=False) parser = Parser(lexicon=lex, policy=policy) # elsewhere from myapp.names import parser name = parser.parse(raw_name) Because :class:`~nameparser.Parser` and its ``lexicon``/``policy`` are immutable and hashable, ``parser`` is safe to import and call from multiple threads with no locking — there is no shared mutable state to protect, unlike v1's module-level ``CONSTANTS``. python-nameparser-2.1.0/docs/design/000077500000000000000000000000001523530431700173725ustar00rootroot00000000000000python-nameparser-2.1.0/docs/design/nameparser-2.0-rfc.md000066400000000000000000000332701523530431700231230ustar00rootroot00000000000000# RFC: nameparser 2.0 — a new core API Status: **open for feedback** — no deadline. Implementation may proceed in parallel; this document will be amended (as commits to this PR) as implementation teaches us things. Comment on specific lines here, or on the discussion issue (#284) for general reactions. ## The short version, and the promise nameparser 2.0 introduces a new, immutable core API and keeps the existing one working: ```python from nameparser import parse name = parse("Dr. Juan Q. Xavier de la Vega III (Doc Vega)") name.given # "Juan" name.family # "de la Vega" name.family_base # "Vega" name.title # "Dr." ``` **The compatibility promise:** code that runs warning-free on nameparser 1.4 keeps running on every 2.x release, with identical results, getting at most `DeprecationWarning`s. `HumanName` continues to exist as a compatibility layer over the new core. The exceptions to "identical results" are (a) parse-output changes that are deliberate bug fixes, each documented in the release log with its issue, and (b) three narrow edge cases listed in the Migration section. Removal of the old API happens in 3.0, not 2.x, and only after warnings have named every replacement. If you only use `HumanName` and never customize configuration, 2.0 should be a non-event for you. The rest of this document is for everyone else — and for anyone who wants to push on the new design while it is still cheap to change. ## Why a new core Fifteen years of issues point at the same architectural limits: - **`HumanName` is four things in one mutable class** — parser, state, vocabulary predicates, and formatter. Every feature grows all four at once (it is currently ~1,600 lines), and mutation semantics are inconsistent: assigning `full_name` re-parses, assigning `last` does not, `capitalize()` mutates in place. - **Parse state is `list[str]`**, so position information is lost and re-derived by value lookups. That is the structural root cause of a whole family of prefix-join bugs (#100 and relatives): when the same string appears twice, the lookup answers the wrong question. - **Configuration is a shared mutable module global** (`CONSTANTS`). Most of `nameparser.config`'s ~850 lines exist to make shared mutation survivable (custom set managers, descriptors, cache invalidation), and it still causes test pollution and cross-thread surprises. - **Equality is broken by design** (#223): case-insensitive equality, equality with plain strings, and hashability are mutually inconsistent promises. 1.3.0 deprecated `==`/`hash()` and shipped `matches()`/`comparison_key()`. - **The roadmap needs structure v1 can't cheaply provide**: family-first name order (#270), per-dataset configuration without global mutation, and honest reporting of genuinely ambiguous parses. The rules-based approach is not changing. nameparser's niche is deterministic, auditable, zero-dependency parsing — the same input always parses the same way. The rewrite reorganizes *how the rules run*, not what the library is. ## The new API ### Parsing produces an immutable value `parse()` returns a `ParsedName`: a frozen dataclass. There is no mutation anywhere in the new API — editing is `replace()`, which returns a new value: ```python name = parse("John Smith") name2 = name.replace(given="Jane") # new object; `name` unchanged ``` `parse()` is **total over `str`**: any string — empty, emoji, garbage — returns a `ParsedName` rather than raising. Content problems are represented in the result; the only exceptions are type errors (`bytes` raises `TypeError` — decode first). Equality is strict structural equality, and the semantic comparisons are explicit and separate — `comparison_key()` for dedup/sorting and `matches()` for component-wise case-insensitive comparison — resolving the #223 trilemma by not conflating the three meanings of "equal". ### Two access layers: strings for the 95% case, tokens for the rest The string properties behave like v1's: always `str`, `""` when empty. Underneath, `ParsedName` carries the actual parse: a tuple of `Token`s, each with its **span** (position in the original string), its role, and tags recording *how it was classified*: ```python name = parse("Juan de la Vega") name.tokens_for(Role.FAMILY) # (Token('de' @5:7 FAMILY {particle}), # Token('la' @8:10 FAMILY {particle}), # Token('Vega' @11:15 FAMILY),) ``` This is the structural fix for the value-lookup bug family: pipeline stages address tokens by index, so "which 'de' did you mean?" cannot arise. It is also the debuggability story — "why is 'Van' in my family name?" is answered by the token's tags, and `repr(name)` prints a compact component-per-line breakdown like v1's. Derived views (`family_base`, `family_particles`, `surnames`, `given_names`) are pure filters over tokens — they can never disagree with the parse. ### `given` and `family`, not `first` and `last` The v1 field names describe Western *position* — they date from before the parser even supported comma formats, when position was all there was. They read wrongly the moment name order varies: under family-first order, `name.last` would return the first word. 2.0 adopts the names the standards world already uses (HTML autocomplete `given-name`/`family-name`, vCard, Unicode CLDR): - `first` → `given`, `last` → `family` - `last_base`/`last_prefixes` → `family_base`/`family_particles` - config: `prefixes` → `particles` (every web standard uses "prefix" for honorifics like "Mr." — v1's usage collides; the linguistics term for van/de la is *particle*) - `title` and `suffix` **stay** — they have ecosystem gravity, and the positional taxonomy ("pre-nominal"/"post-nominal") is documented rather than spelled into identifiers. - `maiden` is a **new** field — a birth surname introduced by a marker like `née` or `geb.`, which v1 folded into `middle`/`last` (#274). `HumanName` keeps every v1 spelling, including `string_format` keys, so existing code is unaffected; only new-API adopters see the new names. ### Configuration: `Lexicon` (vocabulary) and `Policy` (behavior) v1's `Constants` mixes three kinds of thing; 2.0 splits them by what varies together: - **`Lexicon`** — vocabulary data (titles, suffixes, particles, conjunctions, …). Immutable; composable by union; varies by *language*. - **`Policy`** — behavior switches (`name_order`, patronymic rules, delimiter pairs, …). Immutable; varies by *data source*. - **Rendering parameters** — `string_format`, initials style, capitalization — move to render-time arguments; they vary by *output destination* and are no longer configuration at all. ```python from nameparser import Parser, Lexicon, Policy, FAMILY_FIRST parser = Parser( lexicon=Lexicon.default().add(titles={"dra"}), policy=Policy(name_order=FAMILY_FIRST), ) parser.parse("Wang Xiuying").family # "Wang" ``` A `Parser` is immutable, thread-safe, and does all its compilation once at construction — build it at startup, share it everywhere. There is no shared mutable configuration in the new API, which retires the entire test-pollution/config-war problem class. `name_order` is an order-spec tuple (`FAMILY_FIRST = (Role.FAMILY, Role.GIVEN, Role.MIDDLE)`) rather than a boolean, because Vietnamese order — family, middle, *then* given — needs a third arrangement (#270). ### Honesty about ambiguity Some parses are genuinely undecidable by rules — "Kelly Michael" could be either order; in "Van Johnson", "Van" is a Dutch particle *and* a common given name. v1 silently picks a reading. 2.0 picks the same reading **and says so**: ```python parse("Van Johnson").ambiguities # (Ambiguity(PARTICLE_OR_GIVEN, "leading 'Van' may be a family-name # particle; read as a given name", tokens=(...)),) ``` `Ambiguity.kind` values are stable API you can filter on. This is also groundwork: a future `parse_all()` returning ranked alternative readings, and `explain()` returning a stage-by-stage parse trace, are designed for (the pipeline supports both) but deliberately not in 2.0. ### Locales: opt-in packs, never auto-detection Language-specific behavior cannot be auto-detected — "Tham Jun Hoe" is token-for-token indistinguishable from a Western three-token name, and "Ali" is Arabic or Italian. So locale support is explicit: ```python from nameparser import parser_for, locales parser_for(locales.RU).parse("Ivanova Anna Sergeevna").given # "Anna" ``` A locale pack is pure data: a vocabulary fragment plus a partial policy patch, folded in at parser construction. 2.0.0 ships `RU` and `TR_AZ` (formalizing the patronymic support added in 1.3.0); Vietnamese and Japanese (via a `nameparser[ja]` extra with a pluggable segmenter) are designed and staged for later 2.x minors. Vocabulary that cannot misfire on other languages' names (e.g. Cyrillic honorifics) goes in the default lexicon instead of packs. *Amended for 2.1 (#271):* the Chinese/Korean work landed, and split along that last line rather than shipping as one pack. Korean surnames and family-first order for native-script CJK cannot misfire on other languages' names, so both became defaults; only the Chinese surname list needed a pack (`ZH`), because it would corrupt the Japanese names written in the same characters. The never-auto-detect rule holds as written for a name's *language*; its **script** is a different signal, and where the script alone settles a convention, behavior may key on it. ## Migration: what actually happens to existing code **`HumanName` in 2.0** is a wrapper over the new core with v1's exact mutation semantics, field spellings, and formatting attributes. The v1 test suite runs against it as the compatibility gate. v1 pickles load throughout 2.x. **`CONSTANTS` keeps working** — mutations are honored exactly (the facade rebuilds its config snapshot when the shared config changes) and emit a `DeprecationWarning` pointing at the replacements. Mutating a *private* `Constants` instance does not warn. **Already-scheduled removals land as warned** (every one was deprecated in 1.3.0 or 1.4 first): `==`/`hash()`, bytes input, slice item access, `empty_attribute_default`, `constants=None`, silent config-typo fallbacks, legacy pickle blobs, `add_with_encoding()`. Python floor becomes 3.11; the last dependency is dropped. **The three narrow behavior exceptions** (everything else identical): 1. `*_list` attributes return snapshots — in-place `name.first_list.append(...)` no longer mutates parse state (assignment still does). 2. Subclasses overriding parsing hooks (`parse_pieces`, `is_title`, …) are detected and warned at construction: the facade delegates to the new core, so such overrides are no longer called (#280). If you do this, we want to hear what your override does — that is a feature request we would rather absorb than break. 3. Assigning to `CONSTANTS.regexes` raises with a pointer to the named `Policy` flag that replaces it. Custom regex injection has no equivalent in the new model; nothing is silently ignored. **Timeline:** 2.0.0's only *new* warning is on shared-`CONSTANTS` mutation. A warning on `HumanName` construction itself comes in a later 2.x minor, once the new API has proven out — and 3.0 (which removes the facade) does not ship before that warning has been out in a release. ## Alternatives considered and rejected - **Auto-detecting name language/order** — impossible in principle, not just hard; see the Locales section. - **A statistical/ML core** — determinism and auditability are the point of this library; the token/ambiguity model leaves room for optional escalation later without changing the core. - **Keeping `first`/`last`** — see the naming section; the facade keeps them for v1 code, which serves the familiarity constituency without wiring position-based names into the new API. - **A boolean `family_name_first`** — cannot express Vietnamese; the order-spec tuple can (#270). - **Renaming `title`/`suffix` to `pre_nominal`/`post_nominal`** — more precise, but no ecosystem uses those as field names; precision lives in the docs instead. - **Vocabulary entries as records with per-word metadata** — rejected under "no vocabulary data without a consuming rule": a flag the parser doesn't read is a promise it doesn't keep. Cross-category words are handled by dual set membership (closed classes) or the two `_ambiguous` sets (closed-vs-open), each of which has a consuming rule. - **A user-replaceable regex escape hatch** — replaced by named policy flags; anything not expressible through them is a feature request we want to see. ## Feedback we are specifically looking for No deadline — comment whenever, on lines here or on the discussion issue. The questions where answers would most change things: 1. **Do you compare `HumanName` objects with `==` (especially against plain strings), or use them in sets/dicts?** 2.0 changes this to object identity (warned since 1.3.0; `matches()`/`comparison_key()` are the replacements). This is the one silent behavior change in the release. 2. **Do you subclass `HumanName` and override parsing methods?** Tell us what your override accomplishes — we would rather absorb it as a feature than break it silently (#280). 3. **Do you mutate `CONSTANTS` after startup** (not just at import time)? Anything beyond startup-time customization we should know about? 4. **Do you assign custom regexes to `CONSTANTS.regexes`?** What for? Each real use case either becomes a named policy flag or stays broken — we would like the list to be complete before 2.0. 5. **`given`/`family` instead of `first`/`last`** in the new API: does this help or annoy you? 6. **Is there anything you build on top of nameparser** (wrappers, pipelines) where the compatibility promise as stated would still break you? python-nameparser-2.1.0/docs/index.rst000066400000000000000000000022071523530431700177630ustar00rootroot00000000000000.. Nameparser documentation master file, created by sphinx-quickstart on Fri May 16 01:29:58 2014. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. Python Human Name Parser ======================== Version |release| nameparser parses human names into seven fields — title, given, middle, family, suffix, nickname, maiden. Results are immutable, configuration is composable (:class:`~nameparser.Lexicon` for vocabulary, :class:`~nameparser.Policy` for behavior), and locale packs are opt-in. Requires Python 3.11+. .. doctest:: >>> from nameparser import parse >>> name = parse("Dr. Juan Q. Xavier de la Vega III") >>> name.given, name.family ('Juan', 'de la Vega') >>> name.render("{family}, {given}") 'de la Vega, Juan' Coming from 1.x? :doc:`migrate`. .. toctree:: :maxdepth: 2 usage concepts customize locales migrate modules release_log resources contributing Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` **GitHub Project**: https://github.com/derek73/python-nameparser python-nameparser-2.1.0/docs/locales.rst000066400000000000000000000364161523530431700203070ustar00rootroot00000000000000Locale packs ============ A locale pack is an opt-in bundle of policy — and, when a naming tradition needs it, vocabulary — for one specific pattern: East Slavic patronymics, Turkic patronymic markers, and so on. Packs apply only when you ask for one by name, and every pack makes the same promise: it never changes a name outside the shapes it declares. Read that precisely — a shape is a pattern, not a language, so a pack can act on a name from a tradition it was not written for. The warning under `Using a pack`_ shows what that costs. What works without a pack -------------------------- Most international names need no pack at all. The default vocabulary covers five scripts — Latin, Cyrillic, Greek, Arabic and Hebrew, plus Devanagari titles — so honorifics, conjunctions and name particles in those scripts are recognized out of the box: .. doctest:: >>> from nameparser import parse >>> name = parse("الشيخ محمد بن سلمان") >>> name.title, name.given, name.family ('الشيخ', 'محمد', 'بن سلمان') >>> parse("عبد الرحمن محمد").given 'عبد الرحمن' Native-script entries are safe to enable by default precisely because they cannot collide with Latin-script names. That rules out the reverse: transliterations like ``sri``/``shri`` are *not* in the default vocabulary, because they are also ordinary given names in Latin script. The same conservatism holds back a handful of native entries that collide within their own script — bare ``سيد`` and ``شيخ`` (common given names), Hebrew ``רב`` (an ordinary word), the Arabic ``د.`` abbreviation (it would swallow initials). If your data is homogeneous enough that a collision can't occur, the reasoning behind the default doesn't apply to you — add the entry: .. doctest:: >>> from nameparser import Lexicon, Parser >>> lex = Lexicon.default().add( ... titles={"سيد"}, given_name_titles={"سيد"}) >>> name = Parser(lexicon=lex).parse("سيد محمد") >>> name.title, name.given ('سيد', 'محمد') Both fields, because ``given_name_titles`` is a marker over ``titles`` rather than a separate vocabulary: ``titles`` makes the word a title at all, and listing it in ``given_name_titles`` says the honorific precedes the *given* name — as Arabic ones do — so the word after it isn't read as a family name. Listing it in ``given_name_titles`` alone raises ``ValueError`` rather than quietly doing nothing. Two East Asian behaviors are on by default for the same reason, except that what selects them is the *script* rather than the word. The background (covered fully under :ref:`east-asian-names` in :doc:`usage`): Chinese, Japanese, and Korean names put the family name first in native script and are usually written with no space between the parts. Both defaults follow from facts the script alone establishes. A name written wholly in Han or Hangul — or one mixing kanji with kana, a combination only Japanese produces — is assigned family-first, because every language written in those scripts orders names that way; no language guess is involved. An unspaced hangul name is additionally split into surname and given name, because hangul is written by nothing but Korean and Korean surnames are a closed census set that ships as default vocabulary. Splitting an unspaced *Han* name is the one behavior the script cannot license — the same characters could be a Chinese or a Japanese name, and a Chinese surname list splits Japanese names in the wrong place — so it is not a default: the ``zh`` and ``ja`` packs below turn it on for data whose language you can declare. A pack is for something different: a *structural* rule, like reordering a patronymic, that vocabulary alone can't express. Using a pack ------------ :func:`~nameparser.parser_for` folds one or more packs onto a base :class:`~nameparser.Parser` (the module default, unless you pass ``base=``). Here the Russian pack reads "Сидоров Иван Петрович" (Sidorov Ivan Petrovich — surname/given/patronymic order) the way a formal Russian document intends: .. doctest:: >>> from nameparser import locales, parser_for >>> ru = parser_for(locales.RU) >>> ru.parse("Сидоров Иван Петрович").given 'Иван' Packs stack: pass more than one pack and their policies fold together in order. .. doctest:: >>> both = parser_for(locales.RU, locales.TR_AZ) >>> sorted(rule.name for rule in both.policy.patronymic_rules) ['EAST_SLAVIC', 'TURKIC'] Find what's shipped with :func:`~nameparser.locales.available`, and look one up dynamically by its lowercase code with :func:`~nameparser.locales.get` — the same code the ``--locale`` flag takes: .. doctest:: >>> locales.available() ('ja', 'ru', 'tr_az', 'zh') >>> locales.get("ru") is locales.RU True The command line accepts the same codes: ``python -m nameparser --locale ru --json "Сидоров Иван Петрович"`` applies the pack before parsing, equivalent to ``parser_for(locales.get("ru"))``. .. list-table:: Shipped packs :header-rows: 1 :widths: 15 85 * - Code - Turns on * - ``ja`` - Japanese segmentation — activates division for unspaced Japanese names, which needs a segmenter to act: install ``nameparser[ja]`` and pass ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` (``山田太郎`` → family ``山田``, given ``太郎``). The pack ships no surname list, because no list divides a kanji name, and no order: a Japanese name already reads family-first by default. * - ``ru`` - East Slavic patronymic order — detects a formal given/patronymic/family shape (Cyrillic and transliterated ``-ovich``/``-ovna``-style endings) and assigns it accordingly. * - ``tr_az`` - Turkic patronymic markers — detects a standalone marker token (``oglu``, ``qizi``, ``uulu``, and their Latin- and Cyrillic-script variants) and reads the name around it as given/middle/family. * - ``zh`` - Chinese surname segmentation — splits an unspaced Han name into surname and given name (``毛泽东`` → family ``毛``, given ``泽东``) against the surname list the pack ships. It sets no name order: native-script Han already reads family-first without a pack. ``ja``, ``ru`` and ``tr_az`` are policy-only — they carry no vocabulary of their own. ``zh`` is both halves at once: a surname list, plus the one policy field that turns segmentation on for the script it covers. See :doc:`concepts` for how that split (language vocabulary vs. behavior) is drawn, and `Contributing a pack to nameparser`_ for which half a new naming rule belongs in. .. warning:: A pack declares a name *shape*, not a language, and it cannot tell whose name it is looking at. Any surname that happens to end in a patronymic suffix matches the East Slavic rule, including names that are not East Slavic at all: .. doctest:: >>> ru = parser_for(locales.RU) >>> name = ru.parse("David Michael Abramovich") >>> name.given, name.family ('Michael', 'David') The default parser reads that as given ``David``, family ``Abramovich``. This is the trade the pack asks you to make, and it is why packs are opt-in rather than automatic: enable one only for data that is predominantly in the order it detects. If your input mixes traditions, parse the subsets separately with different parsers rather than enabling a pack over all of it. .. _segmenter-contract: Segmenters ----------- ``ja`` is policy-only in a second sense: it turns division on for Japanese text without supplying anything to divide with. That job goes to a **segmenter**, which is passed to :func:`~nameparser.parser_for` rather than carried by the pack — a :class:`~nameparser.Locale` is pure data, and a third-party callable is neither pure nor data. :func:`~nameparser.locales.ja_segmenter` is the shipped one; writing your own is worth it for any script whose divisions you know better than a surname list does. A :data:`~nameparser.Segmenter` is any callable taking a token's text and returning a :class:`~nameparser.Segmentation` — the interior offsets to cut at, plus how confident you are — or ``None`` to decline, leaving the token whole. Declining is the load-bearing half of the contract, because ``segment_scripts`` unions across packs: your segmenter is offered every token of every activated script, not only the ones its own pack turned on. Recognize the text you can actually read and return ``None`` for the rest, rather than answering for a script you never meant to handle. Exceptions are the one thing that does not stay inside the parse — a segmenter is your code, so its errors propagate out of ``parse()`` instead of being absorbed as content errors. The pack half of that arrangement carries no words at all, which makes it the shortest kind of pack there is: .. doctest:: >>> from nameparser import Lexicon, Locale, PolicyPatch, Script >>> mine = Locale(code="myscript", lexicon=Lexicon.empty(), ... policy=PolicyPatch( ... segment_scripts=frozenset({Script.HAN}))) ``nameparser/locales/ja.py`` is the shipped example of exactly that shape: activation is the pack's entire contribution, and a pack applied without a segmenter simply divides nothing. Creating your own Locale ------------------------- You don't need to touch nameparser's registry to use your own pack — :class:`~nameparser.Locale` is a plain, constructible value: ``Locale(code=..., lexicon=..., policy=PolicyPatch(...))``. A :class:`~nameparser.PolicyPatch` is a :class:`~nameparser.Policy`-shaped patch: every field defaults to :data:`~nameparser.UNSET` (leave it alone) instead of to a concrete value, so a pack only ever states what it changes. A patch can also be applied directly, without a pack — see :meth:`Policy.patched() `. The ``policy`` half works that way, but the ``lexicon`` half does not. A pack's :class:`~nameparser.Lexicon` is a complete value in its own right and is validated on its own, before it is unioned onto the base — so a fragment that marks a word must also carry the word it marks. To make an existing base title precede the given name, restate the title in the fragment rather than listing it in ``given_name_titles`` alone. ``zh`` is the shipped worked example: its ``Lexicon(surnames=...)`` has to satisfy every ``Lexicon`` rule standing alone, before anything unions it onto the base. .. doctest:: >>> from nameparser import Lexicon, Locale, PolicyPatch, parser_for >>> lex = Lexicon.empty().add(titles={"kapitan"}) >>> mine = Locale(code="mycorp", lexicon=lex, ... policy=PolicyPatch(middle_as_family=True)) >>> name = parser_for(mine).parse("Kapitan Anna Maria Schmidt") >>> name.title, name.given, name.family ('Kapitan', 'Anna', 'Maria Schmidt') That pack does two things at once: the :class:`~nameparser.Lexicon` fragment teaches the parser that ``kapitan`` is a title, and the :class:`~nameparser.PolicyPatch` turns on ``middle_as_family`` so any remaining given-position words after the first fold into ``family`` instead of ``middle`` — compare this to the default parser's reading of the same string, which has no title and splits ``given='Kapitan'``, ``middle='Anna Maria'``, ``family='Schmidt'``. When ``parser_for`` folds one or more packs onto a base, lexicons union (a pack's words are added to the base's, never removed); policy fields declared as set-valued in :class:`~nameparser.PolicyPatch` (``patronymic_rules`` and the delimiter fields) union the same way; and every other, scalar field is later-wins — if two packs (or a pack and an explicit conflicting value) set the same scalar field, the last one applied wins and a ``UserWarning`` is raised so the conflict isn't silent. Contributing a pack to nameparser ---------------------------------- Shipping a pack in nameparser itself (rather than keeping it local to your own code) means meeting the in-repo contract, checked mechanically by ``tests/v2/test_locales.py``: #. Add a registry entry in ``nameparser/locales/__init__.py`` — a ``"CODE": ("module.path", "ATTR")`` row in ``_REGISTRY``, so the pack loads lazily on first access (importing ``nameparser.locales`` never imports pack modules). #. Declare a module-level ``DEVIATES(name)`` predicate: given a name string, return whether *this pack alone* might parse it differently from the default parser. Over-declaring is safe; under-declaring is not — when in doubt, ``DEVIATES`` should say yes. A pack whose scope is a *script* builds the predicate with ``_script_matcher`` from ``nameparser/_policy.py``, the way ``zh`` and ``ja`` do — never by compiling its own character ranges: the factory's docstring explains how the pack-contract test enforces this. #. Add a rotator list to ``tests/v2/test_locales.py``. Every pack needs one, but what it has to contain follows from how the pack declares its scope. A pack declaring by *marker regex* (``ru``, ``tr_az``) needs at least one name exercising every alternation branch of every regex it defines — ``test_rotators_cover_every_marker_branch`` fails until each branch is hit. A pack declaring by *codepoint range* (``zh``, ``ja``) has no branches to sweep and drops out of that test, so its rotators have to carry the same weight by hand: the unspaced names the pack must split, one per shape of the vocabulary it ships — single surname, compound surname, and any spelling variant it means to cover. A pack that ships no vocabulary lists the shapes its *segmenter* must divide instead, and marks the rotator tests to skip when the optional dependency is absent, so the contract tests still run everywhere. #. Keep the non-interference gate green over the shared corpus plus your rotators: every name the packed parser parses differently from the default must be one your ``DEVIATES`` predicate flags — no silent, undeclared side effects on names outside the pack's stated scope. #. Decide which layer the vocabulary belongs in, if the pack carries any. Vocabulary that is *self-selecting* — able to match only text of the tradition it came from, the way a hangul surname can only ever match hangul — is default-safe, and belongs in the default lexicon (``nameparser/config/``) rather than in a pack: a pack nobody knows to ask for is vocabulary nobody gets. Vocabulary that *declares a language its script does not* — a Chinese surname list, which silently mangles the Japanese names written in the same characters — belongs in the pack, where asking for it is the declaration. ``ja``, ``ru`` and ``tr_az`` need no vocabulary at all and ship an empty :class:`~nameparser.Lexicon`; ``nameparser/locales/zh.py`` is the template for one that does. #. Curate vocabulary conservatively, the same rule as :doc:`customize`: when you're unsure whether a word or a marker belongs, leave it out. ``nameparser/locales/ru.py`` is the reference implementation for a policy-only pack, ``nameparser/locales/zh.py`` for one that carries vocabulary, and ``nameparser/locales/ja.py`` for one whose whole contribution is turning a stage on. Packs still in progress are tracked in issue `#146 `_ (Vietnamese). python-nameparser-2.1.0/docs/migrate.rst000066400000000000000000000423161523530431700203110ustar00rootroot00000000000000Migrating from HumanName ========================= ``HumanName`` and ``CONSTANTS`` keep working in 2.0 — same imports, same attributes, same mutation API (``name.C.titles.add(...)``, ``name.first = "..."``, and so on). Upgrading to 2.0 and migrating your code to the new :class:`~nameparser.Parser`/:class:`~nameparser.Lexicon`/ :class:`~nameparser.Policy` API are two separate decisions — you can do the former today and the latter whenever it's convenient. The compatibility layer (``HumanName`` and ``nameparser.config``) is removed in 3.0; that release is not scheduled. The full 1.x documentation remains the canonical reference for ``HumanName`` and stays online at the readthedocs ``stable`` build, currently the 1.4.0 release: https://nameparser.readthedocs.io/en/stable/ This page exists for the other direction: translating a v1 customization or a v1-shaped comparison into the 2.0 API, one row per old name. Before you upgrade ------------------ What 2.0 removes is the batch of deprecations 1.3 and 1.4 announced. If your test suite runs clean on 1.4 under ``python -W error::DeprecationWarning``, it will run on 2.0 — with four exceptions that 1.4 never warned about. The first three raise the first time you hit them; the fourth only warns, so read it carefully: * ``CONSTANTS.regexes. = ...`` raises ``TypeError``. This includes ``CONSTANTS.regexes.bidi = False``, the opt-out 1.3.1 recommended for keeping bidirectional marks; the 2.0 spellings are ``Policy(strip_bidi=False)`` and ``Policy(strip_emoji=False)`` * assigning a ``*_list`` attribute (``name.first_list = [...]``) raises ``AttributeError`` — the lists are read-only snapshots in 2.0 * positional ``Constants(...)`` construction raises ``TypeError``; the constructor is keyword-only * a subclass overriding a v1 parsing hook (``pre_process``, ``parse_pieces``, ``is_title``, and the rest) gets a ``DeprecationWarning`` at construction naming the hooks it overrode, because 2.0 delegates parsing to the core parser and never calls them One removal changes results without saying anything. ``HumanName`` no longer defines ``__eq__``, so ``name == "John Smith"`` is now ``False`` where 1.x returned ``True``. That one *did* warn on 1.4, but nothing will tell you on 2.0. If you compare names anywhere, grep for ``==`` before upgrading and move to ``matches()`` — see `Comparison`_. One step has to happen *before* you upgrade, because the fix is only available on the version you're leaving: a ``Constants`` pickle written by nameparser 1.2.x or earlier must be re-pickled under 1.3 or 1.4. 2.0 refuses to load one, and by then the code that could rewrite it is gone. If your suite did *not* run clean, these are the replacements for the warned removals: .. list-table:: :header-rows: 1 :widths: 45 55 * - 1.4 spelling - 2.0 replacement * - ``HumanName(b"...")``, ``manager.add(b"...")`` - Decode first: ``HumanName(raw.decode("utf-8"))`` * - ``SetManager.add_with_encoding(b"...")`` - ``add()``, on already-decoded ``str`` * - ``manager()`` (calling a set manager) - ``set(manager)``, or iterate it directly * - ``manager.remove(missing)`` (was tolerant) - ``discard()`` to ignore missing; ``remove()`` now raises ``KeyError`` like ``set.remove`` * - ``HumanName(..., constants=None)`` - ``Constants()`` for library defaults, or ``CONSTANTS.copy()`` for a private snapshot of the current shared config * - ``name['first'] = value`` - ``name.first = value`` * - ``CONSTANTS.regexes.typo`` (returned ``EMPTY_REGEX``) - ``.get("typo")`` for intentional soft access; attribute access now raises ``AttributeError``. The same applies to ``capitalization_exceptions``, whose error also lists the known keys * - ``CONSTANTS.empty_attribute_default`` - Gone; empty fields are always ``''`` Finally, if you want to see the difference on your own data rather than ours, ``tools/differential/`` in the source repository diffs 1.4 and 2.0 parses over a corpus of names you supply. It is development tooling, not part of the installed package — see its README. Attribute map ------------- ``HumanName``'s seven fields and their aggregates map onto :class:`~nameparser.ParsedName` like this: .. list-table:: :header-rows: 1 :widths: 30 40 30 * - ``HumanName`` - ``ParsedName`` - Note * - ``title`` - ``title`` - * - ``first`` - ``given`` - * - ``middle`` - ``middle`` - * - ``last`` - ``family`` - * - ``suffix`` - ``suffix`` - * - ``nickname`` - ``nickname`` - * - ``maiden`` - ``maiden`` - New field (added 1.3) * - ``title_list``, ``first_list``, ``middle_list``, ``last_list``, ``suffix_list``, ``nickname_list``, ``maiden_list`` - ``tokens_for(Role.TITLE)``, ``tokens_for(Role.GIVEN)``, ... - Returns the raw :class:`~nameparser.Token` tuple for that role; read ``.text`` off each token for the string a ``_list`` attribute gave you * - ``given_names`` / ``given_names_list`` - ``given_names`` - Unchanged name; ``middle`` folded into ``first`` * - ``surnames`` / ``surnames_list`` - ``surnames`` - Unchanged name; ``middle`` folded into ``last`` * - ``last_base`` / ``last_base_list`` - ``family_base`` - The surname with leading particles split off * - ``last_prefixes`` / ``last_prefixes_list`` - ``family_particles`` - The particles ``family_base`` was split from (e.g. ``"de la"``) * - ``string_format`` - ``render(spec)`` - A per-call argument now, not stored config — see :doc:`customize` * - ``initials_format``, ``initials_delimiter``, ``initials_separator`` - ``initials(spec, delimiter, separator)`` - Same three knobs, now call-site arguments to :meth:`~nameparser.ParsedName.initials` * - ``suffix_delimiter`` - ``Policy(extra_suffix_delimiters={...})`` - Moves from a ``HumanName``/``Constants`` scalar to a ``Policy`` set field, so more than one custom delimiter can be active at once. It is the *set* that moved, not just the name: passing the old scalar through (``extra_suffix_delimiters=" - "``) raises, rather than silently registering three one-character delimiters * - ``capitalize(force=...)`` - ``capitalized(force=...)`` - :meth:`~nameparser.ParsedName.capitalized` returns a new value rather than mutating in place. Its optional first argument takes a :class:`~nameparser.Lexicon`, if you need custom capitalization exceptions Side by side: .. doctest:: >>> from nameparser import HumanName, parse, Role >>> hn = HumanName("Dr. Juan Q. Xavier de la Vega III") >>> n = parse("Dr. Juan Q. Xavier de la Vega III") >>> hn.title == n.title, hn.first == n.given, hn.last == n.family (True, True, True) >>> hn.first_list == [t.text for t in n.tokens_for(Role.GIVEN)] True >>> hn.given_names == n.given_names, hn.surnames == n.surnames (True, True) >>> hn.last_base == n.family_base, hn.last_prefixes == n.family_particles (True, True) Config map ---------- ``CONSTANTS``' vocabulary sets map onto :class:`~nameparser.Lexicon` fields: .. list-table:: :header-rows: 1 :widths: 30 40 30 * - ``CONSTANTS`` - ``Lexicon`` - Note * - ``titles`` - ``titles`` - * - ``first_name_titles`` - ``given_name_titles`` - * - ``suffix_acronyms`` - ``suffix_acronyms`` - * - ``suffix_not_acronyms`` - ``suffix_words`` - * - ``suffix_acronyms_ambiguous`` - ``suffix_acronyms_ambiguous`` - * - ``prefixes`` - ``particles`` - * - ``non_first_name_prefixes`` - ``particles_ambiguous`` - **Flipped** — see the warning below * - ``conjunctions`` - ``conjunctions`` - * - ``bound_first_names`` - ``bound_given_names`` - * - ``capitalization_exceptions`` - ``capitalization_exceptions`` - Pair-valued; set it via ``dataclasses.replace(lexicon, capitalization_exceptions={...})``, not ``add()``/``remove()`` And behavior/render scalars map onto :class:`~nameparser.Policy` (or a rendering argument, where the 2.0 equivalent isn't config at all): .. list-table:: :header-rows: 1 :widths: 30 40 30 * - ``CONSTANTS`` - 2.0 equivalent - Note * - ``patronymic_name_order`` - ``Policy(patronymic_rules={PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC})`` - v1's single flag enabled both detectors at once; pick one rule (or a locale pack, see :doc:`locales`) if you only want one tradition * - ``middle_name_as_last`` - ``Policy.middle_as_family`` - * - ``nickname_delimiters`` - ``Policy.nickname_delimiters`` - Was a dict of named sentinels; now a plain ``frozenset`` of ``(open, close)`` pairs. Both APIs gained the #273 typographic defaults (smart quotes, guillemets, CJK brackets, ...) in 2.0 * - ``maiden_delimiters`` - ``Policy.maiden_delimiters`` - Same shape change as ``nickname_delimiters``. Precedence differs: in the 2.0 API a pair listed here wins over ``nickname_delimiters``; through the 1.x facade a pair in both buckets keeps parsing as ``nickname`` (v1 behavior) * - ``regexes.bidi`` - ``Policy.strip_bidi`` - ``regexes.bidi = False`` becomes ``Policy(strip_bidi=False)`` * - ``regexes.emoji`` - ``Policy.strip_emoji`` - ``regexes.emoji = False`` becomes ``Policy(strip_emoji=False)`` * - ``force_mixed_case_capitalization`` - ``capitalized(force=True)`` - The stored default is gone; pass ``force`` at the call site. v1's ``capitalize(force=True)`` already worked that way — this constant only supplied the default for calls that omitted it * - ``capitalize_name`` - *(no equivalent)* - 2.0 never capitalizes automatically during ``parse()``; call ``.capitalized()`` explicitly on the result instead Every other ``regexes.*`` entry (``word``, ``spaces``, and the rest of the compiled-pattern proxy) has no 2.0 replacement — parsing behavior is configured entirely through named ``Policy`` fields now, not by handing the parser a regex. .. warning:: ``non_first_name_prefixes`` and ``particles_ambiguous`` mark **complementary** sets, not the same set under a new name. ``non_first_name_prefixes`` lists particles that are *never* read as a given name; ``particles_ambiguous`` lists the particles that *may* be read as one. Translating a customization means flipping the set: ``particles_ambiguous = lexicon.particles - constants.non_first_name_prefixes``. Copying ``non_first_name_prefixes`` straight into ``particles_ambiguous`` silently inverts which particles are allowed to double as a given name. Comparison ---------- ``HumanName.__eq__``/``__hash__`` were deprecated in 1.3.0 and are gone in 2.0's core API; use ``matches()`` for "is this the same name?" and ``comparison_key()`` for dedup, dict keys, and sorting — both exist on ``HumanName`` and on :class:`~nameparser.ParsedName` with the same behavior: .. doctest:: >>> from nameparser import HumanName, parse >>> hn = HumanName("de la Vega, Juan") >>> n = parse("de la Vega, Juan") >>> hn.matches("Juan de la Vega"), n.matches("Juan de la Vega") (True, True) >>> hn.comparison_key() == n.comparison_key() True One behavior changed underneath both methods: components now fold with ``str.casefold()`` instead of ``str.lower()``, so more Unicode case-pairs compare equal than did under 1.4. The change is strictly more permissive — anything 1.4 matched still matches: .. doctest:: >>> parse("Anna STRASSE").matches("Anna Straße") True Behavior changes ----------------- Beyond the API surface mapped above, a handful of parse *outputs* differ between 1.4 and 2.0 for specific input shapes. The full list, with reasoning, is in the 2.0.0 section of :doc:`release_log`. These are the shapes worth grepping your own fixtures for, because a recognized suffix or title now stays in its own field instead of landing in ``first``/``last``: .. doctest:: >>> HumanName("Andrews, M.D.").last, HumanName("Andrews, M.D.").suffix ('Andrews', 'M.D.') >>> HumanName("Johnson PhD").first, HumanName("Johnson PhD").suffix ('Johnson', 'PhD') Under 1.4 those read ``first="M.D."``/``last="Andrews"`` and ``first="Johnson"``/``last="PhD"`` respectively. Two more to check for: a maiden marker now fills the ``maiden`` field rather than being folded into ``middle``/``last`` (``"Jane Smith née Jones"``), and with a custom suffix delimiter configured, a no-space delimiter group renders whole (``"RN/CRNA"``) where 1.x split it (``"RN, CRNA"``) — the role assignment is identical, only the rendered string differs. 2.1 adds three more, and unlike most of the 2.0 API these do reach ``HumanName``: a name written in East Asian script is read family-first, an unspaced Korean name is split into surname and given name, and the katakana middle dot separates tokens the way a space does. .. doctest:: >>> HumanName("毛 泽东").last '毛' >>> HumanName("김민준").last, HumanName("김민준").first ('김', '민준') 1.4 read the first as ``first="毛"``/``last="泽东"``, and left the second whole in ``first``. The Korean one also changes what the name *renders* as — ``str(HumanName("김민준"))`` was ``"김민준"`` and is now ``"민준 김"``, because the split inserts a token boundary that the default format then writes given-name-first. A third shape changes even though nothing splits it. 1.4 routed a lone token to ``first`` whatever it was, so an unspaced Chinese or Japanese name landed there entire; 2.1 reads it as native-script CJK and puts it in ``last`` instead. Nothing is segmented — Han splitting is opt-in through a locale pack and ``HumanName`` cannot apply one — but the field the whole string arrives in is different: .. doctest:: >>> HumanName("毛泽东").last, HumanName("毛泽东").first ('毛泽东', '') >>> HumanName("山田太郎").last, HumanName("山田太郎").first ('山田太郎', '') Both were ``first`` under 1.4. If you feed unspaced CJK through ``HumanName`` and read ``first``, that is the change most likely to reach you, and it is silent — the string is intact, just in the other field. Japanese kana carries the same order rule, which widens both shapes past the wholly-Han text described above. A name that mixes kanji with hiragana or katakana is read family-first, and a lone kana-bearing token moves from ``first`` to ``last`` exactly as ``毛泽东`` does: .. doctest:: >>> HumanName("高橋 みなみ").last, HumanName("高橋 みなみ").first ('高橋', 'みなみ') >>> HumanName("山田 エミ").last '山田' >>> HumanName("高橋みなみ").last, HumanName("高橋みなみ").first ('高橋みなみ', '') 1.4 read the spaced ones as ``first="高橋"``/``last="みなみ"`` and ``first="山田"``/``last="エミ"``, and put the unspaced one whole in ``first``. The spaced shapes change what the name renders as too: ``str(HumanName("高橋 みなみ"))`` was ``"高橋 みなみ"`` and is now ``"みなみ 高橋"``. A name written *wholly* in katakana is deliberately left alone — it is usually a transcribed foreign name already in given-first order — so ``HumanName("マイケル ジャクソン")`` reads ``first="マイケル"``/``last="ジャクソン"`` on both versions. One more shape changes for a different reason: the katakana middle dot ``・``, which divides the parts of such a transcription, is now a token separator rather than an ordinary character. 1.4 saw one token and put it in ``first``; 2.1 sees two: .. doctest:: >>> HumanName("マイケル・ジャクソン").first 'マイケル' >>> HumanName("マイケル・ジャクソン").last 'ジャクソン' >>> HumanName("高橋・一郎").last, HumanName("高橋・一郎").first ('高橋', '一郎') The two divide the same way and land in opposite fields, because the katakana pair keeps its source order while the kanji pair takes the family-first rule. Separating also changes the rendered string, the same way the Korean split does: the dot comes back as a space, so ``str(HumanName("マイケル・ジャクソン"))`` was ``"マイケル・ジャクソン"`` and is now ``"マイケル ジャクソン"``. That reaches delimited content too — the nickname in ``"山田 太郎 (マイケル・ジャクソン)"`` was ``"マイケル・ジャクソン"`` under 1.4 and is ``"マイケル ジャクソン"`` now. ``Constants`` has no switch for any of this — the v1 configuration surface is frozen for 2.x — so the way out is the 2.0 API: ``Parser(policy=Policy(script_orders=(), segment_scripts=frozenset()))`` restores 1.4's reading of every shape above that turns on order or splitting. The middle dots are the exception: both the katakana dot and the Chinese interpunct ``·`` (U+00B7, dividing a transcription like ``威廉·莎士比亚``, and a separator only between classified-script characters — each side judged on its own — where the katakana dot is unconditional) are decided in tokenization rather than by policy, so a name written with either still divides at the dot, and still renders with a space, whatever those two fields are set to. python-nameparser-2.1.0/docs/modules.rst000066400000000000000000000161561523530431700203340ustar00rootroot00000000000000API reference ============= The 2.0 API ----------- Parsing ~~~~~~~ .. autofunction:: nameparser.parse .. autoclass:: nameparser.Parser :members: .. autofunction:: nameparser.parser_for .. autoclass:: nameparser.Segmentation :members: .. py:data:: nameparser.Segmenter :value: Callable[[str], Segmentation | None] The type of the optional ``Parser(segmenter=...)`` hook: a callable given one token's text, returning a :class:`~nameparser.Segmentation` that divides it, or ``None`` to decline and leave it whole. An alias, not a class — any callable of that shape qualifies, and nothing needs to be subclassed or registered. It is consulted only for tokens whose script is listed in :attr:`Policy.segment_scripts `, and only where the surname vocabulary declined first; see :ref:`segmenter-contract` for what a segmenter owes its caller. :func:`~nameparser.locales.ja_segmenter` is the shipped implementation. Results ~~~~~~~ .. autoclass:: nameparser.ParsedName :members: .. autoclass:: nameparser.Token :members: .. py:data:: nameparser.STABLE_TAGS :value: frozenset({"particle", "conjunction", "initial", "joined"}) The four :attr:`Token.tags ` values that are stable API: ``particle`` (a word from the particle vocabulary, "de"/"van", wherever it lands — combine with ``Role.FAMILY`` for actual family particles), ``conjunction`` (a joining word, "and"/"y"), ``initial`` (an initial-shaped word in a script that HAS initials — "J." or "А.", never "씨."), and ``joined`` (a continuation of the previous token within one merged piece, so the suffix view renders "Ph. D." as one credential). Every other tag is namespaced (``vocab:...``) and unstable — never match against those. .. autoclass:: nameparser.Span :members: .. autoclass:: nameparser.Role :members: .. autoclass:: nameparser.Ambiguity :members: .. autoclass:: nameparser.AmbiguityKind :members: Configuration ~~~~~~~~~~~~~ .. autoclass:: nameparser.Lexicon :members: .. autoclass:: nameparser.Policy :members: .. autoclass:: nameparser.PolicyPatch :members: .. py:data:: nameparser.UNSET Sentinel meaning "this patch does not set this field" — the default of every :class:`~nameparser.PolicyPatch` field, distinguishable from every real value including ``None`` and ``False``. You rarely need it: omit a field instead of passing it. Import it to test whether a patch sets a field (``patch.name_order is UNSET``) or to leave a field conditionally unset when building patches programmatically. .. autoclass:: nameparser.PatronymicRule :members: .. autoclass:: nameparser.Script :members: .. _name-order-constants: Name-order constants ^^^^^^^^^^^^^^^^^^^^ The three valid values for ``Policy(name_order=...)``. ``name_order`` is deliberately restricted to these exported constants — an arbitrary tuple of :class:`~nameparser.Role` members raises ``ValueError``, because only these three orders have defined assignment semantics. .. py:data:: nameparser.GIVEN_FIRST :value: (Role.GIVEN, Role.MIDDLE, Role.FAMILY) Western order (the default): the first word of positional input is the given name, the last is the family name, everything between is middle. .. py:data:: nameparser.FAMILY_FIRST :value: (Role.FAMILY, Role.GIVEN, Role.MIDDLE) Family name first, given name second, remaining words middle — e.g. Hungarian, or East Asian order. .. py:data:: nameparser.FAMILY_FIRST_GIVEN_LAST :value: (Role.FAMILY, Role.MIDDLE, Role.GIVEN) Family name first, given name *last*, the words between middle — e.g. Vietnamese full-name order. .. py:data:: nameparser.DEFAULT_SCRIPT_ORDERS :value: ((Script.HAN, FAMILY_FIRST), (Script.HANGUL, FAMILY_FIRST), (Script.HIRAGANA, FAMILY_FIRST)) The default :attr:`~nameparser.Policy.script_orders` table: a name written wholly in Han or Hangul reads family-first, and so does a Japanese name mixing kanji with kana, which resolves to the ``HIRAGANA`` entry whichever of the two syllabaries it actually uses — that member is the license's carrier key, not a claim about the characters present. A name written wholly in katakana has no entry on purpose and stays positional. A matching entry in this table takes precedence over ``name_order``, including a ``name_order`` you set explicitly — ``name_order`` governs only the names no entry matches. The values are drawn from the same three constants above, and the same restriction applies. Build on it for additive customization — ``script_orders=(*DEFAULT_SCRIPT_ORDERS, (Script.HAN, GIVEN_FIRST))``, where a later entry for a script REPLACES an earlier one, so appending is how you override — and pass ``script_orders=()`` to opt out entirely and get the purely positional read back. Latin-script and mixed-script names are never affected either way. Delimiter defaults ^^^^^^^^^^^^^^^^^^ .. py:data:: nameparser.DEFAULT_NICKNAME_DELIMITERS :value: frozenset({("'", "'"), ('"', '"'), ("(", ")"), ("“", "”"), ("„", "“"), ("”", "”"), ("«", "»"), ("»", "«"), ("「", "」"), ("『", "』"), ("（", "）")}) The default :attr:`~nameparser.Policy.nickname_delimiters` set: straight quotes and parentheses plus the typographic conventions — smart quotes, German/Polish low-high quotes, Swedish right-right quotes, guillemets in both directions, CJK corner brackets, and fullwidth parentheses (#273). Curly *single* quotes are deliberately absent: U+2019 is the typographic apostrophe ("O’Connor"). Build on the constant for additive customizations, e.g. ``nickname_delimiters=DEFAULT_NICKNAME_DELIMITERS | {("｟", "｠")}``; to *reroute* a pair to ``maiden``, just list it in :attr:`~nameparser.Policy.maiden_delimiters` (it is dropped from the effective nickname set automatically). Locales ~~~~~~~ .. autoclass:: nameparser.Locale :members: .. automodule:: nameparser.locales :members: get, available .. autofunction:: nameparser.locales.ja_segmenter 1.x compatibility layer ------------------------ .. note:: ``HumanName`` and ``nameparser.config`` are the 1.x API, kept working through 2.x and removed in 3.0. New code should use the 2.0 API above; see :doc:`migrate`. HumanName.parser ~~~~~~~~~~~~~~~~ .. py:module:: nameparser.parser .. py:class:: HumanName :noindex: .. autoclass:: HumanName :members: :special-members: __eq__, __init__ HumanName.config ~~~~~~~~~~~~~~~~ .. automodule:: nameparser.config :members: HumanName.config Defaults ~~~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: nameparser.config.titles :members: .. automodule:: nameparser.config.suffixes :members: .. automodule:: nameparser.config.prefixes :members: .. automodule:: nameparser.config.bound_first_names :members: .. automodule:: nameparser.config.conjunctions :members: .. automodule:: nameparser.config.maiden_markers :members: .. automodule:: nameparser.config.surnames :members: .. automodule:: nameparser.config.capitalization :members: .. automodule:: nameparser.config.regexes :members: python-nameparser-2.1.0/docs/release_log.rst000066400000000000000000001741561523530431700211520ustar00rootroot00000000000000Release Log =========== * 2.1.0 - August 7, 2026 nameparser 2.1 makes East Asian names work without configuration. A name written wholly in Han or hangul, or in kanji with kana, is read family-first. An unspaced Korean name is split against the census surname list. CJK honorifics are recognized whether they are spaced or written against the name. The two conventions that need you to declare a language, Han segmentation and kana-aware division, ship as the opt-in ``locales.ZH`` and ``locales.JA`` packs. See :ref:`east-asian-names` for how it fits together, and :doc:`customize` for the switches that turn it off. Most of this is **default-on**, deliberately: wherever nameparser acts unasked, the script itself settles the convention and no language detection is involved. Latin-script names are unaffected. To restore 2.0's reading of CJK text, use ``Parser(policy=Policy( script_orders=(), segment_scripts=frozenset()))``. **East Asian name support** - Add the Chinese locale pack ``locales.ZH``: opt-in Han segmentation for unspaced names like ``毛泽东``, with the surname vocabulary it needs. A pack rather than a default because a Chinese surname list corrupts Japanese names written in the same characters (``高橋一郎`` would split ``高`` + ``橋一郎``). Japanese data goes through ``locales.JA`` instead. See :doc:`locales` (#271) - Add ``Lexicon.surnames``, ``Policy.script_orders``, ``Policy.segment_scripts``, the ``Script`` enum and the ``DEFAULT_SCRIPT_ORDERS`` constant to the public API. This is the first behavior nameparser keys on the script a name is written in, allowed only where the script itself settles a convention and never as a proxy for guessing the language. See :doc:`modules` (#271) - Add ``Lexicon.honorific_tails``, the vocabulary the glued-honorific peel matches: entries that may be split off the end of a name token. It is a narrower set than the spaced honorific vocabulary, since a glued tail has no token boundary to lean on, and every entry must also be a ``suffix_words`` entry. Extend both in one call, or adding to ``honorific_tails`` alone raises ``ValueError``. See :doc:`customize` (#308) - Add ``AmbiguityKind.SEGMENTATION``, reported when a surname split had a vocabulary-supported alternative: ``"남궁민수"`` is 남궁 + 민수 by the compound surname but 남 + 궁민수 by the single-syllable one, and longest-match had to pick. A name with only one possible split reports nothing (#271) - Add the Japanese locale pack ``locales.JA`` and the segmenter factory ``locales.ja_segmenter()``, which together divide an unspaced Japanese name: ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` reads ``山田太郎`` as family ``山田``, given ``太郎``. They are separate because no surname list can do this job, so the pack activates the stage and a third-party divider performs it. ``ja_segmenter()`` wraps `namedivider-python `_, installed with the new ``nameparser[ja]`` extra; the core stays dependency-free. ``locales.available()`` is now ``('ja', 'ru', 'tr_az', 'zh')``. See :doc:`locales` (closes #272) - Add ``Segmentation`` and the ``Segmenter`` type alias to the public API, plus the keyword-only ``Parser(segmenter=...)`` hook: any callable from a token's text to a ``Segmentation`` or to ``None`` to decline. It is consulted only for scripts in ``Policy.segment_scripts``, and only where the surname vocabulary declined first. Note what that ordering means when packs are stacked: a Japanese name opening on a listed Chinese surname never reaches the segmenter, so ``高橋一郎`` still splits ``高`` + ``橋一郎``. The packs are alternatives, one per corpus. See :ref:`segmenter-contract` (#272) - Add a construction-time ``UserWarning`` when a parser activates segmentation for scripts nothing can divide. ``parser_for(locales.JA)`` without ``segmenter=`` used to build a parser that behaved like a working one minus the feature, silently. It now names the dead scripts and the call to pass. Any configured segmenter or covering surname vocabulary silences it, so the default parser and the ``zh`` pack never warn. A from-scratch lexicon with no hangul surnames warns under the default policy, with ``Policy(segment_scripts=())`` offered as the deactivation. See :ref:`east-asian-names` - Add the ``Script`` members ``HIRAGANA`` and ``KATAKANA``. Two members rather than one ``KANA`` because the parser treats them differently: hiragana never transcribes a foreign name, while a wholly-katakana name usually is one (#272) **Breaking Changes** - Change the pickle compatibility of ``Policy`` and ``Lexicon``: the new fields change the field layout their guarded ``__setstate__`` checks, so a pickle written by 2.0.0 raises ``ValueError`` naming the missing fields instead of loading. Re-pickle after upgrading. ``HumanName`` pickles are unaffected, since the facade pickles v1-shaped component state rather than these objects - Change the pickle compatibility of ``Parser`` the same way, for the new ``segmenter`` field. Re-pickle after upgrading. A ``Parser`` carrying a segmenter pickles only if that segmenter does, which a module-level function does and a closure or lambda does not (#272) - Change one thing about parse totality: ``parse()`` still never raises on any input, but a user-supplied ``Parser(segmenter=...)`` runs inside the parse and its own exceptions propagate rather than being absorbed. A failure there is a bug in your callable, not a fact about the name (#272) **Behavior Changes** - Fix names written wholly in Han or hangul parsing given-first. Native-script CJK now reads family-first through the new ``Policy.script_orders`` table, so ``"毛 泽东"`` gives family ``毛`` where 1.x gave family ``泽东``. A single unspaced token moves the same way, which is the easiest form of this to miss: ``"毛泽东"`` renders identically whichever field holds it, while the spaced form shows the change (``str(HumanName("毛 泽东"))`` is now ``"泽东 毛"``). No language detection is involved, and an explicit comma still wins. **Default-on**, and it reaches ``HumanName``. See :ref:`east-asian-names` (closes #271) - Fix unspaced Korean names not splitting. The census surname list now ships as default vocabulary with hangul segmentation on by default, so ``"김민준"`` parses family ``김``, given ``민준`` where 1.x returned the whole string as ``first``. Rendering follows the split. Nothing but Korean is written in hangul and its surnames are a closed set, which is what makes this safe as a default rather than a pack. **Default-on**. See :doc:`customize` for the two switches (#271) - Fix Japanese names carrying kana parsing given-first. The family-first rule extends to any name whose characters stay within kanji and kana while carrying at least one kana character, so ``"高橋 みなみ"`` gives family ``高橋``. The reasoning is the one hangul already uses: hiragana never transcribes a foreign name, and a transcription is kana alone, so kanji-plus-kana is a Japanese name in Japanese order. A name written **wholly in katakana** is excluded and stays positional, being predominantly a transcribed foreign name. **Default-on**. See :ref:`east-asian-names` (#272) - Fix names containing 〆 (U+3006, the shime mark opening Japanese surnames like 〆木) parsing given-first. The script classifier now counts it as Han, so these names take the family-first reading like any other wholly-Han name (#303) - Fix the katakana middle dot ``・`` (U+30FB, and its halfwidth twin U+FF65) being read as part of a name rather than as a divider. It now separates tokens exactly as a space does, so ``"マイケル・ジャクソン"`` gives given ``マイケル``, family ``ジャクソン`` where 1.x left the whole string in ``first``. Native Japanese names never contain this character, so the separation is unconditional and the policy opt-outs do **not** cover it. Rendering returns the dot as a space (#272) - Fix 间隔号-divided transcriptions parsing as one unsplit token. U+00B7, the interpunct Chinese text divides a transcribed foreign name with, is now a token separator between characters of a classified script, and a name it divides keeps its source order and is never segmented. It divides only between classified-script characters, so the Catalan punt volat in ``Gal·la`` is untouched. The Japanese nakaguro is deliberately not a transcription marker, so ``高橋・一郎`` keeps its family-first reading (#298) - Fix spaced CJK postnominal honorifics parsing as name parts. 씨, 박사, 선생님, 교수님, 군, 양, 先生, 女士, 小姐, 博士, 教授, 様 and 氏 now route to ``suffix``, so ``王小明 先生`` reads family ``王小明`` where the family-first default had made 先生 the given name. See :ref:`east-asian-names` (closes #307) - Fix glued CJK honorifics parsing as part of the name. ``田中さん``, ``김민준씨`` and ``王小明先生`` now split the honorific off the end of the name into ``suffix``. Previously it stayed in the name: ``田中さん`` was entirely the family name, and ``김민준씨`` gave given 민준씨. The peel runs before the name is split or ordered, so ``김민준씨`` still divides into family 김, given 민준. Only entries that can never end a name peel; 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, since 김지양 is a given name and some ninety Japanese surnames end in 殿. **Default-on**, and a lone family name written with a glued honorific now divides where it did not. See :ref:`east-asian-names` for the full set and :doc:`customize` for the off-switch (closes #308) - Fix a comma or a 间隔号 stopping the glued-honorific peel. ``김, 민준씨`` now gives family 김, given 민준, suffix 씨, the same as the spaced ``김 민준씨``, and likewise ``田中, 太郎さん``. Previously each left the honorific inside the name. Both marks say where a name divides into surname and given, and an honorific is not part of the name in either reading. What a comma does instead is say which runs to look in: the two around a family comma. Anything past those is out of reach, so ``김, 민준 지훈씨`` peels while ``김, 민준, 지훈씨`` does not. **Default-on**. See :ref:`east-asian-names` (closes #312) - Fix a glued honorific staying inside the name when the whole post-comma remainder is a credential. ``田中さん, V.`` and ``田中さん, Ph. D.`` now give up さん to ``suffix`` the way ``田中さん, PhD`` already did. The peel had been taking the post-comma run for name text on the strength of the comma alone, walking into the credentials and abandoning the peel there. That run is now tested with the same rule that decides the comma structure, and declined where it is credentials, provided the part before the comma offers a peel site of its own. Where the credential itself lands is still the comma's business and still differs by spelling. **Default-on** (closes #319) - Fix an ASCII period after a CJK honorific stopping it being recognized. ``씨.``, ``様.``, ``氏.``, ``님.``, ``군.``, ``양.`` and ``殿.`` now route to ``suffix`` like their periodless spellings, where the trailing period had left them inside the name. The cause was v1's initial regex, whose ``\w`` is Unicode-aware and matched a hangul syllable or Han ideograph as readily as a letter; a veto written for Latin was being asked of scripts it was never about. Alphabets keep their initials untouched, and ``"А. С. Пушкин"`` is unaffected. Read *period* strictly: only the ASCII full stop is covered, so ``"김민준 씨．"`` written with the fullwidth stop still reads the honorific as the family name. **Default-on** (#320) - Fix NFD-decomposed input missing the East Asian defaults entirely. Script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS, where decomposed text is routine, gets the same order rule as its composed twin. Segmentation matching deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name. Compare NFC-normalized text when comparing across encodings. See :ref:`east-asian-names` (#272) - Fix the Ukrainian conjunction ``й`` not joining the pieces around it. It is the euphonic alternate of ``і``, chosen by the surrounding sounds rather than by meaning, so real Ukrainian data carries both spellings. ``"Олесь й Олена Коваленки"`` now gives given ``"Олесь й Олена"`` where the ``й`` previously landed in ``middle``. Same treatment as the ``и``/``і`` entries added in 2.0.0: the conjunction joins only once the name has enough pieces, and a punctuated initial still wins, so ``"Й. Сліпий"`` is unaffected. Raised in a comment on #267 - Add the Japanese maiden-name marker ``旧姓`` to the default vocabulary. ``"山田花子 旧姓 佐藤"`` now gives family ``山田花子`` and maiden ``佐藤``, where 1.4.0 left the marker in the name. It sits beside the Cyrillic ``урожд.`` and German ``geb.`` entries rather than in ``locales.JA``, on the rule that admitted those: a native-script marker cannot collide with a Latin-script name, so it is safe as a default. Matching is whole-token, so the marker has to *be* a token, which for Japanese means a space or a configured delimiter must divide it from the name. The fullwidth colon does not, so ``"山田（旧姓：佐藤）"`` still returns maiden ``"旧姓：佐藤"``; that one wants the head-peel #317 tracks. **Default-on**. See :doc:`customize` (#309) - Fix a maiden marker inside bracketed content staying in the ``maiden`` value. Where a delimiter pair is routed to ``maiden`` by ``Policy(maiden_delimiters=...)``, a marker at the head of the clause is now dropped the way it always has been in the bare form, so ``"Jane Smith (née Jones)"`` gives maiden ``Jones``, the same answer as the unbracketed spelling. Each clause loses its own leading marker, and only where the clause holds more than one token, so ``"Jane Smith (Nee) (Jones)"`` still gives maiden ``Nee Jones`` (``Nee`` is a real surname). Scope before you count on it: ``Policy.maiden_delimiters`` is **empty by default**, so under the default policy brackets route to ``nickname`` and none of this applies. See :doc:`customize` (closes #329) **Documentation** - Correct the documented scope of the period-abbreviation title rule. It was described as applying to "a leading word", which was never true of any comma form: the rule runs at the front of the part that carries the given name, which after a family comma is the part *after* the comma, so ``"Morse, Det. Insp. Jane"`` gives title ``Det. Insp.``. Behavior is unchanged and matches 1.4.0; only the description was wrong * 2.0.0 - July 27, 2026 Two release candidates preceded this release (rc1 on 2026-07-23, rc2 on 2026-07-26). Please report anything the migration missed on `issue #284 `_. The notes below describe 2.0.0 as a whole. nameparser 2.0 adds a new parsing API alongside ``HumanName``. ``parse()`` returns an immutable ``ParsedName`` whose seven fields are named for what they are (``given``, ``family``) rather than where they sit in a Western name, configured by two frozen value objects -- a ``Lexicon`` of vocabulary and a ``Policy`` of behavior -- instead of a mutable global. ``HumanName`` keeps working: it is now a compatibility facade over the same pipeline, and it stays through 2.x. The removals below are the deprecations announced in 1.3.0 and 1.4.0 coming due; if your code runs warning-free on 1.4.0, most of them will not affect you. See :doc:`migrate` for the field-by-field map. **The 2.0 API** - Add ``parse(text)``, returning an immutable ``ParsedName`` with seven fields -- ``title``, ``given``, ``middle``, ``family``, ``suffix``, ``nickname``, ``maiden`` -- plus the derived views ``given_names``, ``surnames``, ``family_base`` and ``family_particles``. Parsing is a pure function of the text, a ``Lexicon`` and a ``Policy``: nothing global is consulted and nothing is mutated - Add ``Lexicon``, the frozen vocabulary object. ``Lexicon.default()`` is the shipped vocabulary and ``Lexicon.empty()`` is a blank one; ``add(**entries)``, ``remove(**entries)`` and field-wise ``|`` all return new instances. Its fields are ``titles``, ``given_name_titles``, ``suffix_acronyms``, ``suffix_words``, ``suffix_acronyms_ambiguous``, ``particles``, ``particles_ambiguous``, ``conjunctions``, ``bound_given_names``, ``maiden_markers`` and ``capitalization_exceptions`` - Add ``Policy``, the frozen behavior object: ``name_order``, ``patronymic_rules``, ``middle_as_family``, ``nickname_delimiters``, ``maiden_delimiters``, ``extra_suffix_delimiters``, ``lenient_comma_suffixes``, ``strip_emoji`` and ``strip_bidi`` - Add ``Parser``, a reusable parser bound to a lexicon and policy (``Parser(lexicon=..., policy=...).parse(text)``), and ``parser_for(*locales, base=None)``, which folds locale packs onto a base parser - Add ``name_order`` and the constants ``GIVEN_FIRST``, ``FAMILY_FIRST`` and ``FAMILY_FIRST_GIVEN_LAST``, so a family-first name can be parsed as written rather than reordered by hand (the configuration half of #270; the locale packs below complete it) - Add ``PatronymicRule`` with the members ``EAST_SLAVIC`` and ``TURKIC``. v1's single ``patronymic_name_order`` flag enabled both detectors at once; ``Policy(patronymic_rules=...)`` lets you enable either one alone - Add ``PolicyPatch`` and the ``UNSET`` sentinel for partial policy deltas that compose -- set-valued fields union, scalar fields override with later winning. This is the mechanism locale packs are built from, and ``UNSET`` is only needed when you must distinguish "not set" from a real ``False`` or ``None`` - Add ``Token``, ``Span`` and ``Role``: every field is backed by tokens carrying exact ``(start, end)`` offsets into the original string, reachable with ``tokens_for(Role.GIVEN)``. This replaces v1's ``*_list`` attributes and makes it possible to highlight or re-slice the input the parse came from. ``Role`` is a ``StrEnum``, so members compare and stringify as their field names (``Role.GIVEN == "given"``), matching ``AmbiguityKind``; ``tokens_for()`` accepts a role's string name too, and raises ``ValueError`` for an unknown role - Add ``STABLE_TAGS`` to the public API: the four documented ``Token.tags`` values (``particle``, ``conjunction``, ``initial``, ``joined``) - Add ``Policy.patched(patch)``, applying a ``PolicyPatch`` directly without wrapping it in a locale pack - Add ``Parser.matches(a, b)`` and ``Parser.capitalized(name)``: the ``ParsedName`` methods of the same names fall back to the default configuration for str/omitted arguments, which is silently wrong for names parsed with a custom ``Parser`` - Add ``Parser.revise(name, **fields)``: ``ParsedName.replace()`` with the replacement text classified by the parser's vocabulary, so particle/initial/suffix-join behavior survives the edit - Add ``Ambiguity`` and the ``AmbiguityKind`` enum, so a parse reports what it had to guess at instead of guessing silently. The kinds emitted today are ``particle-or-given``, ``suffix-or-name``, ``suffix-or-nickname``, ``unbalanced-delimiter`` and ``comma-structure``; ``order`` is reserved and not yet emitted. The two suffix kinds cover the post-nominals that are also ordinary words: ``"John Smith MA"`` reports that ``MA`` was read as a credential rather than a surname, and ``"JEFFREY (JD) BRICKEN"`` that the delimited ``JD`` was read as a nickname rather than a suffix. A reading the vocabulary settles on its own — ``"John Smith M.A."``, ``"Andrew Perkins (MBA)"`` — is not a guess and reports nothing - Add ``ParsedName`` output and comparison methods: ``render(spec)``, ``initials()``, ``capitalized()`` (which returns a new value rather than mutating in place), ``as_dict()`` (whose ``include_empty`` flag is keyword-only, unlike ``HumanName.as_dict()``'s), ``replace(**fields)``, ``matches()`` and ``comparison_key()`` - Add ``Locale``, the public pack type. Writing your own needs no registration -- construct a ``Locale`` and pass it to ``parser_for()`` - Ship a fully typed public API (PEP 561): the core modules are checked under strict mypy settings, and nameparser 2.0 has no runtime dependencies **Breaking Changes** - Raise the minimum Python to 3.11 and drop the last runtime dependency, ``typing_extensions`` (#257). Python 3.10 is no longer supported - Remove ``HumanName.__eq__`` and ``__hash__`` (deprecated in 1.3.0, #223): instances now compare and hash by identity, so ``HumanName("John Smith") == "John Smith"`` is ``False`` where 1.x returned ``True``. **This changes result silently rather than raising** -- it is the one removal that can pass unnoticed into production. Use ``matches()`` to ask whether two names are the same, and ``comparison_key()`` as a dict key or sort key - Remove every v1 parsing hook and subclass extension point: ``pre_process``, ``post_process``, ``parse_pieces``, ``parse_nicknames``, ``join_on_conjunctions``, the ``is_*`` predicates, ``cap_word``/``cap_piece``, ``handle_firstnames``, ``fix_phd`` and the rest. A subclass that overrides one gets a ``DeprecationWarning`` at construction naming the hooks, because the facade delegates to the core ``Parser`` and never calls them (closes #280). Customize through ``Lexicon``/``Policy`` instead - Remove regex configuration: ``CONSTANTS.regexes`` is now a read-only proxy. Reads still work, but ``CONSTANTS.regexes.bidi = False``, item assignment and ``Constants(regexes=...)`` all raise ``TypeError``. **If you followed 1.3.1's advice to keep bidi marks with** ``CONSTANTS.regexes.bidi = False``, that opt-out is now ``Policy(strip_bidi=False)`` on the 2.0 API; the same applies to ``regexes.emoji`` and ``Policy(strip_emoji=False)`` - Remove ``bytes`` input and the ``encoding`` argument (#245): passing ``bytes`` to ``HumanName`` or to a set manager raises ``TypeError`` with a decode hint, and ``SetManager.add_with_encoding()`` and ``DEFAULT_ENCODING`` are gone. Decode first, then use ``add()`` - Remove ``SetManager.__call__`` (#243); iterate the manager or call ``set(manager)``. ``remove()`` of a missing member now raises ``KeyError`` like ``set.remove`` -- ``discard()`` is the ignore-missing form -- and the set operators ``|``, ``&``, ``-`` and ``^`` return a plain ``set`` - Remove ``HumanName`` slice access and item assignment (#258): ``name[1:-3]`` and ``name['first'] = value`` raise ``TypeError``. String-key reads (``name['first']``) and iteration are unchanged; assign fields as plain attributes - Remove ``Constants.empty_attribute_default`` (#255): empty fields are always ``''``. Assigning it raises ``AttributeError``; a pickle carrying the key still loads, with the value ignored - Remove ``constants=None`` (#261): both ``HumanName(..., constants=None)`` and ``hn.C = None`` raise ``TypeError``. Use ``Constants()`` for library defaults or ``CONSTANTS.copy()`` for a private snapshot - Remove silent unknown-key access on the mapping managers (#256): ``CONSTANTS.capitalization_exceptions.typo`` and ``CONSTANTS.regexes.typo`` raise ``AttributeError`` naming the miss, instead of returning ``None``/``EMPTY_REGEX`` (``capitalization_exceptions`` also lists the known keys). ``.get()`` remains available on both for intentional soft access - Remove support for ``Constants`` pickles written by nameparser 1.2.x or earlier (#279): loading one raises ``ValueError`` telling you to re-pickle under 1.3/1.4 first - Remove the dead ``regexes.no_vowels`` pattern (#268) and the ``Constants.suffixes_prefixes_titles`` cached union; neither was read by the parser - Change ``HumanName``'s ``*_list`` attributes to read-only properties: they remain readable snapshots, but ``name.first_list = [...]`` now raises ``AttributeError`` - Change the ``Constants`` constructor to keyword-only; positional construction no longer works **Behavior Changes** - Recognize maiden-name markers -- ``née``, ``nee``, ``geb.``, ``roz.`` and the Scandinavian participle forms -- and route the following name to the new ``maiden`` field. 1.x folded them into ``middle``/``last``, so ``"Jane Smith née Jones"`` parsed as ``middle="Smith née"`` (closes #274) - Recognize typographic nickname delimiters by default in both APIs (closes #273): smart quotes (``“Jack”``), German and Polish low-high quotes (``„Hansi“``), Swedish right-right quotes (``”Ann”``), guillemets in either direction (``«Petit»``, ``»Hansi«``), CJK corner brackets (``「タロ」``, ``『ハナ』``) and fullwidth parentheses. In 1.x these leaked into ``middle`` as literal text. Curly *single* quotes stay excluded, because U+2019 is the apostrophe in "O’Connor" - Fix the pre-comma piece being routed to ``first`` when everything after the comma is a suffix or title: ``"Andrews, M.D."`` now reads family ``Andrews`` / suffix ``M.D.`` where 1.x read given ``M.D.`` / family ``Andrews``, and ``"Smith, Dr."`` moves ``Smith`` from ``first`` to ``family`` (the title was already correct in 1.x). The piece before a comma is definitionally the family name - Fix a lone recognized trailing suffix with no comma being routed to ``first``/``last``: ``"Johnson PhD"`` and ``"Mr. Johnson PhD"`` now keep the suffix in ``suffix`` - Fix a split ``"Ph. D."`` credential being read as two tokens; it now classifies as one suffix, replacing v1's ``fix_phd`` hook. This now holds wherever the credential sits: 1.x healed the pair only when it trailed, so ``"Ph. D. John Smith"`` parsed as title ``Ph.`` / given ``D.`` with the real given name pushed to ``middle``; it now reads given ``John``, family ``Smith``, suffix ``Ph. D.`` - Fix ``chargé d'affaires``: shipped as one unmatchable ``TITLES`` entry since it was added, it is now two chainable entries (``chargé``, ``d'affaires``), so the title is recognized; the unaccented spelling ``charge`` ships too, like ``attaché``/``attache`` - Remove seven multi-word ``SUFFIX_ACRONYMS`` entries (``leed ap``, ``nicet i``–``nicet iv``, ``psm i``, ``psm ii``) that could never match in any release; splitting them would swallow real names ("John Leed", "Smith, A.P."), so they are dropped instead - Add a ``UserWarning`` when a multi-word entry is stored in a per-word ``Lexicon`` field or as a ``capitalization_exceptions`` key -- such entries can never match - Parse an input with no alphanumeric character to an empty name in both APIs. 1.x kept pure punctuation as a name part, so ``"."`` gave ``first="."`` and ``bool()`` was ``True``; 2.0 empties it, keeping ``bool(parse(x))`` an honest "did I get a name?" test. The check is Unicode-aware, so names in any script are unaffected; only inputs that are entirely punctuation or symbols (``"."``, ``"- -"``) change. Junk embedded in a name with real content -- the stray dot in ``"John . Smith"`` -- is still kept, since that parse is already truthy - Fold a leading never-given particle into the family name. Note that ``Lexicon.particles_ambiguous`` is the **complement** of v1's ``non_first_name_prefixes``, not a rename -- it lists the particles that *may* double as a given name, where v1 listed the ones that may not. Copying a v1 customization across without inverting it silently reverses the behavior; see :doc:`migrate` - Add ``ma`` and ``do`` to ``suffix_acronyms_ambiguous``, the set of post-nominals that are also ordinary surnames. An entry there is read as a credential when the name can spare it — written with periods (``"John Smith M.A."``), or when removing it still leaves a given *and* a family name (``"John Smith MA"`` → suffix ``MA``). With only two pieces to go around, the surname reading wins instead: ``"Jack Ma"`` and ``"Anh Do"`` keep their family names. As a side effect, a parenthesized or quoted ``"(MA)"``/``"(DO)"`` now falls through to nickname parsing rather than escaping to ``suffix``, since inside delimiters the nickname reading is the plausible one - Change ``comparison_key()`` and ``matches()`` in both APIs to fold with ``str.casefold()`` where 1.4 used ``str.lower()``, so Unicode case pairs compare equal -- ``"STRASSE"`` matches ``"Straße"``, and a Greek final sigma matches its regular form. This is strictly more permissive: anything 1.4 matched still matches. Vocabulary normalization deliberately still uses ``lower()``, for v1 parity - Change the 2.0 API's default ``render()``/``str()`` spec to show every non-empty field: ``'{title} {given} "{nickname}" {middle} {family} ({maiden}) {suffix}'``. The quoted nickname round-trips exactly; the parenthesized maiden re-parses as a nickname, a deliberate choice of presentation over lossless round-trip -- use ``née {maiden}`` in a custom spec if you need it to survive a reparse. ``HumanName`` keeps v1's own ``string_format`` default, unchanged - Change delimiter-overlap precedence in the 2.0 API: a pair listed in ``Policy.maiden_delimiters`` is dropped from the effective nickname set, so ``Policy(maiden_delimiters={("(", ")")})`` alone routes parenthesized content to ``maiden``. The default nickname set is exported as ``DEFAULT_NICKNAME_DELIMITERS``. ``HumanName`` keeps v1's nickname-wins precedence, so no existing behavior changes - Change suffix-delimiter rendering when a custom suffix delimiter is configured: for ``suffix_delimiter="/"`` and ``"John Smith, RN/CRNA"``, 1.x split the token and rendered ``suffix="RN, CRNA"`` where 2.0 keeps it whole as ``"RN/CRNA"``. Role assignment is unchanged; only rendering differs - Correct a long-standing typo in the shipped vocabulary that 1.x carried: ``actor`` and ``television`` had trailing spaces in ``TITLES``, so ``"actor" in TITLES`` was ``False``. **Parse output is unchanged** -- the parser normalizes entries on ingest, which is exactly what let the typo go unnoticed -- but code that tests membership against the exported constants directly will see corrected results. The data modules now assert their invariants at import time, alongside the ones ``prefixes.py`` already checked; entries must be stored lowercase and whitespace-free, so this class of typo now fails the build **International name support** - Add ``nameparser.locales`` with the first two packs: ``locales.RU`` (East Slavic patronymic order) and ``locales.TR_AZ`` (Turkic patronymic markers). Packs are pure data folded in at ``parser_for(locales.RU)``, they compose (``parser_for(locales.RU, locales.TR_AZ)`` unions the rules), and they are never auto-detected -- there is no reliable way to detect a name's language from the name alone. ``locales.available()`` and ``locales.get(code)`` look packs up by code; loading is lazy, so importing the package imports no pack (completes #270; #271/#272/#146 stay staged for 2.x) - Add non-Latin vocabulary to the default lexicon (#269): Cyrillic, Greek, Arabic and Hebrew titles, conjunctions and name particles. Native-script entries cannot collide with Latin-script names, which is what makes them safe to enable by default. Deferred pending vetting: the Cyrillic ``мл``/``ст`` suffixes and the bare Greek ``κ`` title, which collides with the initial-plus-surname shape. Behavior note: ``محمد بن سلمان`` now chains ``بن`` onto the family name where 1.x read it as a middle name - Add Arabic-script bound given names (#269): ``عبد``, the kunya pair ``أبو``/``ابو``, and ``أم``/``ام`` join the following word into the given name exactly as their transliterations (``abdul``, ``abu``, ``umm``) always did, so ``عبد الرحمن محمد`` parses given ``عبد الرحمن``, family ``محمد`` where 1.x split it into given plus middle - Add Arabic honorific titles and the conjunction ``و`` (#269): the doctor, professor, hajj, sheikha and engineer forms (``الدكتور``/``الدكتورة``/``دكتور``/``دكتورة``, ``الأستاذ``/``الأستاذة``/``أستاذ``/``أستاذة``, ``الحاج``/``الحاجة``, ``الشيخة``, ``مهندس``) as given-name titles, since Arabic honorifics precede the given name like ``الشيخ``. Deferred under the collision rule: bare ``سيد``/``شيخ``/``أمير``/``سلطان`` (all common given names), the ``د.`` abbreviation (bare ``د`` would swallow initials), and the Ottoman post-nominals ``باشا``/``بك``/``أفندي`` (which survive as family names) - Add Hebrew honorifics and post-nominals, and Devanagari titles (#269): the Israeli honorifics ``גברת``, ``פרופ'``/``פרופ׳``, ``פרופסור``, ``עו"ד``/``עו״ד`` and ``הרב`` as titles; ``ז"ל``/``ז״ל`` and ``שליט"א``/``שליט״א`` as suffixes, in both gershayim spellings; and Devanagari ``श्री``, ``श्रीमती`` and ``डॉ``. Latin ``sri``/``shri`` were deliberately not added, because they collide with real given names where the native script cannot. Deferred: bare ``רב`` (an ordinary word meaning "many") and ``בר`` as a particle (Bar is a common modern given name) **Compatibility layer** - Reimplement ``HumanName`` as a facade over the 2.0 pipeline, and ``Constants`` as a shim resolving to a ``(Lexicon, Policy)`` snapshot with a shared parser cache. Fields, aggregates, mutation through ``name.C.titles.add(...)``, rendering defaults, ``capitalize()``, ``matches()``, ``comparison_key()``, iteration, ``as_dict()`` and pickling are all preserved, and ``nameparser.parser`` and ``nameparser.config`` remain importable. The compatibility layer ships through 2.x and is removed in 3.0 - Note that ``CONSTANTS.capitalize_name`` and ``force_mixed_case_capitalization`` are still honored through the facade, but the 2.0 API never capitalizes during ``parse()`` -- call ``capitalized()`` when you want it - Add ``Role`` members (and their string values ``given``/``family``) as valid ``HumanName`` subscript keys: ``hn[Role.GIVEN]`` returns ``hn.first`` - Add a ``UserWarning`` when assigning ``HumanName.given`` or ``.family``: the facade spells those attributes ``first``/``last``, so the assignment creates an inert stray attribute while the parse keeps the old value. The assignment still happens (v1-legal ad-hoc attributes keep working); the warning names the v1 spelling to use **Command line** - Rewrite ``python -m nameparser`` over the 2.0 API with a real argument parser. It prints the parse plus its capitalized form and initials by default, takes ``--json`` to emit the fields as JSON (``python -m nameparser --json "Doe, John"``), takes ``--locale CODE`` to apply a pack, and exits with a usage message on bad arguments **Documentation** - Rewrite the documentation new-API-first: a new front page and README, ``usage.rst`` as a tour of the 2.0 API, a principle-first ``customize.rst``, a reference split into the 2.0 API and the compatibility layer, and new pages for :doc:`concepts`, :doc:`locales` and :doc:`migrate` -- the last carrying full attribute and configuration maps from v1 names to 2.0 names (#262). The 1.x documentation remains online as the readthedocs ``stable`` build Parsing changes were checked against a 652-name differential corpus -- names harvested from the v1 test banks, plus names reported in the issue tracker -- and everything not listed above parses identically between 1.4.0 and 2.0. The harness lives in ``tools/differential/`` in the source repository (it is development tooling, not part of the installed package) -- see its README to reproduce the comparison against your own names. **Changed since 2.0.0rc1** (for anyone who tested the release candidate) - ``Role`` became a ``StrEnum``; ``str(Role.GIVEN)`` is now ``"given"`` - ``ParsedName.tokens_for()`` raises ``ValueError`` for unknown roles instead of returning no tokens; it also accepts role-name strings - ``ParsedName.as_dict()``'s ``include_empty`` is keyword-only - ``HumanName`` subscripting accepts ``Role`` members - Assigning ``HumanName.given``/``.family`` warns (the facade spells them ``first``/``last``; the assignment was and remains an inert stray attribute) - The eight multi-word vocabulary entries that could never match were repaired (``chargé d'affaires`` split; seven credential acronyms removed), and storing a new multi-word entry now warns; those eight are dropped silently, not warned about, when a restored ``Constants`` pickle carries all eight of them (the pre-2.0 signature) - ``PolicyPatch``'s repr shows only the fields a patch sets, instead of all nine with UNSET sentinels - New since rc1: ``STABLE_TAGS``, ``Policy.patched()``, ``Parser.matches()``, ``Parser.capitalized()``, and ``Parser.revise()`` -- see the API section above * 1.4.0 - July 12, 2026 - Add ``Constants.copy()``, a detached deep copy that preserves the source instance's current customizations (unlike ``Constants()``, which always starts from library defaults) -- useful as ``CONSTANTS.copy()`` for a private snapshot of the shared config (#260) - Deprecate passing ``constants=None`` to ``HumanName`` (or assigning ``hn.C = None``): it silently builds a fresh ``Constants()``, discarding any customizations the caller may have expected to carry over from the shared ``CONSTANTS``. Emits ``DeprecationWarning``; will raise ``TypeError`` in 2.0. Use ``constants=Constants()`` for fresh library defaults or ``constants=CONSTANTS.copy()`` for a private snapshot instead (closes #260) - Deprecate assigning ``Constants.empty_attribute_default`` for removal in 2.0 (#255): once ``None`` support goes, the only legal value left is the default ``''``, so a dial with one position isn't configuration. Emits ``DeprecationWarning``; reading the attribute is unaffected - Deprecate unknown-key attribute access on ``TupleManager``/``RegexTupleManager`` (``CONSTANTS.regexes.typo``, ``CONSTANTS.capitalization_exceptions.typo``, etc.) for removal in 2.0 (#256): a misspelled or omitted key currently degrades silently (``None``/``EMPTY_REGEX``) with no traceback pointing at the typo. Emits ``DeprecationWarning`` naming the miss and the known keys; will raise ``AttributeError`` in 2.0. ``.get()`` remains available for intentional soft access - Deprecate ``HumanName`` slice access (``name[1:-3]``) and item assignment (``name['first'] = value``) for removal in 2.0 (#258): field access by position has no real use case, and item assignment duplicates plain attribute assignment. Both emit ``DeprecationWarning``; string-key access (``name['first']``) is unaffected - Deprecate ``SetManager.add_with_encoding()`` itself for removal in 2.0 (#245), regardless of argument type: use ``add()`` instead (decoding bytes first). Previously only the ``bytes`` path warned; the ``str`` path was silent even though the whole method goes away - Deprecate loading a legacy-format ``Constants`` pickle (written by nameparser <= 1.2.x, before the 1.3.0 pickle fix) for removal in 2.0 (#279): ``__setstate__``'s migration shim currently skips the stale computed-property key silently. Emits ``DeprecationWarning`` once per call telling users to re-pickle; will raise ``ValueError`` in 2.0 - Fix the ``"Lastname, Firstname"`` comma format not being recognized when the input uses the Arabic comma ``،`` (U+060C, the standard comma in Arabic/Persian/Urdu text) or the fullwidth CJK comma ``，`` (U+FF0C) instead of the ASCII comma: both variants now also split the format and no longer leak into the parsed output (closes #265) * 1.3.1 - July 11, 2026 - Fix invisible Unicode bidirectional control characters (LRM/RLM/ALM, the embedding/override marks, and the isolates U+2066–U+2069) surviving parsing and sticking to ``first``/``last``/etc., so a copy-pasted right-to-left name silently failed equality and dedup. They are now stripped in preprocessing like emoji; disable via ``CONSTANTS.regexes.bidi = False`` (closes #266) - Fix ``str()`` corrupting name text containing the substring ``"None"`` when ``empty_attribute_default`` is ``None`` (e.g. ``"Nonez Smith"`` rendered as ``"z Smith"``): empty attributes are now substituted as ``''`` before the format string is applied, instead of scrubbing the interpolated ``"None"`` from the output afterward (closes #254) * 1.3.0 - July 5, 2026 **Breaking Changes & Deprecations** - Deprecate ``HumanName.__eq__`` and ``__hash__`` for removal in 2.0 (#223): the current design's three promises — case-insensitive equality, equality with plain strings, and hashability — are mutually inconsistent (equal objects can hash differently), equality depends on ``string_format``, and ``maiden`` is invisible to it. Both now emit ``DeprecationWarning`` naming the replacement; behavior is otherwise unchanged until 2.0 (closes #224) - Deprecate ``bytes`` input for removal in 2.0 (#245): passing ``bytes`` to ``HumanName``/``full_name`` or to ``SetManager.add()``/``add_with_encoding()`` now emits ``DeprecationWarning`` — decode first, e.g. ``value.decode('utf-8')``. The ``encoding`` constructor argument is deprecated with it - Deprecate ``SetManager.__call__`` for removal in 2.0 (#243): calling a manager returns the raw underlying set, so mutating the result bypasses normalization and cache invalidation; iterate the manager or copy with ``set(manager)`` instead - Add ``SetManager.discard()``, and deprecate ``remove()`` of a *missing* member (#243): it currently does nothing but will raise ``KeyError`` in 2.0, matching ``set.remove``; use ``discard()`` for intentional ignore-missing removal. Removing present members is unchanged and does not warn - Fix ``HumanName`` acting as its own iterator with a stored cursor: breaking out of a loop, iterating in nested loops, or calling ``len(name)`` mid-loop corrupted subsequent iteration; ``iter(name)`` now returns a fresh independent iterator each time. ``next(name)`` on the instance itself (undocumented) now raises ``TypeError`` — call ``next(iter(name))`` instead (closes #225) - Remove the vestigial ``unparsable`` attribute: the guard that was meant to set it has been unreachable since 2013 (v0.2.9), so it has reported ``False`` for every parsed name for over a decade; check ``len(name) == 0`` to detect an empty parse - Remove ``__ne__``; Python 3 derives ``!=`` from ``__eq__`` automatically - Change internal initials helper ``__process_initial__`` to ``_process_initial``: double-underscore-both-sides names are reserved for Python special methods; subclasses overriding the old name must rename their override - Change ``REGEXES`` from a ``set`` of ``(name, pattern)`` tuples to a ``dict``, so a duplicate name is a visible overwrite in the source instead of a nondeterministic winner at import time; code iterating ``REGEXES`` directly now gets keys instead of pairs — use ``.items()`` (#227) - Change ``CAPITALIZATION_EXCEPTIONS`` from a tuple of ``(key, value)`` tuples to a ``dict``; code iterating it directly now gets keys instead of pairs — use ``.items()`` (#233) **Behavior Changes (affect existing parse output)** - Add ``bound_first_names`` set to ``Constants``; bound Arabic given-name prefixes (``abdul``, ``abu``, etc.) now join forward to form a single first name (e.g. ``"abdul salam ahmed salem"`` → ``first="abdul salam"``, ``middle="ahmed"``, ``last="salem"``). Disable via ``CONSTANTS.bound_first_names.clear()``. **Default-on: changes parsing output for names with these prefixes.** (#150) - Treat an unrecognized, multi-letter token ending in a period in the leading title run (before the first name is set), e.g. ``"Major."``, as a ``title`` instead of a ``first`` name; internal-period abbreviations (``"E.T."``) and single-letter initials (``"J."``) are unaffected. **Default-on: changes parsing of names with a leading unknown period-abbreviation** (closes #109) - Fix parsing writing back into the ``Constants`` it reads (usually the shared module-level ``CONSTANTS``): pieces derived while parsing a name — period-joined titles/suffixes like ``"Lt.Gov."`` and conjunction-joined pieces like ``"Mr. and Mrs."`` or ``"von und zu"`` — are now tracked per parse instead of being permanently ``add()``-ed to the config, so parse results no longer depend on which names were parsed earlier in the process and parsing no longer mutates shared state across threads - Fix ``__hash__`` to lowercase the name like ``__eq__`` does, so equal ``HumanName`` instances hash equal and behave correctly in sets and dicts **New comparison methods** - Add ``matches()`` and ``comparison_key()`` for explicit name comparison: ``matches()`` compares parsed components case-insensitively (parsing ``str`` arguments first, so ``name.matches("Smith, John")`` and ``name.matches("John Smith")`` both match) and ``comparison_key()`` returns a hashable tuple of the seven components for dedup, dict keys, and sorting (#224) **New name fields** - Add a first-class ``maiden`` field and ``maiden_delimiters`` to ``Constants``, so a delimiter (e.g. parenthesis) can be routed to ``maiden`` instead of ``nickname`` for alternate/maiden surnames, e.g. ``"Baker (Johnson), Jenny"`` (closes #22) - Add ``given_names`` (and ``given_names_list``) attribute as aggregate of first and middle names, mirroring ``surnames`` (closes #157) - Add ``last_base``, ``last_prefixes`` (and ``_list`` variants) for splitting last-name prefix particles (tussenvoegsels) from the core surname (#130, #132) **New customization options** - Add ``initials_separator`` to ``Constants`` and ``HumanName`` to control spacing between consecutive initials within a name group (#171) - Add ``suffix_delimiter`` to ``Constants`` and ``HumanName`` for parsing suffixes separated by arbitrary delimiters, e.g. ``"RN - CRNA"`` (#156) - Add ``nickname_delimiters`` to ``Constants`` for registering additional nickname-delimiter regex patterns at runtime, without subclassing (closes #110, #112) - Add ``suffix_acronyms_ambiguous`` to ``Constants`` for acronym suffixes that also read as given-name nicknames (e.g. ``"JD"``, ``"Ed"``), used when disambiguating parenthesized/quoted content (#111) **International name support** - Add ``patronymic_name_order`` flag to ``Constants`` and ``HumanName`` for opt-in detection and reordering of Russian formal-order names (Surname GivenName Patronymic) (#85) - Add Turkic (Azerbaijani/Central-Asian) patronymic detection to ``patronymic_name_order``, rotating the reversed 4-token formal shape (``Surname GivenName PatronymicRoot Marker``, e.g. ``oglu``/``qizi``) into Western order (#185) - Add ``middle_name_as_last`` flag to ``Constants`` and ``HumanName`` for opt-in folding of middle names into the last name, for naming systems with no middle-name concept (e.g. Arabic patronymic chaining) (#133) - Add ``non_first_name_prefixes`` to ``Constants``: a leading particle that is never a first name (e.g. ``"de Mesnil"``, ``"dos Santos"``) now parses as a surname with an empty first name, instead of treating the particle as the first name (closes #121) - Add international honorifics to ``TITLES`` (#187) - Add German/Austrian nobility and ecclesiastical titles to ``TITLES`` (closes #101) - Add German/Dutch last-name prefixes and title/degree suffixes; fix ``join_on_conjunctions()`` to register multi-word prefix chains (e.g. ``"von und zu"``) as prefixes, mirroring existing title handling (closes #18) **Parsing fixes** - Fix suffix boundary lookup for prefixed last names with a title before and after (e.g. ``"dr Vincent van Gogh dr"`` producing a corrupted middle name) (closes #100) - Fix a repeated prefix word in a prefix chain (e.g. ``"Juan de la de la Vega"``) silently dropping the earlier occurrence in ``join_on_conjunctions()``: value-based ``pieces.index(prefix)`` lookups re-found the wrong occurrence once the list had already been mutated by prior joins; prefix positions are now tracked positionally instead of re-derived by value (closes #208) - Fix a trailing suffix being silently dropped after an empty comma segment, e.g. ``"Doe, John,, Jr."`` losing the ``"Jr."`` - Fix degenerate comma input (a bare ``","`` or an empty comma segment, e.g. ``"Doe,, Jr."``, ``"John Doe, Jr.,,"``) leaving an empty-string member in ``first_list``, ``last_list``, or ``suffix_list``; whitespace-only tokens assigned via the setters are dropped the same way - Fix suffix-shaped parenthesized/quoted content (e.g. ``"(Ret)"``, ``"(MBA)"``) being misclassified as a nickname instead of a suffix (closes #111) - Fix single-character symbol conjunctions (e.g. ``"&"``, ``"/"``) being ignored in short names (#173) - Fix recognition of single-letter roman numeral suffixes (e.g. ``"I"``, ``"V"``) in suffix-comma format (closes #136) - Fix recognition of trailing ``suffix_not_acronyms`` (e.g. ``"Jr."``) in lastname-comma format (closes #144) - Fix missing comma between ``'msc'`` and ``'mscmsm'`` in ``suffix_acronyms``, which silently concatenated them into a bogus ``'mscmscmsm'`` entry (#111) - Fix ``'apn aprn'`` split into separate ``suffix_acronyms`` entries so each is recognized independently (closes #155) **Formatting and output fixes** - Fix ``IndexError`` in ``initials()``/``initials_list()`` when a ``*_list`` attribute was assigned directly with an element containing unnormalized whitespace (e.g. ``name.middle_list = ['Q R']``), bypassing the parser's whitespace normalization (closes #232) - Fix ``initials()`` emitting a stray empty initial (e.g. ``"J. . V."``) -- or raising ``TypeError`` when ``empty_attribute_default`` is ``None`` -- for name parts with no initialable words, e.g. a prefix-only middle name like ``"de la"`` - Fix capitalization of suffix acronyms written with dots, e.g. ``"M.D."`` (closes #141) - Fix extra whitespace before punctuation in ``str()`` output when a ``string_format`` field is empty (closes #139) - Fix spurious leading space in surnames and empty token in suffix list after ``capitalize()`` with an empty middle or suffix (#164) **API correctness and cleanup** - Fix the five non-cached-union ``SetManager``-backed ``Constants`` attributes (``first_name_titles``, ``conjunctions``, ``bound_first_names``, ``non_first_name_prefixes``, ``suffix_acronyms_ambiguous``) accepting non-``SetManager`` assignment silently (e.g. ``constants.conjunctions = 'and'``), degrading membership checks into substring tests with no error; assignment now raises ``TypeError`` like the four cached-union attributes already did (closes #241) - Fix ``HumanName.C`` accepting an invalid ``constants`` value on post-construction assignment (e.g. ``hn.C = 'garbage'``), bypassing the constructor's validation and failing later with an unrelated ``AttributeError``; ``C`` is now a property that validates on assignment too (closes #239) - Fix ``TupleManager`` (and ``RegexTupleManager``) accepting a bare string/bytes argument (raising a cryptic ``dict``-internals ``ValueError``) or an iterable of 2-character strings (silently shredding each into a key/value pair, e.g. ``Constants(capitalization_exceptions=['ii'])`` becoming ``{'i': 'i'}``); both now raise ``TypeError`` with a clear message (closes #242) - Fix ``SetManager.__contains__`` being the one operation that didn't normalize (lowercase, strip leading/trailing periods) its operand, so e.g. ``'Dr.' in constants.titles`` could return ``False`` even though the title was correctly configured; membership checks now normalize like ``add()``/``remove()``/the constructor/the set operators (closes #244) - Fix a bare string passed to a set-backed ``Constants`` argument (e.g. ``Constants(titles='dr')``), to ``SetManager``, or as a ``SetManager`` set-operator operand (e.g. ``constants.titles |= 'esq'``) being silently split into single characters, replacing or polluting the set and producing wrong parses with no error; it now raises ``TypeError`` with the suggested fix — wrap strings in a list, decode ``bytes`` first (closes #238) - Fix ``SetManager`` set operators and the constructor skipping the lowercase/strip-edge-periods normalization that ``add()`` applies: ``constants.titles |= ['Esq.']`` kept a raw ``'Esq.'`` the parser's lookups could never match, ``titles & ['Dr.']`` missed ``'dr'``, and ``Constants(titles=[...])`` stored raw elements that silently never matched; elements and operands are now normalized everywhere, and non-``str`` elements (``bytes``, ``None``, numbers) raise ``TypeError`` instead of crashing cryptically or being coerced - Fix the ``constants`` constructor argument silently discarding ``Constants`` *subclass* instances: the exact-type check replaced them with fresh defaults, throwing away the caller's configuration. Subclass instances are now used as given; anything that is neither ``None`` nor a ``Constants`` instance now raises ``TypeError`` instead of being silently swapped for defaults (closes #226) - Fix ``Constants`` customizations, singleton identity, and ``TupleManager`` subclass being lost across ``pickle``/``deepcopy`` round-trips (#167, #168, #169) - Fix ``is_rootname()`` returning stale results after ``add()``/``remove()`` on ``titles``, ``prefixes``, ``suffix_acronyms``, or ``suffix_not_acronyms`` (#166) - Fix the library logger calling ``setLevel(logging.ERROR)`` on import, which silently discarded log records regardless of an application's own logging configuration; the logger now leaves its level at ``NOTSET`` and lets the application control verbosity (closes #228) - Minor internal cleanups: drop a dead length check in the initials helper, simplify double-wrapped ``len(list(...))`` calls, and other small parser tidy-ups with no behavior change (closes #229) - Change ``Constants.__repr__`` to report collection sizes and non-default scalar config, replacing the uninformative ```` (#221) * 1.2.1 - June 19, 2026 - Fix ``initials()`` interpolating the literal ``None`` for empty name parts when ``empty_attribute_default = None`` (e.g. ``"J. None D."``); empty parts now render as an empty string and a fully-empty result returns ``empty_attribute_default`` - Add ``python -m nameparser "Name String"`` command-line helper that prints a parsed name - Reorganize the test suite from a single ``tests.py`` into a ``tests/`` pytest package * 1.2.0 - June 11, 2026 - Drop Python 2 and Python < 3.10 support; Python 3.10–3.14 now required - Add type hints and type declarations (PEP 561 ``py.typed`` marker) - Migrate build tooling to ``pyproject.toml``, drop ``setup.py`` - Remove dead Python 2 compatibility shims (``ENCODING`` constant, ``next()`` aliases) - Modernize CI: uv-based workflow, trusted publishing to PyPI, Dependabot * 1.1.3 - September 20, 2023 - Fix case when we have two same prefixes in the name ()#147) * 1.1.2 - November 13, 2022 - Add support for attributes in constructor (#140) - Make HumanName instances hashable (#138) - Update repr for names with single quotes (#137) * 1.1.1 - January 28, 2022 - Fix bug in is_suffix handling of lists (#129) * 1.1.0 - January 3, 2022 - Add initials support (#128) - Add more titles and prefixes (#120, #127, #128, #119) * 1.0.6 - February 8, 2020 - Fix Python 3.8 syntax error (#104) * 1.0.5 - Dec 12, 2019 - Fix suffix parsing bug in comma parts (#98) - Fix deprecation warning on Python 3.7 (#94) - Improved capitalization support of mixed case names (#90) - Remove "elder" from titles (#96) - Add post-nominal list from Wikipedia to suffixes (#93) * 1.0.4 - June 26, 2019 - Better nickname handling of multiple single quotes (#86) - full_name attribute now returns formatted string output instead of original string (#87) * 1.0.3 - April 18, 2019 - fix sys.stdin usage when stdin doesn't exist (#82) - support for escaping log entry arguments (#84) * 1.0.2 - Oct 26, 2018 - Fix handling of only nickname and last name (#78) * 1.0.1 - August 30, 2018 - Fix overzealous regex for "Ph. D." (#43) - Add `surnames` attribute as aggregate of middle and last names * 1.0.0 - August 30, 2018 - Fix support for nicknames in single quotes (#74) - Change prefix handling to support prefixes on first names (#60) - Fix prefix capitalization when not part of lastname (#70) - Handle erroneous space in "Ph. D." (#43) * 0.5.8 - August 19, 2018 - Add "Junior" to suffixes (#76) - Add "dra" and "srta" to titles (#77) * 0.5.7 - June 16, 2018 - Fix doc link (#73) - Fix handling of "do" and "dos" Portuguese prefixes (#71, #72) * 0.5.6 - January 15, 2018 - Fix python version check (#64) * 0.5.5 - January 10, 2018 - Support J.D. as suffix and Wm. as title * 0.5.4 - December 10, 2017 - Add Dr to suffixes (#62) - Add the full set of Italian derivatives from "di" (#59) - Add parameter to specify the encoding of strings added to constants, use 'UTF-8' as fallback (#67) - Fix handling of names composed entirely of conjunctions (#66) * 0.5.3 - June 27, 2017 - Remove emojis from initial string by default with option to include emojis (#58) * 0.5.2 - March 19, 2017 - Added names scrapped from VIAF data, thanks daryanypl (#57) * 0.5.1 - August 12, 2016 - Fix error for names that end with conjunction (#54) * 0.5.0 - August 4, 2016 - Refactor join_on_conjunctions(), fix #53 * 0.4.1 - July 25, 2016 - Remove "bishop" from titles because it also could be a first name - Fix handling of lastname prefixes with periods, e.g. "Jane St. John" (#50) * 0.4.0 - June 2, 2016 - Remove "CONSTANTS.suffixes", replaced by "suffix_acronyms" and "suffix_not_acronyms" (#49) - Add "du" to prefixes - Add "sheikh" variations to titles - Add parameter to force capitalization of mixed case strings * 0.3.16 - March 24, 2016 - Clarify LGPL licence version (#47) - Skip pickle tests if pickle not installed (#48) * 0.3.15 - March 21, 2016 - Fix string format when `empty_attribute_default = None` (#45) - Include tests in release source tarball (#46) * 0.3.14 - March 18, 2016 - Add `CONSTANTS.empty_attribute_default` to customize value returned for empty attributes (#44) * 0.3.13 - March 14, 2016 - Improve string format handling (#41) * 0.3.12 - March 13, 2016 - Fix first name clash with suffixes (#42) - Fix encoding of constants added via the python shell - Add "MSC" to suffixes, fix #41 * 0.3.11 - October 17, 2015 - Fix bug capitalization exceptions (#39) * 0.3.10 - September 19, 2015 - Fix encoding of byte strings on python 2.x (#37) * 0.3.9 - September 5, 2015 - Separate suffixes that are acronyms to handle periods differently, fixes #29, #21 - Don't find titles after first name is filled, fixes (#27) - Add "chair" titles (#37) * 0.3.8 - September 2, 2015 - Use regex to check for roman numerals at end of name (#36) - Add DVM to suffixes * 0.3.7 - August 30, 2015 - Speed improvement, 3x faster - Make HumanName instances pickleable * 0.3.6 - August 6, 2015 - Fix strings that start with conjunctions (#20) - handle assigning lists of names to a name attribute - support dictionary-like assignment of name attributes * 0.3.5 - August 4, 2015 - Fix handling of string encoding in python 2.x (#34) - Add support for dictionary key access, e.g. name['first'] - add 'santa' to prefixes, add 'cpa', 'csm', 'phr', 'pmp' to suffixes (#35) - Fix prefixes before multi-part last names (#23) - Fix capitalization bug (#30) * 0.3.4 - March 1, 2015 - Fix #24, handle first name also a prefix - Fix #26, last name comma format when lastname is also a title * 0.3.3 - Aug 4, 2014 - Allow suffixes to be chained (#8) - Handle trailing suffix in last name comma format (#3). Removes support for titles with periods but no spaces in them, e.g. "Lt.Gen.". (#21) * 0.3.2 - July 16, 2014 - Retain original string in "original" attribute. - Collapse white space when using custom string format. - Fix #19, single comma name format may have trailing suffix * 0.3.1 - July 5, 2014 - Fix Pypi package, include new config module. * 0.3.0 - July 4, 2014 - Refactor configuration to simplify modifications to constants (backwards incompatible) - use unicode_literals to simplify Python 2 & 3 support. - Generate documentation using sphinx and host on readthedocs. * 0.2.10 - May 6, 2014 - If name is only a title and one part, assume it's a last name instead of a first name, with exceptions for some titles like 'Sir'. (`#7 `_). - Add some judicial and other common titles. (#9) * 0.2.9 - Apr 1, 2014 - Add a new nickname attribute containing anything in parenthesis or double quotes (`Issue 33 `_). * 0.2.8 - Oct 25, 2013 - Add support for Python 3.3+. Thanks to @corbinbs. * 0.2.7 - Feb 13, 2013 - Fix bug with multiple conjunctions in title - add legal and crown titles * 0.2.6 - Feb 12, 2013 - Fix python 2.6 import error on logging.NullHandler * 0.2.5 - Feb 11, 2013 - Set logging handler to NullHandler - Remove 'ben' from PREFIXES because it's more common as a name than a prefix. - Deprecate BlankHumanNameError. Do not raise exceptions if full_name is empty string. * 0.2.4 - Feb 10, 2013 - Adjust logging, don't set basicConfig. Fix `Issue 10 `_ and `Issue 26 `_. - Fix handling of single lower case initials that are also conjunctions, e.g. "john e smith". Re `Issue 11 `_. - Fix handling of initials with no space separation, e.g. "E.T. Jones". Fix #11. - Do not remove period from first name, when present. - Remove 'e' from PREFIXES because it is handled as a conjunction. - Python 2.7+ required to run the tests. Mark known failures. - tests/test.py can now take an optional name argument that will return repr() for that name. * 0.2.3 - Fix overzealous "Mac" regex * 0.2.2 - Fix parsing error * 0.2.0 - Significant refactor of parsing logic. Handle conjunctions and prefixes before parsing into attribute buckets. - Support attribute overriding by assignment. - Support multiple titles. - Lowercase titles constants to fix bug with comparison. - Move documentation to README.rst, add release log. * 0.1.4 - Use set() in constants for improved speed. setuptools compatibility - sketerpot * 0.1.3 - Add capitalization feature - twotwo * 0.1.2 - Add slice support python-nameparser-2.1.0/docs/requirements.txt000066400000000000000000000000141523530431700214000ustar00rootroot00000000000000Sphinx furo python-nameparser-2.1.0/docs/resources.rst000066400000000000000000000020141523530431700206620ustar00rootroot00000000000000Naming Practices and Resources ============================== * US_Census_Surname_Data_2000_ * US_Social_Security_Administration_Baby_Names_Index_ * Naming_practice_guide_UK_2006_ * Wikipedia_Anthroponymy_ * Wikipedia_Naming_conventions_ * Wikipedia_List_Of_Titles_ * Tussenvoegsel_ * Family_Name_Affixes_ .. _US_Census_Surname_Data_2000: https://www.census.gov/data/developers/data-sets/surnames/2000.html .. _US_Social_Security_Administration_Baby_Names_Index: https://www.ssa.gov/oact/babynames/limits.html .. _Naming_practice_guide_UK_2006: https://www.fbiic.gov/public/2008/nov/Naming_practice_guide_UK_2006.pdf .. _Wikipedia_Anthroponymy: https://en.wikipedia.org/wiki/Anthroponymy .. _Wikipedia_Naming_conventions: http://en.wikipedia.org/wiki/Wikipedia:Naming_conventions_(people) .. _Wikipedia_List_Of_Titles: https://en.wikipedia.org/wiki/Title .. _Tussenvoegsel: https://en.wikipedia.org/wiki/Tussenvoegsel .. _Family_Name_Affixes : https://en.wikipedia.org/wiki/List_of_family_name_affixes python-nameparser-2.1.0/docs/usage.rst000066400000000000000000001036751523530431700177730ustar00rootroot00000000000000Using the parser ================ Requires Python 3.11+. ``pip install nameparser`` Parse a name ------------ .. doctest:: >>> from nameparser import parse >>> name = parse("Dr. Juan Q. Xavier de la Vega III") >>> name.given, name.family ('Juan', 'de la Vega') >>> name.title, name.middle, name.suffix ('Dr.', 'Q. Xavier', 'III') A parsed name has seven fields: ``title``, ``given``, ``middle``, ``family``, ``suffix``, ``nickname``, and ``maiden``. Parsing never raises; unparseable input yields a :class:`~nameparser.ParsedName` with empty fields plus any ``ambiguities`` the parser noticed along the way (see `When the parser had to guess`_ below, and :doc:`concepts` for why they exist). A name with no fields set is falsy, which is how you tell "nothing parsed" from "parsed to something": .. doctest:: >>> bool(parse("")), bool(parse(" ")), bool(parse("John")) (False, False, True) Input shapes ------------- Three arrangements are understood, and every piece of each is optional: 1. ``Title Given "Nickname" Middle Middle Family Suffix`` 2. ``Family [Suffix], Title Given (Nickname) Middle Middle[,] Suffix [, Suffix]`` 3. ``Title Given Middle Family [Suffix], Suffix [, Suffix]`` The last two differ in what the comma is doing. In form 2 it separates the family name from the rest, so the family name comes first; in form 3 it only sets off suffixes, and the name before it is still given-then-family: .. doctest:: >>> parse("de la Vega, Juan Q. Xavier III").family # form 2 'de la Vega' >>> parse("Doe Jr., John").suffix # form 2, suffix before the comma 'Jr.' >>> parse("John Doe, Jr.").family # form 3 'Doe' For family-first input *without* a comma — common outside Europe — set ``name_order``; see :doc:`customize`. Names written in Han or Hangul, and Japanese names written in kanji and kana, are the exception that needs no setting at all: see `East Asian names`_ below. Words that attach to their neighbors -------------------------------------- Input shapes tell you where the fields sit. This tells you which words merge into one field instead of standing alone — between them, that is most of what decides a parse. Most words stand alone. A few pull in a neighbor, and each kind pulls into one particular field — so this table is also a map of which fields get built by attachment rather than by position. Titles and suffixes attach only to *adjacent words of their own kind* (a run of titles chains into one, but a title never swallows a plain name); the rest reach forward to pull in the next word, whatever it is. Follow a row's name to its full vocabulary set: .. list-table:: :header-rows: 1 :widths: 22 26 12 40 * - Words - Attach to - Field - Example * - :mod:`Titles ` - adjacent titles - ``title`` - ``Asst. Vice Chancellor John`` → ``Asst. Vice Chancellor`` * - :mod:`Suffixes ` - adjacent suffixes - ``suffix`` - ``John Smith PhD MD`` → ``PhD, MD`` * - :mod:`Bound given names ` - the following word - ``given`` - ``abdul salam ahmed`` → ``abdul salam`` * - :mod:`Particles ` - the following surname - ``family`` - ``Juan de la Vega`` → ``de la Vega`` * - :mod:`Maiden markers ` - the following name - ``maiden`` - ``Jane Smith née Jones`` → ``Jones`` * - :mod:`Conjunctions ` - the words on both sides - any - ``John and Jane Smith`` → ``John and Jane`` Conjunctions are the exception the last row names: they take the field of whatever sits on both sides, so the same ``and`` pulls two given names together as easily as two surnames: .. doctest:: >>> parse("John and Jane Smith").given 'John and Jane' >>> parse("Juan de la Vega y Rodriguez").family 'de la Vega y Rodriguez' Position matters in exactly one place: the start of a name. A particle there has no surname to attach to yet, so it either becomes the given name or turns the whole name into a surname, depending on whether it is one that can double as a given name: .. doctest:: >>> parse("van Gogh").given # 'van' can be a given name 'van' >>> parse("de Mesnil").given # 'de' cannot '' >>> parse("de Mesnil").family 'de Mesnil' :doc:`customize` covers how to change which words are in each of these sets, including which particles may double as given names. One shipped vocabulary works the other way round and so is not in the table above: :mod:`surnames ` *splits* a word instead of merging two, and is covered next. .. _east-asian-names: East Asian names ----------------- A Chinese, Japanese, or Korean name written in its own script puts the family name first: 毛泽东 is MAO Zedong, 山田太郎 is YAMADA Taro, 김민준 is KIM Minjun. The family name is short — one Han character or one hangul syllable, occasionally two — and comes from a small closed set, while given names are open-ended. And in native writing the parts are usually not separated at all: the whole name is one unbroken run of characters. A parser therefore has two distinct jobs here: assign family and given to the right fields, and, when the name arrives as a single token, find the boundary inside it. Field assignment is automatic. A name written wholly in Han characters or hangul is assigned family-first, because every language written in those scripts orders names that way — Chinese and Japanese share little else, but they agree on this — so the assignment requires no knowledge of which language the name is in: .. doctest:: >>> parse("毛 泽东").family '毛' >>> parse("山田 太郎").family '山田' Splitting an unspaced name is also automatic, but only for Korean. Hangul is written by exactly one language, and Korean family names are limited to a small closed set: the census surname list is part of the default vocabulary, and the longest listed surname at the start of an unspaced hangul token becomes the family name: .. doctest:: >>> minjun = parse("김민준") >>> minjun.family, minjun.given ('김', '민준') The same split is not automatic for Han text, because there the script does not identify the language: 高橋一郎 is a Japanese name whose family name is 高橋, but 高 alone is a common Chinese surname, so a Chinese surname list would split it in the wrong place. Declaring the language is up to you. When you know the data is Chinese, apply the ``zh`` locale pack, which carries the surname list the split needs: .. doctest:: >>> from nameparser import locales, parser_for >>> parser_for(locales.ZH).parse("毛泽东").family '毛' Chinese also has a transcription convention, and it is written in the punctuation: a foreign name transcribed into Han characters keeps its source order and divides its parts with the 间隔号, the interpunct ``·`` (U+00B7) — 威廉·莎士比亚 is William Shakespeare, given name first. The dot itself is the marker, so nameparser reads U+00B7 as a token separator when it sits between classified-script characters (each side judged on its own — a hangul character beside a Han one qualifies), and a dot anywhere in the name reads the whole name as a transcription listing: it keeps the order it was written in and is never segmented — the role pure katakana plays for Japanese transcriptions, played here by the divider instead of the script. Because only classified characters on both sides make it a divider, the same codepoint interior to a Latin-script name — the Catalan punt volat in ``Gal·la`` — is untouched, and a dot with a classified character on just one side (``王·Smith``) stays part of the word undivided. The Japanese middle dot ・ (covered next) is a different mark carrying a different convention, and keeps its own reading. Japanese ~~~~~~~~~ Japanese writes a name in three scripts at once. Family names and most given names are kanji — the same characters Chinese uses — but a given name is often written in one of the two kana syllabaries instead: hiragana (高橋みなみ) or katakana (山田エミ). The two syllabaries carry different information about whose name it is. Hiragana never transcribes a foreign name, so a name that mixes kanji and kana is a Japanese person's name, written in Japanese order. Katakana is ambiguous: native given names use it, but katakana is also how Japanese text writes a *foreign* name — マイケル・ジャクソン is Michael Jackson — and a transcription keeps the source language's order, given name first, its parts divided by the middle dot ・ (the nakaguro, U+30FB) rather than by a space. Two behaviors follow from that without any configuration. A name whose characters stay within kanji and kana and carry at least one kana is assigned family-first, like any other native-script East Asian name: it cannot be Chinese, and it is not a transcription, because a transcription would have been kana alone. .. doctest:: >>> minami = parse("高橋 みなみ") >>> minami.family, minami.given ('高橋', 'みなみ') >>> parse("山田 エミ").family '山田' And the middle dot separates tokens the way a space does, so a transcribed foreign name divides into its parts — which, being wholly katakana, keep the order they were written in: .. doctest:: >>> michael = parse("マイケル・ジャクソン") >>> michael.given, michael.family ('マイケル', 'ジャクソン') Dividing an *unspaced* Japanese name is a separate matter, and one no surname list can settle: family and given names draw on the same kanji, both sides run one to four characters, and the reading rather than the spelling decides most divisions. nameparser therefore takes a **segmenter** — a callable from token text to a division — and ships a factory wrapping `namedivider-python `_, an optional dependency installed with the ``ja`` extra: .. code-block:: console $ pip install "nameparser[ja]" .. code-block:: python from nameparser import locales, parser_for parser = parser_for(locales.JA, segmenter=locales.ja_segmenter()) parser.parse("山田太郎").family # '山田' Both halves are required, and they do different jobs: the ``ja`` pack activates division for Japanese text, and the segmenter performs it. ``ja_segmenter()`` wraps namedivider's ``BasicNameDivider``, which reads data bundled in the installed package; ``ja_segmenter(gbdt=True)`` selects its gradient-boosted divider instead, which is more accurate and downloads its model and surname files from the network on first use, worth knowing before deploying it somewhere sandboxed or air-gapped. Forgetting the segmenter is loud: building the parser emits a ``UserWarning`` naming the scripts that could never divide and the call to pass, because the misconfigured parser would otherwise behave exactly like a working one minus the feature. The same check guards any configuration whose activated scripts nothing can serve. A from-scratch lexicon with no hangul surnames warns under the default policy, and ``Policy(segment_scripts=())`` is the deactivation the message offers. Decomposed text ^^^^^^^^^^^^^^^ Korean and Japanese text is sometimes stored decomposed, with a syllable held as its separate jamo rather than as one codepoint. macOS filenames are the common source. Everything above works on decomposed input: script classification normalizes to NFC before deciding, so a decomposed name gets the same order rule as its composed twin. Splitting is the exception. An unspaced decomposed hangul name is ordered correctly but not split, because surname matching runs against the text as given and a decomposed name matches no entry in the census list. That is the deliberate choice: being unsplit is recoverable, whereas splitting in the wrong place is not. One consequence is worth stating outright, because it looks like a bug. Parse output preserves the encoding it was given, so a field from a decomposed name is decomposed too, and comparing it against a composed literal fails even where the parse was correct:: decomposed = unicodedata.normalize("NFD", "김 민준") parse(decomposed).family == "김" # False unicodedata.normalize("NFC", parse(decomposed).family) # '김' Normalize both sides before comparing across encodings. Boundaries ~~~~~~~~~~~ Several boundaries apply to all of the above. When the script rules don't apply ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Romanized names ("Kim Min-jun", "Yamada Taro") are Latin script and follow the ordinary positional rules. Order genuinely varies in romanized data, so nothing script-based applies. A name written wholly in katakana stays positional for the reason given above, pack or no pack: it is predominantly a transcription, and a transcription is already in the order it should be read in. A Han transcription written with a space instead of the 间隔号 (威廉 莎士比亚) carries nothing to distinguish it from a native two-token name, and keeps the family-first reading. The dot is the marker, and without it there is no signal. The same holds for a transcription typed with the Japanese middle dot (威廉・莎士比亚): each dot carries its own script's convention, the nakaguro's is Japanese roster formatting rather than transcription, so only the Chinese dot rescues the source order. A comma disables the script behaviors that decide where a name divides, on the reasoning ``name_order`` already follows: whoever wrote the comma has already said where the family name ends. Honorifics come off first ^^^^^^^^^^^^^^^^^^^^^^^^^ Honorifics and degrees follow a CJK name, and both the spaced and the glued forms are recognized as suffixes: .. doctest:: >>> parse("王小明 先生").suffix '先生' >>> glued = parse("김민준씨") >>> glued.family, glued.given, glued.suffix ('김', '민준', '씨') The honorific is split off the end of the last token of the name before the name is split or ordered, so those rules see the name without it. That is why ``김민준씨`` still divides into family 김 and given 민준, and why a configured Japanese segmenter is handed 山田太郎 rather than 山田太郎様. Commas and dots ^^^^^^^^^^^^^^^ Neither a comma nor a 间隔号 switches the peel off. Both say where a name divides: the comma that the writer has already given the family name, the dot that the pieces are a transcription's syllable groups. An honorific is not part of the name in either reading. .. doctest:: >>> parse("김, 민준씨").given == parse("김 민준씨").given True What a comma does instead is say which runs are the name: the two around a family comma, an honorific being as often glued to the given name as to the family. Nothing past those two is in reach. .. doctest:: >>> parse("김, 민준 지훈씨").suffix # second run, still in reach '씨' >>> parse("김, 민준, 지훈씨").suffix # a third run, left whole '지훈씨' What those marks do stop is the *split*, which is a different question and still theirs to answer. ``田中さん, 太郎`` is unchanged, and not because of its comma: the honorific there is not at the end of the name, 太郎 is. Credentials after a comma ^^^^^^^^^^^^^^^^^^^^^^^^^ The second of those two runs counts only where it is name text. A run that is nothing but credentials is not, whatever the comma looked like, and the same test that decides the comma structure decides this too: the run is left alone and the honorific is found in the name. Being credentials is necessary but not enough, because the test cannot help counting the honorific itself. A glued honorific is a suffix word, so it is part of what makes its own run read as credentials. The run is passed over only when the part *before* the comma has an honorific of its own to give up; otherwise passing it over would discard the only one there is. .. doctest:: >>> credential = parse("田中さん, V.") >>> credential.family, credential.suffix ('田中', 'さん') So ``田中さん, V.`` and ``田中さん, Ph. D.`` give さん up exactly as ``田中さん, PhD`` does. Where the credential itself lands is the comma's business rather than the peel's, and still differs by spelling: ``title`` for ``PhD``, ``given`` for ``V.``, and ``Ph. D.`` in ``suffix`` beside さん. Spacing, and where the name divides ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Where a segmenter divides the name, the two spellings part company. A spaced honorific is a token boundary the writer typed, and the segmenter is asked only where an undivided name divides, so anything standing beside the name calls it off, honorific or not. Under the Japanese pack ``佐藤 氏`` keeps family 佐藤, where bare ``佐藤`` would have been divided 佐 + 藤. That is a conservative reading rather than a principled one. A spaced honorific cannot be told apart from a spaced given name by position, and treating it as one keeps four real surnames whole (``佐藤 氏``, ``田中 様``, ``鈴木 先生``, ``中村 教授``) at the price of one division it declines to make (``山田太郎 様``). A glued honorific carries no boundary at all, its writer having drawn none anywhere, so ``田中さん`` divides the way bare ``田中`` does, into family 田 and given 中, with さん in ``suffix``. Writing the honorific spaced is therefore also how you ask for a family name to be kept whole on this path, short of declining the pack. Where the VOCABULARY divides the name the two spellings never part company, because the peel hands the same remainder to the same surname match either way: .. doctest:: >>> parse("김민준 씨").family, parse("김민준씨").family ('김', '김') Under the Chinese pack ``王小明 先生`` and ``王小明先生`` both give family 王, given 小明 and suffix 先生 for the same reason. Spacing the honorific is no lever there, and for Korean data there is no pack to decline either, hangul segmentation being on by default. Which honorifics peel when glued ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The glued reading is deliberately narrower than the spaced one. A spaced honorific sits behind a boundary its writer drew; a glued one has only itself to go on, so a word peels off the end of a name only if it could never BE the end of a name. 씨, 님, 박사, 박사님, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士 and 小姐 qualify. 양, 군, 氏, 博士 and 殿 do not, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿). Those stay recognized in their spaced form, where position settles what the glued form leaves ambiguous. 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Exactly one honorific peels off a token, and the entries are whole honorifics rather than parts: ``김민준박사님`` gives up 박사님 entire, not 님 with 박사 left behind. When a division was a judgment call ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A division the parser had to choose is reported rather than hidden. When an unspaced name has more than one vocabulary-supported split, the longest surname wins and the parse records the decision as an ``AmbiguityKind.SEGMENTATION``, described under `When the parser had to guess`_. ``남궁민수`` is 남궁 + 민수 by the two-syllable surname but 남 + 궁민수 by the single-syllable one: .. doctest:: >>> parse("남궁민수").ambiguities (Ambiguity('segmentation': '남궁'/'민수'),) A name with only one possible split reports nothing. A segmenter's answer is reported on the same kind whenever its confidence falls below the stage's floor, naming the division and the score in the report's ``detail``:: "'山田太郎' splits as '山田' + '太郎' on a segmenter answer scoring 0.44, under the 0.9 confidence floor" With namedivider that line separates its two kinds of answer. A division it states as a rule, such as the kanji-to-kana boundary in 高橋みなみ, scores 1.0 and reports nothing; a division read off kanji statistics scores far below the floor and always reports. Read the report as a statement about the *kind* of answer, not as a measure of how likely this particular one is to be wrong. A lone two-character kanji name divides one character to each side, on namedivider's rule for that length. A name that short carries no evidence of where its own boundary falls, and one character each way is the presumption Japanese practice makes. That is an accepted presumption rather than a measurement, so a two-character token is the shape to check first if a division looks wrong. When a segmenter misbehaves ^^^^^^^^^^^^^^^^^^^^^^^^^^^ A segmenter that answers outside the token it was given, with a cut at or past the end of the text, has violated the protocol, and the parse says so rather than hiding it. It raises ``ValueError`` naming the offending offset and the token's length, the way an answer of the wrong type raises ``TypeError``. Declining silently would leave an off-by-one segmenter invisible: every answer it gave would vanish and the name would merely look undivided. The shipped ``ja_segmenter()`` does decline, returning ``None`` and leaving the token whole, for the cases that are *not* protocol violations: text outside the Japanese repertoire, text too short to divide, an answer that fails to reconstruct its input, and a score outside [0, 1]. Exceptions are the one thing that does *not* stay inside the parse: a segmenter is your code, so its errors propagate rather than being absorbed as content errors. The command line takes the pack but not the segmenter: ``python -m nameparser --locale ja`` has no way to attach one, so it activates nothing by itself. :doc:`customize` covers turning any of these behaviors off. Aggregate views ---------------- .. doctest:: >>> name.given_names # given + middle 'Juan Q. Xavier' >>> name.family_base, name.family_particles # family, split apart ('Vega', 'de la') ``surnames`` (``middle`` + ``family``) is the mirror-image aggregate. The plural is the tell: ``given`` and ``family`` are single fields, while ``given_names`` and ``surnames`` roll several fields together — the same sense in which a passport form asks for your "given names" as one blank that can hold more than one word. ``family_base`` is the one to sort on. Dutch and Belgian directories file a name under the base surname and ignore the *tussenvoegsel* — the ``van``, ``de`` or ``van der`` in front of it — and the same convention applies wherever particles are common: .. doctest:: >>> names = [parse(s) for s in ... ["Vincent van Gogh", "Juan de la Vega", "John Smith"]] >>> [n.family_base for n in sorted(names, key=lambda n: n.family_base.lower())] ['Gogh', 'Smith', 'Vega'] Sorting on ``family`` instead files those under "d", "S" and "v", which is the problem the split exists to solve: .. doctest:: >>> [n.family for n in sorted(names, key=lambda n: n.family.lower())] ['de la Vega', 'Smith', 'van Gogh'] Dicts and strings ------------------ .. doctest:: >>> name.as_dict(include_empty=False) {'title': 'Dr.', 'given': 'Juan', 'middle': 'Q. Xavier', 'family': 'de la Vega', 'suffix': 'III'} >>> str(name) 'Dr. Juan Q. Xavier de la Vega III' >>> name.render("{family}, {given}") 'de la Vega, Juan' >>> name.initials() 'J. Q. X. V.' Fixing case ----------- .. doctest:: >>> str(parse("juan de la vega").capitalized()) 'Juan de la Vega' The no-argument form above uses the DEFAULT lexicon; for a name parsed with a custom :class:`~nameparser.Parser`, call :meth:`Parser.capitalized() ` so the parser's own vocabulary decides the exceptions. Nicknames and maiden names ---------------------------- .. doctest:: >>> parse("Jonathan 'Jack' Kennedy").nickname 'Jack' >>> parse("Jane Smith née Jones").maiden 'Jones' Both fields appear in the default ``str()`` rendering — the nickname quoted after the given name, the maiden name parenthesized after the family name: .. doctest:: >>> str(parse("Jane (Janie) Smith née Jones")) 'Jane "Janie" Smith (Jones)' .. _maiden-roundtrip: That default is built for display, and the two fields differ in what survives it. The quoted nickname reparses as a nickname; the parenthesized maiden name reparses as a *nickname* too, so a parse-render-reparse round trip silently loses it: .. doctest:: >>> parse('Jane "Janie" Smith (Jones)').maiden '' >>> parse('Jane "Janie" Smith (Jones)').nickname 'Janie Jones' If the rendered string has to survive a reparse — storing names as text and reading them back, for instance — render the marker explicitly instead of relying on the default: .. doctest:: >>> name = parse("Jane Smith née Jones") >>> text = name.render("{given} {family} née {maiden}") >>> text 'Jane Smith née Jones' >>> parse(text).maiden 'Jones' Delimited content is not always a nickname. If what's inside is a known suffix, or simply ends in a period, it is read as a suffix instead — parenthesized credentials and retired ranks are far more common than parenthesized nicknames that happen to be credentials: .. doctest:: >>> parse("Andrew Perkins (MBA)").suffix 'MBA' >>> parse("Andrew Perkins (Ret.)").suffix 'Ret.' The exception to that exception is an *ambiguous* acronym — one that is also a plausible name or set of initials. Standing alone inside delimiters, it keeps the nickname reading: .. doctest:: >>> parse("JEFFREY (JD) BRICKEN").nickname 'JD' ``JD`` is in ``suffix_acronyms_ambiguous``; see :doc:`customize` for what that field marks and how to add to it. .. _abbreviated-titles: Titles you didn't configure ----------------------------- A trailing period marks an abbreviation, and the parts of a name that get abbreviated are the ones standing outside it — titles and post-nominals. So a word ending in a period, in no vocabulary list at all, is read as a title when it stands where a title stands: at the front of the part that carries the given name. That is what lets unfamiliar ranks, honorifics and abbreviations work without configuring anything: .. doctest:: >>> parse("Insp. Jane Morse").title 'Insp.' >>> parse("Det. Insp. Jane Morse").title # chains 'Det. Insp.' Neither ``det`` nor ``insp`` is in the shipped vocabulary; the periods are doing all the work. The part that carries the given name is not always the front of the string. After a family comma it is the part *after* the comma, and the rule applies there in exactly the same way: .. doctest:: >>> parse("Morse, Det. Insp. Jane").title 'Det. Insp.' The rule is bounded in three ways, so it doesn't swallow ordinary names. Single initials are left alone, so are abbreviations with interior periods, and it applies only to that leading run — the same word after the given name is a middle name: .. doctest:: >>> parse("J. Smith").given 'J.' >>> parse("E.T. Jones").given 'E.T.' >>> parse("Jane Insp. Morse").middle 'Insp.' Because this is structural rather than vocabulary-driven, emptying ``titles`` does not switch it off; see :doc:`customize`. Only the title direction is inferred this way. A *trailing* abbreviation is matched against the suffix vocabulary and nothing more, so an abbreviated post-nominal is recognized only if it is a word the parser already knows. Comparing names ---------------- ``==`` is strict value equality — two :class:`~nameparser.ParsedName` instances are equal only if every field matches exactly. For "is this the same name, allowing for order and case?" use ``matches()`` or ``comparison_key()`` instead. ``matches()`` parses a str argument with the DEFAULT parser; to compare against a string using a custom :class:`~nameparser.Parser`'s vocabulary, call :meth:`Parser.matches() `. .. doctest:: >>> parse("de la Vega, Juan").matches("Juan de la Vega") True >>> parse("JUAN DE LA VEGA").comparison_key() == parse("Juan de la Vega").comparison_key() True When the parser had to guess ----------------------------- Some names have no single correct reading. A leading ``Van`` could be a given name — it really is for the actor Van Johnson — or the start of a family name, as it is for President Van Buren. Both are the same shape, so no rule can tell them apart. 2.0 takes the more likely reading and *records* the choice on ``ambiguities`` rather than deciding silently: .. doctest:: >>> name = parse("Van Buren") >>> name.given, name.family ('Van', 'Buren') >>> for a in name.ambiguities: ... print(a.kind.value, "-", a.detail) particle-or-given - leading 'Van' may be a family-name particle; read as a given name :class:`~nameparser.AmbiguityKind` members are their own string values, so branching on a kind needs no import: .. doctest:: >>> name.ambiguities[0].kind == "particle-or-given" True >>> [t.text for t in name.ambiguities[0].tokens] ['Van'] The post-nominals that double as ordinary surnames report the same way. ``MA`` after a full name is read as a credential, but after a single given name it stays the surname — either way the choice is recorded: .. doctest:: >>> parse("John Smith MA").suffix 'MA' >>> [a.kind.value for a in parse("John Smith MA").ambiguities] ['suffix-or-name'] >>> parse("Jack MA").family 'MA' A reading the vocabulary settles on its own is not a guess and reports nothing — periods make ``M.A.`` unambiguously a credential: .. doctest:: >>> parse("John Smith M.A.").ambiguities () Most names report none. A non-empty ``ambiguities`` is a useful signal for routing a record to human review instead of trusting it silently. Tokens and spans ------------------ The seven fields are the convenient view. Underneath, each one is backed by tokens carrying their exact offsets into the original string, so you can always get back to the text a field came from: .. doctest:: >>> name = parse("Juan de la Vega") >>> for tok in name.tokens: ... print(f"{tok.text!r:8} {tok.role.value:8} {tuple(tok.span)}") 'Juan' given (0, 4) 'de' family (5, 7) 'la' family (8, 10) 'Vega' family (11, 15) >>> tok = name.tokens[1] >>> name.original[tok.span.start:tok.span.end] 'de' ``tokens_for()`` narrows that to a single role: .. doctest:: >>> from nameparser import Role >>> [t.text for t in name.tokens_for(Role.FAMILY)] ['de', 'la', 'Vega'] A role's string name works too: ``name.tokens_for("family")``. Correcting a parse -------------------- :class:`~nameparser.ParsedName` is immutable, so a correction is a new value: ``replace()`` returns a copy with the given fields changed. Untouched fields keep their tokens (and ``original`` is preserved), with one deliberate exception: an ambiguity that pointed into a replaced field is dropped — correcting the field that was flagged clears the flag, while correcting an unrelated field keeps it. .. doctest:: >>> name = parse("Juan de la Vega") >>> corrected = name.replace(title="Dr.") >>> str(corrected) 'Dr. Juan de la Vega' >>> name.title '' >>> flagged = parse("Van Buren") >>> flagged.replace(given="Martin").ambiguities () >>> [a.kind.value for a in flagged.replace(family="Harrison").ambiguities] ['particle-or-given'] ``replace()`` splits values on whitespace into plain, untagged tokens — the vocabulary knowledge a parse would have about the new text is not there. The views that depend on tags degrade: the parser no longer knows ``de la`` are particles, so ``family_particles`` empties and the particles start contributing initials. .. doctest:: >>> name.family_particles 'de la' >>> replaced = name.replace(family="de la Vega Smith") >>> replaced.family_particles '' >>> replaced.initials() 'J. d. l. V. S.' :meth:`Parser.revise() ` is the same operation with each value classified by the parser's vocabulary, so the correction behaves like a fresh parse of the corrected name (which also means delimiters and marker words in the value are consumed as they would be in a parse): .. doctest:: >>> from nameparser import Parser >>> parser = Parser() >>> revised = parser.revise(name, family="de la Vega Smith") >>> revised.family_particles 'de la' >>> revised.initials() 'J. V. S.' ``revise()`` has two siblings on :class:`~nameparser.Parser`: :meth:`Parser.matches() ` and :meth:`Parser.capitalized() `. Those two matter when you have built a custom parser — the :class:`~nameparser.ParsedName` methods of the same names fall back to the *default* configuration for str or omitted arguments. Command line ------------ ``python -m nameparser`` parses one name and prints what it found. It is the quickest way to see how a particular string comes out — worth reaching for when a name parsed unexpectedly and you want to check a variation of it, or when trying a locale pack before wiring one into code: :: $ python -m nameparser "dr. juan de la vega iii" Parsed: Capitalized: Initials: j. v. ``--json`` prints the fields as a single line instead, which is the form to pipe somewhere: :: $ python -m nameparser --json "Doe, John" {"title": "", "given": "John", "middle": "", "family": "Doe", "suffix": "", "nickname": "", "maiden": ""} Add ``--locale`` to parse with a locale pack (for example ``--locale ru``); see :doc:`locales`. Where next ---------- * :doc:`concepts` — how the parser works * :doc:`customize` — your own vocabulary and behavior * :doc:`locales` — locale packs * :doc:`migrate` — coming from 1.x ``HumanName`` python-nameparser-2.1.0/nameparser/000077500000000000000000000000001523530431700173265ustar00rootroot00000000000000python-nameparser-2.1.0/nameparser/__init__.py000066400000000000000000000024101523530431700214340ustar00rootroot00000000000000from nameparser._version import VERSION as VERSION from nameparser._version import __version__ as __version__ from nameparser.parser import HumanName as HumanName __author__ = "Derek Gulbranson" __author_email__ = 'derek73@gmail.com' __license__ = "LGPL" __url__ = "https://github.com/derek73/python-nameparser" from nameparser._lexicon import Lexicon from nameparser._locale import Locale from nameparser._parser import Parser, parse, parser_for from nameparser._policy import ( DEFAULT_NICKNAME_DELIMITERS, DEFAULT_SCRIPT_ORDERS, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, UNSET, PatronymicRule, Policy, PolicyPatch, Script, ) from nameparser._types import ( STABLE_TAGS, Ambiguity, AmbiguityKind, ParsedName, Role, Segmentation, Segmenter, Span, Token, ) __all__ = [ # v1 (compatibility layer) "HumanName", # v2 core "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "STABLE_TAGS", "Segmentation", "Segmenter", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "Script", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "DEFAULT_NICKNAME_DELIMITERS", "DEFAULT_SCRIPT_ORDERS", "Locale", "Parser", "parse", "parser_for", ] python-nameparser-2.1.0/nameparser/__main__.py000066400000000000000000000032761523530431700214300ustar00rootroot00000000000000"""Command-line debug helper over the 2.0 API (migration spec §6). python -m nameparser "Dr. Juan Q. Xavier de la Vega III" python -m nameparser --json "Doe, John" """ import argparse import json from nameparser import parse def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser( prog="nameparser", description="Parse a personal name.") ap.add_argument("name", help="the name string to parse") ap.add_argument("--json", action="store_true", help="print the component dict as JSON") ap.add_argument("--locale", metavar="CODE", help="parse with a locale pack (e.g. 'ru'); see " "nameparser.locales") args = ap.parse_args(argv) # `is not None`, not truthiness: --locale "" must reach get() and # exit 2 listing the codes, not silently fall back to the default if args.locale is not None: from nameparser import locales, parser_for try: parser = parser_for(locales.get(args.locale)) except KeyError as exc: ap.error(exc.args[0]) # exits 2, message to stderr n = parser.parse(args.name) else: n = parse(args.name) if args.json: print(json.dumps(n.as_dict(), ensure_ascii=False)) return 0 # Label each section: the two reprs are byte-identical for input # that is already correctly cased, so without labels there is no # telling which is the parse and which is the capitalized view. print("Parsed:") print(repr(n)) print("Capitalized:") print(repr(n.capitalized())) print("Initials:", n.initials()) return 0 if __name__ == "__main__": raise SystemExit(main()) python-nameparser-2.1.0/nameparser/_config_shim.py000066400000000000000000001526711523530431700223400ustar00rootroot00000000000000"""v1 ``Constants`` compatibility shim over Lexicon/Policy (migration spec §3). ``nameparser.config`` re-exports these names from the swap commit onward; the whole module is deleted in 3.0 with the facade. Layering: facade layer -- may import anything public; here that's ``nameparser.util`` for ``lc()``, ``nameparser.config.regexes`` for the read-only regexes proxy's underlying compiled patterns, and ``nameparser._lexicon``/``_policy``/``_parser`` for the ``_snapshot()`` translation and the shared parser cache. All ``nameparser.config.`` imports below are deferred (function-local), never module-level: ``nameparser/config/__init__.py`` itself imports ``CONSTANTS``/``Constants``/etc. from this module, so a module-level ``from nameparser.config.regexes import REGEXES`` here would need the ``nameparser.config`` package's ``__init__.py`` to run to completion first -- which needs this module to already be fully initialized. Importing the data submodules lazily, on first use, breaks that cycle. """ from __future__ import annotations import functools import warnings from collections.abc import ( Callable, ItemsView, Iterable, Iterator, KeysView, Mapping, Set, ValuesView, ) from typing import NamedTuple, Self from nameparser._lexicon import Lexicon, _title_key from nameparser._parser import Parser from nameparser._policy import PatronymicRule, Policy from nameparser.util import lc #: The eight multi-word entries the pre-2.0 DEFAULT vocabulary shipped. #: Provably inert in every release (they can never match; see the #: 2.0.0 release log), so dropping them from a restored legacy pickle #: changes no parse -- and keeps the multi-word warning from firing #: eight times, with wrong advice, at library-internal lines, on the #: first parse after a supported 1.3/1.4 pickle upgrade. #: #: Gated on ALL EIGHT being present across the two fields -- the #: signature of a pre-2.0 blob, which froze the complete shipped set. #: A round-trip of 2.0-era state that carries FEWER than all eight #: keeps them (a user who deliberately re-added one or two does not #: match the signature, and .copy() never subtracts either). The trade: #: a 2.0 user who re-adds ALL eight exact strings is indistinguishable #: from a legacy blob and loses them on the next unpickle. _LEGACY_DEAD_ENTRIES = { "titles": frozenset({"chargé d'affaires"}), "suffix_acronyms": frozenset({ "leed ap", "nicet i", "nicet ii", "nicet iii", "nicet iv", "psm i", "psm ii"}), } def _reject_bare_str_or_bytes(value: object, expected: str) -> None: # A bare string is an iterable of its characters, so e.g. SetManager('dr') # would silently shred it into {'d', 'r'} instead of raising -- shared by # SetManager's constructor/operands (#238/#241) and TupleManager's # constructor (#242). Ported from v1's `_reject_bare_str_or_bytes`. if isinstance(value, bytes): raise TypeError( f"expected {expected}, got a single bytes; decode it first: " f"{value!r}.decode()" ) if isinstance(value, str): raise TypeError( f"expected {expected}, got a single str; wrap it in a list: " f"[{value!r}]" ) def _lc_validated(s: object) -> str: # Validates and lc()-normalizes a single element -- shared by bulk # iterable normalization (constructor/operands) and add()'s per-string # loop, so every path raises the same #238/#245-shaped TypeError instead # of lc() crashing cryptically (bytes) or silently transmuting (None). if isinstance(s, bytes): raise TypeError( f"expected a str, got bytes; decode it first: {s!r}.decode()" ) if not isinstance(s, str): raise TypeError(f"expected a str, got {type(s).__name__}: {s!r}") return lc(s) def _normalize_iterable_of_strings( elements: object, expected: str = "an iterable of strings") -> set[str]: # a SetManager's elements were already validated/normalized when it was # built, so copy them instead of re-validating (v1's fast path) if isinstance(elements, SetManager): return set(elements._elements) _reject_bare_str_or_bytes(elements, expected) return {_lc_validated(s) for s in elements} # type: ignore[attr-defined] class SetManager: """v1 ``SetManager`` surface over a plain set of ``lc()``-normalized strings. Mutations call ``_on_change`` (the owning Constants' generation bump, wired by a later task). ``__call__`` and the missing-member-tolerant ``remove()`` are gone per the #243 schedule (warned 1.3.0, removed 2.0): ``remove()`` of a missing member raises ``KeyError``, matching ``set.remove``. """ _elements: set[str] _on_change: Callable[[], None] | None def __init__(self, elements: Iterable[str] = (), _on_change: Callable[[], None] | None = None, _field: str | None = None) -> None: # _field carries a Constants field name (e.g. "titles") through to # the bare-str/bytes guard's message, when this SetManager is being # built on behalf of a named Constants field (constructor kwarg or # __setattr__ auto-wrap); a direct SetManager(...) call gets the # generic v1 message instead. expected = f"{_field} to be an iterable of strings" if _field \ else "an iterable of strings" self._elements = _normalize_iterable_of_strings(elements, expected) self._on_change = _on_change def _changed(self) -> None: if self._on_change is not None: self._on_change() def add(self, *strings: str) -> SetManager: """Add the normalized string arguments to the set. Returns ``self`` for chaining.""" # notify only on real change (v1 parity): a no-op add must not # bump the owner's generation changed = False try: for s in strings: normalized = _lc_validated(s) # TypeError on bytes (#245) if normalized not in self._elements: self._elements.add(normalized) changed = True finally: # a TypeError mid-list still leaves earlier additions # applied, so the owner must hear about them or its cache # goes stale (same rule as remove() below) if changed: self._changed() return self def remove(self, *strings: str) -> SetManager: """Remove the normalized string arguments from the set. Raises ``KeyError`` if any argument is not a member. Returns ``self`` for chaining.""" changed = False try: for s in strings: # _lc_validated: bytes get the same decode-hint TypeError # as add() (#245) normalized = _lc_validated(s) self._elements.remove(normalized) # KeyError on missing (#243) changed = True finally: # a KeyError mid-list still leaves earlier removals applied, # so the owner must hear about them or its cache goes stale if changed: self._changed() return self def discard(self, *strings: str) -> SetManager: """Remove the normalized string arguments from the set if present; missing members are ignored, like ``set.discard``. Returns ``self`` for chaining.""" changed = False for s in strings: normalized = _lc_validated(s) # bytes decode hint (#245) if normalized in self._elements: self._elements.discard(normalized) changed = True if changed: self._changed() return self def clear(self) -> SetManager: """Remove all entries from the set. Returns ``self`` for chaining.""" if self._elements: self._elements.clear() self._changed() return self def __contains__(self, item: object) -> bool: return isinstance(item, str) and lc(item) in self._elements def __iter__(self) -> Iterator[str]: return iter(self._elements) def __len__(self) -> int: return len(self._elements) def __eq__(self, other: object) -> bool: if isinstance(other, SetManager): return self._elements == other._elements if isinstance(other, (set, frozenset)): return self._elements == other return NotImplemented __hash__ = None # type: ignore[assignment] # mutable; v1 parity # -- set operators: accept ANY iterable (v1.3 normalize-everywhere) ----- # A bare str/bytes operand raises TypeError via _normalize_iterable_of_ # strings rather than iterating its characters (#238/#241); everything # else (list, generator, set, another SetManager, ...) is normalized # through lc() before the plain set op runs. def __or__(self, other: object) -> set[str]: return self._elements | _normalize_iterable_of_strings(other) __ror__ = __or__ def __and__(self, other: object) -> set[str]: return self._elements & _normalize_iterable_of_strings(other) __rand__ = __and__ def __sub__(self, other: object) -> set[str]: return self._elements - _normalize_iterable_of_strings(other) def __rsub__(self, other: object) -> set[str]: return _normalize_iterable_of_strings(other) - self._elements def __xor__(self, other: object) -> set[str]: return self._elements ^ _normalize_iterable_of_strings(other) __rxor__ = __xor__ # symmetric difference is commutative, like v1 # -- comparisons: v1 subclassed collections.abc.Set, whose __le__/__lt__/ # __ge__/__gt__ mixins only accept another Set-registered operand (set, # frozenset, or another Set subclass) -- NOT an arbitrary iterable like # list, unlike the operators above. Mirrored here since this SetManager # doesn't itself subclass the ABC. def __le__(self, other: object) -> bool: if not isinstance(other, (SetManager, Set)): return NotImplemented if len(self) > len(other): return False return all(elem in other for elem in self) def __lt__(self, other: object) -> bool: if not isinstance(other, (SetManager, Set)): return NotImplemented return len(self) < len(other) and self.__le__(other) def __ge__(self, other: object) -> bool: if not isinstance(other, (SetManager, Set)): return NotImplemented if len(self) < len(other): return False return all(elem in self for elem in other) def __gt__(self, other: object) -> bool: if not isinstance(other, (SetManager, Set)): return NotImplemented return len(self) > len(other) and self.__ge__(other) def __repr__(self) -> str: # Sorted so repr is stable across runs -- set() iteration order # depends on per-process string hash randomization. elements = ", ".join(repr(e) for e in sorted(self._elements)) return f"SetManager({{{elements}}})" if self._elements else "SetManager(set())" # -- pickle interop with v1 blobs --------------------------------------- def __getstate__(self) -> dict[str, object]: return {"_elements": set(self._elements)} def __setstate__(self, state: dict[str, object]) -> None: # v1 SetManager stored its set under `elements` (plain __dict__ # pickling); the shim stores `_elements`. Accept both, so a # v1.3/1.4 Constants blob's embedded managers unpickle straight # into shim instances. Re-normalize: nothing guarantees an # incoming blob's elements passed through lc(). elements: Iterable[str] = state.get( # type: ignore[assignment] "_elements", state.get("elements", ())) self._elements = {lc(e) for e in elements} self._on_change = None # rewired by the owning Constants def _validated_mapping_args(args: tuple[object, ...]) -> tuple[object, ...]: """The #242 constructor guard, shared by TupleManager and _DelimiterManager: a bare str/bytes silently shreds into a garbage mapping (dict's own error), and an iterable of short strings silently splits each one into a (key, value) pair -- ported from v1's TupleManager.__init__ guard.""" if not args: return args arg = args[0] _reject_bare_str_or_bytes( arg, "a mapping or iterable of (key, value) pairs") if not isinstance(arg, Mapping): checked = [] for item in arg: # type: ignore[attr-defined] if isinstance(item, (str, bytes)): kind = "bytes" if isinstance(item, bytes) else "str" raise TypeError( f"expected (key, value) pairs, got a {kind} " f"element {item!r}; a 2-character string " "silently splits into a key and a value" ) checked.append(item) args = (checked, *args[1:]) return args class TupleManager(dict[str, object]): """v1 ``TupleManager``: a dict with dot-notation access. Backs ``capitalization_exceptions``. Unknown-key attribute access raises ``AttributeError`` naming the key (#256, warned 1.4, enforced 2.0 -- the v1 ``DeprecationWarning`` is gone, this shim only speaks 2.0). Mutations call ``_on_change`` (the owning Constants' generation bump, wired by a later task). """ _on_change: Callable[[], None] | None def __init__(self, *args: object, _on_change: Callable[[], None] | None = None, **kwargs: object) -> None: args = _validated_mapping_args(args) super().__init__(*args, **kwargs) self._on_change = _on_change def _changed(self) -> None: if self._on_change is not None: self._on_change() def __getattr__(self, name: str) -> object: # Only reached for a missing attribute -- real instance attrs # (_on_change) and dict methods (keys, get, ...) resolve first # without ever hitting this. Dunder/underscore probes (pickling, # copy.deepcopy, IPython's _repr_html_) are never config keys. if name.startswith("_"): raise AttributeError(name) try: return self[name] except KeyError: # #256: name the known keys, like v1's 1.4 deprecation warning # did -- this shim only speaks 2.0, so what was a warning there # is a hard AttributeError here. raise AttributeError( f"{name!r} is not a known key " f"({', '.join(sorted(self))}); use .get() for intentional " "soft access." ) from None def __setattr__(self, name: str, value: object) -> None: # v1 parity: dunder probes (typing's __orig_class__, etc.) and this # shim's own _on_change hook get real object-attribute storage; # every other name -- including single-underscore ones, per v1 -- # routes to the dict so `t.mcdonald = 'x'` and `t['mcdonald'] = 'x'` # are the same operation. if name == "_on_change" or (name.startswith("__") and name.endswith("__")): object.__setattr__(self, name, value) else: self[name] = value def __delattr__(self, name: str) -> None: if name == "_on_change" or (name.startswith("__") and name.endswith("__")): object.__delattr__(self, name) else: del self[name] def __setitem__(self, key: str, value: object) -> None: super().__setitem__(key, value) self._changed() def __delitem__(self, key: str) -> None: super().__delitem__(key) self._changed() def pop(self, *args: object) -> object: # bump only on a real removal -- pop(key, default) on a missing # key is a no-op read, not a mutation, same rule as SetManager's # no-op add() present = bool(args) and args[0] in self result = super().pop(*args) # type: ignore[call-overload] if present: self._changed() return result def popitem(self) -> tuple[str, object]: result = super().popitem() # KeyError when empty: no bump self._changed() return result def clear(self) -> None: had_items = bool(self) super().clear() if had_items: # clearing an empty dict is a no-op, not a change self._changed() def update(self, *args: object, **kwargs: object) -> None: # dict.update's C path skips a subclass __setitem__; route every # item through it so subclass validation (_DelimiterManager's # sentinel rule) and the owner notification hold here too for key, value in dict(*args, **kwargs).items(): self[key] = value def setdefault(self, key: str, default: object = None) -> object: if key in self: return self[key] # existing key: a read, not a mutation self[key] = default # validated + notifying path return default # in-place |= must validate/notify like update; dict's C path would # skip both. mypy flags any non-overloaded __ior__ as inconsistent # with dict.__or__'s overloads -- the runtime behavior is the plain # dict |= contract, so the ignore is about typeshed shape only. def __ior__(self, other: object) -> Self: # type: ignore[override, misc] self.update(other) return self # -- pickle interop ------------------------------------------------- def __reduce__(self) -> tuple[type[TupleManager], tuple[()], dict[str, object]]: return (type(self), (), dict(self)) def __setstate__(self, state: dict[str, object]) -> None: # routes through __setitem__ (validated for _DelimiterManager); # _on_change is still None here, so no spurious bumps self.update(state) self._on_change = None # rewired by the owning Constants #: The named delimiter buckets, translated to the ``Policy`` #: (open, close) pairs they stand for (spec §3). The first three are #: v1's; the rest are the #273 typographic conventions, named so the #: v1 keyed idioms (pop/move/del) work on them like the originals. #: Keep in sync with DEFAULT_NICKNAME_DELIMITERS in _policy.py (pinned #: by the default-Constants equality test). _SENTINEL_PAIRS = { "quoted_word": ("'", "'"), "double_quotes": ('"', '"'), "parenthesis": ("(", ")"), "smart_double_quotes": ("“", "”"), "low_high_quotes": ("„", "“"), "right_double_quotes": ("”", "”"), "guillemets": ("«", "»"), "reversed_guillemets": ("»", "«"), "corner_brackets": ("「", "」"), "white_corner_brackets": ("『", "』"), "fullwidth_parenthesis": ("（", "）"), } #: derived, so the manager's accepted keys and _snapshot()'s #: translation table can never drift apart _DELIMITER_SENTINELS = tuple(_SENTINEL_PAIRS) class RegexTupleManager(TupleManager): # pickle-compat: do NOT delete """Pickle-compat alias only: v1.4's ``Constants.regexes`` field was a ``nameparser.config.RegexTupleManager`` instance (a ``TupleManager`` subclass whose ``__getattr__`` fell back to ``EMPTY_REGEX`` for an unknown key). Unpickling a v1.4 blob resolves and constructs this class -- via ``TupleManager.__reduce__``'s ``(type(self), (), state)`` -- before ``Constants.__setstate__`` runs, so the name must exist here even though the shim's ``Constants._snapshot()`` never reads it: ``regexes`` is a read-only ``_RegexesProxy`` in 2.0 (see above), and ``Constants.__setstate__`` below deliberately ignores an incoming ``regexes`` key rather than restoring it from this reconstructed (and otherwise unused) instance. """ class _DelimiterManager(TupleManager): """v1 ``nickname_delimiters``/``maiden_delimiters`` bucket. In 2.0 only the named sentinels in ``_DELIMITER_SENTINELS`` exist (spec §3; the v1 trio plus the #273 typographic pairs) -- assigning any other key raises so a caller reaches for a custom-delimiter Policy kwarg instead of a dict entry that silently does nothing. ``pop()``/ ``__setitem__``/``__delitem__`` stay open (inherited) for the documented bucket-move idiom, e.g. ``maiden_delimiters['parenthesis'] = nickname_delimiters.pop('parenthesis')``. """ def __init__(self, *args: object, _on_change: Callable[[], None] | None = None, **kwargs: object) -> None: # dict's C-level __init__ never calls a subclass __setitem__, so # collect and validate the initial items here -- BEFORE any item # lands -- or the sentinel rule silently misses the constructor. # The parent's #242 guard runs first: bare str/bytes gets the # friendly TypeError, not dict's cryptic one. args = _validated_mapping_args(args) items: dict[str, object] = dict(*args, **kwargs) for key in items: self._reject_non_sentinel(key) super().__init__(items, _on_change=_on_change) @staticmethod def _reject_non_sentinel(key: str) -> None: if key not in _DELIMITER_SENTINELS: raise TypeError( f"2.0 delimiter managers accept only the named sentinels " f"{_DELIMITER_SENTINELS}; for custom delimiter pairs use " f"Policy(nickname_delimiters=...) / maiden_delimiters" ) def __setitem__(self, key: str, value: object) -> None: self._reject_non_sentinel(key) super().__setitem__(key, value) # update/setdefault/|= inherit TupleManager's routing through # __setitem__, so they validate (and notify) for free class _RegexesProxy: """Read-only view over the v1 compiled patterns (``nameparser.config.regexes.REGEXES``). Reads keep working -- ``CONSTANTS.regexes.word`` stays informational -- but 2.0 configures parsing behavior through named ``Policy`` flags, not by mutating a regex, so any attribute *or* item assignment raises ``TypeError`` (spec §3's uniform read-only rule). """ @staticmethod def _regexes() -> Mapping[str, object]: # deferred import: see the module docstring's note on why every # nameparser.config. import in this file is lazy from nameparser.config.regexes import REGEXES return REGEXES #: dict methods this read-only proxy deliberately does not carry. #: Without this, __getattr__ reports them as missing regexes and #: sends the reader hunting for a vocabulary entry. _DICT_ONLY = frozenset({ "pop", "popitem", "setdefault", "update", "clear", "fromkeys", }) def __getattr__(self, name: str) -> object: if name.startswith("_"): raise AttributeError(name) if name in _RegexesProxy._DICT_ONLY: raise AttributeError( f"{name!r} is not supported on CONSTANTS.regexes: it is a " f"read-only view in 2.0, and parsing behavior is configured " f"through named Policy flags" ) try: return self._regexes()[name] except KeyError: raise AttributeError(f"no regex named {name!r}") from None def __getitem__(self, name: str) -> object: return self._regexes()[name] def __contains__(self, name: object) -> bool: return name in self._regexes() def __iter__(self) -> Iterator[str]: return iter(self._regexes()) def keys(self) -> KeysView[str]: return self._regexes().keys() # Defined explicitly because __getattr__ would otherwise claim each # of these names as a regex lookup and raise "no regex named # 'items'". The sibling managers inherit the whole surface from # dict; this proxy has to spell out the read half it supports. # #256's deprecation text promised .get() on both managers. def get(self, name: str, default: object = None) -> object: return self._regexes().get(name, default) def items(self) -> ItemsView[str, object]: return self._regexes().items() def values(self) -> ValuesView[object]: return self._regexes().values() def __len__(self) -> int: return len(self._regexes()) def copy(self) -> dict[str, object]: # v1's regexes was a dict subclass, so copy() returned a plain # mutable dict. Mutating the copy never affected parsing there # either, so this is a faithful read-only carry-over. return dict(self._regexes()) def __setattr__(self, name: str, value: object) -> None: self._raise_readonly(name) def __setitem__(self, name: str, value: object) -> None: self._raise_readonly(name) @staticmethod def _raise_readonly(name: str) -> None: # bidi/emoji are the two regexes v1 code toggled directly # (`CONSTANTS.regexes.bidi = False`) to opt out of stripping; # point those two at their named Policy replacement, everything # else gets the generic pointer. hints = { "bidi": "use Policy(strip_bidi=False) to keep bidi marks", "emoji": "use Policy(strip_emoji=False) to keep emoji", } hint = hints.get( name, "parsing behavior is configured through named Policy " "flags in 2.0; if none fits, open an issue") raise TypeError( f"assigning CONSTANTS.regexes.{name} is not supported in " f"2.0: {hint}" ) _SET_FIELDS = ( "prefixes", "suffix_acronyms", "suffix_not_acronyms", "suffix_acronyms_ambiguous", "titles", "first_name_titles", "conjunctions", "bound_first_names", "non_first_name_prefixes", ) _MANAGER_FIELDS = _SET_FIELDS + ( "capitalization_exceptions", "nickname_delimiters", "maiden_delimiters", ) #: v1's Constants.__repr__ field order (#221) -- kept as its own tuple #: rather than reusing _SET_FIELDS, whose order differs (v1 lists #: suffix_acronyms_ambiguous last, not fourth). _REPR_COLLECTION_ATTRS = ( "prefixes", "suffix_acronyms", "suffix_not_acronyms", "titles", "first_name_titles", "conjunctions", "bound_first_names", "non_first_name_prefixes", "suffix_acronyms_ambiguous", ) #: v1's repr scalar order, minus empty_attribute_default -- removed in 2.0 #: (#255), so there's no such attribute on this shim's Constants to show. _REPR_SCALAR_ATTRS = ( "string_format", "initials_format", "initials_delimiter", "initials_separator", "suffix_delimiter", "capitalize_name", "force_mixed_case_capitalization", "patronymic_name_order", "middle_name_as_last", ) _SCALAR_DEFAULTS: dict[str, object] = { "patronymic_name_order": False, "middle_name_as_last": False, "capitalize_name": False, "force_mixed_case_capitalization": False, "string_format": "{title} {first} {middle} {last} {suffix} ({nickname})", "initials_format": "{first} {middle} {last}", "initials_delimiter": ".", "initials_separator": " ", "suffix_delimiter": None, } # distinguishes "attribute not set yet" from any real scalar value # (None is a legitimate value for string_format/suffix_delimiter) _UNSET = object() # distinguishes "kwarg not passed to Constants()" (use library defaults) # from any real value a caller might pass, including a falsy one like "" _UNSET_KWARG = object() _SHARED_MUTATION_MESSAGE = ( "mutating the shared CONSTANTS singleton is deprecated and will be " "removed in 3.0; build a Lexicon/Policy (or a private Constants " "passed as HumanName(constants=...)) instead. See the migration " "guide." ) def _default_vocab() -> dict[str, set[str]]: # v1 data modules stay the single vocabulary source through 2.x # (same rule as Lexicon.default()). from nameparser.config.bound_first_names import BOUND_FIRST_NAMES from nameparser.config.conjunctions import CONJUNCTIONS from nameparser.config.prefixes import ( NON_FIRST_NAME_PREFIXES, PREFIXES, ) from nameparser.config.suffixes import ( SUFFIX_ACRONYMS, SUFFIX_ACRONYMS_AMBIGUOUS, SUFFIX_NOT_ACRONYMS, ) from nameparser.config.titles import FIRST_NAME_TITLES, TITLES return { "prefixes": PREFIXES, "suffix_acronyms": SUFFIX_ACRONYMS, "suffix_not_acronyms": SUFFIX_NOT_ACRONYMS, "suffix_acronyms_ambiguous": SUFFIX_ACRONYMS_AMBIGUOUS, "titles": TITLES, "first_name_titles": FIRST_NAME_TITLES, "conjunctions": CONJUNCTIONS, "bound_first_names": BOUND_FIRST_NAMES, "non_first_name_prefixes": NON_FIRST_NAME_PREFIXES, } class _RenderDefaults(NamedTuple): """v1 scalar rendering knobs that have no home on ``Policy`` (spec §3): ``__str__``/initials formatting and capitalization stay per-Constants defaults, layered onto a shared ``Parser`` by the facade (a later task) rather than folded into the cache key.""" string_format: str | None initials_format: str initials_delimiter: str initials_separator: str suffix_delimiter: str | None capitalize_name: bool force_mixed_case_capitalization: bool @functools.lru_cache(maxsize=64) def _cached_parser(lexicon: Lexicon, policy: Policy) -> Parser: # keyed on hashable value objects: shared across every facade whose # Constants resolve to the same snapshot (spec §3) return Parser(lexicon=lexicon, policy=policy) class Constants: """v1 ``Constants`` shim: a mutable container whose state resolves to a frozen ``(Lexicon, Policy, _RenderDefaults)`` snapshot via ``_snapshot()``. ``_generation`` increments on every mutation; facades compare it against a cached value to decide whether their snapshot is stale (dirty-tracking, spec §3 -- the facade itself is a later task). The module-level ``CONSTANTS`` singleton (below) has ``_shared`` flipped to ``True``: any mutation reached through it emits ``DeprecationWarning`` pointing at ``Lexicon``/``Policy`` and ``HumanName(constants=...)``. A private ``Constants()`` never warns -- only the shared instance is on the 3.0 removal path. """ _shared = False # the CONSTANTS singleton flips this to True _generation: int prefixes: SetManager suffix_acronyms: SetManager suffix_not_acronyms: SetManager suffix_acronyms_ambiguous: SetManager titles: SetManager first_name_titles: SetManager conjunctions: SetManager bound_first_names: SetManager non_first_name_prefixes: SetManager capitalization_exceptions: TupleManager nickname_delimiters: _DelimiterManager maiden_delimiters: _DelimiterManager regexes: _RegexesProxy patronymic_name_order: bool middle_name_as_last: bool capitalize_name: bool force_mixed_case_capitalization: bool string_format: str | None initials_format: str initials_delimiter: str initials_separator: str suffix_delimiter: str | None def __init__( self, *, prefixes: Iterable[str] | object = _UNSET_KWARG, suffix_acronyms: Iterable[str] | object = _UNSET_KWARG, suffix_not_acronyms: Iterable[str] | object = _UNSET_KWARG, suffix_acronyms_ambiguous: Iterable[str] | object = _UNSET_KWARG, titles: Iterable[str] | object = _UNSET_KWARG, first_name_titles: Iterable[str] | object = _UNSET_KWARG, conjunctions: Iterable[str] | object = _UNSET_KWARG, bound_first_names: Iterable[str] | object = _UNSET_KWARG, non_first_name_prefixes: Iterable[str] | object = _UNSET_KWARG, capitalization_exceptions: Mapping[str, str] | Iterable[tuple[str, str]] | object = _UNSET_KWARG, regexes: object = _UNSET_KWARG, patronymic_name_order: bool = False, middle_name_as_last: bool = False, ) -> None: # v1.4 parity constructor kwargs (#238/#242/#244 hardening); the # signature is spelled out rather than **kwargs so an unknown # keyword raises Python's own TypeError with no help from here. # `regexes` is the one deliberate 2.0 divergence: v1.4 accepted it # (RegexTupleManager(regexes)), but constructor injection is # assignment by another door, and __setattr__ above already # forbids `c.regexes = ...` post-construction -- the uniform 2.0 # rule is that parsing behavior is configured through Policy, not # by handing Constants a compiled-pattern mapping either way. if regexes is not _UNSET_KWARG: raise TypeError( "Constants(regexes=...) is not supported in 2.0; parsing " "behavior is configured through named Policy flags, not " "by constructing Constants with a regex mapping. See the " "migration guide." ) overrides = { "prefixes": prefixes, "suffix_acronyms": suffix_acronyms, "suffix_not_acronyms": suffix_not_acronyms, "suffix_acronyms_ambiguous": suffix_acronyms_ambiguous, "titles": titles, "first_name_titles": first_name_titles, "conjunctions": conjunctions, "bound_first_names": bound_first_names, "non_first_name_prefixes": non_first_name_prefixes, } vocab = _default_vocab() object.__setattr__(self, "_generation", 0) for name in _SET_FIELDS: value = overrides[name] if value is _UNSET_KWARG: value = vocab[name] # a caller-supplied value REPLACES that field's default # vocabulary wholesale (v1 parity); SetManager itself validates/ # normalizes and rejects a bare str/bytes (#238/#241), naming # this field in the message via _field= object.__setattr__( self, name, SetManager(value, _on_change=self._bump, _field=name)) # type: ignore[arg-type] if capitalization_exceptions is _UNSET_KWARG: from nameparser.config.capitalization import ( CAPITALIZATION_EXCEPTIONS, ) capitalization_exceptions = CAPITALIZATION_EXCEPTIONS object.__setattr__(self, "capitalization_exceptions", TupleManager( capitalization_exceptions, # type: ignore[arg-type] _on_change=self._bump)) object.__setattr__(self, "nickname_delimiters", _DelimiterManager( {name: name for name in _DELIMITER_SENTINELS}, _on_change=self._bump)) object.__setattr__(self, "maiden_delimiters", _DelimiterManager( _on_change=self._bump)) object.__setattr__(self, "regexes", _RegexesProxy()) for name, value in _SCALAR_DEFAULTS.items(): object.__setattr__(self, name, value) # the two behavior bools were v1.4 constructor kwargs too # (docs/customize.rst doctests use them); truthiness matches v1, # storage goes through the plain scalar slot if patronymic_name_order: object.__setattr__(self, "patronymic_name_order", True) if middle_name_as_last: object.__setattr__(self, "middle_name_as_last", True) def _invalidate_pst(self) -> None: """Pickle-compat alias only, never called at runtime: v1's four cached-union ``SetManager`` fields (``prefixes``, ``suffix_acronyms``, ``suffix_not_acronyms``, ``titles``) stored their ``_on_change`` as the bound method ``Constants._invalidate_pst``. A pickled bound method serializes as a back-reference to its ``__self__`` (this same ``Constants`` instance, mid-unpickle) plus the method NAME -- reconstructed via ``getattr(constants_obj, '_invalidate_pst')`` before ``Constants.__setstate__`` ever runs (same two-phase-unpickling story as ``RegexTupleManager`` above). Without this name, loading a v1.4 blob raises ``AttributeError`` looking up the method. ``SetManager.__setstate__`` (shim) never restores ``_on_change`` from pickled state regardless -- it always resets to ``None`` and is rewired by ``Constants.__setstate__`` below -- so this bound method value is reconstructed only to satisfy the pickle format; it is discarded immediately and never invoked. """ def _bump(self) -> None: # stacklevel=3 is exact for the direct scalar-assignment path # (user code -> Constants.__setattr__ -> here) and lands one # frame short -- inside the manager's own add()/remove()/ # __setitem__ -- for the indirect manager-mutation path (user # code -> manager method -> _changed() -> here), since a # single warn() call can't be exact for both call depths at # once. Either way the warning still fires from this module, # not the manager's true caller, which is enough: only the # DeprecationWarning's presence/category/message are load- # bearing (see the specified test), not the reported line. if self._shared: warnings.warn(_SHARED_MUTATION_MESSAGE, DeprecationWarning, stacklevel=3) object.__setattr__(self, "_generation", self._generation + 1) def __setattr__(self, name: str, value: object) -> None: if name == "_shared": # the flag the whole shared-mutation DeprecationWarning # mechanism hinges on: only the module-level singleton flip # (object.__setattr__ below) may set it raise AttributeError( "Constants._shared is read-only; it marks the module " "singleton and is set once at import" ) if name == "empty_attribute_default": raise AttributeError( "empty_attribute_default was removed in 2.0 (#255): " "empty attributes are always ''" ) if name == "regexes": raise TypeError( "replacing CONSTANTS.regexes is not supported in 2.0; " "parsing behavior is configured through named Policy " "flags" ) if name in _SET_FIELDS: # v1 allowed wholesale reassignment (c.titles = {...}); same # bare-str/bytes guard as the constructor kwarg path (#238/#241) value = SetManager( value, _on_change=self._bump, _field=name) # type: ignore[arg-type] elif name == "capitalization_exceptions": value = TupleManager(value, _on_change=self._bump) # type: ignore[arg-type] elif name in ("nickname_delimiters", "maiden_delimiters"): value = _DelimiterManager(value, _on_change=self._bump) # type: ignore[arg-type] elif name in _SCALAR_DEFAULTS and \ getattr(self, name, _UNSET) == value: # no-op scalar assignment: managers already suppress no-op # mutations, so re-assigning the current scalar value must # not bump the generation (or warn on the shared singleton) # either. __init__ writes via object.__setattr__, so this # only runs on real user assignments -- _UNSET never # actually matches, it just keeps a not-yet-set attribute # from raising here. Manager-field reassignment above stays # an unconditional bump: comparing manager contents isn't # worth it. object.__setattr__(self, name, value) return object.__setattr__(self, name, value) if name in _MANAGER_FIELDS or name in _SCALAR_DEFAULTS: self._bump() def copy(self) -> Constants: """Independent copy (#260), subclass-preserving. Divergence from v1 (which deepcopied __dict__): attributes OUTSIDE the known field surface -- e.g. ad-hoc names stashed on the instance -- are not carried by copy() or pickling; only the enumerated fields survive.""" # #260 # An independent instance with its own generation counter and # its own manager callbacks -- not a shared-state alias like a # naive attribute-for-attribute copy would produce. v1's copy() # was `copy.deepcopy(self)`, which builds the new object via # `type(self).__new__(type(self))` -- NOT by calling `type(self)()` # -- so a Constants subclass copies as itself without its __init__ # running (and without needing to satisfy whatever signature that # __init__ might require). Mirrored here with an explicit __new__ # bypass rather than type(self)(). new = object.__new__(type(self)) object.__setattr__(new, "_generation", 0) for name in _SET_FIELDS: object.__setattr__( new, name, SetManager(getattr(self, name), _on_change=new._bump)) object.__setattr__(new, "capitalization_exceptions", TupleManager( dict(self.capitalization_exceptions), _on_change=new._bump)) for bucket in ("nickname_delimiters", "maiden_delimiters"): object.__setattr__(new, bucket, _DelimiterManager( dict(getattr(self, bucket)), _on_change=new._bump)) object.__setattr__(new, "regexes", _RegexesProxy()) for name in _SCALAR_DEFAULTS: object.__setattr__(new, name, getattr(self, name)) return new def __repr__(self) -> str: # #221 # Collections (some with hundreds of entries, e.g. titles/prefixes) # are summarized as counts rather than dumped in full, like v1. # Scalars are only shown when they differ from the library default # -- _SCALAR_DEFAULTS stands in for v1's `getattr(type(self), name)` # class-level default, since this shim's scalar defaults are # instance attributes set in __init__, not class attributes. lines = [f" {name}: {len(getattr(self, name))}" for name in _REPR_COLLECTION_ATTRS] lines += [ f" {name}: {value!r}" for name in _REPR_SCALAR_ATTRS if (value := getattr(self, name)) != _SCALAR_DEFAULTS[name] ] return "" # -- snapshot ----------------------------------------------------------- def _snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: # generation-keyed cache: rebuilding the Lexicon re-normalizes # ~1400 vocabulary entries (~185us) -- the same dirty-tracking # the facade uses, applied one level up. A pure read either way # (no bump, no warning). cached = getattr(self, "_snapshot_cache", None) if cached is not None and cached[0] == self._generation: return cached[1] # type: ignore[no-any-return] snapshot = self._build_snapshot() object.__setattr__( self, "_snapshot_cache", (self._generation, snapshot)) return snapshot def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: """Resolve this v1-shaped, mutable Constants into the frozen 2.0 value objects it corresponds to (spec §3). A pure read: no generation bump, no deprecation warning even on the shared singleton -- only direct attribute mutation is on the 3.0 removal path. """ from nameparser.config.maiden_markers import MAIDEN_MARKERS from nameparser.config.suffixes import GLUED_HONORIFICS from nameparser.config.surnames import KOREAN_SURNAMES acronyms = frozenset(self.suffix_acronyms) particles = frozenset(self.prefixes) bound = frozenset(self.bound_first_names) ambiguous_acronyms = frozenset(self.suffix_acronyms_ambiguous) & acronyms # Drop any ambiguous acronym from the word set rather than the # other way round. Lexicon forbids the overlap because the word # branch bypasses the period gate, and adding an ambiguous # acronym to suffix_not_acronyms is INERT in v1 anyway: # is_suffix already accepts it via the acronym branch, and # reserve_last keeps it as the surname. So ignoring the # addition reproduces v1 ("Jack Ma" keeps last='Ma'), where # dropping it from the AMBIGUOUS set instead ungated the word # and lost the family name -- a silent misparse worse than the # raise it avoided. suffix_words = frozenset(self.suffix_not_acronyms) - ambiguous_acronyms # keep in sync with _lexicon._default_lexicon() (pinned by # tests/v2/test_config_shim.py::test_snapshot_field_translation) lexicon = Lexicon( titles=frozenset(self.titles), # TRANSLATE, do not filter. The two versions build the same # lookup key differently: v1 joins the raw title run and # then applies lc(), which strips only the whole string's # edge periods, so an interior word keeps its own ("lt. # col"). v2 normalizes each token and then joins ("lt col"). # Re-folding per word converts a v1 entry into the v2 # spelling; filtering instead dropped every multi-word # honorific containing an abbreviation or a conjunction and # silently swapped given and family. # Only entries v1 could actually match: its key is the # joined title run, so always single-spaced and never # empty. An entry holding a whitespace run was inert there # (translating it would start matching), and one that folds # away entirely would trip _normset's empty-entry check on # a config v1 simply ignored. given_name_titles=frozenset( t for t in ( _title_key(e.split()) for e in self.first_name_titles if e == " ".join(e.split()) ) if t), suffix_acronyms=acronyms, suffix_words=suffix_words, # Intersect with acronyms: Lexicon enforces ambiguous <= # acronyms; v1 behaves the same when an acronym is deleted # but its ambiguous entry lingers (the entry stops # mattering). suffix_acronyms_ambiguous=ambiguous_acronyms, particles=particles, # complement translation: v1 marks the never-given subset; # v2 marks the may-be-given subset. The trailing union keeps # a config v1 accepted: prefixes.py asserts its own data has # no word in both non_first_name_prefixes and # bound_first_names, but nothing stops a caller adding one at # runtime, and v1 then lets the bound rule win (leading "dos # Santos Silva" parses first="dos Santos"). Treating such a # word as may-be-given reproduces that rather than raising. # # KNOWN DEVIATION, pinned by # test_bound_never_given_prefix_deviates_on_two_pieces: v1's # join has a reserve_last guard, so with only two pieces it # does NOT fire and the word stays never-given ("dos Santos" # -> last="dos Santos"). Promotion here is unconditional, so # that case reads given="dos", family="Santos". v1's rule is # piece-count dependent and a static vocabulary set cannot # express it; the alternative is raising on a config v1 # accepted, which is worse. Only reachable via a runtime # config the shipped data forbids. particles_ambiguous=( particles - frozenset(self.non_first_name_prefixes)) | (bound & particles), conjunctions=frozenset(self.conjunctions), bound_given_names=bound, # v1 Constants has no manager for these (#274 is 2.0 # behavior); the data module is the only source maiden_markers=frozenset(MAIDEN_MARKERS), # likewise no v1 manager: the unspaced-name segmentation # vocabulary is 2.0 behavior (#271), so it rides in the # snapshot only -- v1's Constants surface stays frozen. # Unwrapped where maiden_markers above is wrapped: this # module is born frozen (#293), so no wrap surnames=KOREAN_SURNAMES, # likewise no v1 manager: the glued-honorific tail set is # 2.1 behavior (#308), so it rides in the snapshot only. # Wrapped, unlike surnames above: suffixes.py is still a # mutable v1 module, not born-frozen like surnames.py # (#293). Intersect with the word set: Lexicon enforces # tails <= suffix_words, and v1 semantics are that deleting # a suffix word turns the behavior off -- a lingering tail # simply stops mattering, the same rule ambiguous_acronyms # gets against suffix_acronyms above. honorific_tails=frozenset(GLUED_HONORIFICS) & suffix_words, # TupleManager is dict[str, object] (v1 parity: values were # never statically str-typed); every real entry is a str, # same assumption _DelimiterManager's sentinel lookup makes capitalization_exceptions=tuple( sorted(self.capitalization_exceptions.items())), # type: ignore[arg-type] ) rules = frozenset({PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) \ if self.patronymic_name_order else frozenset() policy = Policy( patronymic_rules=rules, middle_as_family=self.middle_name_as_last, nickname_delimiters=frozenset( _SENTINEL_PAIRS[k] for k in self.nickname_delimiters), # v1 precedence: a pair in BOTH v1 buckets parses as nickname. # Policy itself resolves overlap the other way (maiden wins), # so pre-subtract here to keep the facade at v1 behavior. maiden_delimiters=frozenset( _SENTINEL_PAIRS[k] for k in self.maiden_delimiters if k not in self.nickname_delimiters), # suffix_delimiter is a _RenderDefaults-only field here; the # facade layers it onto extra_suffix_delimiters per instance # (a later task) -- _snapshot() itself stays pure translation ) defaults = _RenderDefaults( self.string_format, self.initials_format, self.initials_delimiter, self.initials_separator, self.suffix_delimiter, self.capitalize_name, self.force_mixed_case_capitalization) return lexicon, policy, defaults # -- pickle ----------------------------------------------------------- def __getstate__(self) -> dict[str, object]: state: dict[str, object] = {} for name in _SET_FIELDS: state[name] = set(getattr(self, name)) state["capitalization_exceptions"] = dict( self.capitalization_exceptions) state["nickname_delimiters"] = dict(self.nickname_delimiters) state["maiden_delimiters"] = dict(self.maiden_delimiters) for name in _SCALAR_DEFAULTS: state[name] = getattr(self, name) return state def __setstate__(self, state: dict[str, object]) -> None: if "suffixes_prefixes_titles" in state: # pre-1.3.0 blob: its dir()-sweep __getstate__ captured this # computed property. The 1.4 DeprecationWarning promised # ValueError in 2.0 (#279). raise ValueError( "this pickle was written by nameparser <= 1.2.x (#279); " "re-pickle under 1.3/1.4 to migrate, or re-create the " "configuration. See " "https://github.com/derek73/python-nameparser/issues/279" ) # Accepts BOTH shapes with a single overlay: the shim's own # state and v1.3/1.4 state (public field names -> manager/ # scalar values) share every key that matters, so no shape # marker is needed. empty_attribute_default is accepted and # DROPPED (#255: empty is always '' in 2.0). state = {k: v for k, v in state.items() if k != "empty_attribute_default"} self.__init__() # type: ignore[misc] # defaults, then overlay # (managers re-wrapped below so _on_change points at THIS # instance, not whatever produced the incoming state) managers: dict[str, SetManager] = {} for name in _SET_FIELDS: if name in state: managers[name] = SetManager( state[name], _on_change=self._bump) # type: ignore[arg-type] # SetManager normalized on construction, so the frozen 1.3/1.4 # vocabulary's dead entries are matchable in their normalized # spelling here. Subtract only when ALL EIGHT are present -- # the pre-2.0 signature; a 2.0 user who re-added one or two # keeps them through a round-trip (see _LEGACY_DEAD_ENTRIES). legacy = all( name in managers and entry in managers[name] for name, entries in _LEGACY_DEAD_ENTRIES.items() for entry in entries ) for name, manager in managers.items(): if legacy: # Reach past the public discard() deliberately: this is # part of restoring the state, not a mutation of it, and # must not bump the generation of an instance that is # still being built. manager._elements -= _LEGACY_DEAD_ENTRIES.get( name, frozenset()) object.__setattr__(self, name, manager) if "capitalization_exceptions" in state: object.__setattr__( self, "capitalization_exceptions", TupleManager( state["capitalization_exceptions"], # type: ignore[arg-type] _on_change=self._bump)) for bucket in ("nickname_delimiters", "maiden_delimiters"): if bucket in state: object.__setattr__(self, bucket, _DelimiterManager( state[bucket], _on_change=self._bump)) # type: ignore[arg-type] for name in _SCALAR_DEFAULTS: if name in state: object.__setattr__(self, name, state[name]) CONSTANTS = Constants() object.__setattr__(CONSTANTS, "_shared", True) python-nameparser-2.1.0/nameparser/_facade.py000066400000000000000000000733071523530431700212540ustar00rootroot00000000000000"""The 2.0 ``HumanName`` facade (migration spec §2): a mutable wrapper over a frozen ParsedName, delegating parsing to the core Parser resolved from the bound Constants shim. Keeps every v1 spelling. Deleted in 3.0. Layering: facade layer -- may import anything public plus _render. """ from __future__ import annotations import dataclasses import warnings from collections.abc import Iterator, Mapping from typing import Any # Import order matters here -- breaks a real import cycle. nameparser. # config's package __init__ re-exports CONSTANTS/Constants/etc. from # _config_shim (the v1 nameparser.config.Constants compat path), while # _config_shim's own default CONSTANTS singleton needs nameparser.config's # DATA submodules (titles, prefixes, ...), imported lazily -- see # _config_shim.py's module docstring. If _config_shim were the first of # the two ever touched, building its CONSTANTS would need to import # nameparser.config, whose __init__ would in turn need _config_shim's # (not-yet-built) CONSTANTS: ImportError. Importing the config package # here first lets its __init__ run to completion; when IT then imports # _config_shim to build the default CONSTANTS, nameparser.config is # already registered in sys.modules, so its data-submodule imports # resolve directly instead of re-entering (and failing on) its own # still-executing __init__. import nameparser.config # noqa: F401 import nameparser._render as _render from nameparser._config_shim import CONSTANTS, Constants, _cached_parser from nameparser._lexicon import _normalize from nameparser._parser import Parser from nameparser._types import FOLDED_TAG, ParsedName, Role, Token _V2_FIELD = {"first": "given", "last": "family"} # v1 name -> v2 name _V1_SPELLING = {v2: v1 for v1, v2 in _V2_FIELD.items()} # derived from Role: declaration order IS the canonical field order # (never restated), rendered in the v1 spellings _MEMBERS = tuple(_V1_SPELLING.get(r.value, r.value) for r in Role) #: v1 parsing hooks the facade never calls (spec §2 exception 2 / #280). _V1_HOOKS = ( "pre_process", "post_process", "parse_full_name", "parse_pieces", "parse_nicknames", "join_on_conjunctions", "squash_emoji", "handle_firstnames", "handle_middle_name_as_last", "is_title", "is_conjunction", "is_prefix", "is_roman_numeral", "is_suffix", "is_suffix_lenient", "is_an_initial", ) # Module-level mutable state (sanctioned exception, AGENTS.md "One # sanctioned global"): strong-references every distinct HumanName # subclass for process lifetime so the hook warning fires once per # class. Fine in practice -- subclasses are statically defined, and the # whole module is deleted with the facade layer in 3.0. _WARNED_SUBCLASSES: set[type] = set() def _empty_parsed() -> ParsedName: return ParsedName(original="", tokens=(), ambiguities=()) class HumanName: """v1 ``HumanName`` facade: a mutable wrapper over a frozen ``ParsedName``, delegating all parsing to the core ``Parser`` resolved from the bound ``Constants`` shim (dirty-tracked via its ``_generation``). Keeps every v1 spelling (``first``/``last`` over the core ``given``/``family``); the v1 parsing hooks are never called (#280). Deleted with the facade layer in 3.0. """ def __init__( self, full_name: str = "", constants: Constants | None = CONSTANTS, string_format: str | None = None, initials_format: str | None = None, initials_delimiter: str | None = None, initials_separator: str | None = None, suffix_delimiter: str | None = None, first: str | list[str] | None = None, middle: str | list[str] | None = None, last: str | list[str] | None = None, title: str | list[str] | None = None, suffix: str | list[str] | None = None, nickname: str | list[str] | None = None, maiden: str | list[str] | None = None, ) -> None: if constants is None: raise TypeError( "constants=None was removed in 2.0 (#261): pass a " "Constants instance, or use the new Parser/Lexicon/" "Policy API for per-call configuration" ) if not isinstance(constants, Constants): raise TypeError( f"constants must be a Constants instance, got {constants!r}" ) self._warn_overridden_hooks() self._C = constants self._snapshot_gen = -1 # forces first resolve _, _, defaults = constants._snapshot() self.string_format = (string_format if string_format is not None else defaults.string_format) self.initials_format = (initials_format if initials_format is not None else defaults.initials_format) self.initials_delimiter = ( initials_delimiter if initials_delimiter is not None else defaults.initials_delimiter) self.initials_separator = ( initials_separator if initials_separator is not None else defaults.initials_separator) # These five assignments route through the validating properties # below. The suffix_delimiter setter resets _snapshot_gen to -1 # on every assignment (including this one, harmlessly -- it's # already -1 above), so reassigning it post-construction (e.g. # n.suffix_delimiter = " - ") correctly forces the next # _resolve() to rebuild the Policy with the new delimiter. self.suffix_delimiter = (suffix_delimiter if suffix_delimiter is not None else defaults.suffix_delimiter) self._full_name = "" self._parsed = _empty_parsed() if first or middle or last or title or suffix or nickname or maiden: # These route through the field-setter properties (None # clears the field); no full-string parse, full_name stays "". self.first = first self.middle = middle self.last = last self.title = title self.suffix = suffix self.nickname = nickname self.maiden = maiden else: self._apply_full_name(full_name) @classmethod def _warn_overridden_hooks(cls) -> None: if cls is HumanName or cls in _WARNED_SUBCLASSES: return overridden = [h for h in _V1_HOOKS if getattr(cls, h, None) is not getattr( HumanName, h, None)] _WARNED_SUBCLASSES.add(cls) if overridden: warnings.warn( f"{cls.__name__} overrides v1 parsing hooks " f"({', '.join(overridden)}) that the 2.0 facade never " f"calls; parsing is delegated to the core Parser. " f"Migrate to the Lexicon/Policy API. See " f"https://github.com/derek73/python-nameparser/issues/280", DeprecationWarning, stacklevel=3) # -- render defaults ----------------------------------------------------- # One-line validating setters (spec §2): assigning a non-str (or, for # the two fields that allow it, non-str-non-None) raises TypeError at # assignment time instead of failing later inside .format(). @property def string_format(self) -> str | None: return self._string_format @string_format.setter def string_format(self, value: str | None) -> None: if value is not None and not isinstance(value, str): raise TypeError( f"string_format must be a str or None, got {value!r}") self._string_format = value @property def initials_format(self) -> str: return self._initials_format @initials_format.setter def initials_format(self, value: str) -> None: if not isinstance(value, str): raise TypeError( f"initials_format must be a str, got {value!r}") self._initials_format = value @property def initials_delimiter(self) -> str: return self._initials_delimiter @initials_delimiter.setter def initials_delimiter(self, value: str) -> None: if not isinstance(value, str): raise TypeError( f"initials_delimiter must be a str, got {value!r}") self._initials_delimiter = value @property def initials_separator(self) -> str: return self._initials_separator @initials_separator.setter def initials_separator(self, value: str) -> None: if not isinstance(value, str): raise TypeError( f"initials_separator must be a str, got {value!r}") self._initials_separator = value @property def suffix_delimiter(self) -> str | None: return self._suffix_delimiter @suffix_delimiter.setter def suffix_delimiter(self, value: str | None) -> None: if value is not None and not isinstance(value, str): raise TypeError( f"suffix_delimiter must be a str or None, got {value!r}") self._suffix_delimiter = value # Invalidate the cached Policy: _resolve() layers suffix_delimiter # onto extra_suffix_delimiters, so a stale snapshot would keep # parsing against the old delimiter. self._snapshot_gen = -1 # -- config / parsing --------------------------------------------------- def _resolve(self) -> Parser: """Dirty-tracked parser resolution (spec §3): rebuild the snapshot only when the bound Constants' generation moved.""" gen = self._C._generation if self._snapshot_gen != gen: lexicon, policy, _ = self._C._snapshot() if self.suffix_delimiter: policy = dataclasses.replace( policy, extra_suffix_delimiters=frozenset( {self.suffix_delimiter})) self._lexicon, self._policy = lexicon, policy self._parser = _cached_parser(lexicon, policy) self._snapshot_gen = gen # the fast path is a plain attribute return: hashing the two # value objects for the lru lookup is the whole fast-path cost return self._parser def parse_full_name(self) -> None: """Re-parse the stored ``full_name`` (v1's documented re-parse trigger, docs/customize.rst): mutate ``name.C`` then call this to force a re-parse without reassigning ``full_name``. The v1 parsing INTERNALS this name evokes live in the core ``Parser``, not here; a subclass overriding this method still triggers the #280 hook-override warning, and full_name assignment never consults it.""" self._apply_full_name(self._full_name) def _apply_full_name(self, value: str) -> None: if isinstance(value, bytes): raise TypeError( "bytes input was removed in 2.0 (#245): decode first, " "e.g. HumanName(raw.decode('utf-8'))" ) if not isinstance(value, str): raise TypeError(f"full_name must be a str, got {value!r}") # parse FIRST: if snapshot resolution raises, the instance must # not be left with a new full_name over the old parsed fields parsed = self._resolve().parse(value) self._full_name = value self._parsed = parsed if self._C.capitalize_name: self.capitalize() # v1 parser.py:1653 parity def capitalize(self, force: bool | None = None) -> None: """Re-capitalize the current parse against the bound lexicon. force=None reads the bound Constants' render default (force_mixed_case_capitalization); the core's capitalized() implements the single-case gate (v1 parity) -- not re-implemented here.""" self._resolve() if force is None: force = self._C.force_mixed_case_capitalization self._parsed = self._parsed.capitalized(self._lexicon, force=force) @property def full_name(self) -> str: return self._full_name @full_name.setter def full_name(self, value: str) -> None: self._apply_full_name(value) @property def original(self) -> str: return self._parsed.original or self._full_name @property def C(self) -> Constants: return self._C @C.setter def C(self, constants: Constants | None) -> None: # v1.4 closed #239 by making C a validating setter that ONLY # stores the new value -- no re-parse (checked against v1.4: # `git show 2d5d8c2:nameparser/parser.py` lines ~204-206, the C # setter body is exactly `self._C = self._validate_constants(...)`). # A caller who wants the new config reflected must still trigger # a re-parse, e.g. via parse_full_name() or a full_name # reassignment -- matched here rather than re-parsing eagerly. if constants is None: raise TypeError( "assigning constants=None to C was removed in 2.0 (#261): " "pass a Constants instance, or use the new Parser/Lexicon/" "Policy API for per-call configuration" ) if not isinstance(constants, Constants): raise TypeError( f"constants must be a Constants instance, got {constants!r}" ) self._C = constants self._snapshot_gen = -1 # invalidate: next _resolve() rebuilds @property def has_own_config(self) -> bool: """True when this instance is not using the shared module-level CONSTANTS.""" return self._C is not CONSTANTS # -- fields --------------------------------------------------------- def _get_field(self, member: str) -> str: return getattr(self._parsed, _V2_FIELD.get(member, member)) def _set_field(self, member: str, value: str | list[str] | None) -> None: if value is None: joined = "" elif isinstance(value, list): for element in value: if not isinstance(element, str): raise TypeError( f"name parts must be strings, got {element!r}") joined = " ".join(value) elif isinstance(value, str): joined = value else: raise TypeError( f"{member} must be a str, list, or None, got {value!r}") # v1 setters stay on replace(): revise()'s vocabulary tags would # change v1 parity self._parsed = self._parsed.replace( **{_V2_FIELD.get(member, member): joined}) def _list_for(self, member: str) -> list[str]: # A "joined" continuation token ("Ph." + "D.") belongs to its # predecessor's part, matching v1's fix_phd (suffix_list had ONE # "Ph. D." element). ParsedName._text_for heals only the suffix # string view (the ", " join); the facade list view heals for # every role -- a continuation is never its own list element. role = Role(_V2_FIELD.get(member, member)) parts: list[str] = [] folded: list[str] = [] for tok in self._parsed.tokens_for(role): if "joined" in tok.tags and parts: parts[-1] += " " + tok.text elif FOLDED_TAG in tok.tags: # middle_as_family fold: v1 PREPENDED middle_list to # last_list -- keep the list view consistent with the # string view (_text_for orders folded-first too) folded.append(tok.text) else: parts.append(tok.text) return folded + parts @property def title(self) -> str: return self._get_field("title") @title.setter def title(self, value: str | list[str] | None) -> None: self._set_field("title", value) @property def title_list(self) -> list[str]: return self._list_for("title") @property def first(self) -> str: return self._get_field("first") @first.setter def first(self, value: str | list[str] | None) -> None: self._set_field("first", value) @property def first_list(self) -> list[str]: return self._list_for("first") @property def middle(self) -> str: return self._get_field("middle") @middle.setter def middle(self, value: str | list[str] | None) -> None: self._set_field("middle", value) @property def middle_list(self) -> list[str]: return self._list_for("middle") @property def last(self) -> str: return self._get_field("last") @last.setter def last(self, value: str | list[str] | None) -> None: self._set_field("last", value) @property def last_list(self) -> list[str]: return self._list_for("last") @property def suffix(self) -> str: return self._get_field("suffix") @suffix.setter def suffix(self, value: str | list[str] | None) -> None: self._set_field("suffix", value) @property def suffix_list(self) -> list[str]: return self._list_for("suffix") @property def nickname(self) -> str: return self._get_field("nickname") @nickname.setter def nickname(self, value: str | list[str] | None) -> None: self._set_field("nickname", value) @property def nickname_list(self) -> list[str]: return self._list_for("nickname") @property def maiden(self) -> str: return self._get_field("maiden") @maiden.setter def maiden(self, value: str | list[str] | None) -> None: self._set_field("maiden", value) @property def maiden_list(self) -> list[str]: return self._list_for("maiden") # -- derived views ---------------------------------------------------- @property def surnames_list(self) -> list[str]: return self.middle_list + self.last_list @property def surnames(self) -> str: return " ".join(self.surnames_list) @property def given_names_list(self) -> list[str]: return self.first_list + self.middle_list @property def given_names(self) -> str: return " ".join(self.given_names_list) def _is_particle(self, text: str) -> bool: self._resolve() return _normalize(text) in self._lexicon.particles def _is_conjunction(self, text: str) -> bool: self._resolve() return _normalize(text) in self._lexicon.conjunctions def _split_last(self) -> tuple[list[str], list[str]]: # v1 parser.py _split_last, verbatim: vocabulary lookup at ACCESS # time (so assigned last names split too), with the all-particle # guard (a family name is assumed not to consist entirely of # particles, e.g. surname "Do" which also appears in PREFIXES) words = " ".join(self.last_list).split() i = 0 while i < len(words) and self._is_particle(words[i]): i += 1 if i == len(words): return [], words return words[:i], words[i:] @property def last_prefixes_list(self) -> list[str]: return self._split_last()[0] @property def last_prefixes(self) -> str: return " ".join(self._split_last()[0]) @property def last_base_list(self) -> list[str]: return self._split_last()[1] @property def last_base(self) -> str: return " ".join(self._split_last()[1]) # -- initials ------------------------------------------------------------- def _process_initial(self, name_part: str, firstname: bool = False) -> str: # v1 parser.py:427 verbatim: particles/conjunctions are filtered # from initials unless the part is a first name. split() rather # than split(" "): *_list attributes assigned directly bypass # whitespace normalization, and split(" ") yields empty strings # for repeated spaces (#232). parts = name_part.split() initials = [] for part in parts: if not (self._is_particle(part) or self._is_conjunction(part)) or firstname: initials.append(part[0]) if len(initials) > 0: return self.initials_separator.join(initials) # Return '' (never empty_attribute_default, which may be None) # when a part has no initialable words, e.g. a middle name # consisting only of prefixes ("de la"). Callers drop these # parts entirely. return "" def _initials_lists(self) -> tuple[list[str], list[str], list[str]]: """Initials for the first, middle and last name groups. Parts that yield no initials (e.g. a prefix-only middle name like "de la") are dropped rather than kept as empty strings. """ def group_initials(names: list[str], firstname: bool = False) -> list[str]: return [i for i in (self._process_initial(n, firstname) for n in names if n) if i] return (group_initials(self.first_list, True), group_initials(self.middle_list), group_initials(self.last_list)) def initials_list(self) -> list[str]: first, middle, last = self._initials_lists() return first + middle + last def initials(self) -> str: first, middle, last = self._initials_lists() joiner = self.initials_delimiter + self.initials_separator def group(items: list[str]) -> str: return joiner.join(items) + self.initials_delimiter \ if items else "" # A fully-empty result renders as "" -- the v1 fallback to # C.empty_attribute_default (which may be None) is dropped per # #255. _s = self.initials_format.format( first=group(first), middle=group(middle), last=group(last)) return self.collapse_whitespace(_s) # -- comparison ----------------------------------------------------------- def matches(self, other: str | HumanName) -> bool: """Component-wise case-insensitive comparison (v1 parity); a str argument is parsed with this instance's resolved parser.""" if not isinstance(other, (str, HumanName)): # pre-check so the error names the facade type a caller # actually passed HumanName.matches(), not the core # ParsedName it delegates to below raise TypeError( f"matches() takes a str or HumanName, got {other!r}") target = other._parsed if isinstance(other, HumanName) else other return self._parsed.matches(target, parser=self._resolve()) def comparison_key(self) -> tuple[str, ...]: """One casefolded component per field in canonical order -- the v1 replacement for ==/hash (#223); see ParsedName.comparison_key.""" return self._parsed.comparison_key() # -- dunders ------------------------------------------------------------ def collapse_whitespace(self, string: str) -> str: # v1 parser.py:976 verbatim, over _render's regexes (the #254 # collapse owns them; this public method keeps v1's narrower # two-step contract for initials() and direct callers) string = _render._SPACES.sub(" ", string.strip()) if string and _render._COMMA_CHAR.fullmatch(string[-1]): string = string[:-1] return string def __str__(self) -> str: if self.string_format is not None: rendered = self.string_format.format( **{k: v or "" for k, v in self.as_dict().items()}) # the full #254 collapse is _render._collapse -- one owner # for the cleanup chain the v1 __str__ spelled inline return _render._collapse(rendered) return " ".join(self) def __repr__(self) -> str: attrs = ( f" title: {self.title or ''!r}\n" f" first: {self.first or ''!r}\n" f" middle: {self.middle or ''!r}\n" f" last: {self.last or ''!r}\n" f" suffix: {self.suffix or ''!r}\n" f" nickname: {self.nickname or ''!r}\n" f" maiden: {self.maiden or ''!r}" ) return f"<{self.__class__.__name__} : [\n{attrs}\n]>" def __iter__(self) -> Iterator[str]: return (value for member in _MEMBERS if (value := getattr(self, member))) def __len__(self) -> int: return sum(1 for member in _MEMBERS if getattr(self, member)) def __getitem__(self, key: str) -> str: if isinstance(key, slice): raise TypeError( "slicing a HumanName was removed in 2.0 (#258); access " "the named attributes instead" ) # Role is a StrEnum, so Role members (and the plain 'given'/ # 'family' strings) reach here too -- translate to the v1 # spelling the facade actually exposes as attributes. return getattr(self, _V1_SPELLING.get(key, key)) def __setattr__(self, name: str, value: object) -> None: # "given"/"family" are the 2.0 spellings of first/last; the # facade has no such attributes, so plain assignment creates a # stray instance attribute while the parse (and .first/.last) # keeps the old value -- a silently forked name. Warn but # still set: ad-hoc attribute stashing is a legal v1 pattern, # so any code that worked keeps working. Only these two names # warn -- the other five 2.0 field names are real properties # whose setters work, and Role members reach here as their # string values (StrEnum). if name in _V1_SPELLING: warnings.warn( f"assigning HumanName.{name} creates an inert attribute; " f"the parse is unchanged -- use .{_V1_SPELLING[name]} " f"(the v1 spelling) to update the name", UserWarning, stacklevel=2) super().__setattr__(name, value) def as_dict(self, include_empty: bool = True) -> dict[str, str]: """The seven v1-named components as a dict; include_empty=False drops empty fields.""" d = {member: getattr(self, member) for member in _MEMBERS} if include_empty: return d return {k: v for k, v in d.items() if v} # -- pickle (v1-shaped state; one path for 1.4 and 2.x blobs) ----------- def __getstate__(self) -> dict[str, Any]: # The emitted key set matches v1.4's pickle shape (minus # encoding/_had_comma/_derived_*, which are v1-internal and # ignored on read), so one __setstate__ path serves both eras. state: dict[str, Any] = { "_full_name": self._full_name, "original": self.original, "C": None if self._C is CONSTANTS else self._C, "string_format": self.string_format, "initials_format": self.initials_format, "initials_delimiter": self.initials_delimiter, "initials_separator": self.initials_separator, "suffix_delimiter": self.suffix_delimiter, } for member in _MEMBERS: state[f"{member}_list"] = getattr(self, f"{member}_list") return state def __setstate__(self, state: dict[str, Any]) -> None: c = state.get("C") self._C = CONSTANTS if c is None else c self._snapshot_gen = -1 defaults = self._C._snapshot()[2] self._string_format = state.get("string_format", defaults.string_format) self._initials_format = state.get("initials_format", defaults.initials_format) self._initials_delimiter = state.get("initials_delimiter", defaults.initials_delimiter) self._initials_separator = state.get("initials_separator", defaults.initials_separator) self._suffix_delimiter = state.get("suffix_delimiter", defaults.suffix_delimiter) self._full_name = state.get("_full_name", "") # Components come back exactly as pickled (spec §2): synthetic # tokens, never a re-parse. Build them per *_list ENTRY rather # than from one joined string -- an entry may hold several words # ("Ph. D.", "Q.C. M.P."), and re-splitting the joined string on # whitespace would promote each word to its own entry, which the # suffix view then renders comma-separated ("Ph., D."). Marking # continuation words "joined" is the inverse of _list_for's heal, # so list -> pickle -> list is the identity v1 gave us. tokens: list[Token] = [] for member in _MEMBERS: role = Role(_V2_FIELD.get(member, member)) entries = state.get(f"{member}_list") or [] # Everything here is iterable but shreds differently: a str # yields characters ("John" -> first "J o h n"), a Mapping # yields keys only, bytes yields ints. v1 stored lists, so # this only guards foreign or hand-built state -- but it # names the field at the load site instead of failing # opaquely later, or not at all. if isinstance(entries, (str, bytes, Mapping)): raise TypeError( f"{member}_list must be a list of strings, not " f"{type(entries).__name__} ({entries!r}); this " f"pickle was not written by nameparser" ) for entry in entries: if not isinstance(entry, str): raise TypeError( f"{member}_list entries must be strings, got " f"{entry!r}; this pickle was not written by " f"nameparser" ) for position, word in enumerate(entry.split()): tokens.append(Token( word, None, role, frozenset({"joined"}) if position else frozenset())) self._parsed = ParsedName( original=str(state.get("original", "")), tokens=tuple(tokens)) python-nameparser-2.1.0/nameparser/_lexicon.py000066400000000000000000001013411523530431700215000ustar00rootroot00000000000000"""Immutable vocabulary configuration for the 2.0 API. Layering: may import nameparser.config DATA modules (the imports in _default_lexicon() are the authoritative list) as the single source of vocabulary during 2.x -- never nameparser.config itself, never nameparser.parser. Enforced by tests/v2/test_layering.py. """ from __future__ import annotations import dataclasses import functools import sys import warnings from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from types import FrameType, MappingProxyType from typing import cast #: Vocabulary set fields, in declaration order. add()/remove() operate #: on exactly these and reject capitalization_exceptions (its entries #: are pairs -- use dataclasses.replace); __or__ unions these AND merges #: capitalization_exceptions right-biased. _VOCAB_FIELDS = ( "titles", "given_name_titles", "suffix_acronyms", "suffix_words", "suffix_acronyms_ambiguous", "particles", "particles_ambiguous", "conjunctions", "bound_given_names", "maiden_markers", "surnames", "honorific_tails", ) #: (marker, base, why) triples. Each marker QUALIFIES how entries of #: its base vocabulary are read and carries no vocabulary of its own, #: so an entry outside the base is a configuration mistake -- but the #: mistake differs per pair, and the reason is recorded here rather #: than generalized, because an orphan is NOT simply inert. Nor is the #: qualification one-directional: the first two NARROW their base (an #: entry is read as vocabulary in fewer places), while honorific_tails #: WIDENS it, granting a suffix word the glued position on top of the #: whole-token match every suffix word already gets. #: #: * particles_ambiguous: _assign keys on the tag alone, so an orphan #: makes the parse emit a spurious particle-or-given ambiguity. #: * suffix_acronyms_ambiguous: _vocab returns True on the ambiguous #: set before testing suffix_acronyms, so an orphan silently turns a #: word into a period-gated suffix. #: * honorific_tails: script_segment peels the tail into its own token #: before classify ever runs, so an orphan splits the name and leaves #: the fragment stranded inside it -- worse than not peeling at all. #: Its base is deliberately NARROWER than what actually claims the #: peeled piece: suffix_as_written ORs suffix_words with the #: non-ambiguous suffix_acronyms, so a tail listed only as an acronym #: would classify fine yet is rejected here. Accepted, and a decision #: rather than an oversight -- the three-term predicate is easy to #: get wrong in the dangerous direction (an ambiguous acronym admitted #: as a tail would peel a period-gated word off a real name), and #: nothing needs the acronym half: the shipped tails are CJK #: honorifics, which are words. #: The same relation is asserted a second time in config/suffixes.py, #: over the raw GLUED_HONORIFICS/SUFFIX_NOT_ACRONYMS constants at #: import. The two are not redundant in the way they look: that one #: is an `assert`, stripped under `python -O`, while the check here #: raises unconditionally -- so under -O this is what still holds the #: SHIPPED vocabulary to the invariant, as it is the only thing that #: ever held a caller's own. #: #: given_name_titles is deliberately NOT here and has no check of its #: own -- see the note in __post_init__ for why every attempt at one #: rejected working configurations. _SUBSET_FIELDS = ( ("particles_ambiguous", "particles", "an orphan emits a spurious particle-or-given ambiguity"), ("suffix_acronyms_ambiguous", "suffix_acronyms", "an orphan silently becomes a period-gated suffix"), ("honorific_tails", "suffix_words", "an orphan splits the name and leaves the tail inside it"), ) def _normalize(word: str) -> str: """Lowercase, strip whitespace and EDGE periods -- v1's lc() semantics. Interior periods survive on purpose: 'J.R.' must not collapse to 'jr' and hit the periodless vocabulary (v1 parity, pinned live 2026-07-17). Suffix-ACRONYM membership alone uses the period-free form (see _vocab.suffix_as_written), mirroring v1's is_suffix, which removed periods only for the acronym test. lower(), NOT casefold(): casefold's caseless-matching folds mutate the stored vocabulary itself -- 'κος' becomes the misspelling 'κοσ' (final sigma flattened) and 'großfürst' becomes 'grossfürst' -- while lower() applies Unicode SpecialCasing contextually and keeps both as authored. This function is the single fold for storage AND match-time lookups, so matching stays symmetric either way; lower() is what v1's lc() used, preserving which cross-spellings match. Strips to a FIXED POINT. A single strip().strip(".") leaves periods-around-whitespace half done ('. a .' -> ' a '), so a value that is re-normalized later -- on unpickle, or by a second add() -- would change under its owner. v1 never re-normalized, so this only matters now that storage and match-time share one fold.""" word = word.lower() while True: stripped = word.strip().strip(".") if stripped == word: return word word = stripped def _title_key(words: Iterable[str]) -> str: """The given_name_titles lookup key for a run of title words. A multi-word title is matched as one key ('lt col'), so the fold has to run per word and rejoin -- _normalize on the whole phrase would leave interior periods. Defined once because it is built at match time (post_rules) and at translation time (the v1 facade's first_name_titles), and a divergence between the two fails silently: the entry simply stops matching. Words that fold away are DROPPED, not joined as empty. Keeping the gap makes the fold non-idempotent -- 'lt .' would store 'lt ', which match time can never build (post_rules joins token texts, and a lone '.' is not a title token), so the entry is inert. Storage re-runs this fold on unpickle and on every dataclasses.replace, so a value that changes under a second pass is one Lexicon later rejects as "not written by this version". _normalize converges for the same reason; so must anything built on top of it.""" return " ".join(filter(None, (_normalize(w) for w in words))) def _reject_buffer(value: object, label: str, plural: str) -> None: """Binary sequences iterate to INTS, so every downstream entry check reports a byte value -- "must be strings, got 100" for b'dr', where 100 is 'd'. That names neither the cause nor the fix. v1 shipped a decode hint for this (#238), and parse() and the facade both carry one, so no config entry point should be the odd one out. """ if isinstance(value, (bytes, bytearray, memoryview)): raise TypeError( f"{label} must be an iterable of {plural}, not " f"{type(value).__name__} -- decode first, e.g. " f"raw.decode('utf-8')" ) def _warn_dead_entry(message: str) -> None: # A fixed stacklevel always lands on library internals: the call # depth differs per entry point (Lexicon(), add(), unpickle, # dataclasses.replace, and the v1 shim's lazy snapshot -- built on # the first parse after a Constants mutation, several facade frames # below the user's own add()). Walk out of this module (and # dataclasses' replace frames, and the facade layer that builds # lexicons on the caller's behalf) so the warning points at the # caller's own line. level = 2 frame: FrameType | None = sys._getframe(1) while frame is not None and frame.f_globals.get("__name__") in ( __name__, "dataclasses", "nameparser._config_shim", "nameparser._facade"): frame, level = frame.f_back, level + 1 warnings.warn(message, UserWarning, stacklevel=level) def _normset( entries: Iterable[str], field_name: str, warn: bool = True, ) -> frozenset[str]: # Reject a bare str before iterating: iterating "dr" would silently # yield the single characters {'d', 'r'} -- the set(str) footgun on # the primary customization surface. if isinstance(entries, str): raise TypeError( f"Lexicon.{field_name} must be an iterable of strings, " f"not a bare string" ) _reject_buffer(entries, f"Lexicon.{field_name}", "strings") # A Mapping would silently contribute only its keys; a dict here # almost always means the caller confused this field with # capitalization_exceptions. if isinstance(entries, Mapping): raise TypeError( f"Lexicon.{field_name} must be an iterable of strings, not a " f"mapping (only capitalization_exceptions holds key->value pairs)" ) try: items = tuple(entries) # materialize once; entries may be a generator except TypeError: raise TypeError( f"Lexicon.{field_name} must be an iterable of strings, " f"got {entries!r}" ) from None normalized = set() for w in items: if not isinstance(w, str): raise TypeError( f"Lexicon.{field_name} entries must be strings, got {w!r}" ) # given_name_titles is the one field whose entries are matched as # a multi-word run, so it folds per word: stored as the same key # post_rules builds, or 'lt. col' would be kept verbatim and # never match anything (a silent no-op on the config surface). # Every other field holds single words, where the two folds # agree. n = _title_key(w.split()) if field_name == "given_name_titles" \ else _normalize(w) # "." or "" is a data bug (stray split artifact, empty CSV # cell); dropping it silently would also let a data-module typo # vanish instead of failing CI. if not n: raise ValueError( f"Lexicon.{field_name} entry {w!r} normalizes to empty " f"(lowercase + strip periods/whitespace leaves nothing)" ) # Every field but given_name_titles is matched one word at a # time, so a multi-word entry can never match -- the library # itself shipped eight such dead entries for years (repaired # 2026-07-26). Warn, never raise: an inert entry produces # nothing, and the given_name_titles precedent says a raise # here costs working configurations (see __post_init__). # warn=False is _edit()'s pass (both ops): add() warns via the # new instance's __post_init__; remove() stores nothing, so # warning there would name entries the caller is trying to get # RID of, with "split it" advice that makes no sense for a # no-op. if (warn and field_name != "given_name_titles" # interior whitespace test; split() covers all Unicode # whitespace and n != "".join(n.split())): _warn_dead_entry( f"Lexicon.{field_name} entries are matched one word at " f"a time; multi-word entry {w!r} can never match. " f"Split it into separate entries") normalized.add(n) return frozenset(normalized) def _normpairs( raw: Mapping[str, str] | Iterable[tuple[str, str]], ) -> tuple[tuple[str, str], ...]: """Canonicalize capitalization_exceptions input: _normset's sibling for the one pair-valued field. Dedupes on the NORMALIZED key so the tuple and the derived map always agree ("Ph.D." and "phd" collide after normalization); last occurrence wins, matching dict semantics and the right-bias rule used elsewhere.""" if isinstance(raw, str): raise TypeError( "capitalization_exceptions must be a mapping or an " "iterable of (key, value) pairs, not a bare string" ) _reject_buffer(raw, "capitalization_exceptions", "(key, value) pairs") pairs = raw.items() if isinstance(raw, Mapping) else raw try: pairs = iter(pairs) except TypeError: raise TypeError( "capitalization_exceptions must be a mapping or an " f"iterable of (key, value) pairs, got {raw!r}" ) from None deduped: dict[str, str] = {} for entry in pairs: # A 2-char str entry would unpack "ab" into ("a", "b") # silently, so reject str outright; other mis-shapes would # otherwise surface as bare unpack errors. if isinstance(entry, str): raise TypeError( f"capitalization_exceptions entries must be " f"(key, value) pairs, got {entry!r}" ) try: k, v = entry except (TypeError, ValueError): raise TypeError( f"capitalization_exceptions entries must be " f"(key, value) pairs, got {entry!r}" ) from None if not isinstance(k, str) or not isinstance(v, str): raise TypeError( f"capitalization_exceptions entries must be " f"str -> str, got {k!r}: {v!r}" ) normalized_key = _normalize(k) if not normalized_key: raise ValueError( f"capitalization_exceptions key {k!r} normalizes to " f"empty (lowercase + strip periods/whitespace leaves " f"nothing)" ) # capitalized() looks words up one at a time (the _WORD regex # never yields spaces), so a multi-word key is unreachable. # interior whitespace test; split() covers all Unicode whitespace if normalized_key != "".join(normalized_key.split()): _warn_dead_entry( f"capitalization_exceptions keys are matched one word " f"at a time; multi-word key {k!r} can never match. " f"Split it into per-word entries") deduped[normalized_key] = v return tuple(sorted(deduped.items())) @dataclass(frozen=True, slots=True) class Lexicon: """The vocabulary a parser matches against: which words are titles, particles, suffixes, and so on. Immutable and hashable. Start from :meth:`default` (the shipped vocabulary) or :meth:`empty`, derive variants with :meth:`add` / :meth:`remove` / ``|`` (union), and pass the result to ``Parser(lexicon=...)``. Entries are normalized at construction -- lowercased, edge periods stripped -- so matching is case-insensitive. Vocabulary entries are single words -- a multi-word entry warns at construction and can never match (``given_name_titles``, matched as a space-joined run, is the one exception). Field docs below show examples, not full contents; inspect any field's shipped vocabulary directly, e.g. ``Lexicon.default().conjunctions``.""" #: Pre-nominal titles ("dr", "sir", "capt", ...). Full default #: list: :data:`~nameparser.config.titles.TITLES`. titles: frozenset[str] = frozenset() #: Titles whose single following name reads as a GIVEN name #: ("sheikh", "sister", ...) rather than a family name. Full #: default list: :data:`~nameparser.config.titles.FIRST_NAME_TITLES`. given_name_titles: frozenset[str] = frozenset() #: Post-nominal acronym suffixes, matched with or without periods #: ("phd" matches "PhD" and "Ph.D."). Full default list: #: :data:`~nameparser.config.suffixes.SUFFIX_ACRONYMS`. suffix_acronyms: frozenset[str] = frozenset() #: Post-nominal word suffixes ("jr", "esquire", "iii", ...). Full #: default list: #: :data:`~nameparser.config.suffixes.SUFFIX_NOT_ACRONYMS`. suffix_words: frozenset[str] = frozenset() #: Subset of suffix_acronyms counted as suffixes only when written #: WITH periods -- their bare forms are common surnames ("ma", #: "do": "Jack Ma" keeps his family name). Full default list: #: :data:`~nameparser.config.suffixes.SUFFIX_ACRONYMS_AMBIGUOUS`. suffix_acronyms_ambiguous: frozenset[str] = frozenset() #: Family-name particles that chain onto the following piece #: ("van", "de", "bin", ...). Full default list: #: :data:`~nameparser.config.prefixes.PREFIXES`. particles: frozenset[str] = frozenset() #: Subset of particles that can also BE a given name: a leading #: one reads as given and records a particle-or-given ambiguity #: ("Van Johnson", but also "Van Buren"). No constant of its own #: -- the default derives #: as particles minus #: :data:`~nameparser.config.prefixes.NON_FIRST_NAME_PREFIXES` #: (which marks the opposite, never-given subset). particles_ambiguous: frozenset[str] = frozenset() #: Words or characters that join surrounding pieces into one #: ("and", "&", "y", "и", ...). Full default list: #: :data:`~nameparser.config.conjunctions.CONJUNCTIONS`. conjunctions: frozenset[str] = frozenset() #: Given-name prefixes that bind to the following word to form one #: given name ("abdul" -> "Abdul Salam"); never standalone names. #: Full default list: #: :data:`~nameparser.config.bound_first_names.BOUND_FIRST_NAMES`. bound_given_names: frozenset[str] = frozenset() #: Marker words introducing a birth surname, routed to the maiden #: field ("née", "geb.", "roz.", ...). Full default list: #: :data:`~nameparser.config.maiden_markers.MAIDEN_MARKERS`. maiden_markers: frozenset[str] = frozenset() #: Family names for the unspaced-name segmentation stage (#271), #: matched longest-first against the start of the FIRST token #: written wholly in a script :attr:`Policy.segment_scripts #: ` activates. The default #: carries the Korean census list #: (:data:`~nameparser.config.surnames.KOREAN_SURNAMES`); Chinese #: surnames ship in locales.ZH because Han segmentation is opt-in. surnames: frozenset[str] = frozenset() #: Honorifics that may be peeled off the END of a name token #: (#308), matched longest-first: 田中さん splits into 田中 and さん #: before the tokens are classified. Every entry must also be a #: :attr:`suffix_words` entry -- the peeled tail is claimed by #: suffix classification like any other post-nominal. Deliberately #: NOT gated on :attr:`Policy.segment_scripts #: ` (unlike :attr:`surnames` #: above): 田中さん peels under the default policy, where HAN is in #: no activation set, because a tail entry carries its own license #: to fire. Entries are matched against the RAW token text, and #: only within a name containing a non-ASCII character, so an ASCII #: or mixed-case entry is at best conditionally active -- a ``"Jr"`` #: entry is stored ``"jr"`` and matches only lowercase text. The #: field is effectively CJK-scoped in 2.1, which is what the shipped #: vocabulary is. Full default list: #: :data:`~nameparser.config.suffixes.GLUED_HONORIFICS`. honorific_tails: frozenset[str] = frozenset() #: Lowercase word -> exact-cased replacement used by capitalized() #: ("phd" -> "Ph.D."). Pair-valued: change it with #: dataclasses.replace(), not add()/remove(); read it as a mapping #: via capitalization_exceptions_map. Full default mapping: #: :data:`~nameparser.config.capitalization.CAPITALIZATION_EXCEPTIONS`. # Canonical storage: sorted tuple of (key, value) pairs. The # constructor tolerates any Mapping (or pair iterable) at runtime and # canonicalizes here; this closes the caller-aliasing hole and keeps # Lexicon hashable. Read via capitalization_exceptions_map. capitalization_exceptions: tuple[tuple[str, str], ...] = () _cap_map: Mapping[str, str] = field( init=False, repr=False, compare=False, hash=False, default_factory=lambda: MappingProxyType({})) def __post_init__(self) -> None: for name in _VOCAB_FIELDS: object.__setattr__(self, name, _normset(getattr(self, name), name)) canonical = _normpairs(self.capitalization_exceptions) object.__setattr__(self, "capitalization_exceptions", canonical) object.__setattr__(self, "_cap_map", MappingProxyType(dict(canonical))) for marker, base, why in _SUBSET_FIELDS: orphans = getattr(self, marker) - getattr(self, base) if orphans: raise ValueError( f"{marker} marks a subset of {base}; " f"not in {base}: {', '.join(sorted(orphans))} " f"({why}). Add them to {base}, or drop them from " f"{marker}" ) # NOT validated: given_name_titles against titles. The lookup # key is the space-joined run of Role.TITLE tokens, built by the # parse rather than drawn from this vocabulary, and a # conjunction inside a run is itself tagged Role.TITLE -- so # "sir and dame" is a matchable key whose middle word is in # conjunctions, not titles. A whole-entry check rejected # multi-word entries; a per-word check rejected that one. An # unreachable entry here is inert: nothing consults it, nothing # misparses. That is the cheap failure, and guarding it proved # the expensive one -- three working configurations broken # across two attempts. Do not add a third. # # The v2 form of prefixes.py's NON_FIRST_NAME_PREFIXES-disjoint- # from-BOUND_FIRST_NAMES assertion. That module guards its own # data at import; this guards vocabulary a caller supplies. contradictory = ( self.bound_given_names & self.particles) - self.particles_ambiguous if contradictory: raise ValueError( f"bound_given_names entries that are also particles must be " f"in particles_ambiguous; not in particles_ambiguous: " f"{', '.join(sorted(contradictory))}. A particle that never " f"starts a given name cannot also bind one — add them to " f"particles_ambiguous, or drop them from bound_given_names" ) # suffix_as_written ORs the acronym and word branches, so a word # membership bypasses the period gate the ambiguous set exists # to impose: listing 'ma' in suffix_words would make "Jack Ma" # read as a suffix and lose the family name. gate_bypassed = self.suffix_acronyms_ambiguous & self.suffix_words if gate_bypassed: raise ValueError( f"an ambiguous suffix acronym must not also be a suffix " f"word; in both: {', '.join(sorted(gate_bypassed))}. The " f"word branch matches without periods, which bypasses the " f"period gate suffix_acronyms_ambiguous exists to impose" ) # -- constructors ---------------------------------------------------- @classmethod def empty(cls) -> Lexicon: return cls() @classmethod def default(cls) -> Lexicon: return _default_lexicon() # -- dunders ---------------------------------------------------------- def _deltas_from(self, baseline: Lexicon) -> list[tuple[str, int, int]]: deltas = [] for name in _VOCAB_FIELDS + ("capitalization_exceptions",): mine = set(getattr(self, name)) theirs = set(getattr(baseline, name)) added, removed = len(mine - theirs), len(theirs - mine) if added or removed: deltas.append((name, added, removed)) return deltas def __repr__(self) -> str: # Bounded: renders only which fields deviate from the nearer of # the two named constructors and by how many entries -- never the # entries themselves (design rule, see nameparser._types module # docstring). Diffing empty()-built lexicons against default() # would tell the wrong story ("default minus the entire # default vocabulary"). if self == Lexicon.default(): return "Lexicon(default)" if self == Lexicon.empty(): return "Lexicon(empty)" candidates = [(label, self._deltas_from(baseline)) for label, baseline in (("default", Lexicon.default()), ("empty", Lexicon.empty()))] label, deltas = min( candidates, key=lambda c: sum(a + r for _, a, r in c[1])) rendered = ", ".join( name + ": " + "".join( part for part, n in ((f"+{a}", a), (f"-{r}", r)) if n) for name, a, r in deltas) return f"Lexicon({label} + {rendered})" def __getstate__(self) -> dict[str, object]: # _cap_map is a MappingProxyType, which pickle rejects; ship every # other slot and rebuild the proxy from the canonical tuple on load. return {f.name: getattr(self, f.name) for f in dataclasses.fields(self) if f.name != "_cap_map"} def __setstate__(self, state: dict[str, object]) -> None: # Fail at the unpickle site if the state comes from a different # Lexicon field layout (version skew) -- silently loading it # would defer the failure to some distant attribute read. # Message kept in sync with _types._guarded_setstate by design # (layering keeps this module import-free of _types). expected = {f.name for f in dataclasses.fields(Lexicon)} - {"_cap_map"} if set(state) != expected: missing = ", ".join(sorted(expected - set(state))) or "none" unexpected = ", ".join(sorted(set(state) - expected)) or "none" raise ValueError( f"incompatible Lexicon pickle: missing fields: {missing}; " f"unexpected fields: {unexpected}" ) for name, value in state.items(): object.__setattr__(self, name, value) # Re-run construction validation rather than trusting the blob. # The layout check above catches SHAPE skew; this catches # CONTENT skew, which is likelier -- particles_ambiguous flipped # meaning between v1's never-given set and v2's may-be-given set # without changing its name, so an old pickle would otherwise # load a semantically inverted lexicon in silence. Rebuilds # _cap_map for free, which this used to do by hand. self.__post_init__() # __post_init__ also normalizes, and quietly accepting a # rewritten value would make this a place where caller data # changes without a word. A pickle this library wrote is always # normalized already (_normalize converges), so a difference # here means the state came from somewhere else -- say so while # the offending entries can still be named. # One pass per field, in _VOCAB_FIELDS declaration order. cast is # safe: __post_init__ just ran _normset over each of these and # raised for anything that was not an iterable of str. drifted = [] for name in _VOCAB_FIELDS: lost = frozenset( cast("Iterable[str]", state[name])) - getattr(self, name) if lost: drifted.append(f"{name}: {', '.join(sorted(lost))}") # the pair field too, or ten fields raise and the eleventh is # quietly re-canonicalized (keys normalized, sorted, deduped) given_pairs = cast("Iterable[tuple[str, str]]", state["capitalization_exceptions"]) if tuple(given_pairs) != self.capitalization_exceptions: drifted.append("capitalization_exceptions") if drifted: raise ValueError( "incompatible Lexicon pickle: entries are not normalized " f"({'; '.join(drifted)}); this state was not written by " "this version of nameparser" ) def __or__(self, other: Lexicon) -> Lexicon: if not isinstance(other, Lexicon): return NotImplemented updates: dict[str, object] = { name: getattr(self, name) | getattr(other, name) for name in _VOCAB_FIELDS } # right-biased on key conflicts, mirroring later-wins for scalars merged = dict(self._cap_map) | dict(other._cap_map) updates["capitalization_exceptions"] = tuple(sorted(merged.items())) return dataclasses.replace(self, **updates) # type: ignore[arg-type] # -- properties ------------------------------------------------------- @property def capitalization_exceptions_map(self) -> Mapping[str, str]: return self._cap_map # -- editing ---------------------------------------------------------- def _edit(self, op: str, entries: Mapping[str, Iterable[str]]) -> Lexicon: updates: dict[str, frozenset[str]] = {} for name, words in entries.items(): if name == "capitalization_exceptions": raise TypeError( "capitalization_exceptions holds key->value pairs; " "use dataclasses.replace(lexicon, " "capitalization_exceptions={...}) instead of " f"{op}()" ) if name not in _VOCAB_FIELDS: raise TypeError( f"unknown Lexicon field {name!r}; valid fields: " f"{', '.join(_VOCAB_FIELDS)}" ) current: frozenset[str] = getattr(self, name) # warn=False: this pass only computes the new set membership, # never stores it directly. add() still warns exactly once, # from the replaced instance's own __post_init__ -> _normset; # remove() never reaches __post_init__ with the dead entry # (it is subtracted out here), so it stays silent -- correct, # since a removal stores nothing a warning could be about. # That silence covers the entry BEING REMOVED only: a # different multi-word entry still stored re-warns from the # derived instance's __post_init__, since the warning is # per-construction by design. normalized = _normset(words, name, warn=False) updates[name] = (current | normalized if op == "add" else current - normalized) # mypy's dataclasses.replace() typing checks a **dict's single # value type against every field's type (it can't see which keys # are actually present behind the unpack), so a homogeneous # frozenset[str] dict is flagged against the tuple/Mapping-typed # capitalization_exceptions/_cap_map fields even though this dict # never contains those keys (guarded above). return dataclasses.replace(self, **updates) # type: ignore[arg-type] def add(self, **entries: Iterable[str]) -> Lexicon: return self._edit("add", entries) def remove(self, **entries: Iterable[str]) -> Lexicon: return self._edit("remove", entries) @functools.cache def _default_lexicon() -> Lexicon: # v1 data modules are the single source of vocabulary through 2.x. from nameparser.config.bound_first_names import BOUND_FIRST_NAMES from nameparser.config.capitalization import CAPITALIZATION_EXCEPTIONS from nameparser.config.conjunctions import CONJUNCTIONS from nameparser.config.maiden_markers import MAIDEN_MARKERS from nameparser.config.prefixes import NON_FIRST_NAME_PREFIXES, PREFIXES from nameparser.config.suffixes import ( GLUED_HONORIFICS, SUFFIX_ACRONYMS, SUFFIX_ACRONYMS_AMBIGUOUS, SUFFIX_NOT_ACRONYMS, ) from nameparser.config.surnames import KOREAN_SURNAMES from nameparser.config.titles import FIRST_NAME_TITLES, TITLES # v1 data modules export plain `set[str]`; wrap each at this call site # so the strictly-typed frozenset[str] fields never see a bare set. # keep in sync with _config_shim.Constants._snapshot() (pinned by the # default-Constants equality test in tests/v2/test_config_shim.py) return Lexicon( titles=frozenset(TITLES), given_name_titles=frozenset(FIRST_NAME_TITLES), suffix_acronyms=frozenset(SUFFIX_ACRONYMS), suffix_words=frozenset(SUFFIX_NOT_ACRONYMS), suffix_acronyms_ambiguous=frozenset(SUFFIX_ACRONYMS_AMBIGUOUS), particles=frozenset(PREFIXES), # FLIPPED from v1: v1 marks the never-given subset; v2 marks the # may-be-given subset (migration: complement translation). particles_ambiguous=frozenset(PREFIXES - NON_FIRST_NAME_PREFIXES), conjunctions=frozenset(CONJUNCTIONS), bound_given_names=frozenset(BOUND_FIRST_NAMES), maiden_markers=frozenset(MAIDEN_MARKERS), # surnames.py is born frozen (#293) -- no call-site wrap needed, # unlike the v1 modules above (their wraps drop when #293 lands) surnames=KOREAN_SURNAMES, honorific_tails=frozenset(GLUED_HONORIFICS), # pass canonical pair-tuples so this strictly-typed call site never # feeds a Mapping to the tuple-annotated field; __post_init__ # still tolerates a Mapping at runtime for interactive use capitalization_exceptions=tuple(sorted(CAPITALIZATION_EXCEPTIONS.items())), ) python-nameparser-2.1.0/nameparser/_locale.py000066400000000000000000000065071523530431700213060ustar00rootroot00000000000000"""The Locale type: a named delta over (Lexicon, Policy). A Locale dissolves at parser construction (parser_for, a later plan): lexicon fragments union onto the base, the PolicyPatch folds via apply_patch. Packs are pure data; they have no privileged capabilities. Layering: imports _lexicon and _policy only (enforced by tests/v2/test_layering.py). """ from __future__ import annotations import dataclasses import re from dataclasses import dataclass from nameparser._lexicon import Lexicon from nameparser._policy import UNSET, PolicyPatch from nameparser._types import _guarded_getstate, _guarded_setstate @dataclass(frozen=True, slots=True) class Locale: """A named, shareable bundle of vocabulary and behavior for a naming tradition: a lexicon fragment plus a policy patch, applied together by :func:`nameparser.parser_for`. The packs shipped with nameparser live in :mod:`nameparser.locales`; building your own needs no registration -- construct one and pass it to ``parser_for``. See the :doc:`locale packs guide ` for using, creating, and contributing packs.""" #: Identifier, lowercase ``[a-z0-9_]+`` (e.g. "ru", "tr_az"). code: str #: Vocabulary ADDED to the base parser's lexicon (unioned; a pack #: never removes base vocabulary). lexicon: Lexicon #: Behavior changes folded onto the base policy (set-valued fields #: union; scalars override, later pack wins). policy: PolicyPatch = PolicyPatch() # in the class body so @dataclass(slots=True) keeps them __getstate__ = _guarded_getstate __setstate__ = _guarded_setstate def __post_init__(self) -> None: if not isinstance(self.code, str): raise TypeError( f"Locale.code must be a str, got {self.code!r}" ) if not self.code.strip(): raise ValueError( f"Locale.code must be a non-empty string, got {self.code!r}" ) if self.code != self.code.lower(): raise ValueError( f"Locale.code must be lowercase, got {self.code!r}" ) # Codes are registry keys (parser_for, third-party packs): every # accepted character is supported forever, so pin the charset # while relaxing later is still compatible. One separator only -- # allowing '-' too would make tr-az and tr_az distinct keys. if not re.fullmatch(r"[a-z0-9_]+", self.code): raise ValueError( f"Locale.code must match [a-z0-9_]+, got {self.code!r}" ) if not isinstance(self.lexicon, Lexicon): raise TypeError( f"Locale.lexicon must be a Lexicon, got {self.lexicon!r}" ) if not isinstance(self.policy, PolicyPatch): raise TypeError( f"Locale.policy must be a PolicyPatch, got {self.policy!r}" ) def __repr__(self) -> str: # Bounded: shows the code and which Policy fields the patch sets, # never the Lexicon contents or the patched values themselves # (design rule, see nameparser._types module docstring). patched = [f.name for f in dataclasses.fields(self.policy) if getattr(self.policy, f.name) is not UNSET] suffix = f": {', '.join(patched)}" if patched else "" return f"Locale({self.code!r}{suffix})" python-nameparser-2.1.0/nameparser/_parser.py000066400000000000000000000364641523530431700213500ustar00rootroot00000000000000"""Parser and the module-level parse() for the 2.0 API. Layering: sits on _types/_lexicon/_policy/_locale/_pipeline; never imports _render or the v1 facade (enforced by tests/v2/test_layering.py). _default_parser is THE one sanctioned module-level global (conventions §8): a functools.cache'd frozen Parser over default config. """ from __future__ import annotations import dataclasses import functools import warnings from dataclasses import dataclass, field from nameparser._lexicon import Lexicon from nameparser._locale import Locale from nameparser._pipeline import run from nameparser._pipeline._assemble import assemble from nameparser._pipeline._state import ParseState from nameparser._pipeline._vocab import _SCRIPT_MATCHERS from nameparser._policy import UNSET, Policy, PolicyPatch, _Unset, apply_patch from nameparser._types import ( FOLDED_TAG, ParsedName, Segmenter, Token, _guarded_getstate, _guarded_setstate, _validated_field_strings, ) @dataclass(frozen=True, slots=True) class Parser: """A configured name parser: a :class:`Lexicon` (vocabulary) plus a :class:`Policy` (behavior), both defaulted when omitted. Build one when you need non-default configuration, build it once, and call :meth:`parse` many times -- it is immutable and thread-safe. An optional keyword-only ``segmenter`` (a :data:`~nameparser.Segmenter`) plugs in outside knowledge of where an unspaced CJK token divides -- Japanese kanji names, which no bundled list can settle. It is consulted only for a token the segmentation stage gates in and the vocabulary DECLINES, so a locale pack's surnames always win where they match; returning None declines in turn and the token stays whole. Two promises narrow when one is supplied (locales spec §4): parse-totality gains its one exception -- an exception raised by the segmenter propagates, because a user-supplied callable's own error is a user-code error, not a content error -- and this Parser pickles only if its segmenter does (a module-level function pickles; a lambda or closure does not). With no segmenter, both promises hold unconditionally: all validity checking happens at construction, so a Parser that constructs successfully cannot fail at parse time on any str content. (The None field defaults resolve in __post_init__; after construction lexicon and policy are always non-None -- the annotations state the steady-state truth, hence the assignment ignores on the defaults.)""" lexicon: Lexicon = None # type: ignore[assignment] # None -> default() policy: Policy = None # type: ignore[assignment] # None -> Policy() #: An optional hook supplying outside knowledge of where an unspaced #: token divides -- see the class docstring; None leaves such tokens #: whole. Keyword-only, so the reserved growth stays additive #: (locales spec §4): positional construction keeps its two-argument #: shape. segmenter: Segmenter | None = field(default=None, kw_only=True) # in the class body so @dataclass(slots=True) keeps them __getstate__ = _guarded_getstate __setstate__ = _guarded_setstate def __post_init__(self) -> None: if self.lexicon is None: object.__setattr__(self, "lexicon", Lexicon.default()) elif not isinstance(self.lexicon, Lexicon): raise TypeError( f"lexicon must be a Lexicon or None, got {self.lexicon!r}") if self.policy is None: object.__setattr__(self, "policy", Policy()) elif not isinstance(self.policy, Policy): raise TypeError( f"policy must be a Policy or None, got {self.policy!r}") if self.segmenter is not None and not callable(self.segmenter): raise TypeError( f"segmenter must be callable or None, got {self.segmenter!r}") # A configuration gap that used to be silent (#272's API, made # loud before 2.1.0): segment_scripts can activate a script # that neither the vocabulary nor a segmenter can ever divide # -- the JA pack's whole shape, when its segmenter is # forgotten. The parser then behaves identically to a working # one minus the feature, which reads as "not working" with no # signal why. Statically decidable here, so say it here; a # warning rather than an error because the inert pack is a # pinned, deliberate property (a JA registration must be safe # without the extra), and warnings are filterable by the rare # caller who wants exactly that. if self.segmenter is None: uncovered = sorted( script.value for script in self.policy.segment_scripts if not any(_SCRIPT_MATCHERS[script](entry) for entry in self.lexicon.surnames)) if uncovered: names = ", ".join(uncovered) one = len(uncovered) == 1 # the ja hint only where a Japanese script is among the # dead ones -- a hangul-only gap (a from-scratch # lexicon under the default policy) has different # remedies, and pointing it at ja_segmenter would be a # non sequitur ja_hint = ( " For Japanese, pass " "segmenter=locales.ja_segmenter() (install with: " "pip install 'nameparser[ja]')." if {"han", "hiragana", "katakana"} & set(uncovered) else "") warnings.warn( f"Policy.segment_scripts activates {names} but the " f"vocabulary has no surnames in " f"{'that script' if one else 'those scripts'} " f"and no segmenter is configured: unspaced names " f"written in {'it' if one else 'them'} will never " f"divide. Supply covering surnames, pass a " f"segmenter, or deactivate with " f"Policy(segment_scripts=frozenset()).{ja_hint}", UserWarning, stacklevel=3) def __repr__(self) -> str: # composes the two bounded component reprs (spec §2 reprs); the # segmenter shows by name, and only when one is set, so the # default Parser's repr is unchanged seg = "" if self.segmenter is not None: # never repr() the callable itself: a partial reprs its # bound arguments and a callable instance its address, both # unbounded -- the class name is the bounded fallback name = (getattr(self.segmenter, "__qualname__", None) or type(self.segmenter).__name__) seg = f", segmenter={name}" return f"Parser({self.lexicon!r}, {self.policy!r}{seg})" def parse(self, text: str) -> ParsedName: """Parse one name string into a :class:`ParsedName`. Never raises on string content (unparseable input yields empty fields plus ambiguities); non-str raises TypeError eagerly, with a decode hint for bytes (bytes support ended with 1.x). The one exception to that totality is a configured ``segmenter``, whose own exceptions propagate (see the class docstring).""" if isinstance(text, bytes): raise TypeError( "parse() takes str, not bytes -- decode first, e.g. " "raw.decode('utf-8')") if not isinstance(text, str): raise TypeError(f"parse() takes str, got {text!r}") state = ParseState(original=text, lexicon=self.lexicon, policy=self.policy, segmenter=self.segmenter) return assemble(run(state)) # -- editing ---------------------------------------------------------- def revise(self, name: ParsedName, **fields: str) -> ParsedName: """:meth:`ParsedName.replace` with this parser's vocabulary: each value is tokenized and classified by a full sub-parse, so the stable tags survive and the tag-driven views (family_particles, initials(), the suffix join) behave as if the text had been parsed. The value is classified ON ITS OWN, though -- a word whose reading depends on surrounding context may classify differently than it would in place (a standalone "B. S." reads as initials, not a suffix run). The sub-parse's role choices and ambiguities are discarded -- every harvested token takes the named field's role -- and its structural behavior applies: delimiter characters do not become tokens, and a maiden marker is consumed as in parsing -- mid-value always, and leading a DELIMITED value under a policy routing that pair to maiden, where "(née Jones)" revises to "Jones" while the bare "née Jones" keeps its marker, a leading marker in an undelimited value being no marker at all (#329). Tokens are synthetic (span=None); original is unchanged; a value with no name content (empty, whitespace, or punctuation only) clears the field; ambiguities referencing replaced tokens are dropped.""" if not isinstance(name, ParsedName): raise TypeError(f"revise() takes a ParsedName, got {name!r}") replaced = _validated_field_strings(fields) harvested = { role: tuple( Token(t.text, None, role, t.tags - {FOLDED_TAG}) for t in self.parse(value).tokens) for role, value in replaced.items() } return name._with_field_tokens(harvested) # -- comparison ------------------------------------------------------- def matches(self, a: str | ParsedName, b: str | ParsedName) -> bool: """Component-wise case-insensitive comparison of two names, parsing str arguments with THIS parser. :meth:`ParsedName.matches` parses its str argument with the DEFAULT parser instead -- for names parsed with a custom Parser, use this method.""" if isinstance(a, str): a = self.parse(a) elif not isinstance(a, ParsedName): raise TypeError(f"matches() takes str or ParsedName, got {a!r}") if isinstance(b, str): b = self.parse(b) elif not isinstance(b, ParsedName): raise TypeError(f"matches() takes str or ParsedName, got {b!r}") return a.comparison_key() == b.comparison_key() # -- rendering delegates ---------------------------------------------- def capitalized(self, name: ParsedName, *, force: bool = False) -> ParsedName: """:meth:`ParsedName.capitalized` under THIS parser's lexicon. The no-argument form of that method uses the DEFAULT lexicon -- for names parsed with a custom Parser, use this method.""" if not isinstance(name, ParsedName): raise TypeError(f"capitalized() takes a ParsedName, got {name!r}") return name.capitalized(self.lexicon, force=force) @functools.cache def _default_parser() -> Parser: return Parser() def parse(text: str) -> ParsedName: """Parse a name with the default configuration and return a :class:`ParsedName`. Equivalent to ``Parser().parse(text)``; build your own :class:`Parser` (or use :func:`parser_for`) for custom vocabulary or behavior. Never raises on string content.""" return _default_parser().parse(text) def parser_for(*locales: Locale, base: Parser | None = None, segmenter: Segmenter | None | _Unset = UNSET) -> Parser: """Lexicon fragments unioned left-to-right onto base's; policy patches applied left-to-right (later wins; set-valued fields union per the patch metadata). Validation errors raised while applying a pack are wrapped with that pack's identity (spec §4 amendment) -- PolicyPatch validates lazily, so with stacked packs the raw error would otherwise point at nothing. Two packs setting the same SCALAR field is a declared conflict: UserWarning, later wins. A ``segmenter`` is passed straight through to the built Parser -- ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` is how a pack and a segmenter combine, since packs are pure data and cannot supply one. The argument has THREE states, the same :data:`~nameparser.UNSET` spelling a PolicyPatch field uses, because None is a meaningful value here and not an absence: omitted (UNSET) carries base's segmenter through unchanged; a callable OVERRIDES base's (later wins, the rule scalar policy fields follow); and an explicit ``None`` CLEARS base's, which is how you derive an unsegmented parser from a segmented one without rebuilding its lexicon and policy by hand.""" if base is not None and not isinstance(base, Parser): raise TypeError(f"base must be a Parser or None, got {base!r}") for loc in locales: if not isinstance(loc, Locale): raise TypeError(f"parser_for() takes Locale packs, got {loc!r}") lexicon = base.lexicon if base is not None else Lexicon.default() policy = base.policy if base is not None else Policy() # Resolved here rather than at the return because the return builds # a FRESH Parser: any field not listed there silently takes its # default, and a dropped segmenter would be invisible. UNSET, not # None, is what "not given" means -- None is the CLEAR request, and # collapsing the two would make an explicit # parser_for(..., segmenter=None) silently inherit the very # segmenter it was asked to drop. if segmenter is UNSET: segmenter = base.segmenter if base is not None else None scalar_setters: dict[str, str] = {} for loc in locales: for f in dataclasses.fields(PolicyPatch): if f.metadata.get("compose") == "union": continue if getattr(loc.policy, f.name) is UNSET: continue if f.name in scalar_setters and scalar_setters[f.name] != loc.code: warnings.warn( f"locale {loc.code!r} overrides scalar policy field " f"{f.name!r} already set by locale " f"{scalar_setters[f.name]!r}; later wins", UserWarning, stacklevel=2) scalar_setters[f.name] = loc.code try: lexicon = lexicon | loc.lexicon policy = apply_patch(policy, loc.policy) except (TypeError, ValueError) as exc: # safe: every raise in the apply path (Policy.__post_init__, # Lexicon.__or__, apply_patch) is a PLAIN TypeError/ValueError -- # a subclass with extra mandatory args would break this rewrap raise type(exc)( f"while applying locale {loc.code!r}: {exc}") from exc # Construction warnings (the segmenterless-activation check in # Parser.__post_init__) re-emit from THIS frame: its stacklevel is # sized for direct Parser(...) construction, and through this # function's extra frame the default single-line rendering would # point into the library instead of at the caller -- the exact # call the message tells them to change. with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") built = Parser(lexicon=lexicon, policy=policy, segmenter=segmenter) for w in caught: warnings.warn(w.message, stacklevel=2) return built python-nameparser-2.1.0/nameparser/_pipeline/000077500000000000000000000000001523530431700212725ustar00rootroot00000000000000python-nameparser-2.1.0/nameparser/_pipeline/__init__.py000066400000000000000000000026541523530431700234120ustar00rootroot00000000000000"""The 2.0 parse pipeline: pure stages folded over ParseState. Stage names and ParseState are NOT public API. The stage list is data; runners other than the plain fold (explain(), parse_all()) arrive in 2.x minors without changing any stage signature. Layering: imports only nameparser._pipeline.* (enforced by tests/v2/test_layering.py). """ from __future__ import annotations from collections.abc import Callable from nameparser._pipeline._assign import assign from nameparser._pipeline._classify import classify from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._group import group from nameparser._pipeline._post_rules import post_rules from nameparser._pipeline._script_segment import script_segment from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState from nameparser._pipeline._tokenize import tokenize #: The full eight-stage fold. STAGES: tuple[Callable[[ParseState], ParseState], ...] = ( extract_delimited, tokenize, segment, script_segment, classify, group, assign, post_rules, ) def run(state: ParseState) -> ParseState: """Fold the stages over the initial state. Pure: each stage returns a new ParseState; parse is total over str input, so no stage may raise on the *content* of a name (malformed policy/config is a separate, allowed failure mode).""" for stage in STAGES: state = stage(state) return state python-nameparser-2.1.0/nameparser/_pipeline/_assemble.py000066400000000000000000000105031523530431700235750ustar00rootroot00000000000000"""Not a stage: converts the final ParseState into a public ParsedName. Consumes: tokens (all roles set), dropped, ambiguities (by index). Produces: a validated ParsedName -- the constructor re-checks every invariant (span order/bounds, ambiguity subset), so a pipeline bug that would produce an invalid result dies HERE, not in a renderer three layers away. Structural tokens (dropped maiden markers) are omitted, like delimiter characters. A main-stream token that somehow reaches here with no role takes Role.GIVEN -- parse must never raise on the *content* of a name, so the fallback is deliberately boring rather than an exception. This should never fire in practice (assign/group must set a role on every main-stream token); it exists as a last-resort safety net against a pipeline bug, not as sanctioned behavior for real input -- see test_assemble_falls_back_to_given_for_unassigned_role in tests/v2/pipeline/test_assemble.py. """ from __future__ import annotations from nameparser._pipeline._state import ParseState from nameparser._types import Ambiguity, ParsedName, Role, Token def assemble(state: ParseState) -> ParsedName: dropped = set(state.dropped) final: dict[int, Token] = {} for i, t in enumerate(state.tokens): if i in dropped: continue role = t.role if t.role is not None else Role.GIVEN final[i] = Token(t.text, t.span, role, t.tags) # No alphanumeric character among the SURVIVING tokens means no # name: a bare '.' or '- -' is not a person. v1 kept such input # (parse('.') -> first '.'); 2.0 empties it so bool() stays an # honest "did I get a name?" check. isalnum() is Unicode-aware, so # every real name in any script has content and only pure # punctuation/symbols empty out. (Embedded junk in a name with # content -- 'John . Smith' -- is left alone: that parse is truthy, # so bool() is not misled.) # # This read "anywhere" until #329, and the two said the same thing # for as long as nothing could take a content-bearing token away. # Dropping a maiden marker inside a delimited clause can, and # "(née —)" under Policy(maiden_delimiters=...) is the input where # the readings part: the marker is the clause's only alnum token, # so once it goes structural the em dash is all that survives and # the whole parse empties -- where 1.4.0 and pre-#329 both gave # maiden 'née —'. Deliberate, not fallout: a dropped marker is # structural like a delimiter character, and brackets plus a marker # plus a dash name no one, exactly as "(-)" already named no one. # A guard keyed on what ELSE is in the clause would be a different # rule, and would leave maiden holding marker-plus-punctuation. # Pinned by cases.py's maiden_marker_delimited_content_free and, # for the other side of it -- the same clause inside a name, where # the em dash survives as the maiden value because Jane Smith # carries the content -- ..._content_free_in_a_name beside it. # # The TOKENS go, not the diagnostics: this drops the name, and # "was the input malformed?" is the one question still worth # answering about it -- most of all here, where there is no parse # left to infer it from. parse('(') keeps its unbalanced-delimiter # report, pointing at no token because no token survived. contentless = not any( c.isalnum() for t in final.values() for c in t.text) if contentless: final = {} ambiguities = [] for pending in state.ambiguities: materialized = tuple(final[i] for i in pending.indices if i in final) if pending.indices and not materialized and not contentless: # every referent was dropped: the ambiguity describes # nothing that survives assembly. Born-empty ambiguities # (unbalanced delimiters) are token-independent and kept. # Not so when the whole name was emptied just above -- the # referents did not lose a contest, they were discarded # wholesale, and the report still describes the input. continue ambiguities.append( Ambiguity(pending.kind, pending.detail, materialized)) return ParsedName(original=state.original, tokens=tuple(final.values()), ambiguities=tuple(ambiguities)) python-nameparser-2.1.0/nameparser/_pipeline/_assign.py000066400000000000000000000413671523530431700233020ustar00rootroot00000000000000"""Stage: assign. Consumes: pieces + piece_tags (grouped), segments, structure, tokens. Produces: tokens with roles set on every main-stream token. Reads: Policy.name_order (#270) and Policy.script_orders (#271, which overrides it when every name piece is written wholly in one script, or in the Han/Hiragana/Katakana repertoire the #272 kana license shares across pieces); token/piece tags; Lexicon only through tags already applied by classify (plus the leading-title period rule). Ports v1's assignment loops. NO_COMMA (per name_order): leading title pieces chain while no given-position name has been seen (a title needs a following piece, unless the whole name is one title); then positional assignment per name_order with the trailing-suffix rule: the piece from which everything after is a strict suffix is the last name-position piece, the rest are suffixes. The v1 single-name+ nickname rule lives here (plan deviation #2): one non-title piece plus a nonempty nickname puts that piece in FAMILY. FAMILY_COMMA: segment 0 wholly FAMILY (v1 parity); segment 1 gets leading titles, then given, then middles with strict-suffix pieces to suffix; segments 2+ are suffixes (lenient -- segment already flagged non-suffixy ones COMMA_STRUCTURE). SUFFIX_COMMA: segment 0 as NO_COMMA; segments 1+ wholly SUFFIX. Emits PARTICLE_OR_GIVEN when the leading name piece is a lone particles_ambiguous token with more pieces following ("Van Johnson") -- whatever role name_order assigns that position. """ from __future__ import annotations import dataclasses import re from nameparser._pipeline._vocab import ( effective_script, is_initial_shaped, is_suffix_lenient, resolve_script_set, ) from nameparser._pipeline._group import ( _is_suffix_piece, _is_title_piece, ) from nameparser._pipeline._state import ( ParseState, PendingAmbiguity, Structure, WorkToken, ) from nameparser._policy import Policy, Script from nameparser._types import AmbiguityKind, Role # Ported verbatim from v1 (nameparser/config/regexes.py # "period_abbreviation" and "roman_numeral") -- layering forbids the # config import; keep in sync by hand. _PERIOD_ABBREV = re.compile(r'^[^\W\d_]{2,}\.$') _ROMAN = re.compile(r'^(X|IX|IV|V?I{0,3})$', re.I) def _set_roles(tokens: list[WorkToken], piece: tuple[int, ...], role: Role) -> None: for i in piece: tokens[i] = dataclasses.replace(tokens[i], role=role) def _is_leading_title(piece: tuple[int, ...], ptags: frozenset[str], tokens: list[WorkToken]) -> bool: if _is_title_piece(piece, ptags, tokens): return True return (len(piece) == 1 and bool(_PERIOD_ABBREV.match(tokens[piece[0]].text))) def _peel_leading_titles(pieces: tuple[tuple[int, ...], ...], ptags: tuple[frozenset[str], ...], tokens: list[WorkToken]) -> int: """Assign TITLE to the leading title pieces and return the first non-title index. A title needs a following piece, unless the whole segment is one title (v1 parity).""" n = 0 while n < len(pieces): if ((n + 1 < len(pieces) or len(pieces) == 1) and _is_leading_title(pieces[n], ptags[n], tokens)): _set_roles(tokens, pieces[n], Role.TITLE) n += 1 continue break return n def _effective_order(policy: Policy, pieces: list[tuple[int, ...]], tokens: list[WorkToken], *, dot_divided: bool) -> tuple[Role, Role, Role]: """script_orders resolution (#271): when every name piece is written wholly in ONE script that has an entry, that script's order governs the positional read; anything else -- Latin, mixed scripts, no entry -- falls back to name_order. A 间隔号-divided name (`dot_divided`, #298) suppresses the whole lookup first: the dot marks a transcription -- playing the role pure katakana plays in the kana license, orthography naming the convention -- so the license yields to name_order. Piece-level, after title/suffix peeling: 'Dr. 毛泽东' is a wholly-Han NAME under a Latin title. Kana-licensed tokens (高橋みなみ, #272) resolve to HIRAGANA the same way a wholly-Han or wholly-Hangul token resolves to its own script -- and so does a kana-licensed NAME split across separately single-script PIECES ('高橋 みなみ', Han piece plus Hiragana piece): resolve_script_set generalizes the license from one token's characters to the whole found-script set below, which is why Han+Hangul ('毛 김') still declines even though both individually read family-first -- the license is specific to the Han/Hiragana/Katakana repertoire, not "the entries happen to agree". Naming note, since the two are easy to conflate: THIS function resolves the ORDER for a whole name; `_vocab.effective_script` resolves the SCRIPT for a single token. This function calls that one per token below. """ # #298 transcription marker -- see the docstring; codepoint-scoped # (only U+00B7 records, spec 2026-07-30 decision 5) if dot_divided: return policy.name_order if not policy.script_orders: return policy.name_order # Collect every token's script rather than comparing pairwise as # tokens are seen: the kana license needs the WHOLE set (a Han # piece and a Hiragana piece only license together, never one at a # time), so resolution is deferred to resolve_script_set below. found: set[Script] = set() for piece in pieces: for i in piece: script = effective_script(tokens[i].text) if script is None: # Latin, mixed, or a script with no entry: never a key return policy.name_order found.add(script) resolved = resolve_script_set(found) if resolved is None: # e.g. Han+Hangul: two scripts, neither the kana license's # Han/Hiragana/Katakana repertoire -- no single tradition return policy.name_order return next((order for s, order in policy.script_orders if s is resolved), policy.name_order) def _name_positions(order: tuple[Role, Role, Role], count: int) -> list[Role]: """Roles for `count` name pieces (titles/suffixes already peeled), per name_order. GIVEN_FIRST: given, middles..., family. FAMILY_FIRST: family, given, middles... FAMILY_FIRST_GIVEN_LAST: family, middles..., given. One piece takes order[0]'s role; two pieces take order[0] and the other primary.""" first, second = order[0], order[1] if count == 1: return [first] if first is Role.GIVEN: # GIVEN_FIRST return ([Role.GIVEN] + [Role.MIDDLE] * (count - 2) + [Role.FAMILY]) if second is Role.GIVEN: # FAMILY_FIRST return ([Role.FAMILY, Role.GIVEN] + [Role.MIDDLE] * (count - 2)) return ([Role.FAMILY] + [Role.MIDDLE] * (count - 2) # F_F_GIVEN_LAST + [Role.GIVEN]) def _assign_main(seg_idx: int, state: ParseState, tokens: list[WorkToken], ambiguities: list[PendingAmbiguity]) -> None: pieces = state.pieces[seg_idx] ptags = state.piece_tags[seg_idx] has_nickname = any(t.role is Role.NICKNAME for t in tokens) n = _peel_leading_titles(pieces, ptags, tokens) rest = list(range(n, len(pieces))) if not rest: return # group-flagged suffix pieces (the ph-d merge) are suffixes at ANY # position -- v1's fix_phd extracted the credential from the string # before parsing, so position never mattered (PR review I3) flagged = [k for k in rest if "suffix" in ptags[k]] for k in flagged: _set_roles(tokens, pieces[k], Role.SUFFIX) rest = [k for k in rest if "suffix" not in ptags[k]] if not rest: return # v1 nickname rule (plan deviation #2): v1's p_len == 1 counted # the WHOLE segment before any title peeling -- 'Xyz. (Bud) Smith' # has two pieces, so the title peel wins and Smith stays the given # name (pinned live 2026-07-17) if len(pieces) == 1 and len(rest) == 1 and has_nickname: _set_roles(tokens, pieces[rest[0]], Role.FAMILY) return # peel the trailing suffix run: k = first index in rest from which # every piece is a strict suffix (v1's are_suffixes tail rule, with # the roman-numeral special: a final roman numeral after a # non-initial piece is a suffix) # every bare ambiguous acronym the peel had to resolve -- one # coin-flip each, in either direction, so this collects rather than # overwrites. Deferred to after assignment because the wording reads # the role back, and which role "not peeled" means depends on # name_order. (The roman-numeral fork needs no such deferral and is # reported at its trigger below.) ambiguous_picks: list[tuple[int, ...]] = [] k = len(rest) while k > 0: piece = pieces[rest[k - 1]] tags = ptags[rest[k - 1]] if _is_suffix_piece(piece, tags, tokens): k -= 1 continue # is_initial_shaped, not the "initial" tag: this asks whether # the preceding piece looks like part of an initial run, which # is a question about layout, and #320 narrowed the tag to # initials that can really stand in for a name. Reading the tag # here made '씨.' stop suppressing the fork and cost 'John 씨. V' # its family name. if (k == len(rest) and k >= 2 and len(piece) == 1 and _ROMAN.match(tokens[piece[0]].text) and not is_initial_shaped( tokens[pieces[rest[k - 2]][0]].text)): # a trailing single letter is a name part unless it happens # to be a roman numeral -- and V/X/I are ordinary middle # initials, so taking it as a suffix is a call, not a fact ambiguities.append(PendingAmbiguity( AmbiguityKind.SUFFIX_OR_NAME, f"{tokens[piece[0]].text!r} is a roman numeral, so it " f"reads as a generational suffix; any other single " f"letter there would be a middle initial", tuple(piece))) k -= 1 continue # A bare ambiguous acronym ("MA", not "M.A.") is a credential # only when peeling it still leaves a given AND a family name. # With two pieces, "one of them is a credential" is the less # likely reading, so it stays the family name -- "Jack MA" is a # person, "John Smith MA" is a person with a degree. This is # v1's reserve_last narrowed to the ambiguous set: 2.0 # deliberately peels UNambiguous suffixes even when nothing is # left ("Smith PhD" -> suffix, a classified fix), because there # the vocabulary is not in doubt. bare_ambiguous = (len(piece) == 1 and "vocab:suffix-ambiguous" in tokens[piece[0]].tags) # k < 2 means it is the only piece left, which is not the fork # this reports. if bare_ambiguous and k >= 2: ambiguous_picks.append(tuple(piece)) if k >= 3: # peeling still leaves given + family k -= 1 continue break name_pieces, suffix_pieces = rest[:k], rest[k:] if not name_pieces and suffix_pieces: # everything suffix-shaped after titles: first one is the name name_pieces, suffix_pieces = suffix_pieces[:1], suffix_pieces[1:] # AFTER both peels, and load-bearing: the script test sees the NAME # pieces only, so a Latin title or suffix ('Dr. 毛 泽东', '毛 泽东, # PhD') cannot make a wholly-CJK name look mixed-script. order = _effective_order(state.policy, [pieces[i] for i in name_pieces], tokens, dot_divided=bool(state.interpunct_offsets)) roles = _name_positions(order, len(name_pieces)) for pos, piece_idx in enumerate(name_pieces): _set_roles(tokens, pieces[piece_idx], roles[pos]) for piece_idx in suffix_pieces: _set_roles(tokens, pieces[piece_idx], Role.SUFFIX) for piece in ambiguous_picks: # every pick is in rest, so the loops above just gave it a role token = tokens[piece[0]] assert token.role is not None taken, declined = ( ("a suffix", "a name part") if token.role is Role.SUFFIX else (f"a {token.role.value} name", "a post-nominal")) ambiguities.append(PendingAmbiguity( AmbiguityKind.SUFFIX_OR_NAME, f"{token.text!r} written without periods is both a " f"post-nominal and an ordinary name; read as {taken} " f"rather than {declined}", piece)) # leading ambiguous particle read as a name (#121 surfaced) if name_pieces: head = pieces[name_pieces[0]] if (len(head) == 1 and len(name_pieces) > 1 and "vocab:particle-ambiguous" in tokens[head[0]].tags): ambiguities.append(PendingAmbiguity( AmbiguityKind.PARTICLE_OR_GIVEN, f"leading {tokens[head[0]].text!r} may be a family-name " f"particle; read as a given name", tuple(head))) def assign(state: ParseState) -> ParseState: tokens = list(state.tokens) ambiguities = list(state.ambiguities) if not state.segments: return state if state.structure is Structure.NO_COMMA: _assign_main(0, state, tokens, ambiguities) tail = len(state.segments) elif state.structure is Structure.SUFFIX_COMMA: _assign_main(0, state, tokens, ambiguities) tail = 1 else: # FAMILY_COMMA # PARTICLE_OR_GIVEN is deliberately not emitted here: after a # comma the family is already fixed, so a leading given-position # particle is not meaningfully ambiguous. script_orders is not # consulted here for the parallel reason -- the comma already # fixed the family, so there is no positional read to override. # v1: "lastname part may have suffixes in it" -- the first # piece is always the family even if suffix-shaped; any later # strict-suffix piece goes to SUFFIX per piece ('Smith Jr., # John' -> family=Smith, suffix=Jr.) fam_pieces = state.pieces[0] fam_tags = state.piece_tags[0] for k, piece in enumerate(fam_pieces): if k > 0 and _is_suffix_piece(piece, fam_tags[k], tokens): _set_roles(tokens, piece, Role.SUFFIX) else: _set_roles(tokens, piece, Role.FAMILY) if len(state.segments) > 1: pieces = state.pieces[1] ptags = state.piece_tags[1] n = _peel_leading_titles(pieces, ptags, tokens) given_done = False for m in range(n, len(pieces)): # v1 walk order: the first non-title piece is ALWAYS # the given, before any suffix check -- # 'Hardman, RN - CRNA' keeps first='RN'. One deliberate # 2.0 deviation, classified fix(comma-family): when that # piece is the segment's ONLY piece and unambiguously # suffix-shaped ('Andrews, M.D.'), it is a suffix -- v1 # made it the given. if not given_done: if (m == len(pieces) - 1 and _is_suffix_piece(pieces[m], ptags[m], tokens)): _set_roles(tokens, pieces[m], Role.SUFFIX) continue _set_roles(tokens, pieces[m], Role.GIVEN) given_done = True continue # trailing piece of a two-part name is unambiguously # positioned: v1 accepts the lenient test there # ('Smith, John V' -> suffix='V', #144); with a third # comma part the trailing token is more likely a middle # initial, so strict only last_of_two = (m == len(pieces) - 1 and len(state.segments) == 2) if _is_suffix_piece(pieces[m], ptags[m], tokens) or ( last_of_two and len(pieces[m]) == 1 and is_suffix_lenient( tokens[pieces[m][0]].text, state.lexicon)): _set_roles(tokens, pieces[m], Role.SUFFIX) else: _set_roles(tokens, pieces[m], Role.MIDDLE) tail = 2 # segments past the structure's name segments are wholly suffixes for seg_idx in range(tail, len(state.segments)): for piece in state.pieces[seg_idx]: _set_roles(tokens, piece, Role.SUFFIX) return dataclasses.replace(state, tokens=tuple(tokens), ambiguities=tuple(ambiguities)) python-nameparser-2.1.0/nameparser/_pipeline/_classify.py000066400000000000000000000077021523530431700236260ustar00rootroot00000000000000"""Stage: classify. Consumes: tokens. Produces: tokens with vocabulary tags added (text/span/role unchanged). Reads: every Lexicon vocabulary field; no Policy FIELD is consulted (is_initial does consult the _policy module's _NO_INITIALS constant, which is not configuration -- nothing here varies by Policy value). Tags emitted -- stable (API): "particle", "conjunction", "initial"; namespaced (unstable): "vocab:title", "vocab:given-title", "vocab:suffix", "vocab:suffix-word", "vocab:suffix-ambiguous", "vocab:particle-ambiguous", "vocab:bound-given", "vocab:maiden-marker". "vocab:suffix" means "counts as a suffix as written": unambiguous suffix vocabulary, or an ambiguous acronym written with periods ('M.A.' yes, 'Ma' no -- 'Ma' gets only "vocab:suffix-ambiguous"). The initial veto is assign's job, not classify's: 'V' carries both "vocab:suffix" and "initial". """ from __future__ import annotations import dataclasses from nameparser._lexicon import _normalize from nameparser._pipeline._state import ( ParseState, PendingAmbiguity, WorkToken, ) from nameparser._types import AmbiguityKind, Role from nameparser._pipeline._vocab import ( is_initial, period_joined_vocab, suffix_as_written, ) def _tags_for(token: WorkToken, state: ParseState) -> frozenset[str]: lex = state.lexicon n = _normalize(token.text) tags = set(token.tags) if n in lex.titles: tags.add("vocab:title") if n in lex.given_name_titles: tags.add("vocab:given-title") if suffix_as_written(n, token.text, lex): tags.add("vocab:suffix") if n in lex.suffix_words: tags.add("vocab:suffix-word") if n in lex.suffix_acronyms_ambiguous: tags.add("vocab:suffix-ambiguous") if n in lex.particles: tags.add("particle") if n in lex.particles_ambiguous: tags.add("vocab:particle-ambiguous") if n in lex.conjunctions and not is_initial(token.text): # v1's is_conjunction excludes initials: 'e.' in 'john e. smith' # is a middle initial, not the Spanish conjunction 'e' tags.add("conjunction") if n in lex.bound_given_names: tags.add("vocab:bound-given") if n in lex.maiden_markers: tags.add("vocab:maiden-marker") if is_initial(token.text): tags.add("initial") # v1's period-joined derivation (parse_pieces): a token with a # period not at the end, ANY of whose period chunks is a title, is # a title as a whole ('Lt.Gov.', and by the ANY rule 'Mr.Smith'); # else ANY suffix chunk makes it a suffix ('JD.CPA'). Title wins # (v1's continue). Skipped when the whole token already matched. if "vocab:title" not in tags and "vocab:suffix" not in tags: derived = period_joined_vocab(token.text, lex) if derived == "title": tags.add("vocab:title") elif derived == "suffix": tags.add("vocab:suffix") return frozenset(tags) def classify(state: ParseState) -> ParseState: tokens = tuple( dataclasses.replace(t, tags=_tags_for(t, state)) for t in state.tokens) # Delimited content whose vocabulary cannot settle it: extract's # escape sends an UNambiguous suffix straight through ("(MBA)" -> # suffix) and keeps everything else as a nickname, so an AMBIGUOUS # acronym in there was a coin the parser had to call. Reported here # rather than at the escape itself, which runs before tokenize and # so has no token index to point at. ambiguities = list(state.ambiguities) for i, token in enumerate(tokens): if (token.role is Role.NICKNAME and "vocab:suffix-ambiguous" in token.tags): ambiguities.append(PendingAmbiguity( AmbiguityKind.SUFFIX_OR_NICKNAME, f"delimited {token.text!r} is also a post-nominal; read " f"as a nickname rather than a suffix", (i,))) return dataclasses.replace(state, tokens=tokens, ambiguities=tuple(ambiguities)) python-nameparser-2.1.0/nameparser/_pipeline/_extract.py000066400000000000000000000264121523530431700234620ustar00rootroot00000000000000"""Stage: extract_delimited. Consumes: ParseState.original. Produces: extracted (role + inner span per delimited region), masked (full regions incl. delimiter chars, skipped by tokenize), UNBALANCED_DELIMITER ambiguities for opens with no close. A Role.MAIDEN region is the WHOLE inner span, marker word included -- nothing here strips one. classify tags a marker inside it like any other token, and group drops it from a multi-token clause (#329). Reads: Policy.nickname_delimiters, Policy.maiden_delimiters, and Lexicon.suffix_words / suffix_acronyms / suffix_acronyms_ambiguous through _suffix_shaped, which lets a clause's CONTENT overrule the delimiter's verdict: 'Andrew Perkins (MBA)' is not a nickname, so only the two delimiter spans are masked and the content rejoins the token stream. Matching rules (the #273 mechanism): one left-to-right scan over the original text, no nesting. At each position the LEFTMOST boundary-valid opener among ALL configured pairs wins -- position order, never pair order, decides between conventions that share a character in opposite roles ('“' closes „…“ but opens “…”; '»' closes «…» but opens »…«), so "Hans „Erster“ und “Zweiter” Müller" extracts both names. For pairs whose open == close (quotes), the open must sit at a word boundary (start of text or after whitespace) and the close before one (end, whitespace, or a comma char) -- this is what keeps the apostrophe in O'Connor literal. Empty enclosures are masked (removed from the token stream) but extract nothing; delimiter characters inside a matched region are literal content for every other pair. Bucket precedence is NOT decided here: Policy canonicalizes overlap away before parsing (a pair listed in maiden_delimiters is dropped from the effective nickname set -- maiden wins; the v1 facade restores v1's nickname-wins reading via a pre-subtraction in _config_shim), so the two buckets are always disjoint by the time this stage runs. The nickname-before-maiden candidate order below is only a same-position tie-break for exotic configs where two pairs share an OPEN character. """ from __future__ import annotations import bisect import dataclasses import functools from nameparser._lexicon import Lexicon, _normalize from nameparser._pipeline._state import ( COMMA_CHARS, ParseState, PendingAmbiguity, ) from nameparser._types import AmbiguityKind, Role, Span def _suffix_shaped(content: str, lexicon: Lexicon) -> bool: """v1 parse_nicknames' escape (parser.py:1125-1141): an unambiguous suffix_words member (edge-normalized), an unambiguous acronym (period-free form), or anything ending in a period. No initial veto -- v1 deliberately skipped it here.""" stripped = _normalize(content) acronym = stripped.replace(".", "") return (stripped in lexicon.suffix_words or (acronym in lexicon.suffix_acronyms and acronym not in lexicon.suffix_acronyms_ambiguous) or content.endswith(".")) def _open_ok(text: str, i: int) -> bool: return i == 0 or text[i - 1].isspace() def _close_ok(text: str, j: int, width: int) -> bool: k = j + width return k >= len(text) or text[k].isspace() or text[k] in COMMA_CHARS # Delimiters that also occur INSIDE and at the end of real name parts, # so a dangling one is literal rather than unbalanced. Only the straight # apostrophe qualifies: quotes do not appear inside names, so the same # position with a quote genuinely is ambiguous. #273 dropped the curly # apostrophe from the defaults outright for this reason; the straight # one has to stay a delimiter (v1's quoted_word), so it is carved out # here instead. Deliberately not widened to a configured delimiter set. WORD_INTERNAL_DELIMITERS = frozenset({"'"}) def _word_internal(text: str, j: int, close: str) -> bool: """A word-internal delimiter directly after a word character is part of the word, not a dangling close -- "Mari' Aube'", "Ali Baba'".""" return (close in WORD_INTERNAL_DELIMITERS and j > 0 and (text[j - 1].isalnum() or text[j - 1] == ".")) def _overlaps(span: Span, taken: list[Span], starts: list[int]) -> bool: """`taken` sorted and non-overlapping (both hold by construction), `starts` its start offsets. Bisect rather than scan: the closer sweep tests one span per delimiter character found, so a linear probe is quadratic in the number of matched pairs (400 pairs spent 5.4ms here, against 2.5ms before the sweep existed). Same idiom, and the same reason, as the origin resolution in _tokenize.""" i = bisect.bisect_right(starts, span.start) - 1 # the only candidates are the last span starting at or before us and # its successor -- anything earlier ends before it, anything later # starts after us for k in (i, i + 1): if 0 <= k < len(taken) and span.start < taken[k].end and ( taken[k].start < span.end): return True return False @functools.lru_cache(maxsize=128) def _delimiter_chars( nickname_pairs: frozenset[tuple[str, str]], maiden_pairs: frozenset[tuple[str, str]], ) -> frozenset[str]: """Every character appearing in any configured delimiter, cached on the (hashable) policy frozensets: the common no-delimiter name pays one isdisjoint() instead of a per-pair scan.""" return frozenset( ch for pairs in (nickname_pairs, maiden_pairs) for pair in pairs for part in pair for ch in part ) def _unmatched(open_: str, offset: int) -> tuple[int, PendingAmbiguity]: return (offset, PendingAmbiguity( AmbiguityKind.UNBALANCED_DELIMITER, f"unmatched {open_!r} at offset {offset}; treated as literal text", origin=offset, )) def extract_delimited(state: ParseState) -> ParseState: text = state.original policy = state.policy if _delimiter_chars(policy.nickname_delimiters, policy.maiden_delimiters).isdisjoint(text): return state # Candidate order matters only as a same-position tie-break (see # module docstring); the scan itself is position-driven. order = tuple( (role, open_, close) for role, pairs in ((Role.NICKNAME, policy.nickname_delimiters), (Role.MAIDEN, policy.maiden_delimiters)) for open_, close in sorted(pairs) ) extracted: list[tuple[Role, Span]] = [] masked: list[Span] = [] # candidates, not final: each carries the offset of the unmatched # open so ones consumed by a later match can be filtered at the end unbalanced: list[tuple[int, PendingAmbiguity]] = [] # per-candidate cursor cache: next boundary-valid open at or after # the position it was computed for. find() calls only ever move # forward, keeping the whole scan linear in len(text) per pair. cursors: dict[tuple[Role, str, str], int] = {} exhausted: set[tuple[Role, str, str]] = set() pos = 0 while pos < len(text): best: tuple[int, Role, str, str] | None = None for key in order: if key in exhausted: continue _, open_, close = key i = cursors.get(key, -2) if i != -1 and i < pos: i = text.find(open_, pos) while (i != -1 and open_ == close and not _open_ok(text, i)): i = text.find(open_, i + 1) cursors[key] = i if i != -1 and (best is None or i < best[0]): best = (i, *key) if best is None: break i, role, open_, close = best j = text.find(close, i + len(open_)) while (open_ == close and j != -1 and not _close_ok(text, j, len(close))): j = text.find(close, j + 1) if j == -1: # No (boundary-valid) close exists anywhere to the right -- # the walk above ran to end of text -- so every remaining # open of this pair is unmatched too. Record them all in # one forward pass and retire the pair; other pairs keep # scanning from the same position. unbalanced.append(_unmatched(open_, i)) scan = i + len(open_) while (k := text.find(open_, scan)) != -1: if open_ != close or _open_ok(text, k): unbalanced.append(_unmatched(open_, k)) scan = k + 1 exhausted.add((role, open_, close)) continue inner = Span(i + len(open_), j) if inner.start < inner.end and _suffix_shaped( text[inner.start:inner.end], state.lexicon): # v1 parse_nicknames: suffix-shaped delimited content is # left IN PLACE (undelimited) for normal downstream parsing # -- 'Andrew Perkins (MBA)' keeps MBA a suffix, not a # nickname. Spans index the original (anti-#100), so the v2 # spelling masks only the two delimiter spans and lets the # inner content join the main token stream. masked.append(Span(i, i + len(open_))) masked.append(Span(j, j + len(close))) else: if inner.start < inner.end: extracted.append((role, inner)) masked.append(Span(i, j + len(close))) # position-driven scanning makes overlapping matches # impossible by construction: every later open is found at or # after this match's end pos = j + len(close) extracted.sort(key=lambda pair: pair[1]) masked.sort() # An unmatched-open candidate whose character was consumed by a # later successful match (the bulk pass above runs ahead of the # main scan) is literal content there, not a dangling delimiter. # offsets already claimed as unbalanced, by an open above or by a # close in the sweep below -- either way, do not report one twice reported = {offset for offset, _ in unbalanced} mask_starts = [s.start for s in masked] ambiguities = [ a for offset, a in unbalanced if not _overlaps(Span(offset, offset + 1), masked, mask_starts)] # The scan above is opener-driven: it searches for an open and then # looks rightward for its close, so a close with no open to its # LEFT is never in its search space. Sweep for those separately -- # they signal the same malformed input, and the kind's contract has # always covered them ("opened without closing, or closed without # opening"). Same boundary test as the matched path, which is what # keeps the apostrophe in "O'connor" out of it. # Distinct closes only: the defaults list '”' twice (from both # ('“','”') and ('”','”')), and a repeat can only rediscover # offsets the first pass already handled. for close in sorted({c for _, _, c in order}): start = 0 while (j := text.find(close, start)) != -1: start = j + 1 if (j in reported or not _close_ok(text, j, len(close)) or _word_internal(text, j, close) or _overlaps(Span(j, j + len(close)), masked, mask_starts)): continue reported.add(j) ambiguities.append(_unmatched(close, j)[1]) return dataclasses.replace( state, extracted=tuple(extracted), masked=tuple(masked), ambiguities=state.ambiguities + tuple(ambiguities)) python-nameparser-2.1.0/nameparser/_pipeline/_group.py000066400000000000000000000461111523530431700231420ustar00rootroot00000000000000"""Stage: group. Consumes: tokens (classified), segments, structure, extracted (the role + inner span per delimited region, for the #329 pass below -- the only stage after tokenize that reads it). Produces: pieces + piece_tags per segment (runs of token indices -- tokens are NEVER joined into strings: the anti-#100 invariant); maiden tail tokens get role=MAIDEN; marker tokens land in dropped. Reads: token tags (from classify), and Policy.extra_suffix_delimiters for the tail-segment handling below -- no other Policy field. The v1 "derived titles/prefixes" registration becomes piece_tags entries -- per-parse state that dissolves with the state (v1 kept per-parse sets for the same reason). Reads Policy.extra_suffix_delimiters: tail segments drop delimiter-core tokens (v1 suffix_delimiter parity). Ports v1's join_on_conjunctions + prefix chains + _join_bound_first_name plus three additions: the "Ph. D."-split merge (v1 fix_phd, recorded plan deviation #1), the maiden-marker consuming rule (#274: marker plus following pieces until a suffix become maiden; the marker itself is structural, like a delimiter char, and is dropped from assembly), and the same marker dropped inside EXTRACTED maiden content (#329), which #274 cannot reach because extract's content never enters pieces. """ from __future__ import annotations import bisect import dataclasses from collections.abc import Sequence, Set from enum import IntEnum from nameparser._pipeline._state import ( ParseState, PendingAmbiguity, Structure, WorkToken, ) from nameparser._pipeline._vocab import D as _D from nameparser._pipeline._vocab import PH as _PH from nameparser._pipeline._vocab import delimiter_cores from nameparser._types import AmbiguityKind, Role # the credential-pair regexes live in _vocab, whose own # is_wholly_suffix merges the same pair -- and since #319 that # predicate has TWO callers to stay in sync with, segment's # suffix-comma structure test and script_segment's decline of a # wholly-suffix post-comma run, both of which see the merged reading Piece = list[int] class BoundJoin(IntEnum): """v1 _join_bound_first_name's reserve_last, as the three states it actually has. IntEnum: the value IS the non_suffix threshold, so the >= comparison below reads unchanged.""" DISABLED = 0 # the FAMILY_COMMA family segment (v1 never joined it) LENIENT = 2 # FAMILY_COMMA's post-comma segment (reserve_last=False) STRICT = 3 # main segments (reserve_last=True: keep a family piece) def _is_title_piece(piece: Sequence[int], ptags: Set[str], tokens: Sequence[WorkToken]) -> bool: if "title" in ptags: return True return len(piece) == 1 and "vocab:title" in tokens[piece[0]].tags def _is_prefix_piece(piece: Sequence[int], ptags: Set[str], tokens: Sequence[WorkToken]) -> bool: if "prefix" in ptags: return True return len(piece) == 1 and "particle" in tokens[piece[0]].tags def _is_suffix_piece(piece: Sequence[int], ptags: Set[str], tokens: Sequence[WorkToken]) -> bool: if "suffix" in ptags: return True if len(piece) != 1: return False tags = tokens[piece[0]].tags return "vocab:suffix" in tags and "initial" not in tags def _is_conj_piece(piece: Sequence[int], ptags: Set[str], tokens: Sequence[WorkToken]) -> bool: if "conjunction" in ptags: return True return len(piece) == 1 and "conjunction" in tokens[piece[0]].tags def _is_rootname(piece: Sequence[int], ptags: Set[str], tokens: Sequence[WorkToken]) -> bool: if len(piece) == 1 and "initial" in tokens[piece[0]].tags: return False return not (_is_title_piece(piece, ptags, tokens) or _is_prefix_piece(piece, ptags, tokens) or _is_suffix_piece(piece, ptags, tokens)) def _group_segment(seg: tuple[int, ...], additional: int, tokens: Sequence[WorkToken], bound_join: BoundJoin = BoundJoin.STRICT, ambiguities: list[PendingAmbiguity] | None = None, ) -> tuple[list[Piece], list[set[str]]]: pieces: list[Piece] = [[i] for i in seg] ptags: list[set[str]] = [set() for _ in seg] # Out-parameter, same shape as _assign_main: forks are reported # where they are decided. A caller that passes None (or a throwaway # list) suppresses reporting -- see group() for when that applies. if ambiguities is None: ambiguities = [] def title(k: int) -> bool: return _is_title_piece(pieces[k], ptags[k], tokens) def prefix(k: int) -> bool: return _is_prefix_piece(pieces[k], ptags[k], tokens) def suffix(k: int) -> bool: return _is_suffix_piece(pieces[k], ptags[k], tokens) def conj(k: int) -> bool: return _is_conj_piece(pieces[k], ptags[k], tokens) def merge(lo: int, hi: int, add: Set[str] = frozenset(), drop: Set[str] = frozenset()) -> None: # pieces/ptags are parallel arrays; every merge must update # both in lockstep. # # Extend the first piece IN PLACE rather than rebuilding the # merged list. The obvious spelling -- # pieces[lo:hi] = [[i for p in pieces[lo:hi] for i in p]] # -- re-flattens everything accumulated so far on every call, so # a chain that merges into the same piece n times copies # 1+2+...+n and the stage goes quadratic in the length of the # chain. A conjunction run ("and " * n) does exactly that: it # measured 2.4x-2.9x per doubling against the 2.0x every other # shape holds. No piece list is aliased outside this function # (each starts as a fresh [i], and the callers only read # pieces[k] before a merge), so mutating is safe; verified # identical token/role/tag/span/ambiguity output over 54,877 # names. tests/v2/test_benchmark.py's "and " shape is the guard. # # Every call site passes lo < hi, and this REQUIRES it: with # lo >= hi the slice assignment would insert rather than # replace, putting a second reference to pieces[lo] into the # array, and the next merge to touch either index would extend # the same list twice. The old rebuild-a-fresh-list spelling # was harmless there. Keep the bound if you add a caller. combined = pieces[lo] for piece in pieces[lo + 1:hi]: combined.extend(piece) pieces[lo:hi] = [combined] ptags[lo:hi] = [(set().union(*ptags[lo:hi]) | add) - drop] # ph-d merge first: "Ph." "D." adjacent -> one suffix piece (plan # deviation #1; v1 fix_phd did this by regex on the raw string) k = 0 while k < len(pieces) - 1: a, b = pieces[k], pieces[k + 1] if (len(a) == 1 and len(b) == 1 and _PH.fullmatch(tokens[a[0]].text) and _D.fullmatch(tokens[b[0]].text)): merge(k, k + 2, add={"suffix"}) else: k += 1 if len(pieces) + additional >= 3: total = sum(_is_rootname(p, t, tokens) for p, t in zip(pieces, ptags)) + additional # contiguous conjunction runs merge first (v1: "of the") k = 0 while k < len(pieces) - 1: if conj(k) and conj(k + 1): merge(k, k + 2, add={"conjunction"}) else: k += 1 # each conjunction joins its neighbors (v1's Google Code issue 11 # carve-out, the "john e smith" bug: # a single-letter alphabetic conjunction in a short name is more # likely an initial) k = 0 while k < len(pieces): if not conj(k): k += 1 continue text = " ".join(tokens[i].text for i in pieces[k]) if len(text) == 1 and total < 4 and text.isalpha(): k += 1 continue start = max(0, k - 1) end = min(len(pieces), k + 2) neighbor = start if start < k else end - 1 derived = set() if title(neighbor): derived.add("title") if prefix(neighbor): derived.add("prefix") merge(start, end, add=derived) k = start + 1 # prefix chains: a non-leading prefix run absorbs everything to # the next prefix or suffix (v1's leading_first_name rule keeps # the first piece a name: "Van Johnson") k = 0 while k < len(pieces): if k == 0 or not prefix(k): k += 1 continue j = k + 1 while j < len(pieces) and prefix(j): j += 1 while j < len(pieces) and not prefix(j) and not suffix(j): j += 1 # The other half of PARTICLE_OR_GIVEN. _assign reports the # fork when an ambiguous particle stays a lone leading piece # ("Van Johnson" -> given); the chain here takes the # opposite branch whenever a title shifts it off index 0 # ("Dr. Van Johnson" -> family "Van Johnson"). A fork whose # two sides are decided in different stages needs an emitter # in each. # # j > k + 1 is what makes this a DECISION rather than a # shape: when the next piece is a suffix the inner scan # never advances, merge(k, k+1) folds a piece into itself, # and the particle stays a lone leading piece -- nothing # was chained, and _assign reports that case instead. # Without this the two emitters both fire on the same token. # (Tag test first: it is a set lookup and almost no name has # an ambiguous particle, while title() is a call per piece.) if (j > k + 1 and "vocab:particle-ambiguous" in tokens[pieces[k][0]].tags and all(title(x) for x in range(k))): i = pieces[k][0] ambiguities.append(PendingAmbiguity( AmbiguityKind.PARTICLE_OR_GIVEN, f"{tokens[i].text!r} was chained onto the following " f"name piece; it is also a given name in other " f"names", (i,))) merge(k, j, drop={"prefix"}) k += 1 # bound given names: the first non-title piece joins the next # ONCE (pairwise, v1 parity: 'Salem, Abdul Rahman Ahmed' keeps # Ahmed a middle name). BoundJoin encodes v1's reserve_last. first_name_k = next( (k for k in range(len(pieces)) if not title(k)), None) if (bound_join is not BoundJoin.DISABLED and first_name_k is not None and first_name_k + 1 < len(pieces) and len(pieces[first_name_k]) == 1 and "vocab:bound-given" in tokens[pieces[first_name_k][0]].tags): non_suffix = sum(1 for k in range(len(pieces)) if not title(k) and not suffix(k)) if non_suffix >= bound_join: merge(first_name_k, first_name_k + 2) return pieces, ptags def group(state: ParseState) -> ParseState: tokens = list(state.tokens) dropped = list(state.dropped) ambiguities = list(state.ambiguities) all_pieces: list[tuple[tuple[int, ...], ...]] = [] all_ptags: list[tuple[frozenset[str], ...]] = [] # v1 parity: additional_parts_count=1 applies only to FAMILY_COMMA # parts; the SUFFIX_COMMA pre-comma segment gets 0. additional = 1 if state.structure is Structure.FAMILY_COMMA else 0 # v1 expand_suffix_delimiter parity (#191): tail segments (wholly # consumed as suffixes by assign) drop delimiter-core tokens, the # same structural mechanism as the maiden marker below cores = delimiter_cores(state.policy.extra_suffix_delimiters) tail_start = {Structure.SUFFIX_COMMA: 1, Structure.FAMILY_COMMA: 2}.get(state.structure) family_comma = state.structure is Structure.FAMILY_COMMA for seg_idx, seg in enumerate(state.segments): if family_comma: bound_join = (BoundJoin.LENIENT if seg_idx == 1 else BoundJoin.DISABLED) else: bound_join = BoundJoin.STRICT # Suppressed after a family comma for the same reason _assign # suppresses it there: the family name is already fixed, so # there is no fork left to report. pieces, ptags = _group_segment( seg, additional, tokens, bound_join, None if family_comma else ambiguities) if tail_start is not None and seg_idx >= tail_start: # v1 renders each tail COMMA SEGMENT as one suffix entry # ('Smith, V MD' -> suffix 'V MD'); a delimiter core inside # a segment separates entries and is dropped, but a segment # that IS only the core stays whole (v1 expand() splits # within a part, never erases a lone part). Continuation # tokens within an entry take the stable "joined" tag so # the suffix view space-joins them (the fix_phd mechanism). entry_open = False kept: list[int] = [] for k in range(len(pieces)): is_core = (len(pieces[k]) == 1 and tokens[pieces[k][0]].text in cores and len(pieces) > 1) if is_core: dropped.extend(pieces[k]) entry_open = False continue kept.append(k) for pos, i in enumerate(pieces[k]): if entry_open or pos > 0: tokens[i] = dataclasses.replace( tokens[i], tags=tokens[i].tags | {"joined"}) # piece-level state: the NEXT piece continues this entry entry_open = True if len(kept) != len(pieces): pieces = [pieces[k] for k in kept] ptags = [ptags[k] for k in kept] # continuation tokens of a suffix-merged piece (the ph-d merge) # carry the stable "joined" tag: the suffix string view joins # SUFFIX tokens with ", ", and the tag lets it heal the split for piece, piece_tags_ in zip(pieces, ptags): if "suffix" in piece_tags_ and len(piece) > 1: for i in piece[1:]: tokens[i] = dataclasses.replace( tokens[i], tags=tokens[i].tags | {"joined"}) # maiden markers: a non-leading marker piece consumes following # pieces until a suffix; consumed tokens become MAIDEN, the # marker is dropped (#274) m = next( (k for k in range(1, len(pieces)) if len(pieces[k]) == 1 and "vocab:maiden-marker" in tokens[pieces[k][0]].tags), None) if m is not None: j = m + 1 consumed: list[int] = [] while j < len(pieces) and not _is_suffix_piece( pieces[j], ptags[j], tokens): consumed.extend(pieces[j]) j += 1 if consumed: dropped.extend(pieces[m]) for i in consumed: tokens[i] = dataclasses.replace( tokens[i], role=Role.MAIDEN) pieces[m:j] = [] ptags[m:j] = [] all_pieces.append(tuple(tuple(p) for p in pieces)) all_ptags.append(tuple(frozenset(t) for t in ptags)) # A marker inside EXTRACTED maiden content (#329). classify tags # such a marker like any other token -- what the #274 rule above # lacks is not the TAG but the token: extract claims a delimited # clause and tokenize gives its tokens Role.MAIDEN up front, so # segment (main stream = role is None) leaves them out of every # segment, they never enter `pieces`, and a rule that walks pieces # cannot reach them. # # Scoped to the CLAUSE, via state.extracted (one role + inner span # per delimited region), rather than to a maiden token's # neighbours. Both reasons are load-bearing: # * Role.MAIDEN is not proof of extraction -- the #274 rule above # sets it too, on the bare form, earlier in this same function. # A neighbour test would fire there and eat the 'Nee' out of # "Jane Smith nee Nee Jones". Keying on extracted spans puts # the bare path out of reach by construction. # * Separate clauses are separate content. In "(Nee) (Jones)" the # two land as one contiguous run of maiden tokens, so only the # clause bound keeps the lone "(Nee)" intact. # Drop the clause's FIRST token only when the clause holds more # than one: `Nee` is a real surname (Irish Ni/Nee, and a Chinese # romanization), so a one-token "(Nee)" is a maiden name, not a # marker. FIRST token and no more, whatever the clause holds past # it: cases.py's maiden_marker_delimited_three_token_clause is the # row that bounds this in both directions, every other delimited # row having a two-token clause where the two readings agree. # Spans index the original string by the anti-#100 # invariant, and script_segment only ever splits a token into # sub-slices, so containment stays exact. # # Bisect rather than scan the token list per clause: that is # quadratic in the number of delimited pairs, and "(a) " * 3200 -- # 4x test_benchmark's base, NOT a doubling -- measured a 14.1x cost # against the 4.1x the same shape holds under a policy with no # maiden_delimiters. The control is what says which unit a ratio is # in: linear is ~4 for 4x the input and ~2 for a doubling, so a 4.1x # control cannot be per doubling. Re-measured 2026-08-03 at 11.2x # against 4.2x (3.2x against 2.1x per doubling) -- the separation # replicates, the exact ratio moves with the runner. Same idiom, # and the same reason, as _extract._overlaps and _tokenize's origin # resolution. test_benchmark's maiden_pairs shape is the guard. if any(role is Role.MAIDEN for role, _ in state.extracted): starts = [t.span.start for t in tokens] for role, clause in state.extracted: if role is not Role.MAIDEN: continue # first token starting at or after the clause opens; tokens # are span-sorted and group never reorders or resizes them first = bisect.bisect_left(starts, clause.start) # Testing the SECOND token's end proves BOTH are inside: # tokens do not overlap, so first.end <= second.start, and # bisect already put first.start at or after clause.start. # That is also the "more than one token" test, since the # tokens inside a clause are contiguous in index order. if (first + 1 < len(tokens) and tokens[first + 1].span.end <= clause.end and "vocab:maiden-marker" in tokens[first].tags): dropped.append(first) return dataclasses.replace( state, tokens=tuple(tokens), pieces=tuple(all_pieces), piece_tags=tuple(all_ptags), dropped=tuple(dropped), ambiguities=tuple(ambiguities)) python-nameparser-2.1.0/nameparser/_pipeline/_post_rules.py000066400000000000000000000136051523530431700242070ustar00rootroot00000000000000"""Stage: post_rules. Consumes: tokens (roles assigned). Produces: tokens with roles adjusted by the post rules. Reads: Policy.patronymic_rules, Policy.middle_as_family; Lexicon.given_name_titles. Rules (each a small pure function over the role-bearing tokens): 1. v1 handle_firstnames: when the parse is exactly a title plus ONE given token (no other roles), and the title is not a given-name title ('Sir'), that token is a family name -- "Mr. Johnson". 2. EAST_SLAVIC (opt-in): positional GIVEN/MIDDLE/FAMILY each exactly one token, the FAMILY-position token carries an East Slavic patronymic ending, and the MIDDLE-position token does NOT (given + patronymic + patronymic-derived surname like Abramovich must not rotate) -> rotate: given<-old MIDDLE, middle<-old FAMILY (the patronymic), family<-old GIVEN (v1 parity, pinned live 2026-07-12). 3. TURKIC (opt-in): exactly 1 GIVEN + 2 MIDDLE + 1 FAMILY tokens and the FAMILY-position token is a standalone Turkic marker -> given<-first MIDDLE, middle<-(second MIDDLE, marker), family<-old GIVEN. Both rotations fire only on Structure.NO_COMMA (v1 gates them on `not self._had_comma`): a comma already established the family. These rules reconstruct token POSITION from roles, which is faithful to v1 only under the default GIVEN_FIRST order; their interaction with other name_order values is an open design question for the locale-pack work (#270). """ from __future__ import annotations import dataclasses import re from nameparser._lexicon import _title_key from nameparser._pipeline._state import ParseState, Structure, WorkToken from nameparser._policy import PatronymicRule from nameparser._types import FOLDED_TAG, Role # Ported verbatim from v1 (nameparser/config/regexes.py) -- layering # forbids the config import; keep in sync by hand. _EAST_SLAVIC = re.compile( r"(ovich|ovna|evich|evna|ichna|ilyich|kuzmich|lukich|fomich|fokich)$", re.I) _EAST_SLAVIC_CYR = re.compile( r"(ович|овна|евич|евна|ична|ильич|кузьмич|лукич|фомич|фокич)$", re.I) _TURKIC = re.compile( r"^(oglu|oğlu|ogly|ogli|o['’ʻ]g['’ʻ]li" r"|qizi|qızı|kizi|kyzy|gyzy|uly|uulu)$", re.I) _TURKIC_CYR = re.compile( r"^(оглу|оглы|оғлу|ўғли|угли|кызы|гызы|қызы|қизи|улы|ұлы|уулу)$", re.I) def _idx(tokens: list[WorkToken], role: Role) -> list[int]: return [i for i, t in enumerate(tokens) if t.role is role] def _retag(tokens: list[WorkToken], i: int, role: Role) -> None: tokens[i] = dataclasses.replace(tokens[i], role=role) def post_rules(state: ParseState) -> ParseState: tokens = list(state.tokens) titles = _idx(tokens, Role.TITLE) givens = _idx(tokens, Role.GIVEN) middles = _idx(tokens, Role.MIDDLE) families = _idx(tokens, Role.FAMILY) others = any(t.role in (Role.SUFFIX, Role.NICKNAME, Role.MAIDEN) for t in tokens) # rule 1: title + lone given -> family (v1 handle_firstnames) if titles and givens and not middles and not families and not others: joined = _title_key(tokens[i].text for i in titles) if joined not in state.lexicon.given_name_titles: for i in givens: _retag(tokens, i, Role.FAMILY) # rule 1b: a leading particle that is NEVER a given name means the # whole name is a surname -- fold given (and middles) into family # (v1 handle_non_first_name_prefix; 'de la Vega' -> family, while # ambiguous 'van Gogh' keeps the given reading). The middle/family # guard leaves a degenerate bare 'de' as given rather than # inventing a surname. if len(givens) == 1 and (middles or families): gtags = tokens[givens[0]].tags if "particle" in gtags and "vocab:particle-ambiguous" not in gtags: for i in givens + middles: _retag(tokens, i, Role.FAMILY) # downstream rules key on the role counts: recompute givens = _idx(tokens, Role.GIVEN) middles = _idx(tokens, Role.MIDDLE) families = _idx(tokens, Role.FAMILY) # v1 gates both rotations on `not self._had_comma`; the # middle_as_family fold below runs comma or not (v1 order: # patronymics first, then handle_middle_name_as_last) rules = state.policy.patronymic_rules rotations_apply = state.structure is Structure.NO_COMMA if rotations_apply and PatronymicRule.EAST_SLAVIC in rules and \ len(givens) == 1 and len(middles) == 1 and len(families) == 1: tail = tokens[families[0]].text mid = tokens[middles[0]].text if (_EAST_SLAVIC.search(tail) or _EAST_SLAVIC_CYR.search(tail)) \ and not (_EAST_SLAVIC.search(mid) or _EAST_SLAVIC_CYR.search(mid)): g, m, f = givens[0], middles[0], families[0] _retag(tokens, m, Role.GIVEN) _retag(tokens, f, Role.MIDDLE) _retag(tokens, g, Role.FAMILY) if rotations_apply and PatronymicRule.TURKIC in rules and \ len(givens) == 1 and len(middles) == 2 and len(families) == 1: tail = tokens[families[0]].text if _TURKIC.match(tail) or _TURKIC_CYR.match(tail): g, m1, m2, f = givens[0], middles[0], middles[1], families[0] _retag(tokens, m1, Role.GIVEN) _retag(tokens, m2, Role.MIDDLE) _retag(tokens, f, Role.MIDDLE) _retag(tokens, g, Role.FAMILY) # rule 4: opt-in fold of middles into family (v1 # handle_middle_name_as_last). v1 PREPENDED middle_list to # last_list; spans cannot reorder (anti-#100), so folded tokens # carry a tag and the family views order them first. if state.policy.middle_as_family: for i in _idx(tokens, Role.MIDDLE): tokens[i] = dataclasses.replace( tokens[i], role=Role.FAMILY, tags=tokens[i].tags | {FOLDED_TAG}) return dataclasses.replace(state, tokens=tuple(tokens)) python-nameparser-2.1.0/nameparser/_pipeline/_script_segment.py000066400000000000000000001266521523530431700250450ustar00rootroot00000000000000"""Stage: script_segment (#271, #272, #308, #312). Consumes: tokens, segments, structure, interpunct_offsets, segmenter. Produces: tokens, by two independent splits into sub-slices -- a listed honorific peeled off the END of the name's last non-post-nominal token, in whichever of the name's runs that falls (the tail token also carries this module's _PEELED_TAG), and the first activated-script token of the name segment split into n+1 -- segments (index runs remapped past the insertions), ambiguities (indices likewise remapped, plus a SEGMENTATION report when more than one split was vocabulary-supported, or when a segmenter's answer scored under the confidence floor). Reads: Policy.segment_scripts, Lexicon.surnames, Lexicon.honorific_tails, ParseState.segmenter, and Lexicon suffix vocabulary through TWO predicates, which are NOT one another's singular and plural. _vocab.is_suffix_strict asks whether a single token is a post-nominal, initial veto included (the peel's scan-back and the surname site). _vocab.is_wholly_suffix asks segment's own suffix-comma question of a whole RUN, through the POLICY-selected token test plus period_joined_vocab, delimiter handling and the Ph./D. merge -- so the run predicate says yes both to tokens the token predicate VETOES ("V.", "V", "I") and to tokens it never sees as suffixes at all ("Msc.Ed.", "J.씨", which reach it through period_joined_vocab). The initial-shaped words are the class #319 was ABOUT, not the whole of the disagreement, and the extra routes listed above are where the rest of it comes from; test_is_wholly_suffix_is_not_the_plural_of_is_post_nominal pins the "V." half. The run predicate is how the peel declines a run that is not name text (#319), but only where the name's own run offers a site, since a glued honorific is itself part of what makes a run read as suffix-shaped; it owns two further Policy fields (lenient_comma_suffixes picks the strict or lenient token test -- it flips "田中さん, V." -- and extra_suffix_delimiters both counts a bare delimiter-core token as a suffix and splits a token on a core, the split being the half that flips "田中さん, Jr./V." under {"/"} and the bare core the half that flips "田中さん, /"). Unspaced CJK names give tokenize no separator to find, so this stage inserts the missing token boundary by vocabulary: the first token written in an activated script is matched longest-first against Lexicon.surnames, and a hit splits it in two. Compound-before-single ("夏侯惇" is 夏侯 + 惇, though 夏 is itself a surname) falls out of longest-first. The split makes sub-slices of the one token, rewriting nothing -- spans still index the original exactly, so the anti-#100 invariant holds by construction. A second, independent split runs first (#308): a listed honorific glued to the END of the name's last non-post-nominal token is peeled off as its own token -- 田中さん -> 田中 + さん -- so that suffix classification can claim it and the surname match or segmenter consult below sees the name rather than the name plus an honorific. No STRUCTURAL gate stands over it -- the only things above it are the stage's own two preconditions, the ASCII bail and non-empty segments. Not segment_scripts either: the vocabulary of tails is licensed by the entries themselves, each of which can never end a name, so no per-script trust question arises. And since #312 not the FAMILY comma or the 间隔号 either -- both of those answer where a name divides into surname and given, which the peel never asks, so a comma or a dot elsewhere in the string cannot change whether a token ends in a word that can never end a name. The ASCII bail is the gate a caller adding a LATIN tail meets -- it sits above everything here, so such a tail fires only on a name carrying at least one non-ASCII character (see the bail's own comment, and honorific_tails' field note). A suffix comma gates nothing either -- "Dr 김민준씨, Jr." peels within its name part like any other. Where the VOCABULARY declines -- no prefix matched -- an optional Parser(segmenter=...) gets the token (#272 amendment 2026-07-29). Vocabulary first, segmenter on decline, so parser_for(ZH, JA, segmenter=...) composes MECHANICALLY: a listed surname is a dictionary certainty and wins, and the segmenter takes what is left. Composing is not a free lunch, and the docs qualify it where they show the stack: the zh pack's own mis-split warning survives unchanged, because a Japanese kanji name opening on a listed Chinese surname never reaches the segmenter at all -- 高橋一郎 still splits 高 + 橋一郎 under ZH+JA, exactly as it does under ZH alone. The two packs are corpus ALTERNATIVES, one per corpus; stacking them is for genuinely mixed data that accepts that trade. Its Segmentation may cut anywhere and any number of times, which is why the split path below takes n cuts. One precondition guards it that the vocabulary has no twin of: a segmenter answers where an UNDIVIDED name divides, so it is consulted only when the gated token is the name part's ONLY script-written one -- "山田 太郎" was divided by its writer and must not have its family divided again (a Latin title or suffix draws no such boundary either, and effective_script gates those out before the test is reached). The neighbour test reads effective_script -- merely non-None -- and exempts exactly one token: the tail the peel above MANUFACTURED (#308), which is a boundary nobody drew. A SPACED honorific is not exempt, and the distinction is provenance rather than vocabulary: glued 山田太郎様 was written undivided, while "佐藤 氏" carries a boundary its writer typed. Not that the writer thereby declared 佐藤 a unit -- in "山田太郎 様" the unit they drew is the whole name -- but that at this position a spaced honorific cannot be told apart from a spaced name ELEMENT, so the precondition counts it rather than guess. The trade that buys is measured: counting spaced honorifics keeps four real surnames whole (佐藤 氏, 田中 様, 鈴木 先生, 中村 教授, all four divided bare under the JA pack) and costs the one division 山田太郎 様. A segmenter's own exceptions PROPAGATE -- the single declared exception to parse totality (locales spec section 4): a user-supplied callable's error is a user-code error, not a content error. Placed AFTER segment, on the comma doctrine that script-conditional behavior DECIDING WHERE A NAME DIVIDES is ignored where a comma already decides the family (the rule script_orders follows -- and the qualifier is load-bearing since #312, which is what the peel crosses the comma on): under FAMILY_COMMA the pre-comma text IS the family by declaration, and splitting it would invent a boundary the writer explicitly did not draw -- "남궁민수, 지훈" must render family "남궁민수", not "남궁 민수". That doctrine is about the input's structure, not the split's source, so it covers the segmenter identically: the opt-out runs before either is consulted. The post-comma side is given-name text, no surname site either, so that structure opts out of the SURNAME SPLIT whole -- of the stage entire until #312 moved the peel in front of the gate, an honorific being no part of the name whichever side of the comma it was glued to. NO_COMMA and SUFFIX_COMMA still split, within segments[0] (the name part) only. Running after segment costs the index remaps below; running BEFORE it would have made the comma structure itself depend on the split -- segment's suffix-comma rule needs more than one word before the comma, so a pre-split "김민준, Jr." would have changed structure on vocabulary alone. As written it stays FAMILY_COMMA, which is why the SUFFIX_COMMA path needs a second word ("Dr 김민준, Jr.") to be reachable at all. Activation is per script because the AMBIGUITY is per script (amendment 2026-07-27): HANGUL is on by default (hangul is unambiguously Korean, and its surname set is closed and default-shipped), while HAN is opt-in via locales.ZH -- a Chinese surname list corrupts Japanese names ("高橋一郎" must not split as 高 + 橋一郎). Japanese divides through the segmenter path below under locales.JA, not through this table's vocabulary. The gate resolves each token through effective_script, the same function order resolution uses, so kana-licensed composites (高橋みなみ -> HIRAGANA) gate in under the JA pack's HIRAGANA entry while pure-katakana tokens (-> KATAKANA, in no activation set by design -- they are predominantly transcribed foreign names) never do. Only the FIRST activated-script token is considered, match or no match: family-first traditions put the surname at the front of the name, and a match deeper in the token stream would be a given name or an ordinary word, not a surname site. Where that first token is one the vocabulary reads as a POST-NOMINAL, the stage DECLINES rather than looking further along: an honorific is not part of the name, and a surname leads, so an honorific in the surname's own position means there is no surname here to find. Deciding includes deciding that. Looking further would reach the given name the rule already refuses to touch -- "양 지훈" would split its own given name, 지 being listed too -- while declining leaves the peel's manufactured tail intact, since in "Anderson선생님" that tail is the first and only script-written token (선생님 opens on the listed surname 선, and a spaced "Anderson 선생님" was mis-split that way before the peel existed). """ from __future__ import annotations import dataclasses import functools from collections.abc import Sequence from nameparser._pipeline._state import ( ParseState, PendingAmbiguity, Structure, WorkToken, ) from nameparser._pipeline._vocab import ( effective_script, is_suffix_strict, is_wholly_suffix, ) from nameparser._types import AmbiguityKind, Segmentation, Span #: Marks the tail token the peel below MANUFACTURED, so the segmenter's #: neighbour test can tell it from a token somebody wrote. Namespaced, #: therefore unstable provenance rather than API (_types.STABLE_TAGS is #: the whole stable set, and FOLDED_TAG is the precedent for a #: structural marker carrying this prefix); it is vocabulary-derived #: besides, since honorific_tails is what licensed the split. Emitter #: and reader are both in this module, so unlike FOLDED_TAG it needs no #: home in _types. _PEELED_TAG = "vocab:peeled-honorific" #: Segmenter answers scoring below this attach a SEGMENTATION report #: (amendment 2026-07-29 section 3). Kept at the drafted 0.9 after #: measuring namedivider 0.4.1 over 112 names (#272 Task 5), which #: found a distribution the amendment did not anticipate: the scores #: are BIMODAL, not clustered near 1. A rule-based division (the kana #: boundary in 高橋みなみ, or a two-character name) scores exactly 1.0; #: a kanji-statistics division scores a softmax over the candidate cut #: positions, observed in 0.23-0.68 and driven more by name LENGTH #: (median 0.61 at three characters, 0.37 at five -- the softmax is #: over len-1 candidates) than by correctness: the wrong answers in #: the sample scored 0.32/0.60/0.60/0.61, straddling the correct #: median of 0.52. So no floor INSIDE that band separates error from #: success, and the only cut the data supports is between a stated #: certainty and a statistical guess -- which is what section 3's #: epistemic argument asks for anyway. 0.9 sits in the empty gap #: (0.68 to 1.0), far from both modes, and reads as "confident" for a #: third-party segmenter with a calibrated score too. Consequence, #: pinned by tests and stated so it can be checked: every #: STATISTICALLY divided name carries a SEGMENTATION report, and every #: RULE-divided one -- the kana boundary, a two-character name, #: namedivider's specific-name rules -- carries none. #: namedivider scores its own two-character rule 1.0, so this floor #: keeps that division silent -- see locales/ja.py for why the #: presumption is accepted as stated. #: Not configurable in 2.x (YAGNI). _SEGMENTER_CONFIDENCE_FLOOR = 0.9 def _remap(run: tuple[int, ...], split_at: int, added: int) -> tuple[int, ...]: """One segment run after token `split_at` split into `added` + 1 pieces: the extra pieces join the run immediately after it, and every later index shifts by as many.""" out: list[int] = [] for j in run: if j == split_at: out.extend(range(split_at, split_at + added + 1)) else: out.append(j + added if j > split_at else j) return tuple(out) def _pieces(text: str, splits: tuple[int, ...]) -> tuple[str, ...]: """`text` cut at every offset in `splits`: n offsets, n+1 pieces.""" cuts = (0, *splits, len(text)) return tuple(text[a:b] for a, b in zip(cuts, cuts[1:])) def _split(state: ParseState, i: int, splits: tuple[int, ...], detail: str | None, tail_tag: str | None = None) -> ParseState: """Cut token `i` at every offset in `splits`, recording `detail` as a SEGMENTATION report when there is one, and adding `tail_tag` to the LAST piece when there is one of those. The offsets arrive non-empty, ascending and interior whatever chose them. From a segmenter: non-empty is the caller's own check, strictly ascending with each >= 1 is Segmentation.__post_init__'s, and the last offset is < len(text) -- checked by the caller. From the vocabulary: the single offset is >= 1 and < len(text) by the range(cap, 0, -1) construction and its len-1 cap. The ONE split path: the vocabulary hit is the single-offset case and the segmenter's answer the general one, so neither can drift from the other's index arithmetic. `tail_tag` is part of keeping it one path: the piece is tagged HERE, where it is built, rather than by the caller, which would have to re-derive where its own tail landed after handing the index arithmetic off. Only the peel passes one, and the piece it wants is the last by construction -- the honorific it cut off the end; the surname path passes nothing and the default leaves it exactly as it was.""" token = state.tokens[i] base = token.span.start parts: list[WorkToken] = [] start = 0 for piece in _pieces(token.text, splits): end = start + len(piece) parts.append(dataclasses.replace( token, text=piece, span=Span(base + start, base + end))) start = end if tail_tag is not None: parts[-1] = dataclasses.replace( parts[-1], tags=parts[-1].tags | {tail_tag}) added = len(splits) tokens = state.tokens[:i] + tuple(parts) + state.tokens[i + 1:] # Every index the earlier stages recorded is now stale past the # split point: the segment runs group's own iteration rests on, and # the ambiguities from extract_delimited (resolved to indices by # tokenize) and segment. An ambiguity ON the split token keeps # pointing at the head. segments = tuple(_remap(run, i, added) for run in state.segments) ambiguities = tuple( dataclasses.replace(a, indices=tuple( j + added if j > i else j for j in a.indices)) for a in state.ambiguities) if detail is not None: ambiguities += (PendingAmbiguity( AmbiguityKind.SEGMENTATION, detail, tuple(range(i, i + added + 1))),) return dataclasses.replace(state, tokens=tokens, segments=segments, ambiguities=ambiguities) @functools.lru_cache(maxsize=16) def _longest_entry(entries: frozenset[str]) -> int: """The longest entry in a vocabulary, cached per-vocabulary rather than recomputed per parse (Lexicon is frozen and slotted, so it cannot carry a cached_property of its own). The frozenset is hashable and a process holds only a handful of distinct vocabularies -- the default one, plus one per constructed pack parser -- so maxsize=16 bounds pathological many-lexicon churn without ever evicting in normal use. Keyed by the frozenset VALUE, not by (lexicon, field): the two callers pass surnames and honorific_tails, but every lexicon that leaves KOREAN_SURNAMES alone shares one entry, so the count is distinct SETS rather than lexicons times fields. Callers must pass a NON-EMPTY vocabulary: max() of an empty set raises, and both call sites sit under a match guard that cannot reach it with one.""" return max(map(len, entries)) def _is_post_nominal(state: ParseState, i: int) -> bool: """Whether token `i` is post-nominal VOCABULARY. Two of this stage's decisions ask it and neither is about script: an honorific is neither a surname SITE nor the token a glued honorific hangs off (#308). It answers a VOCABULARY question, not a positional one, and the two callers do not spend the answer the same way. In TRAILING position vocabulary and position agree, so the peel site steps over a True and goes on scanning. In LEADING position they disagree -- 양 is the family name there, whatever the suffix set says -- so the surname site reads a True as an ANSWER and declines, rather than as a token to step past. STRICT, not lenient: the initial veto applies, so "V." is not a post-nominal here though bare "v" is a suffix word. That agrees with what classify does with the same token downstream -- "V." is a middle initial -- and the difference is reachable under the default lexicon: "田中さん V." stops its scan-back at the initial and does not peel, while "田中さん II" steps over II and does. That pair is the case table's ja_honorific_glued_before_an_initial and ja_honorific_glued_before_a_roman_suffix, added because the clause could NAME the discriminating input while swapping in is_suffix_lenient still failed no test -- the shape a "unify the two suffix predicates" refactor would have walked straight past. Suffixes only, deliberately. Should a CJK entry ever join titles, the surname site would want it excluded while the peel site must not: a title never trails, so widening the peel's scan-back would move it off the real last name token. Split the predicate then, not before.""" return is_suffix_strict(state.tokens[i].text, state.lexicon) def _peel_site(state: ParseState, flat: Sequence[int], tails: frozenset[str]) -> tuple[int, int] | None: """Where a peel would land in the token run `flat`: the index of the token to cut and the LENGTH of the listed tail to take off it, or None where that run offers no peel. Two callers, one answer, which is the point of naming it. The peel itself asks it once, of the runs it decided to scan. The gate above that decision asks it of segments[0] ALONE, to find out whether declining the second run would cost the only site. Sharing the scan with the peel rather than approximating it there is what makes the gate's answer mean what it says: a gate that merely looked for a token ENDING in a listed tail would count a token that is a tail entire, which the scan-back skips as a site -- "선생님, J.씨" would decline on a site the peel then cannot use, and lose the peel in exactly the way the gate exists to prevent. (Only the lone-token shape of that divergence is reachable from the gate, and the gate's own other conjunct is what makes that so: SUFFIX_COMMA wants BOTH a suffix-shaped second run and more than one word before the comma, and the gate is asked only where the first of those is already true -- so a run this call ever sees under FAMILY_COMMA failed on the second, and segments[0] holds at most one token. The word-count alone would not say that: "Dr 김민준, 지훈" has two words before the comma and is FAMILY_COMMA.) Callers must pass a NON-EMPTY `tails` -- _longest_entry's precondition, which the stage's own early return supplies.""" i = next((j for j in reversed(flat) if not _is_post_nominal(state, j)), None) if i is None: # nothing but post-nominals, or no tokens at all return None text = state.tokens[i].text # range/cap construction identical to the surname match below, and # for the same two reasons: longest-first, and a len-1 cap that # makes the offset interior by construction (_split's contract). cap = min(_longest_entry(tails), len(text) - 1) for length in range(cap, 0, -1): if text[-length:] in tails: return i, length return None def _peel_honorific_tail(state: ParseState) -> ParseState: """#308: split a listed honorific off the END of the name's last NON-POST-NOMINAL token -- 田中さん -> 田中 + さん -- and let the existing machinery do the rest. Suffix classification claims the tail downstream (every honorific_tails entry is a suffix word too, enforced by Lexicon), and the segmentation half below then sees the remainder rather than the glued whole, so 김민준씨 splits 김 + 민준 and a configured segmenter is handed 山田太郎 rather than 山田太郎様. Scanning back over post-nominals rather than taking the last token outright does three things at once. An unrelated trailing suffix cannot hide the peel site, so "김민준씨 Jr." answers as the comma-written "Dr 김민준씨, Jr." does -- one name, two spellings, one parse. A token that IS a tail (씨, さん, and the nested 선생님, which the cap alone would peel to 선생 + 님) is skipped as a site and stays whole: every tail is a suffix word by the Lexicon invariant, which is what makes that guard hold, and the cap below only keeps the offset interior for _split. And the scan answers None rather than indexing anything, which two reachable inputs need: a name that is nothing but post-nominals ("씨"), and one whose name runs are empty (", , 씨" scopes to two empty ones). Nothing here rests on a structural gate landing first, then, which is what let #312 move both of the stage's gates below it. WHICH tokens are scanned is a separate decision, and the one a reader is likeliest to undo: the NAME's segment runs, flattened, and never state.tokens or every segment. The alternatives agree except where extract_delimited has already claimed a token or a comma has closed the name, both of which this stage can still see -- so "김민준씨 (Jimmy)" and "Dr 김민준씨, V." are the inputs that tell the three apart. See the note at the scan itself. That last token is the last of the NAME, which reaches into a maiden clause: maiden tokens are still main-stream here (extract_delimited has masked only bracketed content), so "김민준 née 박씨" peels 씨 off the MAIDEN name 박씨 and hands it to the person's suffix list -- "née Ms. Park". Intended rather than incidental in that direction: the honorific is the reader's regardless of which of her names it was glued to, and a name-final honorific is exactly what this peels. Since #312 that reach extends to FAMILY_COMMA along with the rest of the crossing -- "김, 민준 née 박씨" now routes 씨 to suffix, where before #312 the stage returned at the family-comma gate above the peel and left it in maiden "박씨". The other direction is a LIMIT, stated rather than fixed: a maiden clause pushes the site off the person's own name, so "김민준씨 née 박" does NOT peel and gives given "민준씨" -- the original bug, intact behind a marker. "김민준씨 née 박씨" shows both at once, two identical honorifics of which only the maiden's is routed. Chasing the marker into the site scan is scope creep for an uncommon input, and it could not be done here anyway: classify has not run, so the marker tokens carry no tag this stage could read. Longest-first, and ONE peel: a remainder that itself ends in a listed tail is accepted rather than chased. No SHIPPED input witnesses that any more -- 박사님 was the last one, and adding it is what removed the case: 김민준박사님 now gives up 박사님 entire, since longest-first reaches the whole honorific. The pin needs a tail whose remainder ends in a DIFFERENT tail, the two not themselves a listed entry, and no pair in the shipped vocabulary has that shape; the stage test test_one_peel_never_a_stack carries it on a synthetic lexicon, which is now its only witness. No script precondition on the remainder either, since the tail alone is the license: Andersonさん peels. Emits no ambiguity, unlike the surname fork below, though longest-first does CHOOSE here too -- 김선생님 gives 선생님 where 님 also matches. The difference is what the runner-up is: a second matching surname is a competing READING of the name, which a caller may prefer, while a shorter tail leaves a remainder that is not a name at all (김선생), so there is nothing to adjudicate.""" tails = state.lexicon.honorific_tails if not tails: return state # The NAME's runs, which is more than segments[0] but never every # segment. A comma is how a writer says which runs are the name: a # FAMILY comma splits the name itself across two of them ("김, # 민준씨" is (0,) and (1,)), and the honorific is as often glued to # the given name as to the family, so the peel has to cross that # boundary (#312). Every OTHER structure keeps the whole name in # segments[0], so no boundary has to be crossed to find the site # there -- which is why the asymmetry is the point rather than an # accident of two cases. Note that "the rest is post-nominals" is # true of the SUFFIX comma only: under NO_COMMA segment returns # exactly one run and any trailing post-nominal is INSIDE it # ("김민준씨 Jr." is one run of two tokens), which is why the # scan-back above steps over such a token rather than simply never # reaching it. # The second run is only NAME text when segment read it as one, # which the structure alone does not say: SUFFIX_COMMA also wants # more than one word before the comma, so a one-word part turns a # wholly suffix-shaped remainder into FAMILY_COMMA anyway ("田中さん, # V." is that input). So ask segment's own predicate instead of # inferring the answer from the structure it produced (#319). # is_wholly_suffix, NOT the plural of _is_post_nominal: the two # disagree on the initial-shaped suffix words ("V.", "V", "I"), # which is the class #319 was reported about, and on everything # the run predicate's extra routes reach and the token predicate # does not ("Msc.Ed." and "J.씨" by period_joined_vocab, both of # which this change also moves). Reaching into such a # run put the site on "V.", which ends in no listed tail, so the # peel silently abandoned and さん stayed glued to the family -- # while "田中さん, PhD" peeled all along, because "PhD" satisfies # the strict test and the scan-back stepped over it. One credential, # two spellings, two answers FROM THE PEEL -- and the peel's answer # is the only one that moved: the peeled remainder is 田中 and lands # in family under every spelling, while where the CREDENTIAL lands # is assign's question and still differs ("PhD" a title, "V." a # given, "Ph. D." a suffix beside さん). # A junk tail is the worse shape of the same reach: in # "김민준씨, J.씨" the site lands on the junk "J.씨", so master # peeled THAT 씨 and left the person's own glued inside family # "김민준씨". Reachable only where the run is genuinely # suffix-shaped, which "J.씨" is by period_joined_vocab; a run of # ordinary name text is scanned on purpose, and a junk tail further # out than the second run is held off by the scope rule instead # ("Dr 김민준씨, Jr., 박씨" is SUFFIX_COMMA with 박씨 in a third # run, so it never reaches here at all). # An EMPTY second run stays in scope and contributes nothing: # is_wholly_suffix is False on it by its own contract (v1 read # "Doe,, Jr." as a family comma), which is the reading this line # wants anyway -- flattening an empty run adds no site. # Policy(lenient_comma_suffixes=False) keeps the old answer for the # INITIAL-shaped suffixes specifically, which is where the # strict/lenient gap lives: the knob drops this call to the strict # predicate too, so is_wholly_suffix(["V."]) is False, the run reads # as name text, it IS scanned, and the peel is abandoned on "V." as # before -- family "田中さん", given "V.". It is not a blanket # freeze of the old behavior, and "田中さん, Ph. D." is the input # that shows the difference: the Ph./D. merge folds that pair to a # form is_suffix_strict accepts, so the run is declined and the peel # fires under the strict knob as well. # Flattening the SEGMENTS rather than state.tokens is load-bearing # too: extracted nickname and maiden content is in tokens but in NO # segment, and scanning tokens would put the peel site on a # nickname ("김민준씨 (Jimmy)" -> the site becomes Jimmy and # nothing peels). ko_honorific_glued_given_nickname pins that. # And declining takes a SECOND condition: segments[0] must hold a # peel site of its own. is_wholly_suffix reaches period_joined_vocab, # which calls a run suffix-shaped when ANY period-chunk is suffix # VOCABULARY -- and every honorific tail is a suffix word by the # Lexicon invariant, so a glued honorific is itself the evidence. # The predicate is circular at THIS call site alone -- not because # segment asks it any earlier (nothing has peeled at either call) # but because segment SPENDS the answer differently: it reads the # run's shape and stops, the answer being the structure, while the # peel reads the same shape and then decides whether to go strip # the very honorific that produced it. "이, J.씨" reads as wholly suffix # only because of the 씨 the peel exists to remove, and declining a # run that holds the only site does not fall back to some other # site -- it loses the peel outright, and with it the given name, # which lands in suffix as "J.씨". Asking for a site in segments[0] # keeps the #319 answer wherever the peel has somewhere else to go # ("田中さん, V." still declines, さん is right there) and gives the # circular case back to master's reading. The two-honorific input # "김민준씨, J.씨" is where the choice is visible and deliberate: # both runs offer a site, so the decline stands and the person's own # 씨 is peeled rather than the junk one behind the comma. runs = state.segments[:1] if state.structure is Structure.FAMILY_COMMA: second = [state.tokens[j].text for j in state.segments[1]] if not (is_wholly_suffix(second, state.lexicon, state.policy) and _peel_site(state, state.segments[0], tails)): runs = state.segments[:2] site = _peel_site(state, [j for seg in runs for j in seg], tails) if site is None: return state i, length = site # The tail carries a tag because this stage MANUFACTURED it. The # segmenter's neighbour test below needs to tell it from a token # somebody wrote, and no vocabulary question can: the two spellings # put the same word in the same place, and only the provenance # differs. return _split(state, i, (len(state.tokens[i].text) - length,), None, tail_tag=_PEELED_TAG) def _split_surname_site(state: ParseState) -> ParseState: """The stage's other split: the first activated-script token of the name part is matched longest-first against Lexicon.surnames, and a hit splits it in two; where the vocabulary declines, an optional Parser(segmenter=...) is consulted instead. A sibling of _peel_honorific_tail rather than a continuation of it. The two answer different questions -- this one asks where a name divides into surname and given, the peel asks whether a token ends in a word that can never end a name -- which is why they carry different gates: the FAMILY comma and the 间隔号 gate this half alone (#312), and segment_scripts below gates it alone too. That is also why the stage entry below interleaves gates with its two calls rather than running one cascade.""" scripts = state.policy.segment_scripts # an empty VOCABULARY deliberately does not bail here -- see below if not scripts: return state # segments[0] is the NAME part under both remaining structures # (everything, under NO_COMMA); later segments are suffixes. Its # members are main-stream token indices by construction, so # extracted nickname/maiden content is unreachable from here. For # this site that is merely tidy -- the first script-written token # is the same either way. It is the PEEL above that reads this run # load-bearingly, and in the two structures that reach here it # reads exactly this one, only backwards; the argument lives at # that scan. i = next((i for i in state.segments[0] if effective_script(state.tokens[i].text) in scripts), None) # A post-nominal in the surname's own position is not a site to # skip past but an answer: a surname LEADS, so if the leading # script-written token is an honorific there is no surname here to # find. Scanning ON would reach the given name, which is exactly # what the first-token rule exists to prevent -- 지 is a listed # surname, so "양 지훈" (양 is a surname AND a shipped honorific) # would have its own given name split in half. Declining also # covers the token the peel above manufactures, which is the first # and only script-written one in "Anderson선생님". if i is None or _is_post_nominal(state, i): return state token = state.tokens[i] text = token.text surnames = state.lexicon.surnames # A token that IS a surname never splits: a bare "남궁" must not # become 남 + 궁 just because the single-syllable surname also # matches -- there is nothing to split off, and a lone token's # role is the order resolution's call. if text in surnames: return state # Longest-first (compound-before-single falls out of it), capped # so the remainder is never empty. Direct membership, no # _normalize: the script gate admits only CJK text, which the # storage fold stores unchanged. An empty vocabulary skips the # match rather than bailing the stage (_longest_entry's max() has # nothing to take): a surname-less lexicon declines every token, # which is exactly the condition the segmenter is consulted on, so # an early bail would make a configured segmenter silently inert # under Lexicon.empty() -- the JA pack's own shape. matches: list[int] = [] if surnames: cap = min(_longest_entry(surnames), len(text) - 1) matches = [length for length in range(cap, 0, -1) if text[:length] in surnames] if matches: take = matches[0] detail = None if len(matches) > 1: # more than one vocabulary-supported split: longest-first # DECIDED a fork, and the deciding stage records it. A # single-match split chose nothing, so it stays silent -- # a dictionary certainty and the statistical guess below # are different epistemic states, and these two emission # rules say so. chosen = " + ".join(map(repr, _pieces(text, (take,)))) other = " + ".join(map(repr, _pieces(text, (matches[1],)))) detail = (f"{text!r} splits as {chosen} on the longest " f"surname; {other} also reads") return _split(state, i, (take,), detail) if state.segmenter is None: return state # the first activated-script token decides # The segmenter's precondition, which the vocabulary has no twin of: # it is asked where an UNDIVIDED name divides, so it may only be # shown a token that is the whole name. Where the name part carries # a second script-written token the writer already drew this # stage's missing boundary -- "山田 太郎" is divided, and dividing # its family again yields 山 + 田 + 太郎 (namedivider answers for # any string, and scores a two-character one 1.0 by rule, so no # confidence check would catch it). The neighbour counts whatever # script it is written in, ACTIVATED or not -- effective_script is # merely non-None -- because a katakana or hangul neighbour is a # boundary its writer drew just as deliberately as a Han one: under # the JA pack "山田太郎 マイケル" declines, though katakana is in no # activation set. A Latin title or suffix is NOT such a boundary: # it says nothing about where the CJK name splits, so "Dr 阿明日, # Jr." still reaches the segmenter. Vocabulary keeps its own rule # -- a listed surname is a certainty about that exact string, # whoever else stands beside it. # The one neighbour that does not count is the one this stage # MANUFACTURED (#308): a glued 山田太郎様 has no writer-drawn # boundary anywhere, so the 様 the peel just cut off cannot be read # as one -- the precondition must see what the writer wrote, which # was a single undivided token. A SPACED 様 does count, and the # reason is weaker than "its writer drew that boundary and chose # to write 山田太郎 as a unit": in "山田太郎 様" the unit the writer # drew is the WHOLE NAME, honorific and all, so that story is false # for the very input it describes. What the code relies on is that # by POSITION a spaced honorific is indistinguishable from a spaced # name element, so the test conservatively counts it. Measured, the # trade is worth it: counting them keeps 佐藤 氏, 田中 様, 鈴木 先生 # and 中村 教授 whole -- all four divide bare under the JA pack (佐 # + 藤, 田 + 中, 鈴 + 木, 中 + 村) -- and costs the single division # 山田太郎 様. Four real surnames against one. # Provenance, not vocabulary: the two spellings put the same word # in the same place, and asking the suffix set instead cannot # separate them. if any(j != i and effective_script(state.tokens[j].text) is not None and _PEELED_TAG not in state.tokens[j].tags for j in state.segments[0]): return state # No try/except around the call: the module docstring's totality # exception. The two checks below are that same doctrine, curated, # and they are where the line this module draws is easiest to state: # a PROTOCOL VIOLATION BY THE SEGMENTER AUTHOR RAISES, while an # ADAPTER'S DEFENSE AGAINST ITS LIBRARY DECLINES. Both checks here # are the first kind -- a wrong answer TYPE and an answer indexing # past the token it was handed are stage-detectable bugs in # user-supplied code, inside the declared totality exception, so # they get the same treatment the callable's own exceptions get. # locales/ja.py is the second kind: its repertoire, length, # reconstruction and score guards all return None, because what # they defend against is namedivider answering a question nobody # asked it, which is a fact about the CONTENT, not a broken # protocol. Bounded like every message here: the type's NAME, never # its contents. answer = state.segmenter(text) if answer is not None and not isinstance(answer, Segmentation): # a duck-typed answer carrying a .splits of its own would # otherwise wander into the split path and surface as a # ValueError naming Token, pointing the reader at nameparser's # insides instead of at their segmenter raise TypeError( f"segmenter must return Segmentation or None, got " f"{type(answer).__name__}") if answer is None or not answer.splits: return state # declined, or confidently one token # splits[-1] is the max -- Segmentation enforced ascending. The # upper bound is the half Segmentation cannot check, since it never # sees the text; an offset at or past the end would make an empty # piece. Declining silently here (as this did before the review) # made an off-by-one segmenter undebuggable: every answer it gave # vanished, and the parse merely looked unsegmented. if answer.splits[-1] >= len(text): raise ValueError( f"segmenter returned splits beyond the token: last offset " f"{answer.splits[-1]}, token length {len(text)}") conf = answer.confidence detail = None if conf is not None and conf < _SEGMENTER_CONFIDENCE_FLOOR: reading = " + ".join(map(repr, _pieces(text, answer.splits))) detail = (f"{text!r} splits as {reading} on a segmenter answer " f"scoring {conf:.2f}, under the " f"{_SEGMENTER_CONFIDENCE_FLOOR} confidence floor") return _split(state, i, answer.splits, detail) def script_segment(state: ParseState) -> ParseState: if state.original.isascii(): # spans index the original exactly (the anti-#100 invariant), # so an ASCII original has only ASCII tokens: nothing here is # in any script's ranges. It also short-circuits the PEEL, # which has no script gate of its own -- so a caller-configured # ASCII tail never fires, and one non-ASCII character anywhere # in the name switches it on. Correct for the CJK vocabulary # that ships, stated because honorific_tails is public: see # that field's own note. Latin orthography SPACES its # post-nominals ("John Smith PhD"), so the glued position this # peel exists for is a CJK one to begin with. What Latin does # glue is PREnominal and period-joined (Mr.Smith, Lt.Gov.), # which _vocab.period_joined_vocab classifies as one token # rather than splitting -- a different mechanism, and no peel # site either way. A glued Latin POST-nominal is spelled the # same way, so it reaches that same mechanism rather than # nothing: period_joined_vocab reads "Smith.Jr." as a title # ('jr' is title vocabulary as well as suffix vocabulary), and # where position allows, that wins -- "Smith.Jr. Anderson" # gives title "Smith.Jr.", family "Anderson". Whatever it # decides, this bail is what settles the question here: it # returns above all of it, so no ASCII input reaches the peel. return state if not state.segments: return state # #312: the peel runs in front of both gates below, because both # answer where a name divides into surname and given, and the peel # does not ask that. It asks whether a token ends in a word that # can never end a name, and a comma or a dot elsewhere in the # string does not change the answer. Placing it above the block # also keeps a future segmentation gate from silently capturing # it: a new gate lands with its siblings, below. state = _peel_honorific_tail(state) if state.structure is Structure.FAMILY_COMMA: return state # the comma already drew the SURNAME boundary if state.interpunct_offsets: # #298: a 间隔号-divided name is a transcription -- its pieces # are syllable groups, not surname+given, so neither the # vocabulary nor the segmenter applies (codepoint-scoped: the # nakaguro records nothing and gates nothing, spec decision 5). # State-global like the FAMILY_COMMA gate above: a marker # anywhere in the name reads the WHOLE name as a transcription # listing, so even an un-dotted hangul token beside a dotted # one stays whole. Scoped to the surname split since #312: an # honorific glued to a transcription is still an honorific. return state return _split_surname_site(state) python-nameparser-2.1.0/nameparser/_pipeline/_segment.py000066400000000000000000000106261523530431700234520ustar00rootroot00000000000000"""Stage: segment. Consumes: tokens (role-None main stream), comma_offsets. Produces: segments (runs of main-token indices; interior segments may be EMPTY -- doubled commas keep their structural position), structure, COMMA_STRUCTURE ambiguities for unrecognized extra segments. Reads: Lexicon suffix vocabulary and Policy, both through _vocab.is_wholly_suffix -- the suffix-comma decision is definitionally vocabulary-dependent (recorded plan deviation #3), and the predicate owns the rest (Policy.lenient_comma_suffixes picks the lenient or strict token test; Policy.extra_suffix_delimiters gives v1 suffix_delimiter parity, a delimiter-core token being transparent). Decision (v1 parity): >=1 comma and the SECOND segment entirely suffix AND >1 word before the first comma -> SUFFIX_COMMA; otherwise FAMILY_COMMA ("Family, Given ..."). Only the second segment decides (v1 parser.py:1318) -- segments beyond it are consumed as suffixes either way, and a NON-EMPTY one that is not entirely suffix is flagged COMMA_STRUCTURE rather than vetoing the structure (an empty one is consumed silently, as v1 consumed it, so "John Smith, MD,, Jr." reports nothing where "John Smith, MD, Bart" does; they are still best-effort consumed as suffixes by assign, since parse must stay total over str input and never raise on content). "Entirely suffix" is is_wholly_suffix's question, so the token test inside it is lenient by default and strict under Policy(lenient_comma_suffixes=False). """ from __future__ import annotations import bisect import dataclasses from nameparser._pipeline._state import ParseState, PendingAmbiguity, Structure from nameparser._pipeline._vocab import is_wholly_suffix from nameparser._types import AmbiguityKind def segment(state: ParseState) -> ParseState: main = [i for i, t in enumerate(state.tokens) if t.role is None] if not main: return dataclasses.replace(state, segments=(), structure=Structure.NO_COMMA) if not state.comma_offsets: return dataclasses.replace(state, segments=(tuple(main),), structure=Structure.NO_COMMA) buckets: list[list[int]] = [[] for _ in range(len(state.comma_offsets) + 1)] for i in main: # comma_offsets is sorted and no offset ever equals a token # start, so bisect_left counts the commas before this token start = state.tokens[i].span.start bucket = bisect.bisect_left(state.comma_offsets, start) buckets[bucket].append(i) groups = [tuple(b) for b in buckets] # v1 strips exactly ONE trailing comma as cosmetic (parser.py's # collapse_whitespace); every other empty bucket is STRUCTURAL and # keeps its position -- in 'Doe,, Jr.' the given segment is empty, # so 'Jr.' stays a tail suffix instead of masquerading as a lone # post-comma title (v1 parity, pinned live 2026-07-16) if len(groups) > 1 and not groups[-1]: groups.pop() if len(groups) <= 1: segs = tuple(groups) if groups and groups[0] else (tuple(main),) return dataclasses.replace(state, segments=segs, structure=Structure.NO_COMMA) def suffixy(seg: tuple[int, ...]) -> bool: return is_wholly_suffix([state.tokens[i].text for i in seg], state.lexicon, state.policy) # v1 parity: only parts[1] decides the suffix-comma structure # (parser.py:1318); parts[2:] are consumed as suffixes # unconditionally either way, so a non-suffix tail segment gets the # COMMA_STRUCTURE flag, not a structure veto structure = (Structure.SUFFIX_COMMA if suffixy(groups[1]) and len(groups[0]) > 1 else Structure.FAMILY_COMMA) ambiguities = list(state.ambiguities) for seg in groups[2:]: # empty segments are consumed silently (v1 skips them without # comment); only non-empty non-suffix tails get flagged if seg and not suffixy(seg): texts = " ".join(state.tokens[i].text for i in seg) ambiguities.append(PendingAmbiguity( AmbiguityKind.COMMA_STRUCTURE, f"segment {texts!r} beyond the recognized comma " f"structures; consumed as suffix best-effort", tuple(seg))) return dataclasses.replace(state, segments=tuple(groups), structure=structure, ambiguities=tuple(ambiguities)) python-nameparser-2.1.0/nameparser/_pipeline/_state.py000066400000000000000000000105711523530431700231270ustar00rootroot00000000000000"""Internal pipeline state: WorkToken and ParseState. WorkTokens are pipeline-internal (no validation -- the tokenizer is the only producer) and are addressed BY INDEX in every stage: pieces and segments are runs of token indices, never joined strings, so value-based lookup (v1's #100 family) is structurally impossible. Layering: imports _types, _lexicon, _policy only (enforced by tests/v2/test_layering.py). """ from __future__ import annotations from dataclasses import dataclass from enum import Enum, auto from nameparser._lexicon import Lexicon from nameparser._policy import Policy from nameparser._types import AmbiguityKind, Role, Segmenter, Span # The comma characters (ASCII/Arabic/fullwidth, #265). Shared here so # tokenize (separators/segmentation) and extract (close-quote # boundaries) cannot drift apart. COMMA_CHARS = frozenset({",", "\u060c", "\uff0c"}) @dataclass(frozen=True, slots=True) class WorkToken: """One tokenized word. role stays None until assign; extracted nickname/maiden tokens arrive with their role pre-set. text is always the exact original slice (tokenize is the sole producer; the anti-#100 invariant depends on it).""" text: str span: Span tags: frozenset[str] = frozenset() role: Role | None = None class Structure(Enum): """segment's comma-structure decision.""" NO_COMMA = auto() FAMILY_COMMA = auto() # "Family, Given ..." (v1 lastname-comma) SUFFIX_COMMA = auto() # "Given Family, Suffix ..." @dataclass(frozen=True, slots=True) class PendingAmbiguity: """An ambiguity recorded mid-pipeline by token INDEX; assemble materializes real Ambiguity objects over the final tokens. ``origin`` is for the one stage that runs BEFORE tokens exist: extract_delimited knows only a character offset, so it records that and tokenize resolves it to the containing token's index. Stages after tokenize set ``indices`` directly and leave ``origin`` None. """ kind: AmbiguityKind detail: str indices: tuple[int, ...] = () origin: int | None = None @dataclass(frozen=True, slots=True) class ParseState: """Carried through the stage fold. Frozen; stages return copies via dataclasses.replace. Fields are filled progressively: extract_delimited -> extracted/masked; tokenize -> tokens (span- sorted)/comma_offsets/interpunct_offsets (the 间隔号 offsets the order and segmentation decisions consult, #298; the nakaguro separators record NOTHING); segment -> segments/structure; script_segment -> tokens and segments again (the one stage that changes the token COUNT: an unspaced CJK token splits into n+1 pieces, still as sub-slices of the original, and every later index in the segment runs shifts by n); classify -> token tags; group -> pieces/piece_tags/dropped AND maiden token roles; assign/post_rules -> the remaining token roles. Ambiguities are recorded by every stage that DECIDES one -- extract (resolved to a token index by tokenize), segment, script_segment, classify, group, and assign -- since a fork whose branches are taken in different stages needs an emitter in each. Post-group, segments may retain indices of dropped tokens -- assign iterates pieces, never segments. This ownership map is pinned by tests/v2/pipeline/test_state.py. segmenter belongs to no stage: like original/lexicon/policy it is passed in at construction by Parser.parse and only ever READ (by script_segment, for a token the vocabulary declined).""" original: str lexicon: Lexicon policy: Policy #: The optional Parser(segmenter=...) hook; None = not configured. segmenter: Segmenter | None = None extracted: tuple[tuple[Role, Span], ...] = () masked: tuple[Span, ...] = () tokens: tuple[WorkToken, ...] = () comma_offsets: tuple[int, ...] = () interpunct_offsets: tuple[int, ...] = () segments: tuple[tuple[int, ...], ...] = () structure: Structure = Structure.NO_COMMA # pieces[s][p] = run of token indices: piece p of segment s. # piece_tags[s][p] = derived flags for that piece ("title", "prefix", # "suffix", "conjunction") set by group's joins. pieces: tuple[tuple[tuple[int, ...], ...], ...] = () piece_tags: tuple[tuple[frozenset[str], ...], ...] = () dropped: tuple[int, ...] = () # structural tokens (maiden markers) ambiguities: tuple[PendingAmbiguity, ...] = () python-nameparser-2.1.0/nameparser/_pipeline/_tokenize.py000066400000000000000000000226501523530431700236400ustar00rootroot00000000000000"""Stage: tokenize. Consumes: original, masked (regions to skip), extracted (regions that tokenize with a pre-set role). Produces: tokens (span-sorted WorkTokens; text always == original slice), comma_offsets (segmentation points; never tokens), interpunct_offsets (间隔号 transcription markers, #298; never tokens). Reads: Policy.strip_emoji, Policy.strip_bidi. There is NO text-rewriting normalize stage: whitespace collapsing, emoji/bidi stripping, and the katakana name-dot split are all character-classification rules here -- ignorable characters act as separators and never enter a token, so spans always index the original exactly as given. Whitespace and the name-dot are unconditional; emoji/bidi stripping alone is policy-gated (Policy.strip_emoji/strip_bidi). The Chinese interpunct U+00B7 is context-sensitive -- see _INTERPUNCT below. v1's squash_emoji/squash_bidi REMOVED the char and joined neighbors ('A\U0001f600B' -> 'AB'); here an ignorable char is a SEPARATOR ('A\U0001f600B' -> 'A', 'B') -- the unavoidable consequence of spans indexing the original exactly. """ from __future__ import annotations import bisect import dataclasses import re from nameparser._pipeline._state import ( COMMA_CHARS, ParseState, WorkToken, ) from nameparser._policy import Policy, _SCRIPT_RANGES, _script_matcher from nameparser._types import Role, Span # Ported from v1 (nameparser/config/regexes.py, "emoji" and "bidi") -- # layering forbids importing the config package here, so the tables are # duplicated by design with this provenance note. When editing, keep # both copies in sync (regexes.py builds its public re_emoji from the # SAME codepoint pairs). Integer ranges, not a regex character class: # the per-char test needs no regex, and CodeQL's py/overly-large-range # false-positives on literal astral ranges (surrogate decomposition). _EMOJI_RANGES = ((0x1F300, 0x1F64F), (0x1F680, 0x1F6FF), (0x2600, 0x26FF), (0x2700, 0x27BF)) _BIDI = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+') # The katakana middle dot and its halfwidth twin divide the parts of # a foreign name transcribed into katakana (マイケル・ジャクソン) -- # native names never contain them, so they separate unconditionally, # like whitespace (amendment 2026-07-29 section 1b). They record no # offset: the nakaguro also divides kanji roster pairs (高橋・一郎, # 姓・名 -- read family-first by the script license, #272), so it is # not a transcription marker. Only the Chinese 间隔号 is (#298), and # it is context-sensitive -- see _INTERPUNCT below. _NAME_DOT_SEPARATORS = frozenset({"\u30FB", "\uFF65"}) _INTERPUNCT = "\u00B7" # Per-CHAR classifier for the interpunct's flank guard: a one-char # string is wholly-classified iff the character is. U+00B7 cannot be # an unconditional separator like the name dots above -- it is also # the Catalan punt volat, INTERIOR to legitimate names (Gal\u00B7la) -- so # it divides only between classified-script characters: the first # context-sensitive separator rule, which is why it lives in # _tokenize_region (where the index exists) and not in _ignorable. _classified_char = _script_matcher(*_SCRIPT_RANGES, whole=True) def _is_emoji(ch: str) -> bool: cp = ord(ch) return any(lo <= cp <= hi for lo, hi in _EMOJI_RANGES) def _stripped(ch: str, policy: Policy) -> bool: """True when the strip policy removes `ch` from the token stream. The ONE definition of that set, shared by _ignorable and _flank: a character that vanishes from tokens must not occupy a flank position either, so a new strip class added here stays transparent to the interpunct guard by construction.""" if policy.strip_bidi and _BIDI.match(ch): return True return bool(policy.strip_emoji and _is_emoji(ch)) def _flank(text: str, indices: range, policy: Policy) -> str | None: """The nearest flank character the strip policy would keep, or None if the range exhausts. Stripped invisibles are TRANSPARENT here: an RTL document quoting a transcription puts U+200F beside the dot in visually identical text. Whitespace and the other separators stay guard-defeating: a dot beside a space is not between characters.""" for i in indices: ch = text[i] if not _stripped(ch, policy): return ch return None def _ignorable(ch: str, state: ParseState) -> bool: if ch.isspace(): return True if ch.isascii(): # both strip classes and the name-dot separators are entirely # non-ASCII (bidi >= U+061C, emoji >= U+2600, dots >= U+30FB): # skip the checks below for every ASCII letter return False # unconditional, like whitespace -- not policy-gated, so this sits # ahead of the policy check rather than in _tokenize_region beside # COMMA_CHARS (commas RECORD an offset; these dots must not -- # only the 间隔号 marks a transcription and records, #298, see # _NAME_DOT_SEPARATORS above). The only load-bearing constraint is # "after the isascii fast path" (both dots are non-ASCII). if ch in _NAME_DOT_SEPARATORS: return True return _stripped(ch, state.policy) def _tokenize_region(state: ParseState, start: int, end: int, role: Role | None, record_offsets: bool, tokens: list[WorkToken], commas: list[int], interpuncts: list[int]) -> None: text = state.original tok_start: int | None = None for i in range(start, end): ch = text[i] is_separator = ch in COMMA_CHARS or _ignorable(ch, state) if ch == _INTERPUNCT: # Region-local flanks: a B7 at a region edge stays token # text, and the bound also stops a custom delimiter's # classified edge character from acting as a flank across # a mask seam. The scan reads raw text like segmentation # matching does (#272's stance): NFD hangul degrades to # no-split, never wrong-split -- classification # NFC-normalizes but the guard does not. left = _flank(text, range(i - 1, start - 1, -1), state.policy) if left is not None and _classified_char(left): right = _flank(text, range(i + 1, end), state.policy) is_separator = (right is not None and _classified_char(right)) if is_separator: if tok_start is not None: tokens.append(WorkToken(text[tok_start:i], Span(tok_start, i), role=role)) tok_start = None if record_offsets: if ch in COMMA_CHARS: commas.append(i) elif ch == _INTERPUNCT: interpuncts.append(i) continue if tok_start is None: tok_start = i if tok_start is not None: tokens.append(WorkToken(text[tok_start:end], Span(tok_start, end), role=role)) def tokenize(state: ParseState) -> ParseState: tokens: list[WorkToken] = [] commas: list[int] = [] interpuncts: list[int] = [] # main stream: everything outside masked regions boundaries = [0] for m in state.masked: boundaries.extend((m.start, m.end)) boundaries.append(len(state.original)) for start, end in zip(boundaries[::2], boundaries[1::2]): _tokenize_region(state, start, end, None, True, tokens, commas, interpuncts) # extracted regions: pre-set role, commas and interpuncts are mere # separators for role, inner in state.extracted: _tokenize_region(state, inner.start, inner.end, role, False, tokens, commas, interpuncts) tokens.sort(key=lambda t: t.span) # extract_delimited runs before tokens exist, so its ambiguities # carry a character offset instead of an index. Resolve them now # that the stray character has landed in a token ('"Nick', 'Smith)') # -- without this the ambiguity is locatable only by parsing the # offset back out of its detail string. An offset inside a masked # region belongs to no token; those keep an empty tuple, which the # kind's contract already allows. # Bisect rather than rescan: the closer sweep can emit one # ambiguity per delimiter character, so a linear scan per ambiguity # is quadratic on pathological input (') ' * 1600 spent 180ms here, # against 9ms before the sweep existed). Spans are non-overlapping # and sorted just above, so the candidate is the last token whose # start is <= the offset. # Nothing to resolve on the overwhelmingly common no-ambiguity # parse, and building starts + the closure is not free. ambiguities = state.ambiguities if any(a.origin is not None for a in ambiguities): starts = [t.span.start for t in tokens] def _containing(offset: int) -> tuple[int, ...]: i = bisect.bisect_right(starts, offset) - 1 if i >= 0 and offset < tokens[i].span.end: return (i,) return () ambiguities = tuple( a if a.origin is None else dataclasses.replace(a, indices=_containing(a.origin)) for a in ambiguities) return dataclasses.replace(state, tokens=tuple(tokens), comma_offsets=tuple(sorted(commas)), interpunct_offsets=tuple(sorted(interpuncts)), ambiguities=ambiguities) python-nameparser-2.1.0/nameparser/_pipeline/_vocab.py000066400000000000000000000411341523530431700231000ustar00rootroot00000000000000"""Shared vocabulary predicates for pipeline stages. Text-level tests used by more than one stage; token/piece-level predicates live with their stage. All take normalized-or-raw text explicitly -- no state. is_wholly_suffix departs from that shape twice, deliberately. It is RUN-level rather than text-level, because the question it answers is genuinely about a run: the Ph./D. merge spans two tokens, so no per-token predicate composed with all() can express it. And it takes the Policy OBJECT, where delimiter_cores takes a pre-extracted frozenset so its caller hands in one field rather than the config -- is_wholly_suffix needs TWO policy fields (lenient_comma_suffixes and extra_suffix_delimiters), and threading both past every caller costs more than the config parameter saves. Still no state: Policy is frozen config, not pipeline state. Layering: imports _lexicon, _types, and _policy only. """ from __future__ import annotations import re import unicodedata from collections.abc import Callable, Iterable, Sequence from nameparser._lexicon import Lexicon, _normalize from nameparser._policy import (Policy, Script, _JA_SCRIPTS, _NO_INITIALS, _SCRIPT_RANGES, _script_matcher) # Ported verbatim from v1 (nameparser/config/regexes.py "initial") minus # its empty-string alternative -- WorkToken text is never empty. Kept in # sync by hand; layering forbids importing the config package here. # "Verbatim" is a promise about the PATTERN, not about the predicate: # since #320 is_initial is this SHAPE test ANDed with a repertoire test # (_in_initialless_script, below), so _INITIAL.fullmatch(text) and # is_initial(text) are no longer the same question -- '씨.' answers yes # to the first and no to the second. Call is_initial; the bare pattern # is not the thing to ask. The narrowing lives in the predicate # precisely so this copy can stay exactly as verbatim as it ever was # -- REGEXES["initial"] is public v1 API and cannot narrow, and the # only difference between the two remains the empty alternative noted # above (config's `?`), which test_regex_sync splices back in. _INITIAL = re.compile(r"^(\w\.|[A-Z])$") # Ported verbatim from v1 (nameparser/config/regexes.py # "period_not_at_end") -- layering forbids the config import; keep in # sync by hand. _PERIOD_NOT_AT_END = re.compile(r".*\..+$", re.I) # The fix_phd credential pair ('Ph.' + 'D.' as adjacent tokens), shared # by is_wholly_suffix below and group's merge (v1 extracted the # credential pre-parse; the predicate and the stage must agree on the # pattern). PH = re.compile(r"^ph\.?$", re.IGNORECASE) D = re.compile(r"^d\.?$", re.IGNORECASE) # The codepoint table lives in _policy beside Script -- one copy # importable from the pipeline and the locale packs alike; everything # here DERIVES from it. (single_script's sweep was first written # per-char on the _EMOJI_RANGES precedent in _tokenize.py, on the # theory that a range test needs no regex; measured at token scale the # compiled regex wins by 3-9x, and by roughly 70x on long tokens.) # Derived, never hand-written -- even the class construction goes # through the shared factory: one wholly-of predicate per script, in # the table's key order -- the FIRST-covering-entry rule _classify # documents. _SCRIPT_MATCHERS: dict[Script, Callable[[str], bool]] = { script: _script_matcher(script, whole=True) for script in _SCRIPT_RANGES } # The whole-token matcher over _policy's _JA_SCRIPTS union, backing # effective_script's kana license. _wholly_ja = _script_matcher(*_JA_SCRIPTS, whole=True) # The repertoire half of is_initial (_policy._NO_INITIALS), kept apart # from _INITIAL's SHAPE half so the pattern itself stays v1-verbatim # and its three copies stay pinned by tests/v2/test_regex_sync.py. # contains-any, not whole=True: the shape half has already admitted the # trailing period, so the text reaching here is '씨.' rather than '씨' # and a wholly-of match would be False for every case this exists for. _in_initialless_script = _script_matcher(*_NO_INITIALS, whole=False) def is_initial_shaped(text: str) -> bool: """v1's is_an_initial verbatim: the SHAPE half alone -- one word character plus a period, or a bare ASCII capital. Callers asking whether a token is STRUCTURALLY part of an initial run want this; callers asking whether it can really stand in for a name want is_initial (#320). The two answers differ only inside _NO_INITIALS scripts, where '씨.' is initial-SHAPED but is not an initial -- see assign's roman-numeral fork, the shape caller, for what picking the wrong one costs.""" return bool(_INITIAL.fullmatch(text)) def is_initial(text: str) -> bool: """'A.' / 'j.' / bare capital -- v1's is_an_initial, narrowed to scripts that HAVE initials (#320). v1's \\w is Unicode-aware and matched CJK too, which made period-written CJK honorifics ('씨.') fail is_suffix_strict -- the veto in _is_suffix_strict_n, NOT the vocabulary: suffix_as_written has no veto, so classify tagged '씨.' 'vocab:suffix' either way, and is_suffix_lenient took it either way too. Downstream of that one strict-test No, the glued honorific in a name carrying such a token went unpeeled ('田中さん 様.').""" return is_initial_shaped(text) and not _in_initialless_script(text) def suffix_as_written(n: str, text: str, lexicon: Lexicon) -> bool: """Counts as a suffix as written, with NO initial veto (the veto differs by caller): unambiguous suffix vocabulary, or an ambiguous acronym written with periods ('M.A.' yes, 'Ma' no). `n` is _normalize(text), passed in so callers normalize once. Single source for classify's "vocab:suffix" tag and the segment/ assign predicates. The ambiguous subset is EXCLUDED from the plain membership test: in the real data suffix_acronyms_ambiguous is a subset of suffix_acronyms, and without the exclusion the period gate is dead code (bare 'Ed'/'Jd' would silently become suffixes). """ # acronyms may be written with periods ('M.B.A.'): the ACRONYM # membership alone uses the period-free form (v1's is_suffix # removed periods only for the suffix_acronyms test); suffix WORDS # match on the plain normalized form a = n.replace(".", "") if "." in text and a in lexicon.suffix_acronyms_ambiguous: return True return (a in lexicon.suffix_acronyms and a not in lexicon.suffix_acronyms_ambiguous) \ or n in lexicon.suffix_words def _is_suffix_strict_n(n: str, text: str, lexicon: Lexicon) -> bool: if is_initial(text): # period-written ambiguous acronyms are exempt from the veto return "." in text and \ n.replace(".", "") in lexicon.suffix_acronyms_ambiguous return suffix_as_written(n, text, lexicon) def is_suffix_strict(text: str, lexicon: Lexicon) -> bool: """v1's is_suffix: suffix_as_written with the initial veto ('V.' in 'John V. Smith' is a middle initial, not roman five).""" return _is_suffix_strict_n(_normalize(text), text, lexicon) def is_suffix_lenient(text: str, lexicon: Lexicon) -> bool: """v1's is_suffix_lenient: suffix_words accepted unconditionally, bypassing the initial veto -- only safe in unambiguous positions (after a comma).""" n = _normalize(text) return n in lexicon.suffix_words \ or _is_suffix_strict_n(n, text, lexicon) def delimiter_cores(policy_delimiters: frozenset[str]) -> frozenset[str]: """Configured suffix delimiters with surrounding whitespace stripped: ' - ' -> '-'. Whitespace-padded delimiters surface as standalone tokens; the stripped core is what tokenize produced.""" return frozenset(d.strip() for d in policy_delimiters if d.strip()) def splits_into_suffixes(text: str, cores: frozenset[str], lexicon: Lexicon) -> bool: """v1 expand_suffix_delimiter parity for delimiters WITHOUT whitespace ('RN/CRNA' with '/'): the token counts as a suffix when some core splits it into >=2 non-empty parts that are all suffixes. The token text is never rewritten (anti-#100): it takes Role.SUFFIX whole, which renders 'RN/CRNA' where v1 rendered 'RN, CRNA' -- the documented divergence, release-log classified.""" for core in cores: if core in text: parts = [part for part in text.split(core) if part] if len(parts) >= 2 and all( is_suffix_lenient(part, lexicon) for part in parts): return True return False def period_joined_vocab(text: str, lexicon: Lexicon) -> str | None: """v1's parse_pieces derivation for interior-period tokens ('Lt.Gov.', 'Msc.Ed.', and by the ANY rule 'Mr.Smith'): ANY title chunk makes the token a title (checked first, v1's continue); else ANY suffix chunk makes it a suffix. Chunk-level suffix membership is v1's is_suffix: bare ambiguous acronyms COUNT ('Msc.Ed.' derives via 'ed') -- the ambiguous period-gate applies to whole tokens only. Returns "title", "suffix", or None.""" if not _PERIOD_NOT_AT_END.match(text): return None chunks = [_normalize(c) for c in text.split(".") if c] if any(c in lexicon.titles for c in chunks): return "title" if any(c in lexicon.suffix_acronyms or c in lexicon.suffix_words for c in chunks): return "suffix" return None def is_wholly_suffix(texts: Sequence[str], lexicon: Lexicon, policy: Policy) -> bool: """Every token in a RUN counts as a suffix -- segment's suffix-comma test, lifted out of it so the peel can ask the same question (#319). NOT the plural of _script_segment._is_post_nominal, which asks is_suffix_strict per token. This asks the POLICY-selected predicate (lenient by default), plus period_joined_vocab, delimiter transparency and the Ph./D. merge. 'V.' is the input that tells them apart: it satisfies this predicate but is not a post-nominal -- and reading one for the other IS the #319 bug. An EMPTY run is False, not vacuously True: v1's suffix-comma detection fails on an empty parts[1] ('John Smith,, MD' is a family-comma parse). The 'wholly' idiom agrees -- _script_matcher's whole=True requires non-empty too -- which is why the name is that one rather than all_suffixes, where Python's all([]) would promise the opposite. An adjacent Ph./D. pair counts as ONE unit (v1's fix_phd extracted the credential pre-parse, so 'Smith, Ph. D.' read as suffix-comma); keep in sync with group's _PH/_D merge. """ if not texts: return False predicate = (is_suffix_lenient if policy.lenient_comma_suffixes else is_suffix_strict) # v1 expand_suffix_delimiter parity (#191): a configured delimiter # is TRANSPARENT in the all-suffix tests -- v1 split the part string # on the delimiter before checking, so the delimiter never counted cores = delimiter_cores(policy.extra_suffix_delimiters) def counts_as_suffix(text: str) -> bool: if text in cores: return True return (predicate(text, lexicon) or period_joined_vocab(text, lexicon) == "suffix" or (bool(cores) and splits_into_suffixes(text, cores, lexicon))) merged = list(texts) k = 0 while k < len(merged) - 1: if PH.fullmatch(merged[k]) and D.fullmatch(merged[k + 1]): merged[k:k + 2] = ["phd"] else: k += 1 return all(counts_as_suffix(t) for t in merged) def _normalized_for_script(text: str) -> str | None: """The guard AND the NFC normalization single_script and effective_script's license path both need, single-sourced so they cannot drift: None for the two shapes neither ever classifies (empty, and the common all-ASCII Latin token -- skipped before normalizing, since ASCII is already NFC and every _SCRIPT_RANGES entry is non-ASCII regardless), else an NFC-normalized copy. NFC, not raw: NFD input decomposes precomposed katakana onto a base character plus a COMBINING mark (U+3099/U+309A, which sit in the HIRAGANA block, not katakana's), so classifying raw NFD text can hand a pure-katakana token the kana license by accident; NFD also decomposes Hangul syllables onto bare jamo (U+1100-U+11FF), entirely outside the HANGUL range, so raw NFD Korean input misses the shipped family-first order rule rather than merely misfiring. Normalizing first fixes both. This is classification-only and read-only: the returned copy is never what gets tokenized, so token text and spans stay exactly what the caller wrote. MATCHING (is_initial, suffix lookups, etc.) deliberately stays on raw text elsewhere in this module -- unlike script classification, NFD only ever costs a match there (a suffix word written NFD fails to match its NFC vocabulary entry), never wrong-matches, so the asymmetry is safe: one direction needs a fix, the other doesn't. """ if not text or text.isascii(): return None return unicodedata.normalize("NFC", text) def _classify(normalized: str) -> Script | None: """The FIRST _SCRIPT_MATCHERS entry covering all of `normalized` (already NFC, via _normalized_for_script), else None. Shared by both public classifiers so each of them normalizes exactly once.""" for script, matcher in _SCRIPT_MATCHERS.items(): if matcher(normalized): return script return None def single_script(text: str) -> Script | None: """The one Script whose ranges cover EVERY char of `text`, else None (mixed-script text has no well-defined convention to apply; the caller falls back to the positional default). Classifies an NFC-normalized copy of `text` -- see _normalized_for_script. Callers wanting the kana-mixed license (a kanji+kana composite resolving to HIRAGANA) want effective_script, not this function.""" normalized = _normalized_for_script(text) if normalized is None: return None return _classify(normalized) def effective_script(text: str) -> Script | None: """single_script, extended by the kana license (#272 amendment): a MIXED token wholly within Han∪hiragana∪katakana is Japanese -- it necessarily contains kana (pure Han is not mixed), cannot be Chinese, and is not a foreign transcription (those are katakana-only: マイケル has no kanji, but さくらエミ -- hiragana plus katakana -- is kana-only AND licensed) -- and resolves to the HIRAGANA carrier entry. Pure-katakana stays KATAKANA (single_script's answer): a lone katakana token is predominantly a transcribed foreign name, so nothing defaults on it.""" # None for both shapes _wholly_ja could never match anyway (empty # text, or all-ASCII text): real work, not a leftover "if text" # guard, since the ASCII case is one a bare emptiness check would # let through. The single normalized copy then serves both the # single-script answer and the license below. normalized = _normalized_for_script(text) if normalized is None: return None script = _classify(normalized) if script is not None: return script if _wholly_ja(normalized): return Script.HIRAGANA return None def resolve_script_set(scripts: Iterable[Script]) -> Script | None: """Generalizes effective_script's kana license from one token's CHARACTERS to a whole name's PIECES (#272): `scripts` is the effective_script of every name token, already resolved individually -- '高橋' (Han) and 'みなみ' (Hiragana) are two separately single-script pieces (split by a space, not mixed within one token), but together are exactly the repertoire effective_script licenses inside a single token (高橋みなみ). A single distinct script is returned as-is (the ordinary case, including a lone wholly-katakana name, which callers key with no table entry); more than one collapses to the HIRAGANA carrier when confined to Han/Hiragana/Katakana, the same set effective_script's license tests; any other mix (Han+Hangul, or no scripts at all -- an empty `scripts`) returns None -- the caller's cue to fall back to the positional default, exactly like effective_script's own None. A non-None result reports what was FOUND, not that a license fired: callers wanting to know whether the kana license specifically was the reason must compare the result against a specific Script (e.g. `is Script.HIRAGANA`), not just its truthiness -- a lone wholly-Han name also returns non-None here, licensing nothing.""" found = frozenset(scripts) if len(found) <= 1: return next(iter(found), None) if found.issubset(_JA_SCRIPTS): return Script.HIRAGANA return None python-nameparser-2.1.0/nameparser/_policy.py000066400000000000000000001327041523530431700213450ustar00rootroot00000000000000"""Immutable behavior configuration for the 2.0 API. Layering: imports nameparser._types only (enforced by tests/v2/test_layering.py). """ from __future__ import annotations import dataclasses import re from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, field from enum import Enum, StrEnum, auto from typing import Any from nameparser._types import Role, _guarded_getstate, _guarded_setstate class PatronymicRule(StrEnum): """Stable rule names (API); implementations live in the pipeline. Enable via ``Policy(patronymic_rules={...})`` or, more commonly, a locale pack (:mod:`nameparser.locales`).""" #: East Slavic formal order: "Sidorov Ivan Petrovich" #: (family, given, patronymic) is detected by the patronymic #: ending and reordered. Enabled by locales.RU. EAST_SLAVIC = "east-slavic" #: Turkic patronymic markers: a standalone "oglu"/"qizi"/"kyzy" #: (etc.) binds to the preceding name as a patronymic. Enabled by #: locales.TR_AZ. TURKIC = "turkic" class Script(StrEnum): """Writing systems the parser can key SCRIPT-CONDITIONAL behavior on: per-script name order (``Policy.script_orders``) and unspaced-name segmentation (``Policy.segment_scripts``). The rule that admits these (amendment 2026-07-27): script-conditional behavior only where the script itself determines the convention -- Latin-script input is never affected. The codepoint table backing these members is internal.""" #: Chinese Hanzi -- and Japanese Kanji: a pure-Han string cannot #: say which language it is, which is fine for ORDER (both write #: family-first natively) and exactly why Han SEGMENTATION is #: opt-in, per language: locales.ZH brings the Chinese surname #: list, locales.JA activates the same stage for a pluggable #: segmenter to divide kanji names with. HAN = "han" #: Korean Hangul (precomposed syllables). Unambiguously Korean. HANGUL = "hangul" #: Japanese hiragana. Never transcribes foreign names, so a mixed #: kanji+kana token (高橋みなみ) is Japanese and resolves HERE -- #: this member is the carrier key in script_orders/segment_scripts. HIRAGANA = "hiragana" #: Japanese katakana. A PURE-katakana token is predominantly a #: transcribed foreign name in its original order (マイケル), so #: no default behavior keys on this member; it exists so the #: classifier can name what it deliberately declines. KATAKANA = "katakana" # Codepoint ranges per Script (#271). This integer table is the single # source of truth for what a script covers; every matcher DERIVES from # it -- _pipeline/_vocab.py compiles its per-script patterns from it, # and it is importable from the pipeline and the locale packs alike, # so the packs' predicates build on it too, through _script_matcher # below (the table lives here rather than in the pipeline because # packs must not import the pipeline). # HAN: the ideographic iteration mark U+3005 and the shime mark # U+3006, the URO plus Extension A, the compatibility block, and the # supplementary-plane block # (Ext B-I + CJK Compat Ideographs Supplement, 0x20000-0x323AF) -- # rare surnames are the biggest real source of supplementary-plane # hanzi in personal names (e.g. 𠮷田's 𠮷, U+20BB7), so leaving them # out silently mis-orders those names; unassigned gaps inside the span # are harmless, since no real name contains an unassigned codepoint. # U+3005 々 is the block-vs-Script case, running the OPPOSITE way to # U+30FB below: 々 already IS Script=Han under UAX #24 (Scripts.txt # reads `3005 ; Han`), but it sits in CJK Symbols and Punctuation, # outside every CJK ideograph block this table spans -- so a # singleton entry was what a BLOCK table needed to reach a character # the Script property would have classified correctly for free. It # earns the reach: 々 repeats the preceding kanji and appears only # inside Han-written names -- 佐々木 (Sasaki, a top-20 Japanese # surname), 野々村, 奈々. Omitting it made 佐々木 a mixed-script token: # the name reversed and never gated into segmentation. # U+3006 〆 (the shime mark) extends that singleton to a two-codepoint # span on a DIFFERENT justification: unlike 々, 〆 is Script=Common # under UAX #24, so this is the table deliberately reaching PAST the # Script property, not around a block boundary -- justified because # within personal names 〆 appears solely in Japanese surnames (〆木 # Shimeki, 〆谷 Shimetani, 〆野) -- its other uses (the envelope # closing mark, 〆切) never reach a name parser -- and it appears in # no other script's names. # HANGUL: precomposed syllables only -- modern Korean # text never writes names as bare jamo. # HIRAGANA/KATAKANA (#272): the two kana blocks, each in full. There # IS a supplementary-plane kana repertoire (Kana Supplement, Kana # Extended-A/B, Small Kana Extension, U+1AFF0-U+1B16F, a few hundred # assigned codepoints -- no exact count here, it moves with the # Unicode version) but none of it is WORTH chasing the way Han's astral # block is: those codepoints are hentaigana and other archaic/ # phonetic-extension forms no modern Japanese name uses, unlike # supplementary Han, which real surnames genuinely need. The Katakana # Phonetic Extensions block (U+31F0-U+31FF, 16 small katakana for Ainu # transcription) is excluded for the same reason -- no modern Japanese # personal name uses them. Halfwidth kana (U+FF65-U+FF9F, including # the voiced/semi-voiced sound marks U+FF9E/U+FF9F) is likewise # deliberately excluded -- legacy bank/CSV data uses it, but it is a # separate normalization problem; #272 Task 2b's separator handling # only touches the halfwidth DOT (U+FF65), not the rest of that block. # This table classifies by Unicode BLOCK, not the UAX #24 Script # property: U+30A0, U+30FB (the middle dot), and U+30FC (the # prolonged sound mark) all carry Script=Common under UAX #24, and the # four kana voicing marks U+3099-U+309C split two and two -- U+3099 # and U+309A are the COMBINING forms (Script=Inherited), U+309B and # U+309C the spacing ones (Script=Common) -- yet every one of them is # needed here, and block membership, not the Script property, is what # puts them in range. The katakana block's upper end (U+30FF) takes in # the middle dot U+30FB, kept rather than carved out for a smaller # reason than it looks: tokenize (#272 Task 2b) turns U+30FB into a # token separator, so no real parse shows the classifier a string # containing one. It is kept so that a DIRECT whole-string call -- # effective_script (_pipeline/_vocab.py) on "マイケル・ジャクソン", which # the unit tests (tests/v2/pipeline/test_vocab.py) make -- still # classifies instead of returning None. The ranges below must # stay mutually disjoint: single_script (_pipeline/_vocab.py) returns # the FIRST covering entry (dict iteration order), so an overlapping # future script would make the result order-dependent instead of # well-defined. _SCRIPT_RANGES: dict[Script, tuple[tuple[int, int], ...]] = { Script.HAN: ((0x3005, 0x3006), (0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), (0x20000, 0x323AF)), Script.HANGUL: ((0xAC00, 0xD7A3),), Script.HIRAGANA: ((0x3040, 0x309F),), Script.KATAKANA: ((0x30A0, 0x30FF),), } #: The Japanese repertoire: the three scripts Japanese names draw on. #: The kana license (_pipeline/_vocab.py's effective_script), the ja #: pack's DEVIATES, and the segmenter adapter's repertoire guard all #: quantify over this one union (HANGUL simply omitted). _JA_SCRIPTS = (Script.HAN, Script.HIRAGANA, Script.KATAKANA) #: Scripts whose characters cannot BE an initial. The criterion is #: orthographic CONVENTION, not what a character is: does the writing #: tradition abbreviate a given name to ONE character plus a period, #: the way "J." stands in for "John"? Han, hangul and kana have no #: such convention, so a lone punctuated 씨/様/김 is not a shortened #: name and the veto has nothing to veto there. Do not restate that #: phonologically ("letters, not syllables") -- Devanagari is an #: abugida and Arabic an abjad, neither has letters in that sense, and #: both abbreviate, so should Script.CYRILLIC or Script.DEVANAGARI #: ever be added neither belongs here; their initials are real and #: pinned as such ("А. С. Пушкин", "م. الفارسي"). #: #: Enumerated rather than derived from _SCRIPT_RANGES' keys: the #: Script enum admits a member so that SOME behavior may key on it #: (see Script), on assorted grounds -- KATAKANA is in it so the #: classifier can name what it deliberately declines, and neither #: DEFAULT_SCRIPT_ORDERS nor segment_scripts' default mentions it. #: Membership therefore settles nothing about abbreviation: the four #: coinciding today is what has been implemented, not a property of #: the enum, and a new member must not inherit this answer. #: #: Decide it from CLDR rather than from the script's typology: count #: the LOCALE-AUTHORED namePattern entries in common/main/.xml's #: personNames block that produce an initial. ja and ko author 29 and #: 32 patterns and use one in none of them; ru uses initials in 7 of 39 #: ("{given-initial} {given2-initial} {surname}"); zh abbreviates but #: overrides initialPattern to "{0}", no period, which is why the #: period is part of the test above. Do NOT read initialPattern alone #: -- root defaults it to "{0}." and nearly every locale inherits it, #: so it reports a period convention for locales that have none. #: Measured 2026-08-02: th is 0 of 20, so Thai (#317) belongs here #: once it earns a member. _NO_INITIALS = (Script.HAN, Script.HANGUL, Script.HIRAGANA, Script.KATAKANA) def _script_matcher(*scripts: Script, whole: bool = False) -> Callable[[str], bool]: """A predicate over strings, compiled once from the union of the named scripts' spans in _SCRIPT_RANGES. whole=False: True when the string CONTAINS any such character -- DEVIATES' contract, where over-declaring is the gate's safe direction. whole=True: True when the string is non-empty and consists WHOLLY of such characters -- the ja adapter's repertoire guard and _vocab's script classifiers. Meant to be called at MODULE scope: "compiled once" is per matcher, and each call compiles a fresh pattern. The compiled pattern lives in the closure ON PURPOSE, and the compilation lives HERE rather than in a pack- local closure: tests/v2/test_locales.py classifies any pack module holding a module-level re.Pattern as a marker pack needing rotator branch coverage, and its registry gate goes further -- a pack that so much as IMPORTS re without exposing such a pattern fails "imports re but exposes no module-level pattern" -- so a range-declaring pack must not import re at all; predicates built here keep the packs invisible to that sweep by construction (and spare _vocab's derived matchers a declaration row in tests/v2/test_regex_sync.py's completeness sweep, which scans the pipeline modules (plus _render) for private module-level patterns).""" if not scripts: raise ValueError("_script_matcher needs at least one Script") cls = "".join(f"\\U{lo:08x}-\\U{hi:08x}" for script in scripts for lo, hi in _SCRIPT_RANGES[script]) # one pattern serves both modes: fullmatch of [cls]+ is wholly-of, # and search over [cls]+ is exactly contains-any pattern = re.compile(f"[{cls}]+") match = pattern.fullmatch if whole else pattern.search def matcher(text: str) -> bool: return match(text) is not None return matcher # Order-spec constants (#270). Each reads as its contents because roles # are named given/family, not first/last. #: Western order (the default): the first word of positional input is #: the given name, the last is the family name, everything between is #: middle. One of the three valid ``Policy(name_order=...)`` values. GIVEN_FIRST = (Role.GIVEN, Role.MIDDLE, Role.FAMILY) #: Family name first, given name second, remaining words middle #: (e.g. Hungarian, or East Asian order). One of the three valid #: ``Policy(name_order=...)`` values. FAMILY_FIRST = (Role.FAMILY, Role.GIVEN, Role.MIDDLE) #: Family name first, given name LAST, words between middle #: (e.g. Vietnamese full-name order). One of the three valid #: ``Policy(name_order=...)`` values. FAMILY_FIRST_GIVEN_LAST = (Role.FAMILY, Role.MIDDLE, Role.GIVEN) _ORDER_CONSTANT_NAMES: dict[tuple[Role, ...], str] = { GIVEN_FIRST: "GIVEN_FIRST", FAMILY_FIRST: "FAMILY_FIRST", FAMILY_FIRST_GIVEN_LAST: "FAMILY_FIRST_GIVEN_LAST", } def _order_repr(value: tuple[Role, ...]) -> str: # Unreachable via Policy's constructor (its __post_init__ restricts # name_order to the three named orders) but REACHABLE via # PolicyPatch, which defers name_order validation to apply time by # design -- value may hold non-Role, even unhashable, elements. A # value smuggled in through __setstate__ (which validates layout, # not values) can also be a non-tuple container or not iterable at # all. repr must never raise, so the named-lookup path is taken # only for a TUPLE whose every element is confirmed a Role; # everything else renders via repr(value). (The annotation states # the Policy-side truth; the PolicyPatch call site passes # getattr-Any.) if isinstance(value, tuple) and all(isinstance(r, Role) for r in value): named = _ORDER_CONSTANT_NAMES.get(value) if named is not None: return named return "(" + ", ".join(r.name for r in value) + ")" return repr(value) # Single source for the migration hint raised by both Policy and # PolicyPatch when patronymic_rules gets a non-iterable (True is the # likeliest wrong value -- v1's flag was a bool that enabled BOTH rules). _PATRONYMIC_MIGRATION_HINT = ( "v1's patronymic_name_order=True enabled both rules -- " "patronymic_rules={PatronymicRule.EAST_SLAVIC, " "PatronymicRule.TURKIC} (or pick one via " "parser_for(locales.RU) / locales.TR_AZ)" ) #: Policy.script_orders' default: wholly-Han, wholly-Hangul, and #: kana-licensed names read family-first. Public and named so opting #: out or extending reads against a documented value (the #: DEFAULT_NICKNAME_DELIMITERS precedent). The HAN entry is safe #: WITHOUT knowing Chinese from Japanese: both write family-first in #: native script -- the languages differ, the convention doesn't. #: HIRAGANA joins by the same rule as HANGUL (the kana license, #: amendment 2026-07-29): a mixed Han-and-kana token cannot be #: Chinese (it contains kana) and is not a foreign transcription #: (transcriptions are katakana-only), so it is Japanese, written #: family-first -- another default change in a minor, release-log- #: classified fix, #294's mechanism. KATAKANA is deliberately absent: #: a PURE-katakana token is predominantly a transcribed foreign name #: kept in its source (usually given-first) order, so nothing should #: default on it. Canonical form: sorted (Script, order) pairs, #: matching the field's storage. DEFAULT_SCRIPT_ORDERS: tuple[ tuple[Script, tuple[Role, Role, Role]], ...] = ( (Script.HAN, FAMILY_FIRST), (Script.HANGUL, FAMILY_FIRST), (Script.HIRAGANA, FAMILY_FIRST), ) #: Policy.nickname_delimiters' default. Public and named so #: customizations read as set math against a documented value -- e.g. #: ``DEFAULT_NICKNAME_DELIMITERS | {("｟", "｠")}`` -- instead of a #: rebuilt literal the user had to go discover. The v1 trio (straight #: quotes + parentheses) plus the typographic conventions (#273): #: smart quotes, low-high and right-right quotes, guillemets both #: directions, CJK corner brackets, fullwidth parentheses. Curly #: SINGLE quotes are deliberately absent: U+2019 is the typographic #: apostrophe ("O’Connor"). DEFAULT_NICKNAME_DELIMITERS = frozenset({ ("'", "'"), ('"', '"'), ("(", ")"), # v1 trio ("“", "”"), # smart quotes (en, zh) ("„", "“"), # low-high (de, pl, cs, hu) ("”", "”"), # right-right (sv, fi) ("«", "»"), # guillemets (fr, ru, it, el) ("»", "«"), # reversed guillemets (de alt) ("「", "」"), ("『", "』"), # CJK corner brackets (ja) ("（", "）"), # fullwidth parentheses (CJK) }) def _reject_bare_string_order(value: object, field_name: str) -> None: # tuple("gmf") would be ("g", "m", "f") -- catch the bare string # with the same TypeError every other iterable field raises. # Single-sourced: called from Policy AND PolicyPatch __post_init__. # field_name is REQUIRED, with no name_order default: script_orders' # values obey the same rule (via _validated_order), and a defaulted # caller that forgot to pass it would silently name the wrong field. if isinstance(value, str): raise TypeError( f"{field_name} must be an iterable of three Roles, " f"not a bare string: {value!r}" ) # A {Role: position} dict iterates to the right three Roles in the # right order, so it would be accepted -- harmlessly today, since # the result is checked against the three exported orders anyway. # Guarded regardless: "iterating this yields something plausible # but not what you wrote" is one bug class, and leaving one field # out of it is how the PolicyPatch hole happened. if isinstance(value, Mapping): raise TypeError( f"{field_name} must be an iterable of three Roles, not a " f"mapping: {value!r}" ) if isinstance(value, (bytes, bytearray, memoryview)): raise TypeError( f"{field_name} must be an iterable of three Roles, not " f"{type(value).__name__} -- decode first, e.g. " f"raw.decode('utf-8')" ) def _reject_str_and_mapping(value: object, field_name: str) -> None: """The two shapes that iterate into something plausible but wrong. A bare string yields its characters (and '' yields nothing at all, so it silently stored an empty set); a Mapping yields only its keys. Both used to be accepted here, storing a value the caller never wrote. Lexicon._normset rejects the same two by name -- the wording is deliberately parallel, since a caller who hits one field's guard should recognize the other's. """ if isinstance(value, str): raise TypeError( f"{field_name} must be an iterable, not a bare string: " f"{value!r}" ) # bytes iterate to ints, so the entry check would report a byte # value and name neither the cause nor the fix; same decode hint # Lexicon and parse() give. if isinstance(value, (bytes, bytearray, memoryview)): raise TypeError( f"{field_name} must be an iterable of strings, not " f"{type(value).__name__} -- decode first, e.g. " f"raw.decode('utf-8')" ) if isinstance(value, Mapping): # Name the way out, not just the harm. "contributes only its # keys" is no help for the two mappings people actually pass: # {} (meant as an empty set -- Python's oldest trap, and the # keys story does not apply because there are none), and an # {open: close} dict, whose keys are only half the pair. raise TypeError( f"{field_name} must be an iterable, not a mapping: " f"{value!r}. A mapping yields only its keys -- write " f"frozenset() for an empty set, or .items() if this is an " f"{{open: close}} pair mapping" ) def _require_iterable(value: Iterable[Any], field_name: str, expected: str = "an iterable") -> Iterable[Any]: """Probe a value's iterability and return its iterator, raising a TypeError naming the field if it has none. `expected` carries the field's own phrasing ("a mapping of Script to order"); the default suits every plain iterable field. Probing with iter() is what makes the message possible: a non-iterable (an int, a bool, ...) is named here, matching the treatment patronymic_rules already gets, instead of a bare "'int' object is not iterable" surfacing from whatever tuple()/frozenset() happens to run first. It is ALSO the reason callers consume the returned iterator OUTSIDE any try of their own: an exception raised inside a caller's generator while it is being consumed is the caller's own error, and must propagate untouched rather than be rewritten as a shape complaint about the field. """ try: return iter(value) except TypeError: raise TypeError( f"{field_name} must be {expected}, got {value!r}" ) from None def _validated_order(value: Iterable[Any], field_name: str) -> tuple[Role, Role, Role]: """name_order's element/permutation check, single-sourced so script_orders values obey the identical rule (only the three exported orders have implemented assignment semantics).""" _reject_bare_string_order(value, field_name) order = tuple(_require_iterable(value, field_name)) # Sole rejection point for plain-string tuples: Role is a StrEnum, # so the named-order membership check below compares EQUAL for # ("given", "middle", "family") -- do not remove this loop as # redundant. for element in order: if not isinstance(element, Role): raise TypeError( f"{field_name} elements must be Role members, " f"got {element!r}" ) # Only the three exported orders have implemented assignment # semantics; the unnamed permutations would silently misassign. # Pre-2.0 strictness is free -- relaxing later is compatible. if order not in (GIVEN_FIRST, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST): raise ValueError( f"{field_name} must be one of the exported orders, got " f"{order!r}; use GIVEN_FIRST, FAMILY_FIRST, or " f"FAMILY_FIRST_GIVEN_LAST" ) return order # type: ignore[return-value] # length checked above def _validated_script(key: object) -> Script: """Coerce one Script key, with the message every script-keyed field shares. Single-sourced deliberately: script_orders and the script-keyed fields that follow it must not each grow their own wording for the same lookup.""" try: # Enum lookup accepts ANY value at runtime and answers a # non-member with ValueError -- which is the contract here, and # the taxonomy's rule for a failed enum lookup whatever the # input type was (stdlib EnumType precedent). return Script(key) # type: ignore[arg-type] except ValueError: valid = ", ".join(v.value for v in Script) raise ValueError( f"unknown script {key!r}; valid scripts: {valid}" ) from None def _validated_script_orders( value: object) -> tuple[tuple[Script, tuple[Role, Role, Role]], ...]: """Policy.script_orders' whole check, from raw input to canonical storage. script_orders is the one MAPPING-shaped field, so the guards read inverted from every other one here: a Mapping is what the caller SHOULD pass, and a bare string is the shape that would otherwise iterate into plausible-looking garbage.""" if isinstance(value, str): raise TypeError( f"script_orders must be a mapping of Script to order, " f"not a bare string: {value!r}" ) if isinstance(value, (bytes, bytearray, memoryview)): raise TypeError( f"script_orders must be a mapping of Script to order, " f"not {type(value).__name__} -- decode first, " f"e.g. raw.decode('utf-8')" ) raw = value.items() if isinstance(value, Mapping) else value raw_iter = _require_iterable( raw, "script_orders", # type: ignore[arg-type] "a mapping of Script to order") canonical: dict[Script, tuple[Role, Role, Role]] = {} for entry in raw_iter: try: key, order = entry except (TypeError, ValueError): raise TypeError( f"script_orders entries must be (Script, order) " f"pairs, got {entry!r}" ) from None canonical[_validated_script(key)] = _validated_order( order, "script_orders") # Sorted pairs, not a dict: Policy is hashable, and two # differently-written but equivalent tables must converge (the # capitalization_exceptions precedent). return tuple(sorted(canonical.items())) def _validated_segment_scripts(value: object) -> frozenset[Script]: """segment_scripts' check: an iterable of Script members (or their string values), coerced via _validated_script so the unknown-script wording stays single-sourced.""" _reject_str_and_mapping(value, "segment_scripts") script_iter = _require_iterable( value, "segment_scripts", # type: ignore[arg-type] "an iterable of Script members") return frozenset(_validated_script(s) for s in script_iter) def _canonical_script_pair(pair: Iterable[Any]) -> tuple[Any, ...]: """Hashability-only canonicalization of one PolicyPatch script_orders entry: tuple-ize the pair AND the order value inside it. Shallow was not enough -- a {Script: [Role, ...]} patch stored the list, and hash() then raised far from the construction site, the same failure name_order's canonicalization exists to prevent. A malformed entry is still tuple-ized (that is the hashability floor); its CONTENTS are left exactly as written so Policy quotes the caller's own value when it raises at apply time.""" out = tuple(pair) if len(out) != 2: return out # not a (Script, order) pair key, value = out # str/bytes are iterable, so tuple() would shred exactly the two # values whose deferred errors ("not a bare string", the decode # hint) need to quote what the caller wrote. if isinstance(value, (str, bytes, bytearray, memoryview)): return out try: return (key, tuple(value)) except TypeError: return out # non-iterable order value def _canonical_patch_script_orders(value: object) -> object: """Canonicalize a PolicyPatch.script_orders value for hashability without validating it: malformed shapes are stored so Policy can quote them at apply time; a caller-generator's own exception propagates from the UNGUARDED materialization below (deferring is impossible once a one-shot iterator is consumed).""" # Excluded HERE rather than delegated: a string's elements are # themselves tuple-izable, so no TypeError ever fires to signal # "leave this alone" -- "han" would shred to (("h",), ("a",), # ("n",)) and Policy's bare-string message would have nothing left # to quote. bytes shred the same way, into ints. if isinstance(value, (str, bytes, bytearray, memoryview)): return value # deferred whole: Policy's guards quote it # Any, not object: a patch defers validation, so anything a caller # wrote can arrive here, and the probe below is precisely the # runtime question mypy has no way to answer statically. pairs: Any = value.items() if isinstance(value, Mapping) else value try: pairs_iter = iter(pairs) # probe only except TypeError: return value # non-iterable: defer to apply items = tuple(pairs_iter) # UNGUARDED: caller errors propagate try: return tuple(map(_canonical_script_pair, items)) except TypeError: return items # malformed entry: materialized, so # Policy can still re-iterate + quote @dataclass(frozen=True, slots=True) class Policy: """The behavior switches a parser runs with: name order, patronymic rules, delimiter routing, input scrubbing. Immutable and hashable; every field has a safe default, so construct with only what you change -- ``Policy(maiden_delimiters=frozenset({("(", ")")}))`` -- and pass the result to ``Parser(policy=...)``.""" #: How positional (no-comma) input maps onto given/middle/family. #: Valid values are exactly the three exported #: :ref:`name-order constants ` -- #: GIVEN_FIRST (the default), FAMILY_FIRST, and #: FAMILY_FIRST_GIVEN_LAST; any other tuple of Roles raises #: ValueError. Ignored when a comma separates family from given: #: "Thomas, John" puts the family name first no matter which words #: could otherwise be either ("Thomas" and "John" both work as #: given or family names). A comma that only sets off suffixes #: ("John Smith, Jr.") leaves name_order governing the name part. name_order: tuple[Role, Role, Role] = GIVEN_FIRST #: Per-script overrides of name_order (#271), consulted when every #: name piece is written wholly in one script, or in the #: Han/Hiragana/Katakana repertoire the #272 kana license shares #: across pieces: {Script: order} (constructor accepts a mapping; #: stored as sorted pairs). The default reads wholly-Han/Hangul #: names, and kana-licensed Japanese names, family-first -- see #: :data:`~nameparser.DEFAULT_SCRIPT_ORDERS`. Opt out with #: ``script_orders=()``. Latin-script and mixed-script input is #: never affected. Like name_order, ignored where a comma already #: decides the family name. script_orders: tuple[tuple[Script, tuple[Role, Role, Role]], ...] = ( DEFAULT_SCRIPT_ORDERS) #: Scripts for which the unspaced-name segmentation stage is #: active (#271): the first token written wholly in an activated #: script is split by longest surname match against #: :attr:`Lexicon.surnames `, and, #: where that vocabulary declines, by a #: :data:`~nameparser.Segmenter` if one was given to the parser #: (#272). Default: {Script.HANGUL} -- hangul is unambiguously #: Korean and Korean surnames are a closed default-shipped set. #: Han is NOT default: a zh surname list corrupts Japanese names #: (高橋一郎 must not split as 高+橋一郎), so it's opt-in via #: locales.ZH for Chinese and locales.JA -- which activates #: Script.HIRAGANA alongside it, the kana license's carrier key -- #: for Japanese. #: Opt out with ``segment_scripts=frozenset()``; note a PolicyPatch unions #: rather than replaces, so a pack can only add scripts, never #: disable one. segment_scripts: frozenset[Script] = frozenset({Script.HANGUL}) #: Opt-in detectors that reorder patronymic-shaped names #: (EAST_SLAVIC, TURKIC); usually set via a locale pack. patronymic_rules: frozenset[PatronymicRule] = frozenset() #: Folds middle into family instead of splitting them (v1's #: middle_name_as_last) -- for data where unrecognized interior #: words are surname parts, not middle names: multi-part surnames #: like Spanish/Portuguese dual surnames ("Gabriel García Márquez" #: -> family "García Márquez" instead of middle "García"). middle_as_family: bool = False # v1's middle_name_as_last #: (open, close) pairs whose enclosed content becomes the nickname #: field. Defaults to #: :data:`~nameparser.DEFAULT_NICKNAME_DELIMITERS` (#273). nickname_delimiters: frozenset[tuple[str, str]] = DEFAULT_NICKNAME_DELIMITERS #: (open, close) pairs whose enclosed content becomes the maiden #: field instead; a pair listed here is dropped from the effective #: nickname set (maiden wins, see __post_init__), so #: maiden_delimiters=frozenset({("(", ")")}) is the whole recipe (#274). #: A maiden_markers word opening the enclosed content is dropped #: from the value, but only where that content holds more than one #: token: a lone "(Nee)" is a maiden NAME, not a marker (#329). maiden_delimiters: frozenset[tuple[str, str]] = frozenset() #: Additional separators that split suffix groups (e.g. " - " for #: "Jane Smith, RN - CRNA"). Additive only: the comma always #: splits suffix groups and cannot be replaced -- comma handling #: is structural (the same comma reading that parses #: "Family, Given" input), not a configurable delimiter. extra_suffix_delimiters: frozenset[str] = frozenset() #: Governs "Family, Suffix"-shaped input where the suffix word is #: also initial-shaped (a single letter, bare or period-written -- #: of the default vocabulary that means the roman numerals "I" and #: "V"): "John Smith, V" reads as John Smith the fifth when True #: (the default, v1 behavior); False reads "V" as a given-name #: initial instead (family "John Smith", given "V"). Multi-letter #: suffixes ("III", "MD") parse the same either way. lenient_comma_suffixes: bool = True #: Excludes emoji from tokenization: they appear in no token, #: field, or rendered view. The original string keeps them (input #: is never modified -- spans stay true). strip_emoji: bool = True #: Excludes bidirectional control characters from tokenization: #: they appear in no token, field, or rendered view; the original #: string keeps them. strip_bidi: bool = True # =False replaces v1's opt-out CONSTANTS.regexes.bidi = False # in the class body so @dataclass(slots=True) keeps them __getstate__ = _guarded_getstate __setstate__ = _guarded_setstate def __post_init__(self) -> None: object.__setattr__( self, "name_order", _validated_order(self.name_order, "name_order")) object.__setattr__( self, "script_orders", _validated_script_orders(self.script_orders)) object.__setattr__( self, "segment_scripts", _validated_segment_scripts(self.segment_scripts)) _reject_str_and_mapping(self.patronymic_rules, "patronymic_rules") # Probe with iter() rather than wrapping tuple(): non-iterables # (True especially -- v1's patronymic_name_order was a bool flag, # so it's the likeliest wrong value here) get the migration- # pointing message, while an exception raised inside a caller's # generator still propagates untouched from the tuple() below # instead of being rewritten. Only the enum lookup itself gets # the unknown-rule message, naming the offender. try: rule_iter = iter(self.patronymic_rules) except TypeError: raise TypeError( f"patronymic_rules must be an iterable of PatronymicRule " f"names, got {self.patronymic_rules!r}; " f"{_PATRONYMIC_MIGRATION_HINT}" ) from None items = tuple(rule_iter) rules = set() for r in items: try: rules.add(PatronymicRule(r)) except ValueError: valid = ", ".join(v.value for v in PatronymicRule) raise ValueError( f"unknown patronymic rule {r!r}; valid rules: {valid}" ) from None object.__setattr__(self, "patronymic_rules", frozenset(rules)) for pairs_name in ("nickname_delimiters", "maiden_delimiters"): _reject_str_and_mapping(getattr(self, pairs_name), pairs_name) pairs = tuple(_require_iterable(getattr(self, pairs_name), pairs_name)) for pair in pairs: if (not isinstance(pair, tuple) or len(pair) != 2 or not all(isinstance(s, str) for s in pair)): raise TypeError( f"{pairs_name} entries must be (open, close) tuples " f"of strings, got {pair!r}" ) if not all(pair): raise ValueError( f"{pairs_name} entries must be pairs of non-empty " f"strings, got {pair!r}" ) object.__setattr__(self, pairs_name, frozenset(pairs)) # Maiden wins: a pair can route to exactly one field, and listing # it in maiden_delimiters is the specific intent, so the effective # nickname set drops it. Canonicalization, not validation (the # name_order coercion precedent): differently-written but # equivalent Policies converge to equal values. The v1 facade # keeps v1's nickname-wins precedence via a pre-subtraction in # _config_shim's snapshot instead. object.__setattr__( self, "nickname_delimiters", self.nickname_delimiters - self.maiden_delimiters) _reject_str_and_mapping(self.extra_suffix_delimiters, "extra_suffix_delimiters") delimiters = tuple(_require_iterable( self.extra_suffix_delimiters, "extra_suffix_delimiters")) for d in delimiters: if not isinstance(d, str): raise TypeError( f"extra_suffix_delimiters entries must be strings, " f"got {d!r}" ) if not d: raise ValueError( "extra_suffix_delimiters entries must be non-empty strings" ) object.__setattr__( self, "extra_suffix_delimiters", frozenset(delimiters) ) # Truthy strings ("no", "false") would silently invert the # caller's intent downstream; bools are the one field kind the # coercing checks above can't cover. for flag in ("middle_as_family", "lenient_comma_suffixes", "strip_emoji", "strip_bidi"): value = getattr(self, flag) if not isinstance(value, bool): raise TypeError( f"{flag} must be a bool, got {value!r}" ) def __repr__(self) -> str: # Bounded: only fields that deviate from the default are shown # (design rule, see nameparser._types module docstring). parts = [] for f in dataclasses.fields(self): value = getattr(self, f.name) if value == f.default: continue if f.name == "name_order": parts.append(f"name_order={_order_repr(value)}") else: parts.append(f"{f.name}={value!r}") return f"Policy({', '.join(parts)})" # -- editing ------------------------------------------------------ def patched(self, patch: PolicyPatch) -> Policy: """Fold a :class:`PolicyPatch` onto this Policy and return the combined Policy. Set-valued fields union with the patch's; scalar fields are overridden by the patch; UNSET fields are left alone. Patch VALUES are validated here (Policy's constructor re-runs on the result), not at patch construction -- see PolicyPatch. The maiden-wins canonicalization applies to the combined result exactly as if it had been constructed directly.""" if not isinstance(patch, PolicyPatch): raise TypeError(f"patched() takes a PolicyPatch, got {patch!r}") return apply_patch(self, patch) class _Unset(Enum): UNSET = auto() #: Sentinel for "this patch does not set this field" (picklable enum #: member, distinguishable from every real value including None/False). UNSET = _Unset.UNSET _UNION = {"compose": "union"} # field metadata: set-valued -> union @dataclass(frozen=True, slots=True) class PolicyPatch: """A partial Policy: one field per Policy field, all defaulting to UNSET. Composition per field is DECLARED via metadata -- set-valued fields union, scalars override (later wins). Kept in lockstep with Policy by the parity test in tests/v2/test_policy.py. Values are validated when the patch is applied (Policy's constructor re-runs), not at patch construction. """ name_order: tuple[Role, Role, Role] | _Unset = UNSET #: Composes as a SCALAR (override, not merge) -- deliberate: nothing #: shipped patches it today, so the simpler rule is the one to #: defend; revisit if a pack ever needs to add one script's entry #: without restating the rest. script_orders: tuple[ tuple[Script, tuple[Role, Role, Role]], ...] | _Unset = UNSET segment_scripts: frozenset[Script] | _Unset = field( default=UNSET, metadata=_UNION) patronymic_rules: frozenset[PatronymicRule] | _Unset = field( default=UNSET, metadata=_UNION) middle_as_family: bool | _Unset = UNSET nickname_delimiters: frozenset[tuple[str, str]] | _Unset = field( default=UNSET, metadata=_UNION) maiden_delimiters: frozenset[tuple[str, str]] | _Unset = field( default=UNSET, metadata=_UNION) extra_suffix_delimiters: frozenset[str] | _Unset = field( default=UNSET, metadata=_UNION) lenient_comma_suffixes: bool | _Unset = UNSET strip_emoji: bool | _Unset = UNSET strip_bidi: bool | _Unset = UNSET # in the class body so @dataclass(slots=True) keeps them __getstate__ = _guarded_getstate __setstate__ = _guarded_setstate def __post_init__(self) -> None: # Canonicalize (but do NOT validate) collection fields so a patch # built from a set/list literal is hashable and unions cleanly in # apply_patch. name_order needs the same treatment: Policy would # coerce a list at apply time, but the patch itself (and any # Locale holding it) must already be hashable. if self.name_order is not UNSET: _reject_bare_string_order(self.name_order, "name_order") object.__setattr__(self, "name_order", tuple(self.name_order)) # Same reason for script_orders, one level deeper: a patch built # from a {Script: order} dict (or a list of pairs) must already # be hashable, since a Locale holds it -- and hashable all the # way down, hence _canonical_script_pair. Validation still # belongs to Policy at apply time, so a shape the canonicalizer # cannot digest is left for it to report; the one-shot and # malformed-entry cases are _canonical_patch_script_orders'. if self.script_orders is not UNSET: object.__setattr__( self, "script_orders", _canonical_patch_script_orders(self.script_orders)) for f in dataclasses.fields(self): if f.metadata.get("compose") != "union": continue value = getattr(self, f.name) if value is UNSET: continue # Shared with Policy, not re-implemented: this used to be an # inline copy of the bare-string half only, so the mapping # half never reached a patch -- and frozenset() below # destroys the evidence, leaving nothing for Policy to catch # at apply time. A Locale pack ships one of these. _reject_str_and_mapping(value, f.name) # same iter() probe as Policy: curated message for # non-iterables (with the v1-flag hint where it applies), # caller-generator exceptions propagate from frozenset() try: iter(value) except TypeError: hint = ("; " + _PATRONYMIC_MIGRATION_HINT if f.name == "patronymic_rules" else "") raise TypeError( f"{f.name} must be an iterable, got {value!r}{hint}" ) from None object.__setattr__(self, f.name, frozenset(value)) # middle_as_family, lenient_comma_suffixes, strip_emoji, and # strip_bidi are scalar (compose="override") fields and # DELIBERATELY get no type check here, unlike name_order and # the union fields above: a PolicyPatch(strip_emoji="off") is # constructible, and only raises once apply_patch runs # Policy.__post_init__'s bool check. This is the one place the # module's eager-validation ethos doesn't apply -- see the # class docstring ("Values are validated when the patch is # applied... not at patch construction"). def __repr__(self) -> str: # Bounded: only fields the patch actually sets are shown; UNSET # fields are omitted (design rule, see nameparser._types module # docstring -- the sibling of Policy's deviation-only repr). parts = [] for f in dataclasses.fields(self): value = getattr(self, f.name) if value is UNSET: continue if f.name == "name_order": parts.append(f"name_order={_order_repr(value)}") else: parts.append(f"{f.name}={value!r}") return f"PolicyPatch({', '.join(parts)})" def apply_patch(policy: Policy, patch: PolicyPatch) -> Policy: """Fold a PolicyPatch onto a Policy. Policy.__post_init__ re-runs via dataclasses.replace, so patched values are revalidated for free -- including the maiden-wins canonicalization: a patch that adds a maiden pair removes that pair from the base's effective nickname set, exactly as if the combined Policy had been constructed directly. Intended (decided 2026-07-19): maiden_delimiters membership IS the routing decision, whoever contributes it.""" updates: dict[str, object] = {} for f in dataclasses.fields(PolicyPatch): value = getattr(patch, f.name) if value is UNSET: continue if f.metadata.get("compose") == "union": value = getattr(policy, f.name) | value updates[f.name] = value if not updates: return policy # Known mypy limitation with **dict-unpacked replace; see the full # explanation at Lexicon._edit in _lexicon.py. return dataclasses.replace(policy, **updates) # type: ignore[arg-type] python-nameparser-2.1.0/nameparser/_render.py000066400000000000000000000203251523530431700213200ustar00rootroot00000000000000"""Rendering for the 2.0 API: ParsedName -> display strings. Layering: imports nameparser._types, and nameparser._lexicon for Lexicon.default() (capitalized() with lexicon=None) and _normalize (enforced by tests/v2/test_layering.py). Parsing code never imports this module; ParsedName's rendering methods delegate here via call-time imports. Malformed str.format specs beyond unknown keys (positional fields, bad conversions) surface the raw str.format error; only unknown KEYS get the enriched KeyError. """ from __future__ import annotations import re from nameparser._lexicon import Lexicon, _normalize from nameparser._types import Ambiguity, ParsedName, Role, Token _SPACES = re.compile(r"\s+") _SPACE_BEFORE_COMMA = re.compile(r"\s+,") _COMMA_CHAR = re.compile(r"[,،，]") # ASCII, Arabic, fullwidth _MAC = re.compile(r"^(ma?c)(\w{2,})", re.IGNORECASE) _WORD = re.compile(r"(\w|\.)+") #: str.format keys render() accepts: the seven role fields in canonical #: order (derived from Role -- never restated) plus the derived views. _DERIVED_VIEWS = ("family_base", "family_particles", "surnames", "given_names") _RENDER_KEYS = tuple(r.value for r in Role) + _DERIVED_VIEWS #: str.format keys initials() accepts: the three name-bearing roles. _INITIALS_KEYS = (Role.GIVEN.value, Role.MIDDLE.value, Role.FAMILY.value) #: Tags whose tokens contribute no initial outside the given group. #: Not STABLE_TAGS -- that also contains "initial", which must contribute. _SKIP_TAGS = frozenset({"particle", "conjunction"}) # Ported verbatim from v1 (nameparser/config/regexes.py "initial", # minus the empty alternative) -- layering forbids importing the # pipeline here; keep in sync with _pipeline/_vocab.py by hand. # Deliberately NOT composed with that module's repertoire test (#320): # layering forbids the import, and nothing here needs it. The only use # is v1's conjunction carve-out in _cap_word below, which this pattern # can only reach once `normalized in lex.conjunctions` already holds -- # and no CJK token reaches that, the shipped vocabulary carrying no CJK # conjunction or particle in the default lexicon or in any locale pack. # That is a property of the shipped DATA, not an invariant -- conjunctions # is public, configurable API -- but the divergence stays harmless if a # user adds one: CJK is caseless, so the carve-out's word.lower() and the # fall-through's word.capitalize() return the same string either way. # So the two copies keep identical PATTERNS and divergent PREDICATES; # test_regex_sync pins the patterns, which is the promise being kept. _INITIAL = re.compile(r"^(\w\.|[A-Z])$") def _collapse(rendered: str) -> str: """The #254 collapse: empty fields substitute '' and every artifact of that is removed -- dangling empty-nickname wrappers, space runs, space-before-comma, one trailing comma character (any script), leading/trailing ', ' debris.""" rendered = (rendered.replace(" ()", "") .replace(" ''", "") .replace(' ""', "")) rendered = _SPACE_BEFORE_COMMA.sub(",", rendered) rendered = _SPACES.sub(" ", rendered.strip()) if rendered and _COMMA_CHAR.fullmatch(rendered[-1]): rendered = rendered[:-1] return rendered.strip(", ") def _format_spec(spec: str, values: dict[str, str], noun: str, keys: tuple[str, ...]) -> str: """Shared tail of render()/initials(): fill the spec, enrich unknown-KEY errors with the valid key list, collapse.""" if not isinstance(spec, str): raise TypeError(f"spec must be a str, got {spec!r}") try: rendered = spec.format(**values) except KeyError as exc: raise KeyError( f"unknown {noun} field {exc.args[0]!r}; valid fields: " f"{', '.join(keys)}" ) from None return _collapse(rendered) def render(name: ParsedName, spec: str) -> str: """Fill the str.format spec from the seven role fields and the derived views (empty fields substitute ''), then apply the #254 collapse. Unknown keys raise KeyError naming the valid fields.""" values = {key: getattr(name, key) for key in _RENDER_KEYS} return _format_spec(spec, values, "render", _RENDER_KEYS) def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str: """First letter of each contributing token per group, v1 semantics: delimiter follows each initial, separator sits between initials within a group. Tokens tagged particle/conjunction contribute no initial in middle/family (given-name tokens always contribute); tags come from the pipeline -- hand-built untagged tokens all contribute. Valid spec keys: given, middle, family.""" if not isinstance(delimiter, str): raise TypeError(f"delimiter must be a str, got {delimiter!r}") if not isinstance(separator, str): raise TypeError(f"separator must be a str, got {separator!r}") values: dict[str, str] = {} for key in _INITIALS_KEYS: role = Role(key) tokens = name.tokens_for(role) if role is not Role.GIVEN: tokens = tuple(t for t in tokens if not (_SKIP_TAGS & t.tags)) values[key] = separator.join( t.text[0] + delimiter for t in tokens) return _format_spec(spec, values, "initials", _INITIALS_KEYS) def _cap_word(word: str, role: Role, lex: Lexicon) -> str: # v1 cap_word order: particle/conjunction rule first, then the # exceptions map, then Mac/Mc, then str.capitalize normalized = _normalize(word) # v1's is_conjunction excludes initials: 'E.' in 'Scott E. Werner' # is an initial, not the conjunction 'e' (pinned live 2026-07-17) if ((normalized in lex.particles and role in (Role.MIDDLE, Role.FAMILY)) or (normalized in lex.conjunctions and not _INITIAL.fullmatch(word))): return word.lower() # v1 cap_word tries the edge-stripped form, then the period-free # form ('Ph.D.' -> 'ph.d' -> 'phd' hits the exceptions map) for key in (normalized, normalized.replace(".", "")): exception = lex.capitalization_exceptions_map.get(key) if exception is not None: return exception if _MAC.match(word): return _MAC.sub( lambda m: m.group(1).capitalize() + m.group(2).capitalize(), word) return word.capitalize() def _cap_text(text: str, role: Role, lex: Lexicon) -> str: # word-by-word within the token text: hyphenated names capitalize # both sides ("macdole-eisenhower" -> "MacDole-Eisenhower") return _WORD.sub(lambda m: _cap_word(m.group(0), role, lex), text) def capitalized(name: ParsedName, lexicon: Lexicon | None, *, force: bool) -> ParsedName: """Case-fixing transform -> new ParsedName, same spans, new token texts. Gate (v1 parity): only single-case input is touched unless force=True; the gate reads the joined token texts (not render() output -- the case gate stays decoupled from spec formatting and the #254 collapse). Idempotent: without force, a capitalized result is mixed-case and the gate returns it unchanged; with force, every _cap_word rule is a fixpoint on its own output.""" if lexicon is not None and not isinstance(lexicon, Lexicon): # eager, before the gate: a garbage argument must not become a # silent no-op on mixed-case input or a deep AttributeError raise TypeError(f"lexicon must be a Lexicon or None, got {lexicon!r}") lex = Lexicon.default() if lexicon is None else lexicon joined = " ".join(t.text for t in name.tokens) if not force and joined not in (joined.upper(), joined.lower()): return name new_tokens = tuple( Token(_cap_text(t.text, t.role, lex), t.span, t.role, t.tags) for t in name.tokens) # equal tokens (possible only for synthetic span=None duplicates) # collapse to one mapping entry -- benign: the rebuilt ambiguity # references an equal token, so the subset invariant still holds replacement = dict(zip(name.tokens, new_tokens)) new_ambiguities = tuple( Ambiguity(a.kind, a.detail, tuple(replacement[t] for t in a.tokens)) for a in name.ambiguities) return ParsedName(original=name.original, tokens=new_tokens, ambiguities=new_ambiguities) python-nameparser-2.1.0/nameparser/_types.py000066400000000000000000001026171523530431700212120ustar00rootroot00000000000000"""Core value types for the 2.0 API. Layering (enforced by tests/v2/test_layering.py): this module imports nothing from nameparser at module level -- it is the bottom of the module-import dependency graph. The rendering delegates import _render and matches() imports _parser at call time; TYPE_CHECKING-only imports supply the Lexicon/Parser annotations. Repr policy (applies to every v2 type's __repr__, across this module and _lexicon.py/_policy.py/_locale.py): bounded output only. No repr may scale with vocabulary size -- collections render as counts or deltas, never contents. """ from __future__ import annotations import dataclasses from collections.abc import Callable, Mapping from dataclasses import dataclass from enum import Enum, StrEnum from typing import TYPE_CHECKING, NamedTuple, NoReturn, TypeVar if TYPE_CHECKING: from nameparser._lexicon import Lexicon from nameparser._parser import Parser class Role(StrEnum): """The seven fields of a parsed name, one per :class:`Token`. Declaration order is the canonical field order everywhere (``as_dict()``, ``comparison_key()``, rendering). A StrEnum, like :class:`AmbiguityKind`: members ARE their string values, so ``token.role == "given"`` compares directly and ``str(Role.GIVEN) == "given"``. Members order as strings, so ``sorted()`` yields alphabetical order -- iterate ``Role`` itself for the canonical order.""" # Declaration order IS the canonical field order (conventions §3): # every listing of the seven fields anywhere derives from this. #: Pre-nominal titles and honorifics ("Dr.", "Sir", "Capt."). TITLE = "title" #: The given (first) name, or its initial. GIVEN = "given" #: Names between given and family -- middle names or initials. MIDDLE = "middle" #: The family (last) name, including any particles ("de la Vega"). FAMILY = "family" #: Post-nominal pieces ("III", "Jr.", "PhD"). SUFFIX = "suffix" #: Delimited nickname content ("Jonathan 'Jack' Kennedy" -> "Jack"). NICKNAME = "nickname" #: A birth surname, from a marker word ("Jane Smith née Jones" -> #: "Jones") or a delimiter pair routed via Policy.maiden_delimiters. MAIDEN = "maiden" class Span(NamedTuple): """Where a :class:`Token` came from: a character range into :attr:`ParsedName.original` such that ``original[start:end]`` is the token's source text (``end`` exclusive). A plain two-int NamedTuple; ``None`` in :attr:`Token.span` marks a synthetic token with no source position.""" #: First character index (0-based). start: int #: One past the last character index. end: int def __add__(self, other: object) -> NoReturn: # type: ignore[override] # Inherited tuple + would concatenate two spans into a 4-tuple. # There is deliberately NO covering-span operation: grouping is # index-run based (the anti-#100 invariant: never resolve by # joining text back together) and never merges spans. raise TypeError( "Span does not support +; tuple concatenation is not a " "covering span" ) #: The four :attr:`Token.tags` values that are stable API. #: "particle" marks a word from the particle vocabulary ("de", "van") #: wherever it lands -- including a given-name "Van" -- so combine it #: with Role.FAMILY (as family_particles does) to get actual family #: particles; "conjunction" a joining word ("and", "y"); "initial" an #: initial-shaped word in a script that HAS initials -- "J." or "А.", #: never "씨." (#320); #: "joined" a continuation of the previous token within one merged #: piece ("Ph." + "D."), which the suffix view joins with a space #: instead of ", ". Every other tag is namespaced ("vocab:...") and is #: unstable debugging provenance -- never match against those. #: This prose is the hand-maintained twin of docs/modules.rst's #: STABLE_TAGS block; nothing pins the two against each other (the #: test only compares the frozenset), so edit both or neither. STABLE_TAGS = frozenset({"particle", "conjunction", "initial", "joined"}) #: The one sanctioned view-reorder marker (namespaced = unstable API). #: Tokens cannot reorder (span order is validated), so a role fold that #: must render BEFORE the role's original tokens tags them with this; #: _text_for and the facade lists prepend carriers. Single-sourced here #: so the emitter (_pipeline/_post_rules) and the consumers cannot #: drift. FOLDED_TAG = "vocab:folded-middle" _E = TypeVar("_E", bound=Enum) def _coerce_enum(value: object, enum_cls: type[_E], noun: str, plural: str) -> _E: """Coerce value to enum_cls, or raise the enriched ValueError listing every valid member (enum lookups stay ValueError for any input -- stdlib EnumType precedent, see AGENTS.md's taxonomy rule).""" if isinstance(value, enum_cls): return value try: return enum_cls(value) except ValueError: valid = ", ".join(str(m.value) for m in enum_cls) raise ValueError( f"unknown {noun} {value!r}; valid {plural}: {valid}" ) from None # Pickle support shared by the frozen slots dataclasses: fail at the # LOAD site when a pickle's field layout does not match this version of # the class (version skew) -- silently loading would defer the failure # to a distant attribute read. Values are deliberately NOT re-validated: # pickle is not a security boundary (arbitrary pickles can execute code # anyway), and canonical state only comes from a validated instance. # These are ASSIGNED IN EACH CLASS BODY (not inherited from a mixin): # @dataclass(slots=True) regenerates the class and installs its own # pickle methods unless __getstate__/__setstate__ are in the class's # own __dict__. Lexicon duplicates this logic by design (its slots also # carry a rebuilt mappingproxy) -- layering keeps _lexicon import-free # of _types. def _guarded_getstate(self: object) -> dict[str, object]: fields = dataclasses.fields(self) # type: ignore[arg-type] return {f.name: getattr(self, f.name) for f in fields} def _guarded_setstate(self: object, state: dict[str, object]) -> None: fields = dataclasses.fields(self) # type: ignore[arg-type] expected = {f.name for f in fields} if set(state) != expected: missing = ", ".join(sorted(expected - set(state))) or "none" unexpected = ", ".join(sorted(set(state) - expected)) or "none" raise ValueError( f"incompatible {type(self).__name__} pickle: missing " f"fields: {missing}; unexpected fields: {unexpected}" ) for name, value in state.items(): object.__setattr__(self, name, value) @dataclass(frozen=True, slots=True) class Token: """One classified word of a parsed name: its text, where it came from, which field it belongs to, and how it was classified. Read tokens off :attr:`ParsedName.tokens` or :meth:`ParsedName.tokens_for`; you only construct one directly when hand-building a :class:`ParsedName`.""" #: The word exactly as written in the input (never empty). text: str #: Position in ParsedName.original; None marks a synthetic token #: (e.g. introduced by replace()) with no source position. span: Span | None #: The field this token belongs to. role: Role #: Classification labels. Exactly the four members of #: :data:`~nameparser.STABLE_TAGS` ("particle", "conjunction", #: "initial", "joined") are API; namespaced tags like "vocab:..." #: are unstable debugging provenance -- never match against them. tags: frozenset[str] = frozenset() # in the class body so @dataclass(slots=True) keeps them __getstate__ = _guarded_getstate __setstate__ = _guarded_setstate def __post_init__(self) -> None: if not isinstance(self.text, str): raise TypeError( f"Token.text must be a str, got {self.text!r}" ) if not self.text: raise ValueError("Token.text must be a non-empty string") object.__setattr__( self, "role", _coerce_enum(self.role, Role, "Role", "roles")) if self.span is not None: if not ( isinstance(self.span, tuple) and len(self.span) == 2 # bool is an int subclass: (False, True) is a comparison # result leaking into a coordinate slot, not a span and all(isinstance(v, int) and not isinstance(v, bool) for v in self.span) ): raise TypeError( f"invalid span {self.span!r}: expected a (start, end) " "pair of ints or None" ) start, end = self.span if start < 0 or end < start: raise ValueError( f"invalid span ({start}, {end}): need 0 <= start <= end" ) object.__setattr__(self, "span", Span(start, end)) # The same guards _normset applies to Lexicon vocabulary: a bare # string would become its character set, a mapping would silently # contribute only its keys. if isinstance(self.tags, str): raise TypeError( "Token.tags must be an iterable of strings, " "not a bare string" ) if isinstance(self.tags, Mapping): raise TypeError( "Token.tags must be an iterable of strings, not a mapping" ) tags = frozenset(self.tags) for tag in tags: if not isinstance(tag, str): raise TypeError( f"Token.tags must contain only strings, got {tag!r}" ) object.__setattr__(self, "tags", tags) def __repr__(self) -> str: # Bounded output: a single token's text/span/role/tags, never # scales with vocabulary size (design rule -- see module docstring). where = (f"@{self.span.start}:{self.span.end}" if self.span is not None else "@synthetic") tags = f" {{{', '.join(sorted(self.tags))}}}" if self.tags else "" return f"Token({self.text!r} {where} {self.role.name}{tags})" @dataclass(frozen=True, slots=True) class Segmentation: """A segmenter's answer for one unspaced token: the interior offsets to split at (each offset begins a new piece, so ``Segmentation((2,))`` cuts a three-character token into ``token[:2]`` and ``token[2:]``; strictly ascending, each >= 1 -- an index protocol, so a segmenter physically cannot invent, drop, or rewrite characters) and an optional confidence in [0, 1]. ``Segmentation(())`` means "confidently one token" -- distinct from returning None, which DECLINES ("I don't know"). The upper bound (< len(token)) is the half this class cannot check, never having seen the text; the consuming stage checks it and RAISES ``ValueError`` on a violation, the same call it makes on an answer of the wrong type -- both are protocol bugs in the segmenter, not facts about the name.""" #: Interior character offsets to split at, ascending. splits: tuple[int, ...] #: How sure the segmenter is, or None for "no opinion". confidence: float | None = None # in the class body so @dataclass(slots=True) keeps them __getstate__ = _guarded_getstate __setstate__ = _guarded_setstate def __post_init__(self) -> None: # Both guards Token.tags carries, for the same two reasons and # a third of this field's own: a bare string is iterable, so # Segmentation("") would sail through as "confidently one # token" -- an opinion nobody stated -- and Segmentation("23") # would fail one character deep naming '2' rather than the # argument. A mapping would silently contribute only its keys, # and a bare int would surface as an uncurated "not iterable" # from the tuple() below. if isinstance(self.splits, str): raise TypeError( "Segmentation.splits must be an iterable of integers, " "not a bare string" ) if isinstance(self.splits, Mapping): raise TypeError( "Segmentation.splits must be an iterable of integers, " "not a mapping" ) try: iter(self.splits) except TypeError: raise TypeError( f"Segmentation.splits must be an iterable of integers, " f"got {self.splits!r}" ) from None # OUTSIDE the try on purpose: iter() cannot run a generator's # body, but tuple() can, and a TypeError raised in there is the # caller's own bug -- relabeling it "must be an iterable" would # send the reader to the wrong place entirely. splits = tuple(self.splits) for offset in splits: # bool is an int subclass: True as an offset is a comparison # result leaking into an index slot, not a split point if isinstance(offset, bool) or not isinstance(offset, int): raise TypeError( f"Segmentation.splits must be integers, got {offset!r}") if offset < 1: raise ValueError( f"Segmentation.splits must be interior offsets " f"(each >= 1), got {offset}") if any(b <= a for a, b in zip(splits, splits[1:])): raise ValueError( f"Segmentation.splits must be strictly ascending, " f"got {splits!r}") object.__setattr__(self, "splits", splits) conf = self.confidence if conf is not None: if isinstance(conf, bool) or not isinstance(conf, (int, float)): raise TypeError( f"Segmentation.confidence must be a float or None, " f"got {conf!r}") # stored as given, not coerced to float: an int 1 is a valid # confidence and the range check is what the callers rely on if not 0.0 <= conf <= 1.0: raise ValueError( f"Segmentation.confidence must be within [0, 1], " f"got {conf!r}") #: The segmenter hook's shape: token text in, :class:`Segmentation` out, #: None to decline. Plug one in via ``Parser(segmenter=...)``. Segmenter = Callable[[str], Segmentation | None] class AmbiguityKind(StrEnum): """The stable vocabulary of :class:`Ambiguity` kinds. A StrEnum: members ARE their string values, so ``kind == "particle-or-given"`` compares directly. New kinds may be added in minor releases; existing values never change meaning. A kind names a FORK THE PARSE HAD TO CALL, not a word that could be read two ways: the same token elsewhere in a name may present no choice at all and is then reported by nothing. Reporting is also partial -- a kind listed here is not necessarily emitted everywhere its fork occurs (the comma paths stay quiet by design), and coverage grows over releases. A non-empty tuple is a signal to act on; an empty one is not a guarantee of certainty.""" #: Reserved: the name's field order itself is uncertain (e.g. a #: two-word name under a non-default name_order). Not yet emitted; #: planned for 2.x. ORDER = "order" #: Delimited content is an ambiguous suffix acronym, so it reads #: plausibly as either a post-nominal or a nickname -- "JEFFREY #: (JD) BRICKEN" keeps the nickname reading, where the #: unambiguous "(MBA)" escapes to suffix on vocabulary alone. SUFFIX_OR_NICKNAME = "suffix-or-nickname" #: A trailing word reads plausibly as either a post-nominal or an #: ordinary name part. Covers an ambiguous acronym written without #: periods ("John Smith MA" takes MA as a credential because a #: family name remains; "Jack MA" keeps it as the name because none #: would) and a trailing roman numeral, which is a suffix where any #: other single letter would be a name ("John Smith V" vs "John #: Smith B"). Which name part was declined depends on position and #: ``name_order``, so ``detail`` names it rather than the kind. SUFFIX_OR_NAME = "suffix-or-name" #: A leading ambiguous particle was read as a given name -- the #: right call for "Van Johnson" (the actor's given name), the #: wrong one for a bare "Van Buren" (the presidential surname); #: the two-word shape cannot distinguish them. PARTICLE_OR_GIVEN = "particle-or-given" #: A nickname/maiden delimiter opened without closing (or closed #: without opening); the text was kept as literal name content, so #: the tokens are the one the stray character ended up inside. #: Two cases leave that tuple empty: a character that lands in no #: token at all (inside a masked region), and an input with no #: alphanumeric content anywhere, which parses to an empty name -- #: the report survives because "was this malformed?" is the only #: question left, but there is no token for it to point at. #: ``parse("(")`` is the second case, not an exotic one. UNBALANCED_DELIMITER = "unbalanced-delimiter" #: More comma-separated segments than any recognized name shape; #: the parse is best-effort over the extra segments. COMMA_STRUCTURE = "comma-structure" #: A division of an unspaced CJK token that the parse had to #: choose, from either of the two things that can divide one. #: A VOCABULARY fork: more than one surname-supported split #: existed ("夏侯惇" was taken as 夏侯 + 惇, while 夏 + 侯惇 also #: matched), longest-match picked, and ``detail`` names both #: readings (#271). Or a SEGMENTER answer scoring under the #: stage's confidence floor: only one reading was offered, but the #: score says it was a statistical guess rather than a stated #: certainty, and ``detail`` names the pieces and the score #: (#272). Either way it points at ALL the tokens the division #: produced -- two for a vocabulary split, n+1 for a segmenter #: answer cutting n times. SEGMENTATION = "segmentation" @dataclass(frozen=True, slots=True) class Ambiguity: """A call the parser made that could legitimately have gone the other way, surfaced on :attr:`ParsedName.ambiguities` instead of silently guessed away. The parse still commits to one reading -- an Ambiguity is a flag for review, not an error.""" #: Which known ambiguity shape this is (stable API values). kind: AmbiguityKind #: Human-readable specifics of this occurrence (wording unstable). detail: str #: The tokens involved -- always a value-equal subset of the owning #: ParsedName's tokens (checked with ==, not identity: two distinct #: Token instances with identical text/span/role/tags satisfy this); #: may be empty (e.g. unbalanced-delimiter). tokens: tuple[Token, ...] # in the class body so @dataclass(slots=True) keeps them __getstate__ = _guarded_getstate __setstate__ = _guarded_setstate def __post_init__(self) -> None: object.__setattr__( self, "kind", _coerce_enum(self.kind, AmbiguityKind, "AmbiguityKind", "kinds")) if not isinstance(self.detail, str): raise TypeError( f"Ambiguity.detail must be a str, got {self.detail!r}" ) if not self.detail: raise ValueError("Ambiguity.detail must be a non-empty string") toks = tuple(self.tokens) for tok in toks: if not isinstance(tok, Token): raise TypeError( f"Ambiguity.tokens must contain only Token instances, " f"got {tok!r}" ) object.__setattr__(self, "tokens", toks) def __repr__(self) -> str: texts = "/".join(repr(t.text) for t in self.tokens) return f"Ambiguity({self.kind.value!r}: {texts})" def _validated_field_strings(fields: dict[str, str]) -> dict[Role, str]: """Shared by ParsedName.replace and Parser.revise: validate a **fields mapping of role names to replacement strings and key it by Role. TypeErrors match replace()'s historical wording.""" by_value = {role.value: role for role in Role} for key, value in fields.items(): if key not in by_value: raise TypeError( f"unknown field {key!r}; expected one of " f"{', '.join(by_value)}" ) if not isinstance(value, str): raise TypeError( f"field {key!r} must be a str, got {value!r}" ) return {by_value[k]: v for k, v in fields.items()} @dataclass(frozen=True, slots=True) class ParsedName: """The immutable result of parsing one name string. Read the seven fields as strings (``.given``, ``.family``, ...); inspect structure through :attr:`tokens` / :meth:`tokens_for`; correct a parse with :meth:`replace` (returns a new value; Parser.revise is the tag-preserving form); produce output with :meth:`render`, :meth:`initials`, :meth:`capitalized`, or ``str()``. Constructor-enforced invariants: spans ascending, non-overlapping, in bounds of `original`; every Ambiguity's tokens are a value-equal subset of `tokens` (see Ambiguity.tokens). Provenance semantics (text == original[span] for parser-produced names) are documented, not enforced -- transforms like replace() legitimately break them. """ #: The input string exactly as passed to parse(). original: str #: Every classified token, in document order. tokens: tuple[Token, ...] #: Judgment calls that could have gone the other way; empty for #: most names (see Ambiguity). ambiguities: tuple[Ambiguity, ...] = () # in the class body so @dataclass(slots=True) keeps them __getstate__ = _guarded_getstate __setstate__ = _guarded_setstate def __post_init__(self) -> None: if not isinstance(self.original, str): raise TypeError( f"ParsedName.original must be a str, got {self.original!r}" ) object.__setattr__(self, "tokens", tuple(self.tokens)) object.__setattr__(self, "ambiguities", tuple(self.ambiguities)) for tok in self.tokens: if not isinstance(tok, Token): raise TypeError( f"ParsedName.tokens must contain only Token instances, " f"got {tok!r}" ) for amb in self.ambiguities: if not isinstance(amb, Ambiguity): raise TypeError( f"ParsedName.ambiguities must contain only Ambiguity " f"instances, got {amb!r}" ) prev_end = 0 for tok in self.tokens: if tok.span is None: continue if tok.span.end > len(self.original): raise ValueError( f"token {tok.text!r} span {tuple(tok.span)} is out of " f"bounds for original of length {len(self.original)}" ) if tok.span.start < prev_end: raise ValueError( f"token spans must be ascending and non-overlapping; " f"token {tok.text!r} at {tuple(tok.span)} begins before " f"offset {prev_end}" ) prev_end = tok.span.end # Hash once rather than rescanning the tuple per referenced # token: a name can carry an ambiguity per token (a string of # stray delimiters does), and the linear form made construction # quadratic in their product. Set membership uses the same value # equality the tuple scan did -- Token is frozen and hashable. if self.ambiguities: known = set(self.tokens) for amb in self.ambiguities: for tok in amb.tokens: # membership is by Token's value equality, not # identity: this only guarantees a value-equal token # exists in self.tokens, not that `tok` IS one of # those objects. if tok not in known: raise ValueError( f"Ambiguity token {tok.text!r} is not a " f"subset of this ParsedName's tokens" ) def __bool__(self) -> bool: return bool(self.tokens) def __str__(self) -> str: return self.render() def __repr__(self) -> str: # 4-space indent, matching HumanName's repr (v1 style) lines = [] for role in Role: text = self._text_for(role) if text: lines.append(f" {role.value}: {text!r}") if self.ambiguities: kinds = [a.kind.value for a in self.ambiguities] lines.append(f" ambiguities: {kinds!r}") body = "\n".join(lines) return f"" if lines else "" # -- string views (canonical order = Role declaration order) -------- def _text_for(self, *roles: Role, tag: str | None = None, without_tag: str | None = None) -> str: suffix_join = roles == (Role.SUFFIX,) parts: list[str] = [] folded: list[str] = [] for tok in self.tokens: if tok.role not in roles: continue if tag is not None and tag not in tok.tags: continue if without_tag is not None and without_tag in tok.tags: continue # "joined" (stable tag) marks a continuation of the previous # token ("Ph." + "D."): attach with a space so the suffix # view's ", " join does not split one credential in two if suffix_join and "joined" in tok.tags and parts: parts[-1] += " " + tok.text elif FOLDED_TAG in tok.tags: # middle_as_family fold: v1 PREPENDED middle_list to # last_list; spans cannot reorder, so the view does folded.append(tok.text) else: parts.append(tok.text) return (", " if suffix_join else " ").join(folded + parts) @property def title(self) -> str: return self._text_for(Role.TITLE) @property def given(self) -> str: return self._text_for(Role.GIVEN) @property def middle(self) -> str: return self._text_for(Role.MIDDLE) @property def family(self) -> str: return self._text_for(Role.FAMILY) @property def suffix(self) -> str: return self._text_for(Role.SUFFIX) @property def nickname(self) -> str: return self._text_for(Role.NICKNAME) @property def maiden(self) -> str: return self._text_for(Role.MAIDEN) # -- derived views (filters over roles + STABLE tags only) ---------- @property def family_particles(self) -> str: return self._text_for(Role.FAMILY, tag="particle") @property def family_base(self) -> str: return self._text_for(Role.FAMILY, without_tag="particle") @property def surnames(self) -> str: return self._text_for(Role.MIDDLE, Role.FAMILY) @property def given_names(self) -> str: return self._text_for(Role.GIVEN, Role.MIDDLE) # -- structured access ---------------------------------------------- def tokens_for(self, role: Role | str) -> tuple[Token, ...]: """The tokens of one field, in document order. Takes a Role member or its string value; anything else raises ValueError naming the valid roles.""" role = _coerce_enum(role, Role, "Role", "roles") return tuple(t for t in self.tokens if t.role is role) def as_dict(self, *, include_empty: bool = True) -> dict[str, str]: # _text_for handles the suffix ", "-join (single-role SUFFIX call) d = {role.value: self._text_for(role) for role in Role} if not include_empty: d = {k: v for k, v in d.items() if v} return d # -- editing ---------------------------------------------------------- def replace(self, **fields: str) -> ParsedName: """Return a new ParsedName with the named fields re-tokenized as synthetic tokens (span=None). Whitespace-splits each value; an empty value clears the field. original is unchanged (provenance). Ambiguities referencing replaced tokens are dropped. Replacement tokens carry NO tags, so tag-driven views degrade: family_particles empties, particles regain their initials, and a multi-word suffix is comma-joined. Parser.revise() is the tag-preserving alternative. """ replaced = _validated_field_strings(fields) synthetic = { role: tuple(Token(word, None, role) for word in value.split()) for role, value in replaced.items() } return self._with_field_tokens(synthetic) def _with_field_tokens( self, replaced: Mapping[Role, tuple[Token, ...]], ) -> ParsedName: """Shared tail of replace()/Parser.revise(): splice each role's replacement tokens in at the role's first position (appended in canonical order when the role had no tokens); drop ambiguities whose referents were replaced.""" # Private contract, made self-enforcing: a token filed under a # key that is not its own role is the one way this shared tail # could build a semantically wrong ParsedName (the splice keys # on the mapping, the views key on the token). for role, toks in replaced.items(): for tok in toks: if tok.role is not role: raise ValueError( f"replacement token {tok.text!r} has role " f"{tok.role.value}, not {role.value}" ) new_tokens: list[Token] = [] emitted: set[Role] = set() for tok in self.tokens: if tok.role in replaced: if tok.role not in emitted: new_tokens.extend(replaced[tok.role]) emitted.add(tok.role) continue new_tokens.append(tok) for role in Role: if role in replaced and role not in emitted: new_tokens.extend(replaced[role]) kept = tuple( amb for amb in self.ambiguities if all(t in new_tokens for t in amb.tokens) ) return ParsedName(self.original, tuple(new_tokens), kept) # -- comparison ------------------------------------------------------- def comparison_key(self) -> tuple[str, ...]: """One casefolded component per Role, in canonical order, for dedup, dict keys, and sorting. The semantic layer; __eq__ stays strict. """ return tuple(self._text_for(role).casefold() for role in Role) def matches(self, other: str | ParsedName, *, parser: Parser | None = None) -> bool: """Component-wise case-insensitive comparison (the semantic layer; __eq__ stays strict). A str argument is parsed with `parser`, or with the DEFAULT parser when None -- if this name came from a custom Parser, pass that parser (or use Parser.matches); otherwise the comparison silently runs under the wrong configuration.""" if isinstance(other, str): import nameparser._parser as _parser active = parser if parser is not None else _parser._default_parser() other = active.parse(other) if not isinstance(other, ParsedName): raise TypeError( f"matches() takes a str or ParsedName, got {other!r}") return self.comparison_key() == other.comparison_key() # -- rendering delegates ---------------------------------------------- # One-line delegation to nameparser._render: parsing code physically # cannot import formatting logic (layering rule, enforced by # tests/v2/test_layering.py), so these import at call time -- module # level stays internal-import-free. def render( self, spec: str = ('{title} {given} "{nickname}" {middle} {family} ' "({maiden}) {suffix}"), ) -> str: """Fill the str.format spec from the seven role fields and the derived views; empty fields collapse (#254), including the default spec's decorations (empty '""' and '()' wrappers). The default shows every non-empty field: the nickname quoted after the given name, the maiden name parenthesized after the family name. Unknown keys raise KeyError naming the valid fields.""" import nameparser._render as _render return _render.render(self, spec) def initials(self, spec: str = "{given} {middle} {family}", delimiter: str = ".", separator: str = " ") -> str: """Initials per group; v1's initials_format/_delimiter/_separator become call-site arguments instead of Config-wide settings. Valid spec keys: given, middle, family.""" import nameparser._render as _render return _render.initials(self, spec, delimiter, separator) def capitalized(self, lexicon: Lexicon | None = None, *, force: bool = False) -> ParsedName: """Case-fixing transform -> new ParsedName, same spans, new token texts. Needs a lexicon for capitalization_exceptions and particle rules; None uses the DEFAULT lexicon -- if this name came from a custom Parser, pass its lexicon or use Parser.capitalized. force=False preserves mixed-case input (v1 parity). Idempotent.""" import nameparser._render as _render return _render.capitalized(self, lexicon, force=force) python-nameparser-2.1.0/nameparser/_version.py000066400000000000000000000007421523530431700215270ustar00rootroot00000000000000#: Release version parts (major, minor, micro). VERSION stays a pure #: numeric tuple so `nameparser.VERSION >= (2, 0, 0)` keeps working; the #: pre-release segment lives separately. VERSION = (2, 1, 0) #: PEP 440 pre-release/dev segment appended to the numeric version, or #: "" for a final release. Joined WITHOUT a dot ("2.0.0rc1", not #: "2.0.0.rc1"); setuptools reads __version__ as the package version. PRE_RELEASE = "" __version__ = ".".join(map(str, VERSION)) + PRE_RELEASE python-nameparser-2.1.0/nameparser/config/000077500000000000000000000000001523530431700205735ustar00rootroot00000000000000python-nameparser-2.1.0/nameparser/config/__init__.py000066400000000000000000000020751523530431700227100ustar00rootroot00000000000000"""v1 import-path preservation (migration spec §3): the Constants shim lives in nameparser._config_shim. The vocabulary data modules in this package (titles, suffixes, ...) remain the single source through 2.x. This package is deleted in 3.0. ``RegexTupleManager`` is re-exported unchanged from the shim purely for pickle compatibility: a v1.4 ``Constants`` blob's ``regexes`` field was pickled as ``nameparser.config.RegexTupleManager(...)``, and unpickling resolves and reconstructs that nested object before ``Constants.__setstate__`` ever runs. Without this name, loading such a blob raises ``AttributeError`` looking up the class, not a clean compatibility failure. """ from nameparser._config_shim import CONSTANTS as CONSTANTS from nameparser._config_shim import Constants as Constants from nameparser._config_shim import RegexTupleManager as RegexTupleManager from nameparser._config_shim import SetManager as SetManager from nameparser._config_shim import TupleManager as TupleManager __all__ = ["CONSTANTS", "Constants", "RegexTupleManager", "SetManager", "TupleManager"] python-nameparser-2.1.0/nameparser/config/_invariants.py000066400000000000000000000023241523530431700234630ustar00rootroot00000000000000"""Import-time hygiene checks shared by the vocabulary data modules. Every constant here is looked up in normalized form, so a stray capital or a surrounding space makes an entry unreachable by a direct membership test -- ``'actor ' in TITLES`` is False -- even though the parser's own ingest (:func:`nameparser._lexicon._normalize`) normalizes it away and papers over the typo. Checking at import turns a silently-inert entry into an immediate failure. Deliberately a weaker fold than ``_normalize``, which also strips edge periods: entries like ``'esq.'`` are legitimate data here, and this module cannot import ``_lexicon`` anyway (``_lexicon`` imports these constants). The relationship checks between constants stay in the modules that own them -- those encode facts about the data, not hygiene. """ from __future__ import annotations from collections.abc import Iterable def assert_normalized(name: str, entries: Iterable[str]) -> None: """Assert every entry is stored lowercase and free of edge whitespace.""" offenders = sorted(w for w in entries if w != w.strip().lower()) assert not offenders, ( f"{name} entries must be stored lowercase and whitespace-free; " f"offending: {offenders}" ) python-nameparser-2.1.0/nameparser/config/bound_first_names.py000066400000000000000000000022031523530431700246430ustar00rootroot00000000000000from nameparser.config._invariants import assert_normalized #: Bound Arabic given-name prefixes that attach to the following word to form #: one first name (e.g. "abdul salam" → first name "abdul salam"). They are #: never standalone names. Join logic runs in the given-name region only, #: mirroring :py:data:`~nameparser.config.prefixes.PREFIXES` for last names. BOUND_FIRST_NAMES: set[str] = { 'abdul', 'abdel', 'abdal', 'abu', 'abou', 'umm', # #269 follow-up: the Arabic-script originals of the entries above. # Script writes "Abdul Rahman" as two words (عبد + الرحمن -- the # article attaches to the following word), so عبد alone covers the # abdul/abdel/abdal variants. Both kunya spellings ship, matching # the أبو/ابو prefix pair. 'عبد', # "abd" (servant of) -- عبد الرحمن -> given "عبد الرحمن" 'أبو', # "abu" (father of), hamza spelling 'ابو', # "abu", hamza-less spelling 'أم', # "umm" (mother of), hamza spelling 'ام', # "umm", hamza-less spelling } assert_normalized("BOUND_FIRST_NAMES", BOUND_FIRST_NAMES) python-nameparser-2.1.0/nameparser/config/capitalization.py000066400000000000000000000006351523530431700241640ustar00rootroot00000000000000from nameparser.config._invariants import assert_normalized CAPITALIZATION_EXCEPTIONS = { 'ii': 'II', 'iii': 'III', 'iv': 'IV', 'md': 'M.D.', 'phd': 'Ph.D.', } """ Any pieces that are not capitalized by capitalizing the first letter. """ # Keys only -- the values are the exact-cased replacements, cased on purpose. assert_normalized("CAPITALIZATION_EXCEPTIONS", CAPITALIZATION_EXCEPTIONS) python-nameparser-2.1.0/nameparser/config/conjunctions.py000066400000000000000000000022501523530431700236600ustar00rootroot00000000000000from nameparser.config._invariants import assert_normalized CONJUNCTIONS = { '&', 'and', 'et', 'e', 'of', 'the', 'und', 'y', # #269: Cyrillic (ru/uk/bg) "and": и, і, та. Ukrainian writes і and # й for the same conjunction, alternating on the surrounding # vowel/consonant for euphony ("Олесь і Олена", "Марія й Петро"), # so real data carries both spellings and neither alone suffices. 'и', 'і', 'й', 'та', # #269: Greek "and": και. 'και', # #269 follow-up: Arabic "and". Formal script attaches و to the # following word (وفاطمة), so a standalone و token appears only in # informal spacing -- common in real data. Single-character like # 'y'/'и': the single-letter carve-out (Google Code issue 11, # the "john e smith" bug) protects short names (joins # only with enough rootname pieces). 'و', } """ Pieces that should join to their neighboring pieces, e.g. "and", "y" and "&". "of" and "the" are also include to facilitate joining multiple titles, e.g. "President of the United States". """ assert_normalized("CONJUNCTIONS", CONJUNCTIONS) python-nameparser-2.1.0/nameparser/config/maiden_markers.py000066400000000000000000000060001523530431700241220ustar00rootroot00000000000000from nameparser.config._invariants import assert_normalized MAIDEN_MARKERS = { 'née', 'né', 'nee', 'geb', 'geborene', 'geboren', 'roz', 'rozená', 'født', 'fødd', 'född', 'урожд', 'урождённая', 'урожденная', 'урождённый', 'урожденный', '旧姓', } """ Marker words that introduce a birth surname, e.g. "Jane Smith née Jones" (#274). French née/né/nee, German geb./geborene, Dutch geboren, Czech/Slovak roz./rozená, Danish/Norwegian født (Nynorsk fødd), Swedish född, Russian урожд./урождённая/урождённый (both ё and е spellings — case normalization does not fold them, and running text routinely writes е). Both grammatical genders are listed where #274 or review attested them (née/né, урождённая/урождённый); Czech masculine rozený awaits the same vetting. Entries are stored normalized: lowercase, no periods. Japanese 旧姓 is here rather than in locales.JA, on the rule that admitted the Cyrillic entries: a native-script marker cannot collide with a Latin-script name, and matching is whole-token, so it is safe as a default. Neither character appears in any shipped surname, title, suffix, conjunction, particle or bound-given vocabulary. locales.JA is for what needs the my-data-is-Japanese declaration -- segmentation, where a pure-Han string cannot say which language wrote it -- and this needs none, since it can only ever match Han text. Matching being whole-token, the marker has to BE a token -- which for Japanese means something has to divide it from the name it marks. A space does, and so does a configured delimiter: extract masks the whole bracketed region, delimiter characters included, before tokenize runs, so a bracket bounds a token exactly as a space does and "山田（旧姓 佐藤）" needs no space in front of 旧姓 at all. The bare "山田花子 旧姓 佐藤" and -- since #329 -- the bracketed "山田 花子（旧姓 佐藤）" under Policy(maiden_delimiters=...) alike give maiden 佐藤. What divides nothing is the fullwidth colon that the form Japanese more often writes puts after the marker: "山田（旧姓：佐藤）" still yields maiden "旧姓：佐藤" with the marker and its colon attached. Not because delimited content escapes classification -- classify tags a marker wherever it is a token -- but because ： is no separator tokenize knows, so marker and name arrive as ONE token and there is nothing to drop. The wholly unspaced "山田花子（旧姓佐藤）" reads as one token for the same reason. Peeling a marker off the head of a token is #317's job. Consumed by the 2.0 parser's default lexicon. The 1.x parser does not read this module. Deliberately absent: Polish "z domu" (a two-token marker; pending the 2.0 pipeline's multi-token matching decision) and the Scandinavian abbreviation "f." (collides with the initial "F." — only the full participles are safe). """ assert_normalized("MAIDEN_MARKERS", MAIDEN_MARKERS) python-nameparser-2.1.0/nameparser/config/prefixes.py000066400000000000000000000107361523530431700230010ustar00rootroot00000000000000from nameparser.config._invariants import assert_normalized from nameparser.config.bound_first_names import BOUND_FIRST_NAMES #: The sub-set of :py:data:`PREFIXES` that are *never* a standalone first name. #: A name that *starts* with one of these has no first name -- the whole thing #: is a surname (e.g. "de Mesnil" -> last name "de Mesnil"). Curated to exclude #: anything that can be a given name in some culture (`al`, `van`, `von`, #: `della`, `di`, `del`, `da`, `vander`, ...) and anything that is also a first #: name prefix (`abu`). When unsure, leave a word out: a missing member just #: means that name is not auto-fixed, whereas a wrong member misparses a real #: person. Must stay a subset of :py:data:`PREFIXES` and disjoint from #: :py:data:`~nameparser.config.bound_first_names.BOUND_FIRST_NAMES`. NON_FIRST_NAME_PREFIXES = { "'t", 'af', 'auf', 'av', 'bint', 'de', "de'", 'degli', 'dei', 'delle', 'delli', 'dello', 'dem', 'der', 'dos', 'het', 'ibn', 'op', 'ter', 'vd', 'vom', 'zu', # #269: Arabic native-script patronymic/clan particles. Unlike their # Latin transliterations, these live in a script namespace with no # collision against an unrelated Latin given name, so each is judged # on its own semantics rather than mirrored blindly: 'بن', # "bin"/"ibn" (son of) -- never a bare given name. Latin # 'bin' is in PREFIXES but not in this set; that judgment # is unchanged by adding the Arabic-script form. 'بنت', # "bint" (daughter of) -- mirrors Latin 'bint' above. 'ابن', # "ibn" (son of, alternate spelling) -- mirrors Latin # 'ibn' above. 'آل', # "aal" (family/clan of, e.g. "Al Saud") -- distinct from # the excluded definite article "ال" (#269 explicitly # excludes standalone "ال"); a clan prefix, never a bare # given name. # #269: Hebrew native-script patronymic particles -- same # reasoning as the Arabic ones above: no Latin-script collision, # and neither functions as a standalone given name in Hebrew usage. # Deferred under the collision rule: 'בר' (Aramaic son-of, as in # Bar-Lev) -- Bar is a common modern Israeli given name, and the # surname spelling is hyphenated anyway. 'בן', # "ben" (son of) 'בת', # "bat" (daughter of) } #: Name pieces that appear before a last name. Prefixes join to the piece #: that follows them to make one new piece. They can be chained together, e.g #: "von der" and "de la". Because they only appear in middle or last names, #: they also signify that all following name pieces should be in the same name #: part, for example, "von" will be joined to all following pieces that are not #: prefixes or suffixes, allowing recognition of double last names when they #: appear after a prefixes. So in "pennie von bergen wessels MD", "von" will #: join with all following name pieces until the suffix "MD", resulting in the #: correct parsing of the last name "von bergen wessels". #: #: Defined as a static union so every :py:data:`NON_FIRST_NAME_PREFIXES` member #: is guaranteed to also be a prefix (and still join forward), with no drift -- #: mirroring ``TITLES = FIRST_NAME_TITLES | {...}`` in #: :py:mod:`nameparser.config.titles`. PREFIXES = NON_FIRST_NAME_PREFIXES | { 'aan', 'aen', 'abu', 'al', 'bar', 'bat', 'bin', 'bon', 'da', 'dal', 'del', 'dela', 'della', 'den', 'di', 'dí', 'do', 'du', 'freiherr', 'freiherrin', 'heer', 'la', 'le', 'mac', 'mc', 'san', 'santa', 'st', 'ste', 'te', 'tho', 'thoe', 'van', 'vande', 'vander', 'vel', 'von', # #269: Arabic "abu" (father of), left ambiguous like its Latin # transliteration 'abu' above (both spellings): "Abu Bakr" reads # "Abu" as a given name, so this stays a PREFIXES-only member, not # NON_FIRST_NAME_PREFIXES. 'أبو', 'ابو', } # Guard the two invariants the docstring above promises, so a future edit that # breaks them fails at import time instead of silently drifting until a test # happens to catch it. assert NON_FIRST_NAME_PREFIXES <= PREFIXES, \ "NON_FIRST_NAME_PREFIXES must stay a subset of PREFIXES" assert not (NON_FIRST_NAME_PREFIXES & BOUND_FIRST_NAMES), \ "NON_FIRST_NAME_PREFIXES must stay disjoint from BOUND_FIRST_NAMES" assert_normalized("PREFIXES", PREFIXES) python-nameparser-2.1.0/nameparser/config/regexes.py000066400000000000000000000050271523530431700226130ustar00rootroot00000000000000import re # emoji ranges from https://stackoverflow.com/questions/26568722/remove-unicode-emoji-using-re-in-python # Built from codepoint pairs rather than a literal character class: # the compiled pattern is identical, but CodeQL's py/overly-large-range # false-positives on literal astral ranges (surrogate decomposition). _EMOJI_RANGES = ((0x1F300, 0x1F64F), (0x1F680, 0x1F6FF), (0x2600, 0x26FF), (0x2700, 0x27BF)) re_emoji = re.compile( '[' + ''.join(f'{chr(lo)}-{chr(hi)}' for lo, hi in _EMOJI_RANGES) + ']+') # Invisible bidirectional formatting characters: ALM, LRM, RLM, the # embedding/override marks (LRE/RLE/PDF/LRO/RLO) and the isolates # (LRI/RLI/FSI/PDI). Copy-pasted RTL names often carry these; they render # as nothing but stick to name parts and break equality/lookups. re_bidi = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+') EMPTY_REGEX = re.compile('') REGEXES = { "spaces": re.compile(r"\s+"), "word": re.compile(r"(\w|\.)+"), "mac": re.compile(r'^(ma?c)(\w{2,})', re.I), "initial": re.compile(r'^(\w\.|[A-Z])?$'), "quoted_word": re.compile(r'(? (module name, module attribute). Codes are the #: lowercase module names; attribute constants are uppercase (spec §2). _REGISTRY = { "JA": ("nameparser.locales.ja", "JA"), "RU": ("nameparser.locales.ru", "RU"), "TR_AZ": ("nameparser.locales.tr_az", "TR_AZ"), "ZH": ("nameparser.locales.zh", "ZH"), } def __getattr__(name: str) -> Locale: entry = _REGISTRY.get(name) if entry is None: raise AttributeError( f"module {__name__!r} has no locale {name!r}; available: " f"{', '.join(sorted(_REGISTRY))}" ) module_name, attr = entry locale: Locale = getattr(importlib.import_module(module_name), attr) # cache: next access skips __getattr__. Benign race under free # threading: two threads racing here both import the same module # object (import machinery serializes/dedupes that) and assign the # same singleton Locale back to the same name, so the last write # wins with no observable difference. globals()[name] = locale return locale def __dir__() -> list[str]: return sorted(set(globals()) | set(_REGISTRY)) def available() -> tuple[str, ...]: """The registered locale codes, lowercase, sorted.""" return tuple(sorted(attr.lower() for attr in _REGISTRY)) def ja_segmenter(*, gbdt: bool = False) -> Segmenter: """The Japanese segmenter factory (#272), needing the optional ``pip install nameparser[ja]``; see :mod:`nameparser.locales.ja` for what it wraps, what it declines, and its licensing. A real function rather than a _REGISTRY entry, because the registry resolves LOCALES: __getattr__ is annotated to return one, so routing a callable through it would type every pack access as "Locale or Segmenter" for no gain. Laziness is preserved here instead -- the pack module (and with it the optional dependency) loads on the call, not on importing this package.""" from nameparser.locales.ja import ja_segmenter as factory return factory(gbdt=gbdt) def get(code: str) -> Locale: """Dynamic lookup by code ('ru'); raises KeyError listing the available codes (spec §2).""" attr = code.upper() if isinstance(code, str) else code if not isinstance(code, str) or attr not in _REGISTRY: raise KeyError( f"unknown locale code {code!r}; available: " f"{', '.join(available())}" ) return __getattr__(attr) python-nameparser-2.1.0/nameparser/locales/ja.py000066400000000000000000000277041523530431700217260ustar00rootroot00000000000000"""The Japanese locale pack (#272): activates segmentation for the Japanese repertoire -- Han and the kana-licensed composites that resolve to HIRAGANA -- so that an unspaced 山田太郎 can be divided at all. Applying this pack IS the "my data is Japanese" declaration, the same role locales.ZH plays for Chinese: a pure-Han string cannot say which language it is written in (林 is Lin or Hayashi), so neither activation may be a default. Unlike zh, this pack ships NO vocabulary and NO order: * No vocabulary, because no surname list divides Japanese names. Family and given names draw on the same kanji, both sides run one to four characters, and the reading -- not the spelling -- is what settles most splits. That is what the pluggable segmenter is for, and it is why this pack alone is inert: it activates a stage that then has nothing to segment with. * No order, because a kana-bearing Japanese name already reads family-first by default (amendment 2026-07-29 §1: hiragana identifies native Japanese as certainly as hangul identifies Korean), and wholly Han names have read family-first since the 2026-07-27 amendment. Pure KATAKANA is deliberately outside both the order rule and the activation set above -- katakana is how Japanese writes FOREIGN names (マイケル・ジャクソン), in their original given-first order. Data sources: none. The pack carries no vocabulary of its own, so there is no list here to cite, extend or keep current -- the data that divides a Japanese name belongs to the segmenter, and namedivider's sources and their terms are set out under Licensing below. The segmenter is a FACTORY, ``ja_segmenter()``, not pack data: a Locale is pure data that dissolves into a Parser at construction, and a third-party callable is neither pure nor data. Combining the two is explicit -- ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` -- which also keeps the optional dependency out of every import path that merely touches this module. Licensing (the optional dependency is never bundled; nameparser's own LGPL is untouched): namedivider-python's source and its GBDT model are MIT. Its surname data -- ``family_name_repository.pickle``, sourced from Myoji-Yurai.net -- carries custom terms permitting commercial and non-commercial use *for dividing names*, which is the only use any caller of this module can put it to. Those terms bind only the opt-in ``ja_segmenter(gbdt=True)`` path, which is what loads that file; the default BasicNameDivider reads namedivider's own bundled kanji.csv and never touches it. Its BERT model for katakana division is CC-BY-SA and is NOT used (katakana division is out of scope, amendment §6). Declared deviations (spec §2 authoring requirement 3): the pack sets one union policy field, self-selecting by script, so it can only change names containing characters of the Japanese repertoire -- DEVIATES below declares exactly that, over-declaring within the repertoire (a name also needs an unspaced token and a segmenter to actually change, but declaring every repertoire-bearing name is the safe direction, zh's precedent). """ from __future__ import annotations from typing import TYPE_CHECKING from nameparser._lexicon import Lexicon from nameparser._locale import Locale from nameparser._policy import (PolicyPatch, Script, _JA_SCRIPTS, _script_matcher) from nameparser._types import Segmentation if TYPE_CHECKING: from nameparser._types import Segmenter JA = Locale( code="ja", lexicon=Lexicon.empty(), policy=PolicyPatch( segment_scripts=frozenset({Script.HAN, Script.HIRAGANA})), ) # Both compiled by _policy's factory from the shared codepoint table # and its _JA_SCRIPTS union, so the pack carries no spans of its own # to drift out of sync: the contains-any form backs DEVIATES, the # whole-token form is the adapter's repertoire guard. Both match # KATAKANA even though the pack never ACTIVATES it: a katakana-bearing # token can still be a kana-licensed composite the pack acts on # (山田エミ) -- DEVIATES must declare it, and the adapter must not # decline it. A pure-katakana token never reaches THE ADAPTER: # KATAKANA is in no activation set, so the stage's own gate stops it # before the segmenter is consulted. DEVIATES still declares one, # and must: it is a predicate over any name at all, called before # any gate runs, and over-declaring is its safe direction by design. _has_japanese = _script_matcher(*_JA_SCRIPTS) _wholly_japanese = _script_matcher(*_JA_SCRIPTS, whole=True) #: How far outside [0, 1] a namedivider score may stray and still count #: as float noise around a real answer rather than a broken divider -- #: see the adapter's own comment for why it is float32-sized. _SCORE_EPSILON = 1e-6 def DEVIATES(name: str) -> bool: """True when this pack may parse `name` differently from the default parser (the declared-deviation predicate the non-interference gate consumes): any name bearing a repertoire character -- see the repertoire note above for why katakana is declared and why over-declaring is safe.""" return _has_japanese(name) def ja_segmenter(*, gbdt: bool = False) -> Segmenter: """A :data:`~nameparser.Segmenter` wrapping namedivider-python, installed with ``pip install nameparser[ja]``: parser = parser_for(locales.JA, segmenter=locales.ja_segmenter()) Wraps ``BasicNameDivider`` (99.3% accurate, ~4k names/sec, pure Python) by default, which needs only data bundled in the installed package; ``gbdt=True`` selects ``GBDTNameDivider`` (99.9%), which asks more of the environment on two counts. It runs namedivider's gradient-boosted model through lightgbm -- a hard dependency of namedivider-python, so the extra installs it, but its wheel does not bundle the native OpenMP runtime it loads at import (a missing ``libomp`` surfaces as lightgbm's own OSError naming the library it could not find). And the first such call DOWNLOADS its model and surname files from the network into ``~/.cache/namedivider-python`` (namedivider prints its own progress to stdout doing so), which matters for offline, sandboxed or air-gapped deployments. The adapter declines -- returns None, leaving the token whole -- for text outside the Japanese repertoire, for text bearing the shime mark 〆 (namedivider 0.4.x's rule path cuts it in the wrong place), for text too short to divide, for any answer that fails to reconstruct its input, and for any score outside [0, 1] by more than float noise (a divider scoring 87.0 is broken, and its division is worth no more than its score). An answer that puts the whole token on one side comes back as "confidently one token" rather than a decline: the divider read it and found nothing to cut, which is not the same as having no opinion. Answers arrive with namedivider's own confidence, which the parsing stage compares against its floor to decide whether to report a SEGMENTATION ambiguity: namedivider scores a rule-based division (the kana boundary in 高橋みなみ, or a two-character name) 1.0 and a kanji-statistics division well below that, so every statistically divided name carries the report and every rule-divided one carries none. The two-character rule is the case to know about: dividing 原恵 one character each way is a presumption Japanese practice makes, not a measurement (usage.rst says so), yet the 1.0 keeps it silent. That is namedivider's own stated certainty, taken as given here; ``DividedName.algorithm`` names which rule answered and is the hook if presumption-reporting is ever wanted. """ try: import namedivider except ImportError as exc: raise ImportError( "the Japanese segmenter needs namedivider-python -- " "install it with: pip install nameparser[ja]" ) from exc divider = (namedivider.GBDTNameDivider() if gbdt else namedivider.BasicNameDivider()) def segment(text: str) -> Segmentation | None: # namedivider RAISES below two characters ("Name length needs # at least 2 chars"), and a segmenter's exceptions propagate by # contract, so a one-character token -- routine in spaced input # like "林 太郎" -- must be declined here, not passed on. if len(text) < 2: return None # segment_scripts UNIONS under pack composition, so this can # receive tokens of any other activated script (hangul, under # the default policy). Decline anything outside the Japanese # repertoire rather than hand it to namedivider, which would # answer for it regardless -- the same union DEVIATES declares. if not _wholly_japanese(text): return None # namedivider 0.4.x has no knowledge of the shime mark: its # rule path cuts 〆木太郎 to 〆 | 木太郎 at confidence 1.0 # (measured -- wrong for every attested 〆 surname: 〆木, 〆谷, # 〆野), so a wrong family would arrive with no SEGMENTATION # report. Decline instead: the family-first ORDER fix for 〆 # stands regardless, and division can return when a divider # knows the mark. The blanket check is deliberately broader # than that rule-path failure: mid-token 〆 (山〆太郎) comes # back 山 | 〆太郎 at 0.375 on the kanji_feature path # (measured), a low-confidence answer that WOULD have carried # a SEGMENTATION report -- swept in anyway, because decline is # the safe direction and mid-token 〆 is vanishingly rare. if "〆" in text: return None result = divider.divide_name(text) if result.family + result.given != text: return None # defensive: never trust reconstruction # float(): the score is a numpy scalar on the statistical path. # Two different failures wear the same shape here, and they get # different answers. Float error at the edge of namedivider's # softmax is arithmetic noise around a real answer, so an # epsilon either side of [0, 1] is tolerated and clamped -- # Segmentation enforces the range and would otherwise raise # mid-parse. A score OUTSIDE that epsilon is not noise: a # divider reporting 87.0 confidence is broken, and its division # is worth nothing either. Declining keeps the parse safe AND # observable by absence -- an unconditional clamp mapped 87.0 # to 1.0, i.e. "certain", silencing the SEGMENTATION report at # exactly the point of maximum brokenness. NaN takes the same # exit for free: every comparison against it is False, so it # fails both bounds below and declines. # # _SCORE_EPSILON is sized for FLOAT32, not float64: numpy # softmax output is commonly float32, whose eps is ~1.2e-7, and # namedivider's rule-based answers score exactly 1.0 -- so a # float64-sized 1e-9 band would decline a correct rule-based # division that came back a single ulp high (1.0000001 is # exactly that). A few float32 ulps is still five orders of # magnitude below any score a broken divider would produce, so # nothing is bought by tightening it. score = float(result.score) if not -_SCORE_EPSILON <= score <= 1.0 + _SCORE_EPSILON: return None confidence = min(1.0, max(0.0, score)) # Defensive, the reconstruction check's twin: no namedivider # 0.4.x path can return an empty side (every answer, rule or # statistical, is built by slicing the input at an interior # offset). If one ever did, it would mean "I read this as # undivided" -- an empty Segmentation, which is a stated # opinion, and not the None that declines to have one. if not result.family or not result.given: return Segmentation((), confidence=confidence) return Segmentation((len(result.family),), confidence=confidence) return segment python-nameparser-2.1.0/nameparser/locales/ru.py000066400000000000000000000051211523530431700217470ustar00rootroot00000000000000"""The Russian locale pack (locales spec §3): policy-only -- it turns on the EAST_SLAVIC patronymic rule. The morphology data (-ovich/-ovna endings, Cyrillic and transliterated) lives inside the rule implementation in nameparser/_pipeline/_post_rules.py, not in the Lexicon (mirrors v1's patronymic_name_order flag design, v1.3.0). Cyrillic titles/conjunctions are default-lexicon vocabulary (#269), not pack data (spec §2 sorting rule). Data sources: the v1.3.0 patronymic rule (PR #154 discussion and the east-slavic test bank); no external lists -- the pack itself carries no vocabulary. Declared deviations (spec §2 authoring requirement 3): applying this pack changes only NO_COMMA names whose final token carries an East Slavic patronymic ending while the middle token does not -- DEVIATES(name) below is the machine-readable declaration the non-interference gate checks. """ from __future__ import annotations import re from nameparser._lexicon import Lexicon from nameparser._locale import Locale from nameparser._policy import PatronymicRule, PolicyPatch RU = Locale( code="ru", lexicon=Lexicon.empty(), policy=PolicyPatch( patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})), ) # Ported by hand from nameparser/_pipeline/_post_rules.py's # _EAST_SLAVIC/_EAST_SLAVIC_CYR (the rule is the source of truth; this # predicate only DECLARES the deviation surface for the non-interference # gate -- layering forbids importing the pipeline from a pack). Keep # both alternations byte-identical to those two patterns. _EAST_SLAVIC = re.compile( r"(ovich|ovna|evich|evna|ichna|ilyich|kuzmich|lukich|fomich|fokich)$", re.I) _EAST_SLAVIC_CYR = re.compile( r"(ович|овна|евич|евна|ична|ильич|кузьмич|лукич|фомич|фокич)$", re.I) def DEVIATES(name: str) -> bool: """True when this pack may parse `name` differently from the default parser (the declared-deviation predicate the non-interference gate consumes).""" # Scan per token: the rule fires on the family-POSITION token, not # the name's final characters, so a whole-string search would miss # 'Ivan Petr Sidorovich Jr.' (a suffix after the ending -> the $ # anchor never lands: under-declaration, the unsafe direction for # the gate). Per-token scanning instead OVER-declares (e.g. # 'Sidorovich Anna' matches though the rule's 1+1+1 shape never # fires) -- the safe direction: DEVIATES may claim more than the # rule changes, never less. return any(_EAST_SLAVIC.search(tok) or _EAST_SLAVIC_CYR.search(tok) for tok in name.split()) python-nameparser-2.1.0/nameparser/locales/tr_az.py000066400000000000000000000040221523530431700224370ustar00rootroot00000000000000"""The Turkish/Azerbaijani locale pack (locales spec §3): policy-only -- it turns on the TURKIC patronymic rule. The marker data (oglu/qizi/ uulu/kyzy and Cyrillic forms) lives inside the rule implementation in nameparser/_pipeline/_post_rules.py (mirrors v1's flag design). Data sources: the v1 Turkic patronymic rule (issue #185 and its test bank, tests/test_turkic_patronymic_order.py); the pack carries no vocabulary. Declared deviations (spec §2 authoring requirement 3): applying this pack changes only NO_COMMA names where some token is a standalone Turkic patronymic marker -- DEVIATES(name) below is the machine-readable declaration the non-interference gate checks. """ from __future__ import annotations import re from nameparser._lexicon import Lexicon from nameparser._locale import Locale from nameparser._policy import PatronymicRule, PolicyPatch TR_AZ = Locale( code="tr_az", lexicon=Lexicon.empty(), policy=PolicyPatch( patronymic_rules=frozenset({PatronymicRule.TURKIC})), ) # Ported by hand from nameparser/_pipeline/_post_rules.py's # _TURKIC/_TURKIC_CYR (the rule is the source of truth; this predicate # only DECLARES the deviation surface for the non-interference gate -- # layering forbids importing the pipeline from a pack). Keep both # alternations byte-identical to those two patterns. Note both are # fullmatch-shaped (^...$): the rule tests a single WHOLE token, so # DEVIATES below scans per-token rather than searching the whole name. _TURKIC = re.compile( r"^(oglu|oğlu|ogly|ogli|o['’ʻ]g['’ʻ]li" r"|qizi|qızı|kizi|kyzy|gyzy|uly|uulu)$", re.I) _TURKIC_CYR = re.compile( r"^(оглу|оглы|оғлу|ўғли|угли|кызы|гызы|қызы|қизи|улы|ұлы|уулу)$", re.I) def DEVIATES(name: str) -> bool: """True when this pack may parse `name` differently from the default parser (the declared-deviation predicate the non-interference gate consumes).""" return any(_TURKIC.match(tok) or _TURKIC_CYR.match(tok) for tok in name.split()) python-nameparser-2.1.0/nameparser/locales/zh.py000066400000000000000000000132301523530431700217420ustar00rootroot00000000000000"""The Chinese locale pack (#271): activates Han segmentation and ships the surname vocabulary for it. Deliberately does NOT set any order -- Policy.script_orders already reads wholly-Han names family-first by default (amendment 2026-07-27); segmentation is the one Han behavior that must stay opt-in, because a pure-Han string cannot say whether it is Chinese or Japanese (林 is Lin or Hayashi) and a zh surname list corrupts Japanese names (高橋一郎 would split 高 + 橋一郎). Applying this pack IS the "my data is Chinese" declaration. Japanese segmentation is locales.JA's job instead: that pack activates the same stage for the Japanese repertoire and defers the division itself to a pluggable segmenter, since no surname list divides a kanji name. Data sources: single-character surnames are the top-100 of the 2020 PRC census; traditional-script variant forms are included for names written in Taiwan/Hong Kong/overseas orthography; two-character compounds are the Hundred Family Surnames compounds still in modern use. A coverage floor, not a census roster: extend by giving parser_for a base whose lexicon already carries the additions -- ``parser_for(locales.ZH, base=Parser(lexicon=Lexicon.default().add( surnames={...})))``. (Adding to the finished parser's lexicon instead is a silent no-op: Lexicon is frozen, so ``.add`` returns a NEW lexicon that no parser is holding.) Declared deviations (spec §2 authoring requirement 3): the pack adds vocabulary and one union policy field, both self-selecting by script, so it can only change names containing Han characters -- DEVIATES below declares exactly that (over-declaring within the script: only segmentation can actually change a parse -- an unspaced surname match, or a ``Parser(segmenter=...)`` answer where the vocabulary declines -- but declaring every Han-bearing name is the safe direction). """ from __future__ import annotations from nameparser._lexicon import Lexicon from nameparser._locale import Locale from nameparser._policy import PolicyPatch, Script, _script_matcher # 2020 census top-100 plus the annotated additions below, simplified # forms, census rank order, 10 per row -- append new entries at the # tail, do not alphabetize (rank order is the only in-file record of # where the coverage floor sits). Two entries are NOT census rows and # are called out so the rank-order record stays honest: # # * 阎 sits off-rank beside 闫 rather than at the tail, because the two # are a reading pair, not a rank neighbour: 闫 is the census row and # 阎 is the distinct surname sharing its reading (Yan). Keeping them # adjacent is worth the one break in rank order. # * 萧 is at the tail: the mainland census records this surname under # the merged 肖 (rank 33 above), but 萧 stays a distinct simplified # spelling in current use and both circulate. The traditional form # 蕭 ships either way, below. _SINGLE = ( "王", "李", "张", "刘", "陈", "杨", "黄", "赵", "吴", "周", "徐", "孙", "马", "朱", "胡", "郭", "何", "林", "高", "罗", "郑", "梁", "谢", "宋", "唐", "许", "韩", "邓", "冯", "曹", "彭", "曾", "肖", "田", "董", "潘", "袁", "蔡", "蒋", "余", "于", "杜", "叶", "程", "魏", "苏", "吕", "丁", "任", "卢", "姚", "沈", "钟", "姜", "崔", "谭", "陆", "范", "汪", "廖", "石", "金", "韦", "贾", "夏", "傅", "方", "邹", "熊", "白", "孟", "秦", "邱", "侯", "江", "尹", "薛", "闫", "阎", "段", "雷", "龙", "黎", "史", "陶", "贺", "毛", "郝", "顾", "龚", "邵", "万", "覃", "武", "钱", "戴", "严", "莫", "孔", "向", "常", "萧", ) # Traditional-script forms of the above where the glyph differs. 蕭 is # the traditional spelling of both 萧 and the merged 肖 (see the note # above); every other row is a plain 1:1 variant. _SINGLE_TRADITIONAL = ( "張", "劉", "陳", "楊", "黃", "趙", "吳", "孫", "馬", "羅", "鄭", "謝", "許", "韓", "鄧", "馮", "蕭", "葉", "蘇", "呂", "盧", "鍾", "譚", "陸", "韋", "賈", "鄒", "閆", "閻", "龍", "賀", "顧", "龔", "萬", "錢", "嚴", "蔣", ) # Hundred Family Surnames compounds still in modern use, simplified. _COMPOUND = ( "欧阳", "司马", "诸葛", "上官", "夏侯", "皇甫", "尉迟", "公孙", "长孙", "慕容", "司徒", "司空", "令狐", "宇文", "东方", "独孤", "南宫", "西门", "澹台", "淳于", "单于", "申屠", "公羊", "仲孙", "轩辕", "呼延", "端木", "百里", "东郭", "闻人", "拓跋", "万俟", "夹谷", "太史", ) # Traditional forms of the compounds where any glyph differs. _COMPOUND_TRADITIONAL = ( "歐陽", "司馬", "諸葛", "尉遲", "公孫", "長孫", "東方", "獨孤", "南宮", "西門", "澹臺", "單于", "仲孫", "軒轅", "東郭", "聞人", "萬俟", "夾谷", ) # Private: ZH.lexicon.surnames is the supported way to read this, and # publishing a name later is compatible where un-publishing is not. _SURNAMES = frozenset( _SINGLE + _SINGLE_TRADITIONAL + _COMPOUND + _COMPOUND_TRADITIONAL) ZH = Locale( code="zh", lexicon=Lexicon(surnames=_SURNAMES), policy=PolicyPatch(segment_scripts=frozenset({Script.HAN})), ) # Compiled by _policy's factory from the shared codepoint table, so # the pack carries no Han spans of its own to drift out of sync. _has_han = _script_matcher(Script.HAN) def DEVIATES(name: str) -> bool: """True when this pack may parse `name` differently from the default parser (the declared-deviation predicate the non-interference gate consumes): any Han-bearing name -- see the module docstring's declared-deviations note for why over-declaring is safe.""" return _has_han(name) python-nameparser-2.1.0/nameparser/parser.py000066400000000000000000000003351523530431700211750ustar00rootroot00000000000000"""v1 import-path preservation (migration spec §3): the 2.0 HumanName facade lives in nameparser._facade. This module is deleted in 3.0. """ from nameparser._facade import HumanName as HumanName __all__ = ["HumanName"] python-nameparser-2.1.0/nameparser/py.typed000066400000000000000000000000001523530431700210130ustar00rootroot00000000000000python-nameparser-2.1.0/nameparser/util.py000066400000000000000000000007001523530431700206520ustar00rootroot00000000000000"""v1 normalization, kept for the 1.x compatibility layer. Removed in 3.0 with the rest of that layer. The 2.0 core does NOT use this: :func:`nameparser._lexicon._normalize` is its fold, which strips edge periods to a fixed point where ``lc()`` strips once. """ def lc(value: str) -> str: """Lowercase and strip leading/trailing periods to normalize for comparison.""" if not value: return '' return value.lower().strip('.') python-nameparser-2.1.0/pyproject.toml000066400000000000000000000064161523530431700201140ustar00rootroot00000000000000[project] name = "nameparser" description = "A simple Python module for parsing human names into their individual components." readme = "README.rst" requires-python = ">=3.11" license = {text = "LGPL"} authors = [{name = "Derek Gulbranson", email = "derek73@gmail.com"}] dynamic = ["version"] keywords = ["names", "parser"] classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Development Status :: 5 - Production/Stable", "Natural Language :: English", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Text Processing :: Linguistic", ] dependencies = [] [project.optional-dependencies] # Japanese kanji-name segmentation (#272), wrapped by # locales.ja_segmenter(). Optional and never bundled: the core stays # zero-dependency. 0.4 is the floor its BasicNameDivider/DividedName # API dates from. ja = ["namedivider-python>=0.4"] [build-system] requires = ["setuptools (>=82.0.1)"] build-backend = "setuptools.build_meta" [tool.setuptools.dynamic] version = {attr = "nameparser._version.__version__"} [tool.setuptools.package-data] nameparser = ["py.typed"] [dependency-groups] dev = [ "pytest (>=8)", "dill (>=0.2.5)", "sphinx (>=8)", "furo", "mypy (>=2.1)", "ruff (>=0.15)", "pytest-timeout>=2.4.0", "pytest-cov>=6", "hypothesis", ] [tool.mypy] packages = ["nameparser", "tests"] [[tool.mypy.overrides]] module = [ "dill.*", # optional extras, unstubbed and absent from the dev environment: # the ja pack imports namedivider lazily, inside the factory "namedivider.*", ] ignore_missing_imports = true [[tool.mypy.overrides]] module = [ "nameparser._types", "nameparser._lexicon", "nameparser._policy", "nameparser._render", "nameparser._locale", "nameparser._pipeline.*", "nameparser._parser", ] disallow_untyped_defs = true disallow_incomplete_defs = true disallow_any_generics = true warn_return_any = true strict_equality = true [[tool.mypy.overrides]] module = ["tests.v2.*"] check_untyped_defs = true [tool.pytest.ini_options] testpaths = ["tests", "nameparser"] addopts = "--doctest-modules" python_classes = ["*Tests", "*TestCase"] # Fail if an xfail-marked test unexpectedly passes, so a fixed/regressed # expectation surfaces instead of silently turning into an xpass. xfail_strict = true # Unexpected warnings are failures: warnings are load-bearing API in 2.0 # (the multi-word-entry UserWarning, the facade deprecation warnings), and # without this a warning regression is invisible to the suite -- a real # bug (eight spurious warnings on restoring a 1.4 Constants pickle) # shipped to review this way. Tests that EXPECT a warning assert it with # pytest.warns; targeted ignores below need a comment justifying each. filterwarnings = [ "error", ] [tool.ruff.lint] extend-select = [ "ANN", # flake8-annotations "UP", # pyupgrade ] ignore = [ "E74", # ambiguous-{variable,class,function}-name (I have trouble believing that these are real rules) ] python-nameparser-2.1.0/tests/000077500000000000000000000000001523530431700163335ustar00rootroot00000000000000python-nameparser-2.1.0/tests/__init__.py000066400000000000000000000000001523530431700204320ustar00rootroot00000000000000python-nameparser-2.1.0/tests/base.py000066400000000000000000000060151523530431700176210ustar00rootroot00000000000000from typing import Any, ClassVar, Generic, TypeVar from nameparser import HumanName from nameparser.config import Constants T = TypeVar('T') class HumanNameTestBase(Generic[T]): """Shared assert helpers for the parsing tests. Formerly subclassed unittest.TestCase. It is now a plain class so pytest can apply the parametrized dual-run fixture in conftest.py — parametrized fixtures do not apply to unittest.TestCase subclasses. The assert* methods are thin shims so existing test bodies move over unchanged. """ def m(self, actual: T, expected: T, hn: HumanName) -> None: """assertEqual with a better message. ``expected or ''`` used to fall back to ``hn.C.empty_attribute_default``, which could be '' or None depending on which of the (formerly dual-run) settings a test ran under. That setting was removed in 2.0 (#255): empty attributes are always ''. """ expected_ = expected or '' try: assert actual == expected_, f"{actual!r} != {expected!r} for {hn.original!r}\n{hn!r}" except UnicodeDecodeError: assert actual == expected_, f"actual={actual!r} != expected={expected_!r}" def assertEqual(self, first: object, second: object, msg: object = None) -> None: assert first == second, msg def assertTrue(self, expr: object, msg: object = None) -> None: assert expr, msg def assertFalse(self, expr: object, msg: object = None) -> None: assert not expr, msg def assertIn(self, member: object, container: object, msg: object = None) -> None: assert member in container, msg # type: ignore[operator] def assertNotEqual(self, first: object, second: object, msg: object = None) -> None: assert first != second, msg def assertNotIn(self, member: object, container: object, msg: object = None) -> None: assert member not in container, msg # type: ignore[operator] def assertIs(self, first: object, second: object, msg: object = None) -> None: assert first is second, msg or f"{first!r} is not {second!r}" def assertIsNot(self, first: object, second: object, msg: object = None) -> None: assert first is not second, msg or f"{first!r} is {second!r}" def assertIsNone(self, expr: object, msg: object = None) -> None: assert expr is None, msg or f"{expr!r} is not None" def assertIsNotNone(self, expr: object, msg: object = None) -> None: assert expr is not None, msg or "unexpectedly None" class FlaggedConstantsTestBase(HumanNameTestBase[T]): """Base for test classes that parse with a dedicated, flagged Constants. Subclasses set ``constants_kwargs``; each test method gets a fresh ``Constants(**constants_kwargs)`` via ``setup_method``, and ``hn()`` parses with it. """ constants_kwargs: ClassVar[dict[str, Any]] = {} def setup_method(self) -> None: self.C = Constants(**self.constants_kwargs) def hn(self, name: str) -> HumanName: return HumanName(name, constants=self.C) python-nameparser-2.1.0/tests/conftest.py000066400000000000000000000063321523530431700205360ustar00rootroot00000000000000import copy import warnings from collections.abc import Iterator import pytest from nameparser.config import CONSTANTS # Scalar (non-collection) config attributes that individual tests mutate on the # global CONSTANTS singleton. Several tests change these without restoring them; # the original suite only survived because unittest happens to run methods in # alphabetical order, so a later test reset the value. pytest runs in definition # order, so we snapshot and restore these around every test to keep tests # isolated regardless of order. # # empty_attribute_default is gone from this list: it was removed in 2.0 (#255, # see nameparser/_config_shim.py's Constants.__setattr__), so there is no # longer a second parsing path to snapshot/restore or dual-run against. Empty # attributes are always ''. _SCALAR_CONFIG_ATTRS = ( "string_format", "initials_format", "initials_delimiter", "initials_separator", "suffix_delimiter", "capitalize_name", "force_mixed_case_capitalization", ) # Collection config attributes (the SetManager / TupleManager constants). Tests # that customize the global CONSTANTS — e.g. adding or removing a title — mutate # these in place, so a shallow snapshot of the reference would not protect later # tests. We snapshot independent copies and restore them, making collection # mutations order-independent too. # # regexes is gone from this list: it is a read-only _RegexesProxy in 2.0 (set # once in Constants.__init__), and reassigning it raises TypeError, so it can # neither be mutated by a test nor restored by this fixture. _COLLECTION_CONFIG_ATTRS = ( "prefixes", "suffix_acronyms", "suffix_not_acronyms", "titles", "first_name_titles", "conjunctions", "bound_first_names", "non_first_name_prefixes", "capitalization_exceptions", "nickname_delimiters", "maiden_delimiters", ) @pytest.fixture(autouse=True) def _isolate_constants() -> Iterator[None]: """Snapshot and restore the shared CONSTANTS singleton around every test. Formerly also drove a parametrized dual-run over empty_attribute_default settings (reproducing the original tests.py __main__ block, which ran the whole suite twice as a regression check that the three parsing code paths agreed). That setting no longer exists in 2.0 (#255: empty attributes are always ''), so only the isolation half of the original fixture remains. """ scalar_snapshot = {attr: getattr(CONSTANTS, attr) for attr in _SCALAR_CONFIG_ATTRS} collection_snapshot = { attr: copy.deepcopy(getattr(CONSTANTS, attr)) for attr in _COLLECTION_CONFIG_ATTRS } yield # Restoring through the shared singleton is itself a mutation and would # otherwise emit the shared-CONSTANTS DeprecationWarning (#262) on every # single test's teardown; this fixture is infrastructure; it is not # exercising that deprecation, so it's silenced narrowly here rather than # at every test. with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) for attr, value in scalar_snapshot.items(): setattr(CONSTANTS, attr, value) for attr, value in collection_snapshot.items(): setattr(CONSTANTS, attr, value) python-nameparser-2.1.0/tests/test_bound_first_names.py000066400000000000000000000134201523530431700234450ustar00rootroot00000000000000from nameparser import HumanName from tests.base import HumanNameTestBase class BoundFirstNamesTestCase(HumanNameTestBase): # The v1 is_bound_first_name predicate is gone with the other v1 parsing # hooks (#280); the vocabulary's behavior is pinned through the parsing # tests below. # --- no-comma: basic joining --- def test_no_comma_basic_join(self) -> None: hn = HumanName("abdul salam ahmed salem") self.m(hn.first, "abdul salam", hn) self.m(hn.middle, "ahmed", hn) self.m(hn.last, "salem", hn) def test_no_comma_three_tokens_no_middle(self) -> None: hn = HumanName("abdul salam salem") self.m(hn.first, "abdul salam", hn) self.m(hn.middle, "", hn) self.m(hn.last, "salem", hn) def test_no_comma_guard_two_tokens_no_join(self) -> None: """Guard: only last name remains after prefix → no join.""" hn = HumanName("abdul salam") self.m(hn.first, "abdul", hn) self.m(hn.last, "salam", hn) def test_no_comma_guard_suffix_not_swallowed(self) -> None: """Guard: prefix + one name + suffix — suffix must not become last.""" hn = HumanName("abdul salam jr") self.m(hn.first, "abdul", hn) self.m(hn.last, "salam", hn) self.m(hn.suffix, "jr", hn) # --- lastname-comma path --- def test_lastname_comma_join(self) -> None: hn = HumanName("salem, abdul salam") self.m(hn.first, "abdul salam", hn) self.m(hn.middle, "", hn) self.m(hn.last, "salem", hn) def test_lastname_comma_join_with_middle(self) -> None: hn = HumanName("salem, abdul salam ahmed") self.m(hn.first, "abdul salam", hn) self.m(hn.middle, "ahmed", hn) self.m(hn.last, "salem", hn) # --- interaction with titles --- def test_title_kept_prefix_joins(self) -> None: hn = HumanName("Dr. abdul salam ahmed salem") self.m(hn.title, "Dr.", hn) self.m(hn.first, "abdul salam", hn) self.m(hn.middle, "ahmed", hn) self.m(hn.last, "salem", hn) # --- interaction with last-name prefixes --- def test_abu_bakr_al_baghdadi(self) -> None: """abu joins forward as first-prefix; al joins forward as last-prefix.""" hn = HumanName("abu bakr al baghdadi") self.m(hn.first, "abu bakr", hn) self.m(hn.last, "al baghdadi", hn) # --- interaction with suffixes --- def test_suffix_kept_prefix_joins(self) -> None: hn = HumanName("abdul salam ahmed salem jr") self.m(hn.first, "abdul salam", hn) self.m(hn.middle, "ahmed", hn) self.m(hn.last, "salem", hn) self.m(hn.suffix, "jr", hn) # --- guard / no-op --- def test_mohamad_unchanged(self) -> None: """mohamad is deliberately not in bound_first_names.""" hn = HumanName("Mohamad Ali Khalil") self.m(hn.first, "Mohamad", hn) self.m(hn.middle, "Ali", hn) self.m(hn.last, "Khalil", hn) def test_single_token_already_joined_unchanged(self) -> None: """abdulsalam is one token — not in the set, no join.""" hn = HumanName("abdulsalam ahmed salem") self.m(hn.first, "abdulsalam", hn) self.m(hn.middle, "ahmed", hn) self.m(hn.last, "salem", hn) def test_prefix_alone_no_join(self) -> None: """Single-word name that is a prefix: nothing to join.""" hn = HumanName("abdul") self.m(hn.first, "abdul", hn) def test_lastname_comma_prefix_only_no_join(self) -> None: """Prefix as sole post-comma token: nothing to join.""" hn = HumanName("salem, abdul") self.m(hn.first, "abdul", hn) self.m(hn.last, "salem", hn) def test_mid_name_prefix_becomes_last_prefix(self) -> None: """abu in non-first position is handled as a last-name prefix, not first-name.""" hn = HumanName("ahmed abu bakr") self.m(hn.first, "ahmed", hn) self.m(hn.last, "abu bakr", hn) # --- suffix-comma path --- def test_suffix_comma_join(self) -> None: hn = HumanName("Abdul Salam Hassan, MD") self.m(hn.first, "Abdul Salam", hn) self.m(hn.middle, "", hn) self.m(hn.last, "Hassan", hn) self.m(hn.suffix, "MD", hn) def test_suffix_comma_join_with_middle(self) -> None: hn = HumanName("Abdul Salam Ahmed Salem, MD") self.m(hn.first, "Abdul Salam", hn) self.m(hn.middle, "Ahmed", hn) self.m(hn.last, "Salem", hn) self.m(hn.suffix, "MD", hn) def test_suffix_comma_guard_two_tokens_no_join(self) -> None: """Guard: only last name remains after prefix → no join, even with suffix comma.""" hn = HumanName("Abdul Salam, MD") self.m(hn.first, "Abdul", hn) self.m(hn.last, "Salam", hn) self.m(hn.suffix, "MD", hn) def test_suffix_comma_title_kept_prefix_joins(self) -> None: hn = HumanName("Dr. Abdul Salam Hassan, MD") self.m(hn.title, "Dr.", hn) self.m(hn.first, "Abdul Salam", hn) self.m(hn.last, "Hassan", hn) self.m(hn.suffix, "MD", hn) def test_suffix_comma_abu_bakr_al_baghdadi(self) -> None: """abu joins forward as first-prefix; al joins forward as last-prefix, even with suffix comma.""" hn = HumanName("Abu Bakr Al Baghdadi, MD") self.m(hn.first, "Abu Bakr", hn) self.m(hn.last, "Al Baghdadi", hn) self.m(hn.suffix, "MD", hn) # --- opt-out --- def test_opt_out_via_clear(self) -> None: """Clearing bound_first_names restores prior behavior.""" from nameparser.config import Constants c = Constants(bound_first_names=set()) hn = HumanName("abdul salam ahmed salem", constants=c) self.m(hn.first, "abdul", hn) self.m(hn.middle, "salam ahmed", hn) self.m(hn.last, "salem", hn) python-nameparser-2.1.0/tests/test_brute_force.py000066400000000000000000000663041523530431700222540ustar00rootroot00000000000000from nameparser import HumanName from tests.base import HumanNameTestBase class HumanNameBruteForceTests(HumanNameTestBase): def test1(self) -> None: hn = HumanName("John Doe") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) def test2(self) -> None: hn = HumanName("John Doe, Jr.") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "Jr.", hn) def test3(self) -> None: hn = HumanName("John Doe III") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "III", hn) def test4(self) -> None: hn = HumanName("Doe, John") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) def test5(self) -> None: hn = HumanName("Doe, John, Jr.") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "Jr.", hn) def test6(self) -> None: hn = HumanName("Doe, John III") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "III", hn) def test7(self) -> None: hn = HumanName("John A. Doe") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A.", hn) def test8(self) -> None: hn = HumanName("John A. Doe, Jr") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A.", hn) self.m(hn.suffix, "Jr", hn) def test9(self) -> None: hn = HumanName("John A. Doe III") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A.", hn) self.m(hn.suffix, "III", hn) def test10(self) -> None: hn = HumanName("Doe, John A.") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A.", hn) def test11(self) -> None: hn = HumanName("Doe, John A., Jr.") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A.", hn) self.m(hn.suffix, "Jr.", hn) def test12(self) -> None: hn = HumanName("Doe, John A., III") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A.", hn) self.m(hn.suffix, "III", hn) def test13(self) -> None: hn = HumanName("John A. Kenneth Doe") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A. Kenneth", hn) def test14(self) -> None: hn = HumanName("John A. Kenneth Doe, Jr.") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A. Kenneth", hn) self.m(hn.suffix, "Jr.", hn) def test15(self) -> None: hn = HumanName("John A. Kenneth Doe III") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A. Kenneth", hn) self.m(hn.suffix, "III", hn) def test16(self) -> None: # "John." is an unrecognized period-abbreviation in the leading # title position of the lastname-comma format, so is_leading_title() # now treats it as a title rather than a first name (see #109). hn = HumanName("Doe, John. A. Kenneth") self.m(hn.title, "John.", hn) self.m(hn.first, "A.", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "Kenneth", hn) def test17(self) -> None: # Same period-abbreviation-as-title behavior as test16 (see #109). hn = HumanName("Doe, John. A. Kenneth, Jr.") self.m(hn.title, "John.", hn) self.m(hn.first, "A.", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "Kenneth", hn) self.m(hn.suffix, "Jr.", hn) def test18(self) -> None: # Same period-abbreviation-as-title behavior as test16 (see #109). hn = HumanName("Doe, John. A. Kenneth III") self.m(hn.title, "John.", hn) self.m(hn.first, "A.", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "Kenneth", hn) self.m(hn.suffix, "III", hn) def test19(self) -> None: hn = HumanName("Dr. John Doe") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.title, "Dr.", hn) def test20(self) -> None: hn = HumanName("Dr. John Doe, Jr.") self.m(hn.title, "Dr.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "Jr.", hn) def test21(self) -> None: hn = HumanName("Dr. John Doe III") self.m(hn.title, "Dr.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "III", hn) def test22(self) -> None: hn = HumanName("Doe, Dr. John") self.m(hn.title, "Dr.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) def test23(self) -> None: hn = HumanName("Doe, Dr. John, Jr.") self.m(hn.title, "Dr.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "Jr.", hn) def test24(self) -> None: hn = HumanName("Doe, Dr. John III") self.m(hn.title, "Dr.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "III", hn) def test25(self) -> None: hn = HumanName("Dr. John A. Doe") self.m(hn.title, "Dr.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A.", hn) def test26(self) -> None: hn = HumanName("Dr. John A. Doe, Jr.") self.m(hn.title, "Dr.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A.", hn) self.m(hn.suffix, "Jr.", hn) def test27(self) -> None: hn = HumanName("Dr. John A. Doe III") self.m(hn.title, "Dr.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A.", hn) self.m(hn.suffix, "III", hn) def test28(self) -> None: hn = HumanName("Doe, Dr. John A.") self.m(hn.title, "Dr.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A.", hn) def test29(self) -> None: hn = HumanName("Doe, Dr. John A. Jr.") self.m(hn.title, "Dr.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A.", hn) self.m(hn.suffix, "Jr.", hn) def test30(self) -> None: hn = HumanName("Doe, Dr. John A. III") self.m(hn.title, "Dr.", hn) self.m(hn.middle, "A.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "III", hn) def test31(self) -> None: hn = HumanName("Dr. John A. Kenneth Doe") self.m(hn.title, "Dr.", hn) self.m(hn.middle, "A. Kenneth", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) def test32(self) -> None: hn = HumanName("Dr. John A. Kenneth Doe, Jr.") self.m(hn.title, "Dr.", hn) self.m(hn.middle, "A. Kenneth", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "Jr.", hn) def test33(self) -> None: hn = HumanName("Al Arnold Gore, Jr.") self.m(hn.middle, "Arnold", hn) self.m(hn.first, "Al", hn) self.m(hn.last, "Gore", hn) self.m(hn.suffix, "Jr.", hn) def test34(self) -> None: hn = HumanName("Dr. John A. Kenneth Doe III") self.m(hn.title, "Dr.", hn) self.m(hn.middle, "A. Kenneth", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "III", hn) def test35(self) -> None: hn = HumanName("Doe, Dr. John A. Kenneth") self.m(hn.title, "Dr.", hn) self.m(hn.middle, "A. Kenneth", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) def test36(self) -> None: hn = HumanName("Doe, Dr. John A. Kenneth Jr.") self.m(hn.title, "Dr.", hn) self.m(hn.middle, "A. Kenneth", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "Jr.", hn) def test37(self) -> None: hn = HumanName("Doe, Dr. John A. Kenneth III") self.m(hn.title, "Dr.", hn) self.m(hn.middle, "A. Kenneth", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "III", hn) def test38(self) -> None: hn = HumanName("Juan de la Vega") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) def test39(self) -> None: hn = HumanName("Juan de la Vega, Jr.") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.suffix, "Jr.", hn) def test40(self) -> None: hn = HumanName("Juan de la Vega III") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.suffix, "III", hn) def test41(self) -> None: hn = HumanName("de la Vega, Juan") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) def test42(self) -> None: hn = HumanName("de la Vega, Juan, Jr.") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.suffix, "Jr.", hn) def test43(self) -> None: hn = HumanName("de la Vega, Juan III") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.suffix, "III", hn) def test44(self) -> None: hn = HumanName("Juan Velasquez y Garcia") self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) def test45(self) -> None: hn = HumanName("Juan Velasquez y Garcia, Jr.") self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "Jr.", hn) def test46(self) -> None: hn = HumanName("Juan Velasquez y Garcia III") self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "III", hn) def test47(self) -> None: hn = HumanName("Velasquez y Garcia, Juan") self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) def test48(self) -> None: hn = HumanName("Velasquez y Garcia, Juan, Jr.") self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "Jr.", hn) def test49(self) -> None: hn = HumanName("Velasquez y Garcia, Juan III") self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "III", hn) def test50(self) -> None: hn = HumanName("Dr. Juan de la Vega") self.m(hn.title, "Dr.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) def test51(self) -> None: hn = HumanName("Dr. Juan de la Vega, Jr.") self.m(hn.title, "Dr.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.suffix, "Jr.", hn) def test52(self) -> None: hn = HumanName("Dr. Juan de la Vega III") self.m(hn.title, "Dr.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.suffix, "III", hn) def test53(self) -> None: hn = HumanName("de la Vega, Dr. Juan") self.m(hn.title, "Dr.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) def test54(self) -> None: hn = HumanName("de la Vega, Dr. Juan, Jr.") self.m(hn.title, "Dr.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.suffix, "Jr.", hn) def test55(self) -> None: hn = HumanName("de la Vega, Dr. Juan III") self.m(hn.title, "Dr.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.suffix, "III", hn) def test56(self) -> None: hn = HumanName("Dr. Juan Velasquez y Garcia") self.m(hn.title, "Dr.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) def test57(self) -> None: hn = HumanName("Dr. Juan Velasquez y Garcia, Jr.") self.m(hn.title, "Dr.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "Jr.", hn) def test58(self) -> None: hn = HumanName("Dr. Juan Velasquez y Garcia III") self.m(hn.title, "Dr.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "III", hn) def test59(self) -> None: hn = HumanName("Velasquez y Garcia, Dr. Juan") self.m(hn.title, "Dr.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) def test60(self) -> None: hn = HumanName("Velasquez y Garcia, Dr. Juan, Jr.") self.m(hn.title, "Dr.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "Jr.", hn) def test61(self) -> None: hn = HumanName("Velasquez y Garcia, Dr. Juan III") self.m(hn.title, "Dr.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "III", hn) def test62(self) -> None: hn = HumanName("Juan Q. de la Vega") self.m(hn.first, "Juan", hn) self.m(hn.middle, "Q.", hn) self.m(hn.last, "de la Vega", hn) def test63(self) -> None: hn = HumanName("Juan Q. de la Vega, Jr.") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.middle, "Q.", hn) self.m(hn.suffix, "Jr.", hn) def test64(self) -> None: hn = HumanName("Juan Q. de la Vega III") self.m(hn.first, "Juan", hn) self.m(hn.middle, "Q.", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.suffix, "III", hn) def test65(self) -> None: hn = HumanName("de la Vega, Juan Q.") self.m(hn.first, "Juan", hn) self.m(hn.middle, "Q.", hn) self.m(hn.last, "de la Vega", hn) def test66(self) -> None: hn = HumanName("de la Vega, Juan Q., Jr.") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.middle, "Q.", hn) self.m(hn.suffix, "Jr.", hn) def test67(self) -> None: hn = HumanName("de la Vega, Juan Q. III") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.middle, "Q.", hn) self.m(hn.suffix, "III", hn) def test68(self) -> None: hn = HumanName("Juan Q. Velasquez y Garcia") self.m(hn.middle, "Q.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) def test69(self) -> None: hn = HumanName("Juan Q. Velasquez y Garcia, Jr.") self.m(hn.middle, "Q.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "Jr.", hn) def test70(self) -> None: hn = HumanName("Juan Q. Velasquez y Garcia III") self.m(hn.middle, "Q.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "III", hn) def test71(self) -> None: hn = HumanName("Velasquez y Garcia, Juan Q.") self.m(hn.middle, "Q.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) def test72(self) -> None: hn = HumanName("Velasquez y Garcia, Juan Q., Jr.") self.m(hn.middle, "Q.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "Jr.", hn) def test73(self) -> None: hn = HumanName("Velasquez y Garcia, Juan Q. III") self.m(hn.middle, "Q.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "III", hn) def test74(self) -> None: hn = HumanName("Dr. Juan Q. de la Vega") self.m(hn.title, "Dr.", hn) self.m(hn.first, "Juan", hn) self.m(hn.middle, "Q.", hn) self.m(hn.last, "de la Vega", hn) def test75(self) -> None: hn = HumanName("Dr. Juan Q. de la Vega, Jr.") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.middle, "Q.", hn) self.m(hn.title, "Dr.", hn) self.m(hn.suffix, "Jr.", hn) def test76(self) -> None: hn = HumanName("Dr. Juan Q. de la Vega III") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.middle, "Q.", hn) self.m(hn.title, "Dr.", hn) self.m(hn.suffix, "III", hn) def test77(self) -> None: hn = HumanName("de la Vega, Dr. Juan Q.") self.m(hn.first, "Juan", hn) self.m(hn.middle, "Q.", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.title, "Dr.", hn) def test78(self) -> None: hn = HumanName("de la Vega, Dr. Juan Q., Jr.") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.middle, "Q.", hn) self.m(hn.suffix, "Jr.", hn) self.m(hn.title, "Dr.", hn) def test79(self) -> None: hn = HumanName("de la Vega, Dr. Juan Q. III") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.middle, "Q.", hn) self.m(hn.suffix, "III", hn) self.m(hn.title, "Dr.", hn) def test80(self) -> None: hn = HumanName("Dr. Juan Q. Velasquez y Garcia") self.m(hn.title, "Dr.", hn) self.m(hn.middle, "Q.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) def test81(self) -> None: hn = HumanName("Dr. Juan Q. Velasquez y Garcia, Jr.") self.m(hn.title, "Dr.", hn) self.m(hn.middle, "Q.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "Jr.", hn) def test82(self) -> None: hn = HumanName("Dr. Juan Q. Velasquez y Garcia III") self.m(hn.middle, "Q.", hn) self.m(hn.title, "Dr.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "III", hn) def test83(self) -> None: hn = HumanName("Velasquez y Garcia, Dr. Juan Q.") self.m(hn.title, "Dr.", hn) self.m(hn.middle, "Q.", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) def test84(self) -> None: hn = HumanName("Velasquez y Garcia, Dr. Juan Q., Jr.") self.m(hn.middle, "Q.", hn) self.m(hn.first, "Juan", hn) self.m(hn.title, "Dr.", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "Jr.", hn) def test85(self) -> None: hn = HumanName("Velasquez y Garcia, Dr. Juan Q. III") self.m(hn.middle, "Q.", hn) self.m(hn.first, "Juan", hn) self.m(hn.title, "Dr.", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "III", hn) def test86(self) -> None: hn = HumanName("Juan Q. Xavier de la Vega") self.m(hn.first, "Juan", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.last, "de la Vega", hn) def test87(self) -> None: hn = HumanName("Juan Q. Xavier de la Vega, Jr.") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.suffix, "Jr.", hn) def test88(self) -> None: hn = HumanName("Juan Q. Xavier de la Vega III") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.suffix, "III", hn) def test89(self) -> None: hn = HumanName("de la Vega, Juan Q. Xavier") self.m(hn.first, "Juan", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.last, "de la Vega", hn) def test90(self) -> None: hn = HumanName("de la Vega, Juan Q. Xavier, Jr.") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.suffix, "Jr.", hn) def test91(self) -> None: hn = HumanName("de la Vega, Juan Q. Xavier III") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.suffix, "III", hn) def test92(self) -> None: hn = HumanName("Dr. Juan Q. Xavier de la Vega") self.m(hn.first, "Juan", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.title, "Dr.", hn) self.m(hn.last, "de la Vega", hn) def test93(self) -> None: hn = HumanName("Dr. Juan Q. Xavier de la Vega, Jr.") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.title, "Dr.", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.suffix, "Jr.", hn) def test94(self) -> None: hn = HumanName("Dr. Juan Q. Xavier de la Vega III") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.title, "Dr.", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.suffix, "III", hn) def test95(self) -> None: hn = HumanName("de la Vega, Dr. Juan Q. Xavier") self.m(hn.first, "Juan", hn) self.m(hn.title, "Dr.", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.last, "de la Vega", hn) def test96(self) -> None: hn = HumanName("de la Vega, Dr. Juan Q. Xavier, Jr.") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.title, "Dr.", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.suffix, "Jr.", hn) def test97(self) -> None: hn = HumanName("de la Vega, Dr. Juan Q. Xavier III") self.m(hn.first, "Juan", hn) self.m(hn.title, "Dr.", hn) self.m(hn.last, "de la Vega", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.suffix, "III", hn) def test98(self) -> None: hn = HumanName("Juan Q. Xavier Velasquez y Garcia") self.m(hn.middle, "Q. Xavier", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) def test99(self) -> None: hn = HumanName("Juan Q. Xavier Velasquez y Garcia, Jr.") self.m(hn.middle, "Q. Xavier", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "Jr.", hn) def test100(self) -> None: hn = HumanName("Juan Q. Xavier Velasquez y Garcia III") self.m(hn.middle, "Q. Xavier", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "III", hn) def test101(self) -> None: hn = HumanName("Velasquez y Garcia, Juan Q. Xavier") self.m(hn.middle, "Q. Xavier", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) def test102(self) -> None: hn = HumanName("Velasquez y Garcia, Juan Q. Xavier, Jr.") self.m(hn.middle, "Q. Xavier", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "Jr.", hn) def test103(self) -> None: hn = HumanName("Velasquez y Garcia, Juan Q. Xavier III") self.m(hn.middle, "Q. Xavier", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "III", hn) def test104(self) -> None: hn = HumanName("Dr. Juan Q. Xavier Velasquez y Garcia") self.m(hn.title, "Dr.", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) def test105(self) -> None: hn = HumanName("Dr. Juan Q. Xavier Velasquez y Garcia, Jr.") self.m(hn.middle, "Q. Xavier", hn) self.m(hn.first, "Juan", hn) self.m(hn.title, "Dr.", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "Jr.", hn) def test106(self) -> None: hn = HumanName("Dr. Juan Q. Xavier Velasquez y Garcia III") self.m(hn.middle, "Q. Xavier", hn) self.m(hn.first, "Juan", hn) self.m(hn.title, "Dr.", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "III", hn) def test107(self) -> None: hn = HumanName("Velasquez y Garcia, Dr. Juan Q. Xavier") self.m(hn.title, "Dr.", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) def test108(self) -> None: hn = HumanName("Velasquez y Garcia, Dr. Juan Q. Xavier, Jr.") self.m(hn.middle, "Q. Xavier", hn) self.m(hn.first, "Juan", hn) self.m(hn.title, "Dr.", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "Jr.", hn) def test109(self) -> None: hn = HumanName("Velasquez y Garcia, Dr. Juan Q. Xavier III") self.m(hn.middle, "Q. Xavier", hn) self.m(hn.first, "Juan", hn) self.m(hn.title, "Dr.", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "III", hn) def test110(self) -> None: hn = HumanName("John Doe, CLU, CFP, LUTC") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "CLU, CFP, LUTC", hn) def test111(self) -> None: hn = HumanName("John P. Doe, CLU, CFP, LUTC") self.m(hn.first, "John", hn) self.m(hn.middle, "P.", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "CLU, CFP, LUTC", hn) def test112(self) -> None: hn = HumanName("Dr. John P. Doe-Ray, CLU, CFP, LUTC") self.m(hn.first, "John", hn) self.m(hn.middle, "P.", hn) self.m(hn.last, "Doe-Ray", hn) self.m(hn.title, "Dr.", hn) self.m(hn.suffix, "CLU, CFP, LUTC", hn) def test113(self) -> None: hn = HumanName("Doe-Ray, Dr. John P., CLU, CFP, LUTC") self.m(hn.title, "Dr.", hn) self.m(hn.middle, "P.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe-Ray", hn) self.m(hn.suffix, "CLU, CFP, LUTC", hn) def test115(self) -> None: hn = HumanName("Hon. Barrington P. Doe-Ray, Jr.") self.m(hn.title, "Hon.", hn) self.m(hn.middle, "P.", hn) self.m(hn.first, "Barrington", hn) self.m(hn.last, "Doe-Ray", hn) def test116(self) -> None: hn = HumanName("Doe-Ray, Hon. Barrington P. Jr., CFP, LUTC") self.m(hn.title, "Hon.", hn) self.m(hn.middle, "P.", hn) self.m(hn.first, "Barrington", hn) self.m(hn.last, "Doe-Ray", hn) self.m(hn.suffix, "Jr., CFP, LUTC", hn) def test117(self) -> None: hn = HumanName("Rt. Hon. Paul E. Mary") self.m(hn.title, "Rt. Hon.", hn) self.m(hn.first, "Paul", hn) self.m(hn.middle, "E.", hn) self.m(hn.last, "Mary", hn) def test119(self) -> None: hn = HumanName("Lord God Almighty") self.m(hn.title, "Lord", hn) self.m(hn.first, "God", hn) self.m(hn.last, "Almighty", hn) python-nameparser-2.1.0/tests/test_capitalization.py000066400000000000000000000134771523530431700227730ustar00rootroot00000000000000import pytest from nameparser import HumanName from tests.base import HumanNameTestBase class HumanNameCapitalizationTestCase(HumanNameTestBase): def test_capitalization_exception_for_III(self) -> None: hn = HumanName('juan q. xavier velasquez y garcia iii') hn.capitalize() self.m(str(hn), 'Juan Q. Xavier Velasquez y Garcia III', hn) # FIXME: this test does not pass due to a known issue # http://code.google.com/p/python-nameparser/issues/detail?id=22 @pytest.mark.xfail def test_capitalization_exception_for_already_capitalized_III_KNOWN_FAILURE(self) -> None: hn = HumanName('juan garcia III') hn.capitalize() self.m(str(hn), 'Juan Garcia III', hn) def test_capitalize_title(self) -> None: hn = HumanName('lt. gen. john a. kenneth doe iv') hn.capitalize() self.m(str(hn), 'Lt. Gen. John A. Kenneth Doe IV', hn) def test_capitalize_title_to_lower(self) -> None: hn = HumanName('LT. GEN. JOHN A. KENNETH DOE IV') hn.capitalize() self.m(str(hn), 'Lt. Gen. John A. Kenneth Doe IV', hn) # Capitalization with M(a)c and hyphenated names def test_capitalization_with_Mac_as_hyphenated_names(self) -> None: hn = HumanName('donovan mcnabb-smith') hn.capitalize() self.m(str(hn), 'Donovan McNabb-Smith', hn) def test_capitization_middle_initial_is_also_a_conjunction(self) -> None: hn = HumanName('scott e. werner') hn.capitalize() self.m(str(hn), 'Scott E. Werner', hn) def test_capitalize_empty_middle_produces_no_leading_space_in_surnames(self) -> None: # str.split(' ') on an empty string returns [''] rather than [], so an # absent middle produced a spurious token that leaked into surnames_list # and caused a leading space in the surnames property (' Doe' not 'Doe'). hn = HumanName('john doe') hn.capitalize() self.m(hn.surnames, 'Doe', hn) self.assertEqual(hn.middle_list, []) self.assertEqual(hn.surnames_list, ['Doe']) def test_capitalize_force_empty_middle_produces_no_leading_space_in_surnames(self) -> None: # Without force=True, capitalize() exits early for mixed-case names and # never reaches the split lines. Confirm the fix covers that path too. hn = HumanName('Jane Doe') hn.capitalize(force=True) self.m(hn.surnames, 'Doe', hn) self.assertEqual(hn.middle_list, []) def test_capitalize_empty_attributes_produce_no_spurious_tokens(self) -> None: # Confirm the fix extends beyond surnames: empty attribute lists are [] # not [''], and non-empty ones contain only real tokens. hn = HumanName('Jane Doe') hn.capitalize(force=True) self.assertEqual(hn.title_list, []) self.assertEqual(hn.first_list, ['Jane']) self.assertEqual(hn.last_list, ['Doe']) def test_capitalize_title_and_last_only_no_spurious_tokens(self) -> None: # title+last with no first or middle leaves first_list and middle_list # both empty. All-caps triggers capitalize() without force=True. hn = HumanName('DR DOE') hn.capitalize() self.assertEqual(hn.first_list, []) self.assertEqual(hn.middle_list, []) self.m(str(hn), 'Dr Doe', hn) def test_capitalize_empty_suffix_produces_no_spurious_tokens(self) -> None: # ''.split(', ') returns [''] just like ''.split(' ') did for the other # attributes — an absent suffix should produce suffix_list == [], not ['']. hn = HumanName('JOHN DOE') hn.capitalize() self.assertEqual(hn.suffix_list, []) def test_capitalize_single_suffix_still_works(self) -> None: hn = HumanName('JOHN DOE PHD') hn.capitalize() self.assertEqual(hn.suffix_list, ['Ph.D.']) def test_capitalize_multiple_suffixes_still_split_correctly(self) -> None: hn = HumanName('JOHN DOE PHD MD') hn.capitalize() self.assertEqual(hn.suffix_list, ['Ph.D.', 'M.D.']) def test_capitalize_suffix_acronym_with_dots(self) -> None: # Suffixes already written with dots (e.g. "M.D.") should capitalize # to their exception form, not title-case to "M.d." (issue #141) hn = HumanName('GREGORY HOUSE M.D.') hn.capitalize() self.assertEqual(hn.suffix, 'M.D.') # Leaving already-capitalized names alone def test_no_change_to_mixed_chase(self) -> None: hn = HumanName('Shirley Maclaine') hn.capitalize() self.m(str(hn), 'Shirley Maclaine', hn) def test_force_capitalization(self) -> None: hn = HumanName('Shirley Maclaine') hn.capitalize(force=True) self.m(str(hn), 'Shirley MacLaine', hn) def test_capitalize_diacritics(self) -> None: hn = HumanName('matthëus schmidt') hn.capitalize() self.m(str(hn), 'Matthëus Schmidt', hn) # http://code.google.com/p/python-nameparser/issues/detail?id=15 def test_downcasing_mac(self) -> None: hn = HumanName('RONALD MACDONALD') hn.capitalize() self.m(str(hn), 'Ronald MacDonald', hn) # http://code.google.com/p/python-nameparser/issues/detail?id=23 def test_downcasing_mc(self) -> None: hn = HumanName('RONALD MCDONALD') hn.capitalize() self.m(str(hn), 'Ronald McDonald', hn) def test_short_names_with_mac(self) -> None: hn = HumanName('mack johnson') hn.capitalize() self.m(str(hn), 'Mack Johnson', hn) def test_portuguese_prefixes(self) -> None: hn = HumanName("joao da silva do amaral de souza") hn.capitalize() self.m(str(hn), 'Joao da Silva do Amaral de Souza', hn) def test_capitalize_prefix_clash_on_first_name(self) -> None: hn = HumanName("van nguyen") hn.capitalize() self.m(str(hn), 'Van Nguyen', hn) python-nameparser-2.1.0/tests/test_comma_variants.py000066400000000000000000000025671523530431700227610ustar00rootroot00000000000000from nameparser import HumanName from tests.base import HumanNameTestBase class HumanNameCommaVariantsTests(HumanNameTestBase): """Non-ASCII comma characters should split "Last, First" the same as ',' (#265).""" def test_arabic_comma_splits_lastname_format(self) -> None: hn = HumanName("سلمان، محمد") self.m(hn.first, "محمد", hn) self.m(hn.last, "سلمان", hn) def test_fullwidth_comma_splits_lastname_format(self) -> None: hn = HumanName("Smith，John") self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) def test_arabic_comma_does_not_pollute_output(self) -> None: hn = HumanName("سلمان، محمد") self.assertNotIn("،", hn.last) self.assertNotIn("،", str(hn)) def test_trailing_arabic_comma_stripped(self) -> None: # matches ASCII behavior: a single word with a trailing comma has # nothing after the comma, so it's a bare name, not "Last," hn = HumanName("سلمان،") self.m(hn.first, "سلمان", hn) # The v1 test for a custom regexes dict omitting the 'commas' key (the # EMPTY_REGEX shatter guard) died with the mechanism: Constants(regexes= # ...) raises TypeError in 2.0 (deliberate divergence, migration spec # section 3 uniform rule) -- pinned in test_python_api.py's # test_override_regex_raises. python-nameparser-2.1.0/tests/test_conjunctions.py000066400000000000000000000266311523530431700224700ustar00rootroot00000000000000import pytest from nameparser import HumanName from tests.base import HumanNameTestBase class HumanNameConjunctionTestCase(HumanNameTestBase): # Last name with conjunction def test_last_name_with_conjunction(self) -> None: hn = HumanName('Jose Aznar y Lopez') self.m(hn.first, "Jose", hn) self.m(hn.last, "Aznar y Lopez", hn) def test_multiple_conjunctions(self) -> None: hn = HumanName("part1 of The part2 of the part3 and part4") self.m(hn.first, "part1 of The part2 of the part3 and part4", hn) def test_multiple_conjunctions2(self) -> None: hn = HumanName("part1 of and The part2 of the part3 And part4") self.m(hn.first, "part1 of and The part2 of the part3 And part4", hn) def test_ends_with_conjunction(self) -> None: hn = HumanName("Jon Dough and") self.m(hn.first, "Jon", hn) self.m(hn.last, "Dough and", hn) def test_ends_with_two_conjunctions(self) -> None: hn = HumanName("Jon Dough and of") self.m(hn.first, "Jon", hn) self.m(hn.last, "Dough and of", hn) def test_starts_with_conjunction(self) -> None: hn = HumanName("and Jon Dough") self.m(hn.first, "and Jon", hn) self.m(hn.last, "Dough", hn) def test_starts_with_two_conjunctions(self) -> None: hn = HumanName("the and Jon Dough") self.m(hn.first, "the and Jon", hn) self.m(hn.last, "Dough", hn) # Potential conjunction/prefix treated as initial (because uppercase) def test_uppercase_middle_initial_conflict_with_conjunction(self) -> None: hn = HumanName('John E Smith') self.m(hn.first, "John", hn) self.m(hn.middle, "E", hn) self.m(hn.last, "Smith", hn) def test_lowercase_middle_initial_with_period_conflict_with_conjunction(self) -> None: hn = HumanName('john e. smith') self.m(hn.first, "john", hn) self.m(hn.middle, "e.", hn) self.m(hn.last, "smith", hn) # The conjunction "e" can also be an initial def test_lowercase_first_initial_conflict_with_conjunction(self) -> None: hn = HumanName('e j smith') self.m(hn.first, "e", hn) self.m(hn.middle, "j", hn) self.m(hn.last, "smith", hn) def test_lowercase_middle_initial_conflict_with_conjunction(self) -> None: hn = HumanName('John e Smith') self.m(hn.first, "John", hn) self.m(hn.middle, "e", hn) self.m(hn.last, "Smith", hn) def test_lowercase_middle_initial_and_suffix_conflict_with_conjunction(self) -> None: hn = HumanName('John e Smith, III') self.m(hn.first, "John", hn) self.m(hn.middle, "e", hn) self.m(hn.last, "Smith", hn) self.m(hn.suffix, "III", hn) def test_lowercase_middle_initial_and_nocomma_suffix_conflict_with_conjunction(self) -> None: hn = HumanName('John e Smith III') self.m(hn.first, "John", hn) self.m(hn.middle, "e", hn) self.m(hn.last, "Smith", hn) self.m(hn.suffix, "III", hn) def test_lowercase_middle_initial_comma_lastname_and_suffix_conflict_with_conjunction(self) -> None: hn = HumanName('Smith, John e, III, Jr') self.m(hn.first, "John", hn) self.m(hn.middle, "e", hn) self.m(hn.last, "Smith", hn) self.m(hn.suffix, "III, Jr", hn) @pytest.mark.xfail def test_two_initials_conflict_with_conjunction(self) -> None: # Supporting this seems to screw up titles with periods in them like M.B.A. hn = HumanName('E.T. Smith') self.m(hn.first, "E.", hn) self.m(hn.middle, "T.", hn) self.m(hn.last, "Smith", hn) def test_couples_names(self) -> None: hn = HumanName('John and Jane Smith') self.m(hn.first, "John and Jane", hn) self.m(hn.last, "Smith", hn) def test_couples_names_with_conjunction_lastname(self) -> None: hn = HumanName('John and Jane Aznar y Lopez') self.m(hn.first, "John and Jane", hn) self.m(hn.last, "Aznar y Lopez", hn) def test_couple_titles(self) -> None: hn = HumanName('Mr. and Mrs. John and Jane Smith') self.m(hn.title, "Mr. and Mrs.", hn) self.m(hn.first, "John and Jane", hn) self.m(hn.last, "Smith", hn) def test_couple_titles_ampersand_conjunction(self) -> None: # issue 151: single-char conjunctions in the conjunctions list should # be honored even when total_length < 4 hn = HumanName('Mr. & Mrs. John Smith') self.m(hn.title, "Mr. & Mrs.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) def test_ampersand_conjunction_short_name_no_titles(self) -> None: # & is non-alpha so it should always be honored as a conjunction, # even when total_length < 4 (no titles to inflate the count) hn = HumanName('John & Jane') self.m(hn.first, "John & Jane", hn) def test_single_char_alpha_conjunction_still_treated_as_initial_when_short(self) -> None: # single-char alpha conjunctions (e, y) are still treated as initials # when total_length < 4; only non-alpha symbols like & bypass this guard hn = HumanName('John y Jane') self.m(hn.first, "John", hn) self.m(hn.middle, "y", hn) self.m(hn.last, "Jane", hn) def test_title_with_three_part_name_last_initial_is_suffix_uppercase_no_period(self) -> None: hn = HumanName("King John Alexander V") self.m(hn.title, "King", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Alexander", hn) self.m(hn.suffix, "V", hn) def test_four_name_parts_with_suffix_that_could_be_initial_lowercase_no_period(self) -> None: hn = HumanName("larry james edward johnson v") self.m(hn.first, "larry", hn) self.m(hn.middle, "james edward", hn) self.m(hn.last, "johnson", hn) self.m(hn.suffix, "v", hn) def test_four_name_parts_with_suffix_that_could_be_initial_uppercase_no_period(self) -> None: hn = HumanName("Larry James Johnson I") self.m(hn.first, "Larry", hn) self.m(hn.middle, "James", hn) self.m(hn.last, "Johnson", hn) self.m(hn.suffix, "I", hn) def test_roman_numeral_initials(self) -> None: hn = HumanName("Larry V I") self.m(hn.first, "Larry", hn) self.m(hn.middle, "V", hn) self.m(hn.last, "I", hn) self.m(hn.suffix, "", hn) def test_roman_numeral_suffix_not_in_suffix_list(self) -> None: # VI-X are not in the suffix word lists, so they reach the # is_roman_numeral(nxt) branch rather than are_suffixes() hn = HumanName("John Smith VI") self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) self.m(hn.suffix, "VI", hn) # tests for Rev. title (Reverend) def test124(self) -> None: hn = HumanName("Rev. John A. Kenneth Doe") self.m(hn.title, "Rev.", hn) self.m(hn.middle, "A. Kenneth", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) def test125(self) -> None: hn = HumanName("Rev John A. Kenneth Doe") self.m(hn.title, "Rev", hn) self.m(hn.middle, "A. Kenneth", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) def test126(self) -> None: hn = HumanName("Doe, Rev. John A. Jr.") self.m(hn.title, "Rev.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A.", hn) self.m(hn.suffix, "Jr.", hn) def test127(self) -> None: hn = HumanName("Buca di Beppo") self.m(hn.first, "Buca", hn) self.m(hn.last, "di Beppo", hn) def test_le_as_last_name(self) -> None: hn = HumanName("Yin Le") self.m(hn.first, "Yin", hn) self.m(hn.last, "Le", hn) def test_le_as_last_name_with_middle_initial(self) -> None: hn = HumanName("Yin a Le") self.m(hn.first, "Yin", hn) self.m(hn.middle, "a", hn) self.m(hn.last, "Le", hn) def test_conjunction_in_an_address_with_a_title(self) -> None: hn = HumanName("His Excellency Lord Duncan") self.m(hn.title, "His Excellency Lord", hn) self.m(hn.last, "Duncan", hn) @pytest.mark.xfail def test_conjunction_in_an_address_with_a_first_name_title(self) -> None: hn = HumanName("Her Majesty Queen Elizabeth") self.m(hn.title, "Her Majesty Queen", hn) # if you want to be technical, Queen is in FIRST_NAME_TITLES self.m(hn.first, "Elizabeth", hn) def test_name_is_conjunctions(self) -> None: hn = HumanName("e and e") self.m(hn.first, "e and e", hn) def test_conjunction_bridges_prefix_chain(self) -> None: # "von" and "zu" are both prefixes, but "und" between them is only a # conjunction. join_on_conjunctions() merges "von und zu" into one # piece before the prefix-joining step runs, so without registering # that merged piece as a prefix too, it's stranded in the middle name # instead of joining to the last name. See German nobility styles # like "von und zu". hn = HumanName("Alois von und zu Liechtenstein") self.m(hn.first, "Alois", hn) self.m(hn.middle, "", hn) self.m(hn.last, "von und zu Liechtenstein", hn) def test_conjunction_bridges_prefix_chain_with_leading_title(self) -> None: # Same bridging, but with extra prefix words on both sides of the # conjunction and a leading title-like word ("Freiherrin") that is # itself a prefix, confirming the chain still joins fully into last. hn = HumanName("Annette Charlotte Freiherrin von und zu der Tann-Rathsamhausen") self.m(hn.first, "Annette", hn) self.m(hn.middle, "Charlotte", hn) self.m(hn.last, "Freiherrin von und zu der Tann-Rathsamhausen", hn) def test_conjunction_prefix_merge_at_start_stays_first_name(self) -> None: # Guards the i == 0 branch of the same fix: when the conjunction is # merged with a following prefix at the very start of the name, the # existing leading-prefix rule (a lone prefix opening the name is # treated as part of the first name, not joined to last) must still # apply to the merged piece. hn = HumanName("and van Buren") self.m(hn.first, "and van", hn) self.m(hn.last, "Buren", hn) def test_conjunction_bridges_word_that_is_both_title_and_prefix(self) -> None: # "freiherr" is registered as both a title and a prefix. When it sits # next to a conjunction, join_on_conjunctions() runs the is_title and # is_prefix checks independently (not elif), so the merged piece # ("freiherr und") is added to both constants sets. Confirms that # dual registration doesn't break the prefix-bridging into last. hn = HumanName("Fritz Freiherr und von Bar") self.m(hn.first, "Fritz", hn) self.m(hn.middle, "", hn) self.m(hn.last, "Freiherr und von Bar", hn) def test_conjunction_bridges_prefix_chain_with_multiple_conjunctions(self) -> None: # Two separate conjunctions ("und" appearing twice, not contiguous) # each bridge their own pair of adjacent prefixes, so both merges # must register as prefixes for the whole chain to join into last. hn = HumanName("Alois von und zu und von Liechtenstein") self.m(hn.first, "Alois", hn) self.m(hn.middle, "", hn) self.m(hn.last, "von und zu und von Liechtenstein", hn) python-nameparser-2.1.0/tests/test_constants.py000066400000000000000000001404371523530431700217710ustar00rootroot00000000000000import copy import pickle import re import warnings from typing import Any import pytest from nameparser import HumanName from nameparser.config import CONSTANTS, Constants, SetManager, TupleManager from nameparser.config.titles import TITLES from tests.base import HumanNameTestBase class ConstantsCustomizationTests(HumanNameTestBase): def test_add_title(self) -> None: hn = HumanName("Te Awanui-a-Rangi Black", constants=Constants()) start_len = len(hn.C.titles) self.assertTrue(start_len > 0) hn.C.titles.add('te') self.assertEqual(start_len + 1, len(hn.C.titles)) hn.parse_full_name() self.m(hn.title, "Te", hn) self.m(hn.first, "Awanui-a-Rangi", hn) self.m(hn.last, "Black", hn) def test_constants_subclass_instance_is_used(self) -> None: class CustomConstants(Constants): pass c = CustomConstants() c.titles.add('chancellor') hn = HumanName("Chancellor Jane Smith", constants=c) self.assertIs(hn.C, c) self.m(hn.title, "Chancellor", hn) self.m(hn.first, "Jane", hn) self.m(hn.last, "Smith", hn) def test_constants_invalid_type_raises_typeerror(self) -> None: with pytest.raises(TypeError, match="constants must be"): HumanName("John Doe", constants="not a Constants") # type: ignore[arg-type] def test_constants_class_instead_of_instance_raises_with_hint(self) -> None: # 2.0's message names the received class rather than v1's "did you # mean Constants()" hint; the class-not-instance mistake still fails # loudly at construction with pytest.raises(TypeError, match=r"constants must be a Constants instance"): HumanName("John Doe", constants=Constants) # type: ignore[arg-type] def test_assigning_invalid_constants_after_construction_raises(self) -> None: # #226 validated the constructor's `constants` argument, but `hn.C = ...` # bypassed it entirely: the bad value was accepted silently and only # surfaced far later, deep inside parsing, with no mention of `C` (#239) hn = HumanName("John Doe") with pytest.raises(TypeError, match="constants must be"): hn.C = "garbage" # type: ignore[assignment] def test_assigning_constants_class_after_construction_raises_with_hint(self) -> None: # same message note as the constructor variant above hn = HumanName("John Doe") with pytest.raises(TypeError, match=r"constants must be a Constants instance"): hn.C = Constants # type: ignore[assignment] def test_assigning_none_to_constants_raises(self) -> None: # constants=None was removed in 2.0 (#261, warned since 1.3.1): the v1 # silently-build-a-fresh-Constants fallback is gone from the C setter # too; pass Constants() or CONSTANTS.copy() explicitly hn = HumanName("John Doe") with pytest.raises(TypeError, match="261"): hn.C = None # type: ignore[assignment] def test_constants_bare_string_kwarg_raises_typeerror(self) -> None: # a bare string is an iterable of its characters, so set('dr') would # silently replace the default titles with {'d', 'r'} (#238); the # type system can't catch this because str satisfies Iterable[str] with pytest.raises(TypeError, match=r"wrap it in a list"): Constants(titles='dr') def test_set_manager_bare_string_raises_typeerror(self) -> None: with pytest.raises(TypeError, match=r"wrap it in a list"): SetManager('dr') def test_set_manager_bytes_raises_with_decode_hint(self) -> None: # a "wrap it in a list" hint would be a trap for bytes: [b'dr'] is # accepted but its elements never match parsed str tokens with pytest.raises(TypeError, match=r"decode it first"): SetManager(b'dr') # type: ignore[arg-type] def test_set_manager_bare_string_operand_raises_typeerror(self) -> None: # Set's mixin __or__/__and__ hand _from_iterable a generator, so the # constructor guard alone never sees a bare-string operand; without # an operand check, c.titles |= 'esq' silently adds 'e', 's', 'q' sm = SetManager(['dr', 'mr']) for op in (lambda: sm | 'abc', lambda: 'abc' | sm, lambda: sm & 'abc', lambda: 'abc' & sm, lambda: sm - 'abc', lambda: sm ^ 'abc'): with pytest.raises(TypeError, match=r"wrap it in a list"): op() def test_set_manager_operators_accept_lists(self) -> None: # end-to-end: an operator-built set wired back into Constants must # behave like an add()-built one, normalization included c = Constants() with pytest.raises(TypeError, match=r"wrap it in a list"): c.titles |= 'esq' # type: ignore[assignment] # |= produces a plain set that Constants.__setattr__ re-wraps in a # SetManager (2.0's auto-wrap; see the plain-iterable assignment # test below) -- mypy sees only the set-for-SetManager assignment c.titles |= ['Esq.'] # type: ignore[assignment] self.assertIn('esq', c.titles) hn = HumanName("Esq Jane Smith", constants=c) self.m(hn.title, "Esq", hn) def test_set_manager_operators_normalize_like_add(self) -> None: # add() lowercases and strips leading/trailing periods; without the # same normalization of operator operands, (titles | ['Esq.']) keeps # a raw 'Esq.', which the parser's lc()-based lookups can never match # — silently broken config, same failure family as the bare-string # shredding (#238). # Compared via set(...) rather than v1's .elements: the raw-set # accessor was internal machinery and is gone in 2.0 (#243 family); # 2.0 operators also return a plain set rather than a SetManager. sm = SetManager(['dr', 'mr']) self.assertEqual(set(sm | ['Esq.']), {'dr', 'mr', 'esq'}) self.assertEqual(set(['Esq.', 'Dr.'] | sm), {'dr', 'mr', 'esq'}) self.assertEqual(set(sm & ['Dr.']), {'dr'}) self.assertEqual(set(['Dr.'] & sm), {'dr'}) self.assertEqual(set(sm - ['Dr.']), {'mr'}) self.assertEqual(set(['Dr.', 'Esq.'] - sm), {'esq'}) self.assertEqual(set(sm ^ ['Dr.', 'Esq.']), {'mr', 'esq'}) # pins __rxor__ separately in case it ever stops aliasing __xor__ self.assertEqual(set(['Dr.', 'Esq.'] ^ sm), {'mr', 'esq'}) def test_set_manager_contains_normalizes_like_add(self) -> None: # add()/remove()/the constructor/the operators all normalize (lowercase, # strip leading/trailing periods) -- __contains__ was the lone holdout, # so `'Dr.' in c.titles` returned False even though every other # operation on the same value succeeded (#244) sm = SetManager(['dr', 'mr']) self.assertIn('Dr.', sm) self.assertIn('MR', sm) self.assertNotIn('Esq.', sm) def test_set_manager_le_uses_normalizing_contains(self) -> None: # the ABC comparison mixins route through __contains__, so an # un-normalized membership check would leak into subset/equality too sm = SetManager(['dr', 'mr']) self.assertTrue(sm <= SetManager(['dr', 'mr', 'esq'])) self.assertTrue({'Dr.', 'Mr.'} <= sm) def test_set_manager_rsub_is_order_sensitive(self) -> None: # __sub__ and __rsub__ are hand-written separately (subtraction # isn't commutative, unlike |/&/^), so a copy-paste operand swap # in __rsub__ would silently flip the result and nothing else # in this file would catch it sm = SetManager(['dr', 'mr']) self.assertEqual(set(['Dr.', 'Esq.'] - sm), {'esq'}) self.assertEqual(set(sm - ['Dr.', 'Esq.']), {'mr'}) def test_set_manager_constructor_normalizes_like_add(self) -> None: # without constructor normalization the operators misfire against # the exact spelling visibly stored in the set: & returns empty # and - silently no-ops sm = SetManager(['Dr.', 'MR']) self.assertEqual(set(sm), {'dr', 'mr'}) self.assertEqual(set(sm & ['Dr.']), {'dr'}) self.assertEqual(set(sm - ['Dr.']), {'mr'}) def test_constants_kwarg_elements_are_normalized(self) -> None: # Constants(titles=[...]) was the last silently-dead config path: # a raw 'Chemistry' element can never match the parser's lc() lookups c = Constants(titles=['chancellor', 'Chemistry']) hn = HumanName("Chemistry Jane Smith", constants=c) self.m(hn.title, "Chemistry", hn) def test_default_constants_construction_does_not_alias_defaults(self) -> None: # Constants() reuses the prebuilt _DEFAULT_TITLES snapshot via an # identity check instead of re-validating ~1,400 entries; if that # fast path ever returned the shared elements set instead of a # copy, mutating one Constants() instance would corrupt every # other instance's (and the module-level default's) titles c1 = Constants() c1.titles.add('zzz_should_not_leak') c2 = Constants() self.assertNotIn('zzz_should_not_leak', c2.titles) self.assertNotIn('zzz_should_not_leak', TITLES) def test_equal_but_not_identical_titles_list_still_validates(self) -> None: # the fast path in Constants.__init__ is an `is` check against the # raw TITLES object, not `==`; an equal-but-copied list must still # go through full normalization rather than accidentally matching c = Constants(titles=list(TITLES) + ['Extra.']) self.assertIn('extra', c.titles) def test_set_manager_non_str_elements_raise_typeerror(self) -> None: # lc() on junk elements either crashes context-free (bytes, int) or # silently transmutes None into '' — raise a curated error instead with pytest.raises(TypeError, match=r"decode it first"): SetManager([b'dr']) # type: ignore[list-item] with pytest.raises(TypeError, match=r"expected a str"): SetManager([None]) # type: ignore[list-item] with pytest.raises(TypeError, match=r"expected a str"): SetManager(['dr']) | [1] # type: ignore[list-item] def test_tuplemanager_bare_string_raises_typeerror(self) -> None: # dict(['ab', 'cd']) shreds each 2-char string into a key/value pair # silently -- and dict('ab') itself raises a cryptic "dictionary update # sequence element #0 has length 1; 2 is required" naming no argument # and suggesting no fix (#242) with pytest.raises(TypeError, match=r"wrap it in a list"): TupleManager('ab') # type: ignore[arg-type] def test_tuplemanager_bytes_raises_with_decode_hint(self) -> None: with pytest.raises(TypeError, match=r"decode it first"): TupleManager(b'ab') # type: ignore[arg-type] def test_tuplemanager_string_element_raises_typeerror(self) -> None: # the silent variant: an iterable of 2-character strings is a valid # dict-update sequence, so each one shreds into a key/value pair with pytest.raises(TypeError, match=r"key and a value"): TupleManager(['ab', 'cd']) # type: ignore[arg-type] def test_constants_capitalization_exceptions_string_elements_raise(self) -> None: with pytest.raises(TypeError, match=r"key and a value"): Constants(capitalization_exceptions=['ii']) # type: ignore[list-item] def test_tuplemanager_accepts_mapping_and_pairs(self) -> None: # the guard must not reject the two legitimate constructor shapes tm = TupleManager({'a': '1'}) self.assertEqual(tm.a, '1') tm2 = TupleManager([('b', '2')]) self.assertEqual(tm2.b, '2') def test_remove_title(self) -> None: hn = HumanName("Hon Solo", constants=Constants()) start_len = len(hn.C.titles) self.assertTrue(start_len > 0) hn.C.titles.remove('hon') self.assertEqual(start_len - 1, len(hn.C.titles)) hn.parse_full_name() self.m(hn.first, "Hon", hn) self.m(hn.last, "Solo", hn) def test_add_multiple_arguments(self) -> None: hn = HumanName("Assoc Dean of Chemistry Robert Johns", constants=Constants()) hn.C.titles.add('dean', 'Chemistry') hn.parse_full_name() self.m(hn.title, "Assoc Dean of Chemistry", hn) self.m(hn.first, "Robert", hn) self.m(hn.last, "Johns", hn) def test_instances_can_have_own_constants(self) -> None: hn = HumanName("", Constants()) hn2 = HumanName("") hn.C.titles.remove('hon') self.assertEqual('hon' in hn.C.titles, False) self.assertEqual(hn.has_own_config, True) self.assertEqual('hon' in hn2.C.titles, True) self.assertEqual(hn2.has_own_config, False) def test_can_change_global_constants(self) -> None: # This test exercises shared-CONSTANTS mutation specifically, which # 2.0 deprecates (removal 3.0; the message points at Lexicon/Policy # and private Constants); deprecated_call() pins the warning while # asserting the mutation is still honored. hn = HumanName("") hn2 = HumanName("") with pytest.deprecated_call(match="Lexicon"): hn.C.titles.remove('hon') self.assertEqual('hon' in hn.C.titles, False) self.assertEqual('hon' in hn2.C.titles, False) self.assertEqual(hn.has_own_config, False) self.assertEqual(hn2.has_own_config, False) # No manual cleanup needed: the autouse fixture in conftest.py snapshots # and restores the global CONSTANTS collections around every test. def test_custom_nickname_delimiter_raises(self) -> None: # Custom (non-sentinel) delimiter additions raise in 2.0 (deliberate # divergence, migration spec section 3 -- same uniform rule as # regexes); Policy(nickname_delimiters=...) is the replacement. The # #112 use case this test used to pin moved to the new API. hn = HumanName("") with pytest.raises(TypeError, match="Policy"): hn.C.nickname_delimiters['curly_braces'] = re.compile(r'\{(.*?)\}') def test_remove_multiple_arguments(self) -> None: hn = HumanName("Ms Hon Solo", constants=Constants()) hn.C.titles.remove('hon', 'ms') hn.parse_full_name() self.m(hn.first, "Ms", hn) self.m(hn.middle, "Hon", hn) self.m(hn.last, "Solo", hn) def test_chain_multiple_arguments(self) -> None: hn = HumanName("Dean Ms Hon Solo", constants=Constants()) hn.C.titles.remove('hon', 'ms').add('dean') hn.parse_full_name() self.m(hn.title, "Dean", hn) self.m(hn.first, "Ms", hn) self.m(hn.middle, "Hon", hn) self.m(hn.last, "Solo", hn) def test_clear_removes_all_entries(self) -> None: hn = HumanName("Ms Hon Solo", constants=Constants()) hn.C.titles.clear() hn.parse_full_name() self.m(hn.first, "Ms", hn) self.m(hn.middle, "Hon", hn) self.m(hn.last, "Solo", hn) def test_empty_attribute_default_removed(self) -> None: # empty_attribute_default was removed in 2.0 (#255, warned since # 1.3.0): empty attributes are always ''. Assignment raises naming # the issue; the attribute no longer exists to read either. c = Constants() with pytest.raises(AttributeError, match="255"): c.empty_attribute_default = None # type: ignore[assignment] self.assertFalse(hasattr(c, 'empty_attribute_default')) hn = HumanName("") self.assertEqual(hn.first, '') self.assertEqual(hn.last, '') def test_add_with_encoding_removed(self) -> None: # add_with_encoding() was removed in 2.0 (#245/#263, warned in 1.4); # decode and use add() instead c = Constants() with pytest.raises(AttributeError, match="add_with_encoding"): c.titles.add_with_encoding(b'b\351ck', encoding='latin_1') # type: ignore[attr-defined] c.titles.add(b'b\351ck'.decode('latin_1')) self.assertIn('béck', c.titles) def test_set_manager_add_bytes_raises_with_decode_hint(self) -> None: # bytes elements were removed in 2.0 (#245, warned since 1.3.0) sm = SetManager(['dr']) with pytest.raises(TypeError, match="decode"): sm.add(b'esq') # type: ignore[arg-type] self.assertNotIn('esq', sm) def test_set_manager_add_str_does_not_warn(self) -> None: sm = SetManager(['dr']) with warnings.catch_warnings(): warnings.simplefilter("error") sm.add('esq') self.assertIn('esq', sm) def test_set_manager_call_removed(self) -> None: # __call__ handed out the raw underlying set, bypassing normalization # and change tracking; removed in 2.0 (#243, warned since 1.3.0) -- # iterate the manager or copy with set(manager) instead sm = SetManager(['dr']) with pytest.raises(TypeError, match="not callable"): sm() # type: ignore[operator] self.assertEqual(set(sm), {'dr'}) def test_set_manager_discard_ignores_missing_without_warning(self) -> None: sm = SetManager(['dr', 'mr']) with warnings.catch_warnings(): warnings.simplefilter("error") result = sm.discard('nope').discard('Dr.') # normalizes like remove() self.assertIs(result, sm) self.assertEqual(set(sm), {'mr'}) def test_set_manager_remove_missing_member_raises(self) -> None: # ignore-missing remove() became KeyError in 2.0 (#243, warned since # 1.3.0); discard() is the intentional ignore-missing spelling sm = SetManager(['dr']) with pytest.raises(KeyError): sm.remove('nope') self.assertEqual(set(sm), {'dr'}) # removing a present member stays silent with warnings.catch_warnings(): warnings.simplefilter("error") sm.remove('dr') self.assertEqual(len(sm), 0) def test_set_manager_remove_mixed_present_and_missing_in_one_call(self) -> None: # a single call mixing a present and a missing member raises for the # missing one (#243) but still applies the present removal, so # config state and change tracking don't silently diverge from what # was removed before the KeyError c = Constants() with pytest.raises(KeyError): c.titles.remove('hon', 'nope') self.assertNotIn('hon', c.titles) def test_set_manager_discard_mixed_present_and_missing_in_one_call(self) -> None: sm = SetManager(['dr', 'mr']) with warnings.catch_warnings(): warnings.simplefilter("error") result = sm.discard('dr', 'nope') self.assertIs(result, sm) self.assertEqual(set(sm), {'mr'}) def test_pickle_roundtrip_preserves_customizations(self) -> None: """A pickled Constants must restore its customized collections. Regression test: __setstate__ previously passed the whole state dict to __init__ as the `prefixes` argument, so every collection silently reverted to its module default on unpickling. """ c = Constants() c.titles.add('customtitle') c.prefixes.add('customprefix') c.titles.remove('hon') # (custom nickname delimiters raise in 2.0 -- see # test_custom_nickname_delimiter_raises -- so the delimiter leg of # this round-trip moved to the sentinel default below) # Safe: round-tripping a Constants the test just built, not untrusted data. restored = pickle.loads(pickle.dumps(c)) self.assertIn('customtitle', restored.titles) self.assertIn('customprefix', restored.prefixes) self.assertNotIn('hon', restored.titles) # The contributing collections must match the original exactly. self.assertEqual(set(restored.titles), set(c.titles)) self.assertEqual(set(restored.prefixes), set(c.prefixes)) # The collections must also keep their manager type, not just # contents (nickname_delimiters is the 2.0 delimiter manager, a # TupleManager subclass, so isinstance rather than exact type). self.assertEqual(type(restored.titles), SetManager) self.assertEqual(type(restored.prefixes), SetManager) self.assertIn('parenthesis', restored.nickname_delimiters) self.assertTrue(isinstance(restored.nickname_delimiters, TupleManager)) def test_pickle_roundtrip_preserves_instance_scalar_override(self) -> None: """An instance-level scalar override must survive a pickle round-trip. Exercised via string_format: the v1 vehicle for this test, empty_attribute_default, was removed in 2.0 (#255). """ c = Constants() c.string_format = "{last}" # Safe: round-tripping a Constants the test just built, not untrusted data. with warnings.catch_warnings(): warnings.simplefilter("error") restored = pickle.loads(pickle.dumps(c)) self.assertEqual(restored.string_format, "{last}") def test_unpickle_legacy_state_raises(self) -> None: """Pre-1.3.0 pickles now raise (#279, warned since 1.3.0). Those blobs are recognizable by the computed ``suffixes_prefixes_titles`` property their dir()-sweep ``__getstate__`` captured; the 1.4 DeprecationWarning promised ValueError in 2.0, pointing at re-pickling under 1.3/1.4. """ legacy_state: dict[str, object] = { 'prefixes': {'van'}, 'titles': {'dr', 'legacytitle'}, 'suffixes_prefixes_titles': {'van', 'dr'}, } restored = Constants.__new__(Constants) with pytest.raises(ValueError, match="279"): restored.__setstate__(legacy_state) def test_setstate_without_legacy_keys_does_not_warn(self) -> None: c = Constants() c.titles.add('legacytitle') state = c.__getstate__() self.assertNotIn('suffixes_prefixes_titles', state) restored = Constants.__new__(Constants) with warnings.catch_warnings(): warnings.simplefilter("error") restored.__setstate__(state) self.assertIn('legacytitle', restored.titles) def test_pickle_roundtrip_keeps_regexes_readable(self) -> None: """The regexes surface must survive a Constants pickle round-trip. In 2.0 ``regexes`` is a read-only proxy over the built-in patterns (not pickled state), so the v1 concern this test carried -- the RegexTupleManager subclass being downgraded by ``__reduce__`` -- no longer exists; what remains contractual is that reads keep working on the restored instance and unknown keys fail loudly (#256). """ c = Constants() # Safe: round-tripping a Constants the test just built, not untrusted data. restored = pickle.loads(pickle.dumps(c)) self.assertEqual(type(restored.regexes), type(c.regexes)) self.assertIsNotNone(restored.regexes.mac) with pytest.raises(AttributeError): # unknown-key access raises (#256) restored.regexes.does_not_exist def test_regexes_deepcopy_roundtrip(self) -> None: """copy.deepcopy of the regexes proxy must round-trip. The v1 bug this pinned: __getattr__ answered every unknown name -- including copy.deepcopy's __deepcopy__ probe -- with the EMPTY_REGEX default, so copy mistook a re.Pattern for a deep-copy hook. The 2.0 proxy must keep ignoring protocol probes. """ c = Constants() dup = copy.deepcopy(c.regexes) self.assertEqual(type(dup), type(c.regexes)) self.assertEqual(set(dup.keys()), set(c.regexes.keys())) # Unknown keys raise in 2.0 (#256) instead of returning EMPTY_REGEX. with pytest.raises(AttributeError): dup.does_not_exist def test_nickname_delimiters_deepcopy_roundtrip(self) -> None: """copy.deepcopy of nickname_delimiters must round-trip. Mirrors test_regexes_deepcopy_roundtrip on the delimiter manager (a TupleManager subclass in 2.0). Custom entries raise in 2.0, so the round-trip is exercised on the three built-in sentinels. """ c = Constants() dup = copy.deepcopy(c.nickname_delimiters) self.assertTrue(isinstance(dup, TupleManager)) self.assertEqual(dict(dup), dict(c.nickname_delimiters)) with pytest.raises(AttributeError): # unknown-key access raises (#256) dup.does_not_exist self.assertIsNotNone(dup.parenthesis) def test_maiden_delimiters_deepcopy_roundtrip(self) -> None: """copy.deepcopy of maiden_delimiters (empty by default) must round-trip.""" c = Constants() dup = copy.deepcopy(c.maiden_delimiters) self.assertTrue(isinstance(dup, TupleManager)) self.assertEqual(dict(dup), {}) with pytest.raises(AttributeError): # unknown-key access raises (#256) dup.does_not_exist def test_nickname_delimiters_default_builtins_resolve_live(self) -> None: # The three built-ins are stored as the *name* of a regexes entry # (a plain string), not a copied pattern, so overriding # self.C.regexes.parenthesis etc. keeps affecting nickname parsing -- # see test_overriding_builtin_regex_still_affects_nickname_parsing in # test_nicknames.py. c = Constants() entries = dict(c.nickname_delimiters) # v1 trio still present, still stored name-as-value for name in ('quoted_word', 'double_quotes', 'parenthesis'): self.assertEqual(entries[name], name) # A pre-#273 pickle restores as exactly its own three keys -- # __setstate__ REPLACES the bucket, never merges defaults in -- # so old configs don't silently gain the typographic pairs. legacy = Constants() state = legacy.__getstate__() state['nickname_delimiters'] = { 'quoted_word': 'quoted_word', 'double_quotes': 'double_quotes', 'parenthesis': 'parenthesis'} restored = Constants.__new__(Constants) restored.__setstate__(state) self.assertEqual(set(restored.nickname_delimiters), {'quoted_word', 'double_quotes', 'parenthesis'}) from nameparser import HumanName assert HumanName('John «Jack» Kennedy', constants=restored).nickname == '' assert HumanName('John (Jack) Kennedy', constants=restored).nickname == 'Jack' # 2.0 adds the #273 typographic sentinels alongside them, same # name-as-value scheme (full list pinned in tests/v2/ # test_config_shim.py against _SENTINEL_PAIRS) assert all(value == name for name, value in entries.items()) assert 'smart_double_quotes' in entries self.assertEqual(dict(c.maiden_delimiters), {}) def test_extra_nickname_delimiters_removed(self) -> None: c = Constants() self.assertFalse(hasattr(c, 'extra_nickname_delimiters')) def test_tuplemanager_setattr_delattr_ignore_dunder_names(self) -> None: """Regression test for the bug that motivated nickname_delimiters' __setattr__/__delattr__ dunder guard. Constructing a subscripted generic, e.g. TupleManager[re.Pattern[str] | str]({...}), makes typing's GenericAlias.__call__ set __orig_class__ on the new instance right after __init__ returns. Before the guard existed, __setattr__ was a bare dict.__setitem__ alias, so that assignment silently inserted a bogus '__orig_class__' entry into the dict itself, corrupting .values()/iteration for every TupleManager instance -- exactly what parse_nicknames() iterates over. This bit nickname_delimiters' construction (#22) before the guard was added. """ # The 2.0 TupleManager is a plain dict subclass, not Generic like # v1's -- but dict's inherited __class_getitem__ still makes the # subscription work at runtime, which is exactly the GenericAlias # __orig_class__ probe this test exists to exercise. tm = TupleManager[re.Pattern[str] | str]({'a': re.compile('x')}) # type: ignore[misc] self.assertNotIn('__orig_class__', tm) self.assertEqual(dict(tm), {'a': re.compile('x')}) # Dunder assignment/deletion still work as normal object attributes, # just routed around dict-backed storage. tm.__custom_dunder__ = 'probe' # type: ignore[attr-defined] self.assertEqual(tm.__custom_dunder__, 'probe') # type: ignore[attr-defined] self.assertNotIn('__custom_dunder__', tm) del tm.__custom_dunder__ # type: ignore[attr-defined] self.assertFalse(hasattr(tm, '__custom_dunder__')) def test_regextuplemanager_ignores_dunder_lookups(self) -> None: """Unknown dunder names report as absent, not as the EMPTY_REGEX default. Dunder names are Python's protocol probes (copy.deepcopy looks up __deepcopy__, inspect.unwrap looks up __wrapped__, ...), never config keys. Answering them with a regex breaks that machinery. """ c = Constants() sentinel = object() self.assertEqual(getattr(c.regexes, '__deepcopy__', sentinel), sentinel) # A normal (non-dunder) unknown key raises in 2.0 (#256, warned # since 1.4) instead of degrading to the EMPTY_REGEX default. with pytest.raises(AttributeError): c.regexes.unknown_key def test_tuplemanager_ignores_dunder_lookups(self) -> None: """Base TupleManager must report unknown dunder names as absent too. It returned None for any missing attribute, so `hasattr(tm, '__x__')` was always True — a landmine for any probe that does hasattr-then-call. Guarding dunders keeps the base consistent with RegexTupleManager. """ c = Constants() tm = c.capitalization_exceptions # a plain TupleManager sentinel = object() self.assertEqual(type(tm), TupleManager) self.assertFalse(hasattr(tm, '__deepcopy__')) self.assertEqual(getattr(tm, '__wrapped__', sentinel), sentinel) # A normal (non-dunder) unknown key raises in 2.0 (#256, warned # since 1.4) instead of returning the None default. with pytest.raises(AttributeError): tm.unknown_key def test_sunder_probe_reports_absent_without_deprecation_warning(self) -> None: # Single-underscore introspection probes (IPython/Jupyter's # _repr_html_, _ipython_canary_method_should_not_exist_, etc.) are # never config keys, just like dunders. In 2.0 they raise a plain # AttributeError -- the protocol-correct "absent" answer, so # hasattr-then-call probes work -- with no typo-flavored noise # (v1.4 returned the manager default silently instead). c = Constants() with warnings.catch_warnings(): warnings.simplefilter("error") self.assertFalse(hasattr(c.regexes, '_repr_html_')) self.assertFalse(hasattr(c.capitalization_exceptions, '_ipython_canary_method_should_not_exist_')) def test_tuplemanager_unknown_key_raises_naming_known_keys(self) -> None: # unknown-key attribute access raises in 2.0 (#256, warned since # 1.4): AttributeError naming the known keys, as the 1.4 warning # promised c = Constants() tm = c.capitalization_exceptions with pytest.raises(AttributeError, match="phd_typo") as excinfo: tm.phd_typo message = str(excinfo.value) for key in tm.keys(): self.assertIn(key, message) def test_tuplemanager_known_key_does_not_warn(self) -> None: c = Constants() with warnings.catch_warnings(): warnings.simplefilter("error") list(c.capitalization_exceptions.keys()) # sanity: has entries first_key = next(iter(c.capitalization_exceptions)) getattr(c.capitalization_exceptions, first_key) def test_regexes_unknown_key_raises(self) -> None: # the read-only regexes proxy raises on unknown keys too (#256); # unlike the tuple managers its message names only the missing key, # not the known-keys listing c = Constants() with pytest.raises(AttributeError, match="parenthesys"): c.regexes.parenthesys def test_regextuplemanager_known_key_does_not_warn(self) -> None: c = Constants() with warnings.catch_warnings(): warnings.simplefilter("error") c.regexes.mac def test_dunder_probe_does_not_emit_deprecation_warning(self) -> None: # dunder protocol probes must stay silent -- only real config typos warn c = Constants() sentinel = object() with warnings.catch_warnings(): warnings.simplefilter("error") self.assertEqual(getattr(c.regexes, '__deepcopy__', sentinel), sentinel) self.assertEqual(getattr(c.capitalization_exceptions, '__deepcopy__', sentinel), sentinel) # The v1 suffixes_prefixes_titles cached-union property and its # invalidation machinery (including the is_rootname predicate that read # it) are gone in 2.0 -- change tracking is the shim's generation # counter, covered in tests/v2/test_config_shim.py. The ~13 tests that # exercised cache priming/invalidation were deleted with it; the two # kept below re-pin their surviving contracts on public parsing surface. def test_pickle_roundtrip_rewires_change_tracking(self) -> None: """Mutations on a deserialized Constants must still reach the parser. The v1 version primed and re-read suffixes_prefixes_titles; that cache is gone, so this pins the same contract -- post-unpickle mutations are honored -- through an actual parse. """ c = Constants() # Safe: round-tripping a Constants the test just built, not untrusted data. restored = pickle.loads(pickle.dumps(c)) hn = HumanName("Posttitle Jane Smith", constants=restored) self.m(hn.title, "", hn) restored.titles.add('posttitle') hn.parse_full_name() self.m(hn.title, "Posttitle", hn) def test_replaced_manager_is_wired_for_change_tracking(self) -> None: """Wholesale manager replacement must wire the new manager's mutations. Covers the config-teardown path where a fresh SetManager is assigned directly (v1 pinned this via the suffixes_prefixes_titles cache; the 2.0 contract is that both the replacement and the new manager's own later mutations are honored by the next parse). """ c = Constants() hn = HumanName("Brandnewtitle Jane Smith", constants=c) self.m(hn.title, "", hn) c.titles = SetManager(['brandnewtitle']) hn.parse_full_name() # The replacement is reflected... self.m(hn.title, "Brandnewtitle", hn) # ...and the new manager's own mutations are tracked too, proving # the change callback was re-wired to the replacement. c.titles.add('secondtitle') hn.full_name = "Secondtitle Jane Smith" self.m(hn.title, "Secondtitle", hn) def test_tuplemanager_delattr_removes_dict_entry(self) -> None: """Deleting a non-dunder attribute must remove the dict entry. TupleManager routes non-dunder attribute deletion to ``del self[attr]`` (the mirror of its dot-notation __setattr__), so ``del tm.key`` and ``del tm['key']`` are the same operation. """ c = Constants() self.assertIn('ii', c.capitalization_exceptions) del c.capitalization_exceptions.ii # type: ignore[attr-defined] self.assertNotIn('ii', c.capitalization_exceptions) def test_assigning_iterable_to_set_attr_wraps_and_normalizes(self) -> None: """Assigning a plain iterable to a set field wraps it in a SetManager. 2.0 divergence from v1's guard: v1 demanded a pre-built SetManager (raising TypeError otherwise) because a bare collection would have broken its cache-invalidation wiring; the shim wraps the value itself (with full element validation and normalization), which protects the same invariant -- change tracking stays wired -- without the ceremony. Bare strings still raise (below), so #241's silent substring-test corruption stays impossible. """ c = Constants() c.titles = ['Mr.', 'ms'] # type: ignore[assignment] self.assertEqual(type(c.titles), SetManager) self.assertEqual(set(c.titles), {'mr', 'ms'}) for name in ('first_name_titles', 'conjunctions', 'bound_first_names', 'non_first_name_prefixes', 'suffix_acronyms_ambiguous'): setattr(c, name, ['X.']) self.assertEqual(type(getattr(c, name)), SetManager) self.assertIn('x', getattr(c, name)) def test_bare_string_assignment_to_conjunctions_raises(self) -> None: # the original #241 repro: 'and' assigned as a bare str would # silently degrade `piece.lower() in self.C.conjunctions` into a # substring test (or, wrapped naively, shred into {'a','n','d'}) c = Constants() with pytest.raises(TypeError, match='wrap it in a list'): c.conjunctions = 'and' # type: ignore[assignment] class ParsingDoesNotMutateConfigTests(HumanNameTestBase): """Parsing a name must never write back into the Constants it reads. The parser derives extra lookup entries while parsing (period-joined titles/suffixes like "Lt.Gov.", conjunction-joined pieces like "Mr. and Mrs." or "von und zu"). Those derivations are needed within the parse, but historically they were add()ed to ``self.C`` — by default the shared module-level CONSTANTS singleton — so parsing one name permanently changed how every later name in the process parsed: parse results depended on input order, and concurrent parsing raced on the shared sets. """ @staticmethod def _config_snapshot(constants: Constants) -> dict: """Snapshot every piece of configuration ``constants`` owns. Enumerated via ``Constants.__getstate__()`` — the canonical listing of an instance's own config (descriptor-backed names mapped to their public form, private caches like ``_pst`` excluded) — so a collection added to ``Constants`` later is watched automatically, with no attribute list to keep in sync. """ snap: dict[str, Any] = {} for attr, value in constants.__getstate__().items(): if isinstance(value, SetManager): snap[attr] = set(value) elif isinstance(value, TupleManager): snap[attr] = dict(value) else: snap[attr] = value # scalar override # Fail loud if the structural discovery ever stops seeing the sets # the parser historically leaked into. assert {'titles', 'prefixes', 'conjunctions', 'suffix_acronyms', 'suffix_not_acronyms'} <= set(snap), \ "config snapshot no longer covers the historically-mutated sets" return snap def _assert_config_unchanged(self, constants: Constants, before: dict, parsed: str) -> None: after = self._config_snapshot(constants) diffs = [] for attr in sorted(set(before) | set(after)): b, a = before.get(attr), after.get(attr) if b != a: if isinstance(b, set) and isinstance(a, set): diffs.append(f"{attr}: added {sorted(a - b)}, removed {sorted(b - a)}") else: diffs.append(f"{attr}: {b!r} -> {a!r}") self.assertEqual(diffs, [], f"parsing {parsed!r} changed the config: {diffs}") def _assert_parse_leaves_config_unchanged(self, name: str) -> HumanName: before = self._config_snapshot(CONSTANTS) hn = HumanName(name) self._assert_config_unchanged(CONSTANTS, before, name) return hn def test_period_joined_title_does_not_leak_into_titles(self) -> None: hn = self._assert_parse_leaves_config_unchanged("Lt.Gov. John Doe") # the within-parse derivation must still work self.m(hn.title, "Lt.Gov.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) def test_period_joined_suffix_does_not_leak_into_suffixes(self) -> None: hn = self._assert_parse_leaves_config_unchanged("John Doe JD.CPA") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "JD.CPA", hn) def test_joined_conjunctions_do_not_leak_into_conjunctions(self) -> None: hn = self._assert_parse_leaves_config_unchanged("Louis of the Netherlands") self.m(hn.first, "Louis of the Netherlands", hn) def test_title_conjunction_join_does_not_leak_into_titles(self) -> None: hn = self._assert_parse_leaves_config_unchanged("Mr. and Mrs. John Smith") self.m(hn.title, "Mr. and Mrs.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) def test_prefix_conjunction_join_does_not_leak_into_prefixes(self) -> None: hn = self._assert_parse_leaves_config_unchanged("Alois von und zu Liechtenstein") self.m(hn.first, "Alois", hn) self.m(hn.last, "von und zu Liechtenstein", hn) def test_instance_owned_constants_not_mutated_by_parsing(self) -> None: hn = HumanName("", constants=Constants()) before = self._config_snapshot(hn.C) hn.full_name = "Lt.Gov. John Doe" self._assert_config_unchanged(hn.C, before, "Lt.Gov. John Doe") self.m(hn.title, "Lt.Gov.", hn) def test_derivations_reset_between_parses_of_same_instance(self) -> None: # Re-assigning full_name re-parses; each parse must re-derive from a # clean slate and still resolve the period-joined title. hn = HumanName("Lt.Gov. John Doe") hn.full_name = "Lt.Gov. Jane Roe" self.m(hn.title, "Lt.Gov.", hn) self.m(hn.first, "Jane", hn) self.m(hn.last, "Roe", hn) class ConstantsReprTests(HumanNameTestBase): # The name lists live here rather than reading v1's # Constants._repr_collection_attrs/_repr_scalar_attrs class attributes, # which were internals and are gone in 2.0 (the shim keeps them as # module-level tuples). empty_attribute_default left the scalar list # with #255. collection_attrs = ( 'prefixes', 'suffix_acronyms', 'suffix_not_acronyms', 'titles', 'first_name_titles', 'conjunctions', 'bound_first_names', 'non_first_name_prefixes', 'suffix_acronyms_ambiguous', ) scalar_attrs = ( 'string_format', 'initials_format', 'initials_delimiter', 'initials_separator', 'suffix_delimiter', 'capitalize_name', 'force_mixed_case_capitalization', 'patronymic_name_order', 'middle_name_as_last', ) def test_repr_reports_actual_collection_sizes(self) -> None: c = Constants() repr_str = repr(c) for name in self.collection_attrs: self.assertIn(f"{name}: {len(getattr(c, name))}", repr_str) def test_repr_omits_scalars_at_default_value(self) -> None: c = Constants() repr_str = repr(c) for name in self.scalar_attrs: self.assertNotIn(name, repr_str) def test_repr_shows_scalar_override_via_constructor(self) -> None: c = Constants(middle_name_as_last=True) self.assertIn("middle_name_as_last: True", repr(c)) def test_repr_shows_scalar_override_via_assignment(self) -> None: # Most _repr_scalar_attrs (e.g. capitalize_name) aren't __init__ kwargs # at all -- they're only ever overridden by direct assignment. c = Constants() c.capitalize_name = True self.assertIn("capitalize_name: True", repr(c)) def test_repr_shows_multiple_simultaneous_scalar_overrides(self) -> None: c = Constants(patronymic_name_order=True) c.capitalize_name = True repr_str = repr(c) self.assertIn("patronymic_name_order: True", repr_str) self.assertIn("capitalize_name: True", repr_str) def test_repr_reflects_mutated_collection_size(self) -> None: c = Constants() before = len(c.titles) c.titles.add('a-brand-new-title-for-repr-test') self.assertIn(f"titles: {before + 1}", repr(c)) def test_repr_reports_empty_collection(self) -> None: c = Constants(titles=[]) self.assertIn("titles: 0", repr(c)) def test_repr_is_bracketed_multiline(self) -> None: repr_str = repr(Constants()) self.assertTrue(repr_str.startswith(" None: c = Constants() dup = c.copy() dup.titles.add('a-brand-new-title-for-copy-test') self.assertNotIn('a-brand-new-title-for-copy-test', c.titles) def test_copy_is_not_the_same_object(self) -> None: c = Constants() dup = c.copy() self.assertIsNot(dup, c) self.assertTrue(isinstance(dup, Constants)) def test_copy_preserves_subclass_type(self) -> None: # copy() is deepcopy-based (restored via __getstate__/__setstate__, # not by re-invoking type(self)(...)), so a naive reimplementation # could silently downgrade a subclass instance to plain Constants. class CustomConstants(Constants): pass c = CustomConstants() dup = c.copy() self.assertTrue(isinstance(dup, CustomConstants)) def test_copy_preserves_scalar_override_without_warning(self) -> None: # copy() restores saved state rather than replaying user # assignments, so it must carry scalar overrides across silently. # (The v1 vehicle for this test, empty_attribute_default, was # removed in 2.0 -- #255.) c = Constants() c.string_format = "{last}" with warnings.catch_warnings(): warnings.simplefilter("error") dup = c.copy() self.assertEqual(dup.string_format, "{last}") def test_copy_snapshots_current_customizations(self) -> None: # Unlike Constants(), which always starts from library defaults, # .copy() preserves whatever customizations the original already has. c = Constants() c.titles.add('zephyrmark') dup = c.copy() self.assertIn('zephyrmark', dup.titles) # and stays a snapshot -- later mutation of the original doesn't leak in c.titles.remove('zephyrmark') self.assertIn('zephyrmark', dup.titles) def test_fresh_constants_does_not_include_source_customizations(self) -> None: # Contrast case for the snapshot test above: Constants() ignores # whatever CONSTANTS has been customized with. c = Constants() c.titles.add('zephyrmark') fresh = Constants() self.assertNotIn('zephyrmark', fresh.titles) class ConstantsNoneRemovalTests(HumanNameTestBase): """constants=None was removed in 2.0 (#261, warned since 1.3.1). The v1 fallback silently built a fresh Constants(), discarding any customizations the caller may have expected to carry over; 2.0 raises TypeError at every entry point instead. """ def test_explicit_none_raises_on_construction(self) -> None: with pytest.raises(TypeError, match="261"): HumanName("John Doe", constants=None) # type: ignore[arg-type] def test_explicit_none_raises_on_positional_argument(self) -> None: with pytest.raises(TypeError, match="261"): HumanName("John Doe", None) # type: ignore[arg-type] def test_explicit_none_raises_on_c_setter(self) -> None: hn = HumanName("John Doe") with pytest.raises(TypeError, match="261"): hn.C = None # type: ignore[assignment] def test_none_error_names_the_replacement(self) -> None: with pytest.raises(TypeError) as excinfo: HumanName("John Doe", constants=None) # type: ignore[arg-type] self.assertIn("Constants", str(excinfo.value)) def test_omitted_constants_argument_does_not_warn(self) -> None: with warnings.catch_warnings(): warnings.simplefilter("error") HumanName("John Doe") def test_explicit_own_constants_instance_does_not_warn(self) -> None: with warnings.catch_warnings(): warnings.simplefilter("error") HumanName("John Doe", constants=Constants()) python-nameparser-2.1.0/tests/test_east_slavic_patronymic_order.py000066400000000000000000000166021523530431700257060ustar00rootroot00000000000000import re from typing import cast from nameparser import HumanName from nameparser.config import Constants from tests.base import FlaggedConstantsTestBase, HumanNameTestBase # The 2.0 regexes proxy types its attributes as ``object`` (reads are # informational); the pattern-inspection tests below cast once here. _LATIN = cast('re.Pattern[str]', Constants().regexes.east_slavic_patronymic) _CYRILLIC = cast('re.Pattern[str]', Constants().regexes.east_slavic_patronymic_cyrillic) def test_latin_patronymic_matches() -> None: # One common suffix and one irregular — the integration tests cover the rest. assert _LATIN.search("Ivanovich") assert _LATIN.search("Ilyich") def test_latin_patronymic_rejects_non_patronymic() -> None: # EMPTY_REGEX (the default for missing keys) matches everything, # so this test is red until the real pattern is in place. assert not _LATIN.search("Smith") def test_latin_patronymic_end_anchored() -> None: # A surname ending in a patronymic suffix matches; the end-anchor does not # prevent this. The parser guard tests verify reordering is suppressed. assert _LATIN.search("Abramovich") def test_cyrillic_patronymic_matches() -> None: # One common suffix and one irregular. assert _CYRILLIC.search("Иванович") assert _CYRILLIC.search("ильич") def test_cyrillic_patronymic_matches_capitalized_irregular_forms() -> None: # The irregular forms (Ильич, Кузьмич, ...) are short enough that the # capitalized first letter falls within the matched suffix itself, unlike # the common suffixes (-ович, -евна, ...) where only the surname root is # capitalized. Case-insensitivity is required for these to match. assert _CYRILLIC.search("Ильич") assert _CYRILLIC.search("Кузьмич") assert _CYRILLIC.search("Лукич") assert _CYRILLIC.search("Фомич") assert _CYRILLIC.search("Фокич") def test_cyrillic_patronymic_rejects_non_patronymic() -> None: assert not _CYRILLIC.search("Иванов") class PatronymicNameOrderReorderTests(FlaggedConstantsTestBase): """Names that SHOULD be rotated when the flag is on.""" constants_kwargs = {"patronymic_name_order": True} def test_canonical_latin(self) -> None: n = self.hn("Ivanov Ivan Ivanovich") assert n.first == "Ivan" assert n.middle == "Ivanovich" assert n.last == "Ivanov" def test_sergeevich(self) -> None: n = self.hn("Zarubkin Alexander Sergeevich") assert n.first == "Alexander" assert n.middle == "Sergeevich" assert n.last == "Zarubkin" def test_hyphenated_surname(self) -> None: # A hyphenated surname counts as one token. n = self.hn("Blokin-Mechtalin Konstantin Yurievich") assert n.first == "Konstantin" assert n.middle == "Yurievich" assert n.last == "Blokin-Mechtalin" def test_surname_looks_like_patronymic(self) -> None: # "Petsevich" ends in -evich but is in the FIRST position. n = self.hn("Petsevich Sergey Vitalyevich") assert n.first == "Sergey" assert n.middle == "Vitalyevich" assert n.last == "Petsevich" def test_cyrillic(self) -> None: n = self.hn("Иванов Иван Иванович") assert n.first == "Иван" assert n.middle == "Иванович" assert n.last == "Иванов" def test_cyrillic_capitalized_irregular_form(self) -> None: # "Ильич" is short enough that the capitalized first letter falls # within the irregular suffix itself; requires case-insensitive match. n = self.hn("Иванов Иван Ильич") assert n.first == "Иван" assert n.middle == "Ильич" assert n.last == "Иванов" def test_title_preserved(self) -> None: n = self.hn("Dr. Ivanov Ivan Ivanovich") assert n.title == "Dr." assert n.first == "Ivan" assert n.middle == "Ivanovich" assert n.last == "Ivanov" def test_suffix_preserved(self) -> None: n = self.hn("Ivanov Ivan Ivanovich Jr.") assert n.first == "Ivan" assert n.middle == "Ivanovich" assert n.last == "Ivanov" assert n.suffix == "Jr." def test_western_patronymic_surname_reordered_when_flag_on(self) -> None: # Documented opt-in tradeoff: a Western name whose last token ends in a # patronymic suffix is reordered incorrectly. Not a bug to fix. n = self.hn("David Michael Abramovich") assert n.first == "Michael" assert n.middle == "Abramovich" assert n.last == "David" class PatronymicNameOrderGuardsTests(FlaggedConstantsTestBase): """Names that must NOT be reordered even when the flag is on.""" constants_kwargs = {"patronymic_name_order": True} def test_already_correct_order(self) -> None: # middle is patronymic → already in Western order, do not rotate n = self.hn("Ivan Ivanovich Ivanov") assert n.first == "Ivan" assert n.middle == "Ivanovich" assert n.last == "Ivanov" def test_middle_is_patronymic_surname_ends_ovich(self) -> None: # "Roman Arkadyevich Abramovich": middle IS patronymic → guard fires n = self.hn("Roman Arkadyevich Abramovich") assert n.first == "Roman" assert n.middle == "Arkadyevich" assert n.last == "Abramovich" def test_two_token_name(self) -> None: # 2-token: middle_list is empty → condition fails n = self.hn("Roman Abramovich") assert n.first == "Roman" assert n.last == "Abramovich" def test_no_patronymic(self) -> None: # Three tokens but no patronymic suffix on last → not reordered n = self.hn("Ivanov Ivan Petrov") assert n.first == "Ivanov" assert n.middle == "Ivan" assert n.last == "Petrov" def test_western_name_unchanged(self) -> None: n = self.hn("John Michael Smith") assert n.first == "John" assert n.middle == "Michael" assert n.last == "Smith" def test_comma_guard_last_first_pat(self) -> None: # "Ivanov, Ivan Ivanovich" — comma means the order was declared n = self.hn("Ivanov, Ivan Ivanovich") assert n.first == "Ivan" assert n.middle == "Ivanovich" assert n.last == "Ivanov" def test_comma_guard_patronymic_form_surname(self) -> None: # Without the comma guard this would wrongly rotate n = self.hn("Sergeevich, Ivan Petrov") assert n.last == "Sergeevich" class PatronymicNameOrderFlagOffTests(HumanNameTestBase): """With default Constants (flag=False) nothing changes.""" def test_canonical_unchanged(self) -> None: n = HumanName("Ivanov Ivan Ivanovich") assert n.first == "Ivanov" assert n.middle == "Ivan" assert n.last == "Ivanovich" class PatronymicNameOrderFlagTests(HumanNameTestBase): def test_default_is_false(self) -> None: C = Constants() assert C.patronymic_name_order is False def test_can_set_true_via_constructor(self) -> None: C = Constants(patronymic_name_order=True) assert C.patronymic_name_order is True def test_does_not_affect_other_instance(self) -> None: C1 = Constants(patronymic_name_order=True) C2 = Constants() assert C1.patronymic_name_order is True assert C2.patronymic_name_order is False python-nameparser-2.1.0/tests/test_first_name.py000066400000000000000000000050221523530431700220720ustar00rootroot00000000000000import pytest from nameparser import HumanName from tests.base import HumanNameTestBase class FirstNameHandlingTests(HumanNameTestBase): def test_first_name(self) -> None: hn = HumanName("Andrew") self.m(hn.first, "Andrew", hn) def test_assume_title_and_one_other_name_is_last_name(self) -> None: hn = HumanName("Rev Andrews") self.m(hn.title, "Rev", hn) self.m(hn.last, "Andrews", hn) def test_assume_suffix_title_and_one_other_name_is_last_name(self) -> None: # xfail in v1 (which parsed first='M.D.'); 2.0's family-comma # suffix peel routes the post-comma suffix piece to suffix and # keeps the pre-comma piece in last, so the long-desired # expectation now holds. 2.0: fix(comma-family/suffix-routing). hn = HumanName("Andrews, M.D.") self.m(hn.suffix, "M.D.", hn) self.m(hn.last, "Andrews", hn) def test_suffix_in_lastname_part_of_lastname_comma_format(self) -> None: hn = HumanName("Smith Jr., John") self.m(hn.last, "Smith", hn) self.m(hn.first, "John", hn) self.m(hn.suffix, "Jr.", hn) def test_sir_exception_to_first_name_rule(self) -> None: hn = HumanName("Sir Gerald") self.m(hn.title, "Sir", hn) self.m(hn.first, "Gerald", hn) def test_king_exception_to_first_name_rule(self) -> None: hn = HumanName("King Henry") self.m(hn.title, "King", hn) self.m(hn.first, "Henry", hn) def test_queen_exception_to_first_name_rule(self) -> None: hn = HumanName("Queen Elizabeth") self.m(hn.title, "Queen", hn) self.m(hn.first, "Elizabeth", hn) def test_dame_exception_to_first_name_rule(self) -> None: hn = HumanName("Dame Mary") self.m(hn.title, "Dame", hn) self.m(hn.first, "Mary", hn) def test_first_name_is_not_prefix_if_only_two_parts(self) -> None: """When there are only two parts, don't join prefixes or conjunctions""" hn = HumanName("Van Nguyen") self.m(hn.first, "Van", hn) self.m(hn.last, "Nguyen", hn) def test_first_name_is_not_prefix_if_only_two_parts_comma(self) -> None: hn = HumanName("Nguyen, Van") self.m(hn.first, "Van", hn) self.m(hn.last, "Nguyen", hn) @pytest.mark.xfail def test_first_name_is_prefix_if_three_parts(self) -> None: """Not sure how to fix this without breaking Mr and Mrs""" hn = HumanName("Mr. Van Nguyen") self.m(hn.first, "Van", hn) self.m(hn.last, "Nguyen", hn) python-nameparser-2.1.0/tests/test_initials.py000066400000000000000000000226551523530431700215720ustar00rootroot00000000000000from nameparser import HumanName from nameparser.config import Constants from tests.base import HumanNameTestBase class InitialsTestCase(HumanNameTestBase): def test_initials(self) -> None: hn = HumanName("Andrew Boris Petersen") self.m(hn.initials(), "A. B. P.", hn) def test_initials_simple_name(self) -> None: hn = HumanName("John Doe") self.m(hn.initials(), "J. D.", hn) hn = HumanName("John Doe", initials_format="{first} {last}") self.m(hn.initials(), "J. D.", hn) hn = HumanName("John Doe", initials_format="{last}") self.m(hn.initials(), "D.", hn) hn = HumanName("John Doe", initials_format="{first}") self.m(hn.initials(), "J.", hn) hn = HumanName("John Doe", initials_format="{middle}") self.m(hn.initials(), "", hn) def test_initials_all_empty_renders_empty_string(self) -> None: # The v1 None display mode (and its literal-"None" interpolation # regressions) died with #255: a fully-empty result is now always # '', matching the first/last accessors. hn = HumanName("", constants=Constants()) self.assertEqual(hn.initials(), "") def test_initials_middle_name_all_prefixes(self) -> None: # "Vega, Juan de la" parses with middle name "de la", which contains # no initialable words (both are prefixes). The part must be skipped # entirely — not emit an empty initial ("J. . V.") and not crash when # empty_attribute_default is None. hn = HumanName("Vega, Juan de la") self.m(hn.middle, "de la", hn) self.assertEqual(hn.initials_list(), ["J", "V"]) self.assertEqual(hn.initials(), "J. V.") def test_initials_complex_name(self) -> None: hn = HumanName("Doe, John A. Kenneth, Jr.") self.m(hn.initials(), "J. A. K. D.", hn) def test_initials_format(self) -> None: hn = HumanName("Doe, John A. Kenneth, Jr.", initials_format="{first} {middle}") self.m(hn.initials(), "J. A. K.", hn) hn = HumanName("Doe, John A. Kenneth, Jr.", initials_format="{first} {last}") self.m(hn.initials(), "J. D.", hn) hn = HumanName("Doe, John A. Kenneth, Jr.", initials_format="{middle} {last}") self.m(hn.initials(), "A. K. D.", hn) hn = HumanName("Doe, John A. Kenneth, Jr.", initials_format="{first}, {last}") self.m(hn.initials(), "J., D.", hn) # The *_constants tests below ran on the shared CONSTANTS singleton in # v1; 2.0 deprecates shared mutation, so they use a private Constants # passed as constants= (the migration-guide idiom) -- the config # attribute under test is the same either way. def test_initials_format_constants(self) -> None: c = Constants() c.initials_format = "{first} {last}" hn = HumanName("Doe, John A. Kenneth, Jr.", constants=c) self.m(hn.initials(), "J. D.", hn) c.initials_format = "{first} {last}" hn = HumanName("Doe, John A. Kenneth, Jr.", constants=c) self.m(hn.initials(), "J. D.", hn) def test_initials_delimiter(self) -> None: hn = HumanName("Doe, John A. Kenneth, Jr.", initials_delimiter=";") self.m(hn.initials(), "J; A; K; D;", hn) def test_initials_delimiter_constants(self) -> None: c = Constants() c.initials_delimiter = ";" hn = HumanName("Doe, John A. Kenneth, Jr.", constants=c) self.m(hn.initials(), "J; A; K; D;", hn) def test_initials_list(self) -> None: hn = HumanName("Andrew Boris Petersen") self.m(hn.initials_list(), ["A", "B", "P"], hn) def test_initials_list_complex_name(self) -> None: hn = HumanName("Doe, John A. Kenneth, Jr.") self.m(hn.initials_list(), ["J", "A", "K", "D"], hn) def test_initials_with_prefix_firstname(self) -> None: hn = HumanName("Van Jeremy Johnson") self.m(hn.initials_list(), ["V", "J", "J"], hn) def test_initials_with_prefix(self) -> None: hn = HumanName("Alex van Johnson") self.m(hn.initials_list(), ["A", "J"], hn) def test_initials_delimiter_empty_string_kwarg(self) -> None: # Regression: initials_delimiter='' was silently ignored due to `or` defaulting hn = HumanName("Doe, John A.", initials_delimiter="") self.m(hn.initials(), "J A D", hn) def test_initials_format_empty_string_kwarg(self) -> None: # Regression: initials_format='' was silently ignored due to `or` defaulting hn = HumanName("Doe, John A.") hn2 = HumanName("Doe, John A.", initials_format="") self.assertNotEqual(hn.initials(), hn2.initials()) # "".format(...) returns ""; collapse_whitespace returns "" which falls through # to empty_attribute_default (may be "" or None depending on config variant). self.assertFalse(hn2.initials()) def test_initials_separator_kwarg(self) -> None: # initials_separator="" with initials_format="{first}{middle}{last}" gives # period-separated initials with no spaces — a common academic citation style hn = HumanName( "Doe, John A. Kenneth", initials_separator="", initials_format="{first}{middle}{last}", ) self.m(hn.initials(), "J.A.K.D.", hn) def test_initials_separator_custom_value(self) -> None: # Non-empty custom separator exercising _process_initial on a multi-word # token. "Van Berg" is a single name part whose two words produce two initials # joined by initials_separator. hn = HumanName("", initials_separator="-", initials_delimiter=".") result = hn._process_initial("Van Berg", firstname=True) self.assertEqual(result, "V-B") def test_str_default_behavior_unchanged(self) -> None: # Regression guard for the `or` → `is not None` change in __str__: # the default path (no string_format kwarg) must still produce the expected string. hn = HumanName("John Doe") self.assertEqual(str(hn), "John Doe") def test_initials_with_doubled_space_in_list_element(self) -> None: # v1's #232 hole -- direct *_list assignment injecting unnormalized # elements that made initials raise IndexError -- is unreachable in # 2.0: *_list attributes are read-only snapshots (spec section 2 # exc. 1) and the field setter normalizes whitespace, splitting a # doubled-space element into clean members. hn = HumanName(first="John") hn.middle = ["Q R"] self.assertEqual(hn.middle_list, ["Q", "R"]) self.assertEqual(hn.initials_list(), ["J", "Q", "R"]) self.assertEqual(hn.initials(), "J. Q. R.") def test_constructor_first(self) -> None: hn = HumanName(first="TheName") self.m(hn.first, "TheName", hn) def test_constructor_middle(self) -> None: hn = HumanName(middle="TheName") self.m(hn.middle, "TheName", hn) def test_constructor_last(self) -> None: hn = HumanName(last="TheName") self.m(hn.last, "TheName", hn) def test_constructor_title(self) -> None: hn = HumanName(title="TheName") self.m(hn.title, "TheName", hn) def test_constructor_suffix(self) -> None: hn = HumanName(suffix="TheName") self.m(hn.suffix, "TheName", hn) def test_constructor_nickname(self) -> None: hn = HumanName(nickname="TheName") self.m(hn.nickname, "TheName", hn) def test_constructor_multiple(self) -> None: hn = HumanName(first="TheName", last="lastname", title="mytitle", full_name="donotparse") self.m(hn.first, "TheName", hn) self.m(hn.last, "lastname", hn) self.m(hn.title, "mytitle", hn) def test_initials_separator_kwarg_multiword_part(self) -> None: # Regression: initials_separator kwarg must flow into _process_initial # for multi-word name parts, not just into the initials() join calls. hn = HumanName("", initials_separator="") result = hn._process_initial("Van Berg", firstname=True) self.assertEqual(result, "VB") def test_string_format_empty_string_kwarg(self) -> None: # Regression: string_format='' was silently ignored due to `or` defaulting hn = HumanName("John Doe", string_format="") self.assertEqual(str(hn), "") def test_initials_separator_empty_multi_part_middle(self) -> None: # Full workflow from issue #152: empty delimiter + separator + compact format # gives fully concatenated initials with no spaces or punctuation. # Spaces between groups come from initials_format, so that must also be set. hn = HumanName( "Doe, John A. Kenneth", initials_delimiter="", initials_separator="", initials_format="{first}{middle}{last}", ) self.m(hn.initials(), "JAKD", hn) def test_initials_separator_constants_multi_part_middle(self) -> None: c = Constants() c.initials_delimiter = "" c.initials_separator = "" c.initials_format = "{first}{middle}{last}" hn = HumanName("Doe, John A. Kenneth", constants=c) self.m(hn.initials(), "JAKD", hn) def test_initials_separator_default_on_constants(self) -> None: # The shared singleton's default must be untouched by the private- # Constants tests above (and, in file order, by anything the autouse # fixture restores). from nameparser.config import CONSTANTS self.assertEqual(CONSTANTS.initials_separator, " ") python-nameparser-2.1.0/tests/test_middle_name_as_last.py000066400000000000000000000114721523530431700237150ustar00rootroot00000000000000from nameparser import HumanName from nameparser.config import Constants from tests.base import FlaggedConstantsTestBase, HumanNameTestBase class MiddleNameAsLastFlagTests(HumanNameTestBase): def test_default_is_false(self) -> None: C = Constants() assert C.middle_name_as_last is False def test_can_set_true_via_constructor(self) -> None: C = Constants(middle_name_as_last=True) assert C.middle_name_as_last is True def test_does_not_affect_other_instance(self) -> None: C1 = Constants(middle_name_as_last=True) C2 = Constants() assert C1.middle_name_as_last is True assert C2.middle_name_as_last is False class MiddleNameAsLastFoldTests(FlaggedConstantsTestBase): constants_kwargs = {"middle_name_as_last": True} def test_fold_no_comma(self) -> None: n = self.hn("Mohamad Ahmad Ali Hassan") self.m(n.first, "Mohamad", n) self.m(n.middle, "", n) self.m(n.last, "Ahmad Ali Hassan", n) def test_fold_comma_converges(self) -> None: no_comma = self.hn("Mohamad Ahmad Ali Hassan") comma = self.hn("Hassan, Mohamad Ahmad Ali") self.m(comma.first, no_comma.first, comma) self.m(comma.last, no_comma.last, comma) def test_title_and_suffix_preserved(self) -> None: n = self.hn("Dr. Mohamad Ahmad Hassan Jr") self.m(n.title, "Dr.", n) self.m(n.last, "Ahmad Hassan", n) self.m(n.suffix, "Jr", n) def test_suffix_preserved_comma_format(self) -> None: # Comma-delimited suffix takes a different code path than the # title/suffix no-comma case above; the fold must still apply. n = self.hn("Hassan, Mohamad Ahmad Ali, Jr.") self.m(n.first, "Mohamad", n) self.m(n.middle, "", n) self.m(n.last, "Ahmad Ali Hassan", n) self.m(n.suffix, "Jr.", n) def test_nickname_preserved(self) -> None: # Nicknames are stripped in pre_process(), before the fold runs. n = self.hn('Mohamad "Mo" Ahmad Ali Hassan') self.m(n.nickname, "Mo", n) self.m(n.middle, "", n) self.m(n.last, "Ahmad Ali Hassan", n) def test_no_middle_is_noop(self) -> None: n = self.hn("John Doe") self.m(n.first, "John", n) self.m(n.middle, "", n) self.m(n.last, "Doe", n) def test_single_token_is_noop(self) -> None: n = self.hn("Cher") self.m(n.first, "Cher", n) self.m(n.middle, "", n) self.m(n.last, "", n) def test_given_names_and_surnames_track_fold(self) -> None: n = self.hn("Mohamad Ahmad Ali Hassan") self.m(n.given_names, n.first, n) self.m(n.surnames, n.last, n) def test_last_prefixes_still_split_after_fold(self) -> None: # Unfolded this is first="Miguel", middle="da Silva do Amaral", # last="de Souza" (last_prefixes="de"). Folded, last_list becomes # ["da","Silva","do","Amaral","de","Souza"]; _split_last() strips # leading contiguous prefix words from the start, so only the # leading "da" is stripped ("Silva" is not a prefix, so scanning # stops there) — last_prefixes="da", not "de". n = self.hn("Miguel da Silva do Amaral de Souza") self.m(n.last_prefixes, "da", n) class MiddleNameAsLastFlagOffTests(HumanNameTestBase): def test_default_constants_unaffected(self) -> None: n = HumanName("Mohamad Ahmad Ali Hassan") self.m(n.middle, "Ahmad Ali", n) self.m(n.last, "Hassan", n) class MiddleNameAsLastWithPatronymicOrderTests(FlaggedConstantsTestBase): """Both localization flags on: patronymic reordering must settle first/middle/last before the fold collapses middle into last, per the design's stated ordering rationale (post_process() runs the patronymic hook before the middle_name_as_last hook).""" constants_kwargs = {"middle_name_as_last": True, "patronymic_name_order": True} def test_rotate_then_fold_no_comma(self) -> None: # patronymic_name_order rotates "Ivanov Petr Sergeyevich" to # first=Petr, middle=Sergeyevich, last=Ivanov; the fold then # collapses that settled middle into last. n = self.hn("Ivanov Petr Sergeyevich") self.m(n.first, "Petr", n) self.m(n.middle, "", n) self.m(n.last, "Sergeyevich Ivanov", n) def test_fold_applies_even_when_comma_suppresses_rotation(self) -> None: # A comma suppresses patronymic_name_order's rotation (_had_comma # guard), so middle stays "Sergeyevich" unrotated going into the # fold. The fold still absorbs it into last, producing the same # first/last as the no-comma case above via a different mechanism. n = self.hn("Ivanov, Petr Sergeyevich") self.m(n.first, "Petr", n) self.m(n.middle, "", n) self.m(n.last, "Sergeyevich Ivanov", n) python-nameparser-2.1.0/tests/test_nicknames.py000066400000000000000000000332511523530431700217200ustar00rootroot00000000000000import re import pytest from nameparser import HumanName from nameparser.config import Constants from tests.base import HumanNameTestBase class NicknameTestCase(HumanNameTestBase): # https://code.google.com/p/python-nameparser/issues/detail?id=33 def test_nickname_in_parenthesis(self) -> None: hn = HumanName("Benjamin (Ben) Franklin") self.m(hn.first, "Benjamin", hn) self.m(hn.middle, "", hn) self.m(hn.last, "Franklin", hn) self.m(hn.nickname, "Ben", hn) # https://github.com/derek73/python-nameparser/issues/112 def test_add_custom_nickname_delimiter_raises(self) -> None: # Custom (non-sentinel) delimiter additions raise in 2.0 (deliberate # divergence, migration spec section 3 -- same uniform rule as # regexes); Policy(nickname_delimiters=...) is the replacement. The # v1 tests for removing custom delimiters, registering several at # once, and the ambiguous-acronym carve-out through a custom # delimiter died with the mechanism. hn = HumanName("Benjamin {Ben} Franklin", constants=Constants()) # curly braces aren't a recognized delimiter by default self.m(hn.nickname, "", hn) with pytest.raises(TypeError, match="Policy"): hn.C.nickname_delimiters['curly_braces'] = re.compile(r'\{(.*?)\}') def test_overriding_builtin_regex_raises(self) -> None: # v1's other customization path -- replacing a regexes entry so the # built-in delimiter sentinels resolve to it live -- is the same # uniform 2.0 removal: any regexes assignment raises, pointing at # Policy. hn = HumanName("Benjamin [Ben] Franklin", constants=Constants()) self.m(hn.nickname, "", hn) with pytest.raises(TypeError, match="Policy"): hn.C.regexes['parenthesis'] = re.compile(r'\[(.*?)\]') def test_two_word_nickname_in_parenthesis(self) -> None: hn = HumanName("Benjamin (Big Ben) Franklin") self.m(hn.first, "Benjamin", hn) self.m(hn.middle, "", hn) self.m(hn.last, "Franklin", hn) self.m(hn.nickname, "Big Ben", hn) def test_two_words_in_quotes(self) -> None: hn = HumanName('Benjamin "Big Ben" Franklin') self.m(hn.first, "Benjamin", hn) self.m(hn.middle, "", hn) self.m(hn.last, "Franklin", hn) self.m(hn.nickname, "Big Ben", hn) def test_nickname_in_parenthesis_with_comma(self) -> None: hn = HumanName("Franklin, Benjamin (Ben)") self.m(hn.first, "Benjamin", hn) self.m(hn.middle, "", hn) self.m(hn.last, "Franklin", hn) self.m(hn.nickname, "Ben", hn) def test_nickname_in_parenthesis_with_comma_and_suffix(self) -> None: hn = HumanName("Franklin, Benjamin (Ben), Jr.") self.m(hn.first, "Benjamin", hn) self.m(hn.middle, "", hn) self.m(hn.last, "Franklin", hn) self.m(hn.suffix, "Jr.", hn) self.m(hn.nickname, "Ben", hn) def test_nickname_in_single_quotes(self) -> None: hn = HumanName("Benjamin 'Ben' Franklin") self.m(hn.first, "Benjamin", hn) self.m(hn.middle, "", hn) self.m(hn.last, "Franklin", hn) self.m(hn.nickname, "Ben", hn) def test_nickname_in_double_quotes(self) -> None: hn = HumanName("Benjamin \"Ben\" Franklin") self.m(hn.first, "Benjamin", hn) self.m(hn.middle, "", hn) self.m(hn.last, "Franklin", hn) self.m(hn.nickname, "Ben", hn) def test_single_quotes_on_first_name_not_treated_as_nickname(self) -> None: hn = HumanName("Brian Andrew O'connor") self.m(hn.first, "Brian", hn) self.m(hn.middle, "Andrew", hn) self.m(hn.last, "O'connor", hn) self.m(hn.nickname, "", hn) def test_single_quotes_on_both_name_not_treated_as_nickname(self) -> None: hn = HumanName("La'tanya O'connor") self.m(hn.first, "La'tanya", hn) self.m(hn.middle, "", hn) self.m(hn.last, "O'connor", hn) self.m(hn.nickname, "", hn) def test_single_quotes_on_end_of_last_name_not_treated_as_nickname(self) -> None: hn = HumanName("Mari' Aube'") self.m(hn.first, "Mari'", hn) self.m(hn.middle, "", hn) self.m(hn.last, "Aube'", hn) self.m(hn.nickname, "", hn) def test_okina_inside_name_not_treated_as_nickname(self) -> None: hn = HumanName("Harrieta Keōpūolani Nāhiʻenaʻena") self.m(hn.first, "Harrieta", hn) self.m(hn.middle, "Keōpūolani", hn) self.m(hn.last, "Nāhiʻenaʻena", hn) self.m(hn.nickname, "", hn) def test_single_quotes_not_treated_as_nickname_Hawaiian_example(self) -> None: hn = HumanName("Harietta Keopuolani Nahi'ena'ena") self.m(hn.first, "Harietta", hn) self.m(hn.middle, "Keopuolani", hn) self.m(hn.last, "Nahi'ena'ena", hn) self.m(hn.nickname, "", hn) def test_single_quotes_not_treated_as_nickname_Kenyan_example(self) -> None: hn = HumanName("Naomi Wambui Ng'ang'a") self.m(hn.first, "Naomi", hn) self.m(hn.middle, "Wambui", hn) self.m(hn.last, "Ng'ang'a", hn) self.m(hn.nickname, "", hn) def test_single_quotes_not_treated_as_nickname_Samoan_example(self) -> None: hn = HumanName("Va'apu'u Vitale") self.m(hn.first, "Va'apu'u", hn) self.m(hn.middle, "", hn) self.m(hn.last, "Vitale", hn) self.m(hn.nickname, "", hn) # http://code.google.com/p/python-nameparser/issues/detail?id=17 def test_parenthesis_are_removed_from_name(self) -> None: hn = HumanName("John Jones (Unknown)") self.m(hn.first, "John", hn) self.m(hn.last, "Jones", hn) # not testing the nicknames because we don't actually care # about Google Docs here def test_duplicate_parenthesis_are_removed_from_name(self) -> None: hn = HumanName("John Jones (Google Docs), Jr. (Unknown)") self.m(hn.first, "John", hn) self.m(hn.last, "Jones", hn) self.m(hn.suffix, "Jr.", hn) def test_nickname_and_last_name(self) -> None: hn = HumanName('"Rick" Edmonds') self.m(hn.first, "", hn) self.m(hn.last, "Edmonds", hn) self.m(hn.nickname, "Rick", hn) @pytest.mark.xfail def test_nickname_and_last_name_with_title(self) -> None: hn = HumanName('Senator "Rick" Edmonds') self.m(hn.title, "Senator", hn) self.m(hn.first, "", hn) self.m(hn.last, "Edmonds", hn) self.m(hn.nickname, "Rick", hn) def test_ambiguous_suffix_acronym_in_parenthesis_stays_nickname(self) -> None: # JD is in SUFFIX_ACRONYMS_AMBIGUOUS: both a law-degree acronym and a # common given-name nickname. Existing behavior (nickname) must be # preserved -- see issue #111. hn = HumanName("JEFFREY (JD) BRICKEN") self.m(hn.nickname, "JD", hn) self.m(hn.suffix, "", hn) class MaidenNameTestCase(HumanNameTestBase): def test_maiden_assignment_and_property(self) -> None: hn = HumanName("Jenny Baker") hn.maiden = "Johnson" self.m(hn.maiden, "Johnson", hn) def test_maiden_defaults_empty(self) -> None: hn = HumanName("Jenny Baker") self.m(hn.maiden, "", hn) def test_maiden_key_always_in_as_dict(self) -> None: # empty attributes are always '' in 2.0 (#255) hn = HumanName("Bob Dole") self.assertEqual(hn.as_dict()['maiden'], '') self.assertNotIn('maiden', hn.as_dict(False)) def test_maiden_appears_in_as_dict_when_populated(self) -> None: hn = HumanName("Jenny Baker") hn.maiden = "Johnson" self.assertEqual(hn.as_dict()['maiden'], "Johnson") self.assertEqual(hn.as_dict(False)['maiden'], "Johnson") def test_maiden_appears_in_slice(self) -> None: # list(hn), not the deprecated hn[:] slice (#258) hn = HumanName("Jenny Baker") hn.maiden = "Johnson" self.assertIn("Johnson", list(hn)) def test_maiden_via_constructor_kwarg(self) -> None: hn = HumanName(first="Jenny", last="Baker", maiden="Johnson") self.m(hn.first, "Jenny", hn) self.m(hn.last, "Baker", hn) self.m(hn.maiden, "Johnson", hn) def test_maiden_name_in_parenthesis_with_comma(self) -> None: C = Constants() C.maiden_delimiters['parenthesis'] = C.nickname_delimiters.pop('parenthesis') hn = HumanName("Baker (Johnson), Jenny", constants=C) self.m(hn.first, "Jenny", hn) self.m(hn.last, "Baker", hn) self.m(hn.maiden, "Johnson", hn) def test_maiden_name_in_parenthesis_no_comma(self) -> None: C = Constants() C.maiden_delimiters['parenthesis'] = C.nickname_delimiters.pop('parenthesis') hn = HumanName("Jenny Baker (Johnson)", constants=C) self.m(hn.first, "Jenny", hn) self.m(hn.last, "Baker", hn) self.m(hn.maiden, "Johnson", hn) def test_quotes_still_nickname_when_parens_routed_to_maiden(self) -> None: C = Constants() C.maiden_delimiters['parenthesis'] = C.nickname_delimiters.pop('parenthesis') hn = HumanName('Jenny "JJ" Baker (Johnson)', constants=C) self.m(hn.first, "Jenny", hn) self.m(hn.last, "Baker", hn) self.m(hn.nickname, "JJ", hn) self.m(hn.maiden, "Johnson", hn) def test_maiden_off_by_default_parenthesis_still_routes_to_nickname(self) -> None: hn = HumanName("Baker (Johnson), Jenny") self.m(hn.first, "Jenny", hn) self.m(hn.last, "Baker", hn) self.m(hn.nickname, "Johnson", hn) self.m(hn.maiden, "", hn) def test_suffix_shaped_content_in_maiden_bucket_stays_in_place(self) -> None: # The #189 suffix-shaped carve-out in handle_match() applies # regardless of which bucket the delimiter routes to. C = Constants() C.maiden_delimiters['parenthesis'] = C.nickname_delimiters.pop('parenthesis') hn = HumanName("Baker (Jr.), Jenny", constants=C) self.m(hn.first, "Jenny", hn) self.m(hn.last, "Baker", hn) self.m(hn.suffix, "Jr.", hn) self.m(hn.maiden, "", hn) def test_marker_inside_maiden_parenthesis_is_consumed(self) -> None: # #329 through the v1 API, which is where the release log # promises it. The facade runner covers the same ground by # translating the shared case table, but it decides for itself # which rows it can express -- when that gate was wrong these # rows silently skipped, so a test that spells the v1 idiom out # directly is what keeps the promise pinned independently of it. # Same value as the bare "Jane Smith née Jones", which is the # agreement #329 was about. C = Constants() C.maiden_delimiters['parenthesis'] = C.nickname_delimiters.pop('parenthesis') hn = HumanName("Jane Smith (née Jones)", constants=C) self.m(hn.first, "Jane", hn) self.m(hn.last, "Smith", hn) self.m(hn.maiden, "Jones", hn) def test_unmarked_maiden_parenthesis_keeps_every_word(self) -> None: # The other side of the same rule: the drop is conditioned on # the first word being a marker, so ordinary two-word content # arrives whole. C = Constants() C.maiden_delimiters['parenthesis'] = C.nickname_delimiters.pop('parenthesis') hn = HumanName("Jane Smith (Mary Jones)", constants=C) self.m(hn.first, "Jane", hn) self.m(hn.last, "Smith", hn) self.m(hn.maiden, "Mary Jones", hn) def test_lone_marker_word_in_its_own_parenthesis_is_a_name(self) -> None: # `Nee` is a real surname (Irish Ní/Nee, and a Chinese # romanization), so a one-word clause is content whatever it # spells -- even with a second clause following that could read # as the name it marks. C = Constants() C.maiden_delimiters['parenthesis'] = C.nickname_delimiters.pop('parenthesis') hn = HumanName("Jane Smith (Nee) (Jones)", constants=C) self.m(hn.first, "Jane", hn) self.m(hn.last, "Smith", hn) self.m(hn.maiden, "Nee Jones", hn) def test_maiden_appears_in_as_dict_via_routing(self) -> None: C = Constants() C.maiden_delimiters['parenthesis'] = C.nickname_delimiters.pop('parenthesis') hn = HumanName("Baker (Johnson), Jenny", constants=C) self.assertEqual(hn.as_dict()['maiden'], "Johnson") def test_unresolvable_string_sentinel_raises(self) -> None: # A string value that doesn't name a real delimiter used to silently # fall back to EMPTY_REGEX in v1 (matching at every position and # corrupting parsing) until 1.3 made the parse raise ValueError. 2.0 # enforces the invariant one step earlier: the delimiter managers # accept only the named sentinels, so the typo fails loudly at # assignment instead of at the next parse. C = Constants() with pytest.raises(TypeError, match="sentinel"): C.nickname_delimiters['typo'] = 'parenthesus' def test_routing_same_delimiter_to_both_buckets_nickname_wins(self) -> None: # Misuse case: assigning the same key into both dicts instead of # moving it with pop() (as the docs instruct). nickname_delimiters is # processed first in parse_nicknames()'s bucket loop, so it consumes # the match via re.sub() before maiden_delimiters ever sees it -- # maiden stays empty. Pinning this precedence so it doesn't silently # change if the bucket processing order is ever reordered. C = Constants() C.maiden_delimiters['parenthesis'] = C.regexes['parenthesis'] hn = HumanName("Baker (Johnson), Jenny", constants=C) self.m(hn.nickname, "Johnson", hn) self.m(hn.maiden, "", hn) python-nameparser-2.1.0/tests/test_output_format.py000066400000000000000000000207611523530431700226620ustar00rootroot00000000000000import pytest from nameparser import HumanName from nameparser.config import Constants from tests.base import HumanNameTestBase class HumanNameOutputFormatTests(HumanNameTestBase): def test_formatting_init_argument(self) -> None: hn = HumanName("Rev John A. Kenneth Doe III (Kenny)", string_format="TEST1") self.assertEqual(str(hn), "TEST1") # The four *_constants_attribute tests below ran on the shared CONSTANTS # singleton in v1; 2.0 deprecates shared mutation, so they use a private # Constants passed as constants= (the migration-guide idiom) -- the # config attribute under test is the same either way. def test_formatting_constants_attribute(self) -> None: c = Constants() c.string_format = "TEST2" hn = HumanName("Rev John A. Kenneth Doe III (Kenny)", constants=c) self.assertEqual(str(hn), "TEST2") def test_capitalize_name_constants_attribute(self) -> None: c = Constants() c.capitalize_name = True hn = HumanName("bob v. de la macdole-eisenhower phd", constants=c) self.assertEqual(str(hn), "Bob V. de la MacDole-Eisenhower Ph.D.") def test_force_mixed_case_capitalization_constants_attribute(self) -> None: c = Constants() c.force_mixed_case_capitalization = True hn = HumanName('Shirley Maclaine', constants=c) hn.capitalize() self.assertEqual(str(hn), "Shirley MacLaine") def test_capitalize_name_and_force_mixed_case_capitalization_constants_attributes(self) -> None: c = Constants() c.capitalize_name = True c.force_mixed_case_capitalization = True hn = HumanName('Shirley Maclaine', constants=c) self.assertEqual(str(hn), "Shirley MacLaine") def test_quote_nickname_formating(self) -> None: hn = HumanName("Rev John A. Kenneth Doe III (Kenny)") hn.string_format = "{title} {first} {middle} {last} {suffix} '{nickname}'" self.assertEqual(str(hn), "Rev John A. Kenneth Doe III 'Kenny'") hn.string_format = "{last}, {title} {first} {middle}, {suffix} '{nickname}'" self.assertEqual(str(hn), "Doe, Rev John A. Kenneth, III 'Kenny'") def test_formating_removing_keys_from_format_string(self) -> None: hn = HumanName("Rev John A. Kenneth Doe III (Kenny)") hn.string_format = "{title} {first} {middle} {last} {suffix} '{nickname}'" self.assertEqual(str(hn), "Rev John A. Kenneth Doe III 'Kenny'") hn.string_format = "{last}, {title} {first} {middle}, {suffix}" self.assertEqual(str(hn), "Doe, Rev John A. Kenneth, III") hn.string_format = "{last}, {title} {first} {middle}" self.assertEqual(str(hn), "Doe, Rev John A. Kenneth") hn.string_format = "{last}, {first} {middle}" self.assertEqual(str(hn), "Doe, John A. Kenneth") hn.string_format = "{last}, {first}" self.assertEqual(str(hn), "Doe, John") hn.string_format = "{first} {last}" self.assertEqual(str(hn), "John Doe") def test_formating_removing_pieces_from_name_buckets(self) -> None: hn = HumanName("Rev John A. Kenneth Doe III (Kenny)") hn.string_format = "{title} {first} {middle} {last} {suffix} '{nickname}'" self.assertEqual(str(hn), "Rev John A. Kenneth Doe III 'Kenny'") hn.string_format = "{title} {first} {middle} {last} {suffix}" self.assertEqual(str(hn), "Rev John A. Kenneth Doe III") hn.middle = '' self.assertEqual(str(hn), "Rev John Doe III") hn.suffix = '' self.assertEqual(str(hn), "Rev John Doe") hn.title = '' self.assertEqual(str(hn), "John Doe") def test_formating_of_nicknames_with_parenthesis(self) -> None: hn = HumanName("Rev John A. Kenneth Doe III (Kenny)") hn.string_format = "{title} {first} {middle} {last} {suffix} ({nickname})" self.assertEqual(str(hn), "Rev John A. Kenneth Doe III (Kenny)") hn.nickname = '' self.assertEqual(str(hn), "Rev John A. Kenneth Doe III") def test_formating_of_nicknames_with_single_quotes(self) -> None: hn = HumanName("Rev John A. Kenneth Doe III (Kenny)") hn.string_format = "{title} {first} {middle} {last} {suffix} '{nickname}'" self.assertEqual(str(hn), "Rev John A. Kenneth Doe III 'Kenny'") hn.nickname = '' self.assertEqual(str(hn), "Rev John A. Kenneth Doe III") def test_formating_of_nicknames_with_double_quotes(self) -> None: hn = HumanName("Rev John A. Kenneth Doe III (Kenny)") hn.string_format = "{title} {first} {middle} {last} {suffix} \"{nickname}\"" self.assertEqual(str(hn), "Rev John A. Kenneth Doe III \"Kenny\"") hn.nickname = '' self.assertEqual(str(hn), "Rev John A. Kenneth Doe III") def test_formating_of_nicknames_in_middle(self) -> None: hn = HumanName("Rev John A. Kenneth Doe III (Kenny)") hn.string_format = "{title} {first} ({nickname}) {middle} {last} {suffix}" self.assertEqual(str(hn), "Rev John (Kenny) A. Kenneth Doe III") hn.nickname = '' self.assertEqual(str(hn), "Rev John A. Kenneth Doe III") def test_name_containing_none_substring_survives_formatting(self) -> None: # Residue of the #254 regression: v1's None-mode __str__ once # scrubbed the literal string 'None' from formatted output. The # None mode is gone in 2.0 (#255), but real name text spelled # 'None'/'Nonez' must still never be scrubbed by formatting. self.assertEqual(str(HumanName("Nonez Smith")), "Nonez Smith") self.assertEqual(str(HumanName("None Smith")), "None Smith") def test_empty_field_drops_surrounding_whitespace(self) -> None: # issue #139: adjacent whitespace/punctuation should be dropped when a field is empty hn = HumanName("John Smith") hn.string_format = "{last} {suffix}, {first}" self.assertEqual(str(hn), "Smith, John") def test_empty_field_present_suffix_unaffected(self) -> None: hn = HumanName("John Smith Jr") hn.string_format = "{last} {suffix}, {first}" self.assertEqual(str(hn), "Smith Jr, John") def test_multiple_empty_fields_before_comma(self) -> None: hn = HumanName("John Smith") hn.string_format = "{title} {suffix}, {first} {last}" self.assertEqual(str(hn), "John Smith") def test_remove_emojis(self) -> None: hn = HumanName("Sam Smith 😊") self.m(hn.first, "Sam", hn) self.m(hn.last, "Smith", hn) self.assertEqual(str(hn), "Sam Smith") def test_keep_non_emojis(self) -> None: hn = HumanName("∫≜⩕ Smith 😊") self.m(hn.first, "∫≜⩕", hn) self.m(hn.last, "Smith", hn) self.assertEqual(str(hn), "∫≜⩕ Smith") def test_keep_emojis_opt_out_moved_to_policy(self) -> None: # v1's regexes.emoji = False opt-out is not supported in 2.0 # (deliberate divergence, migration spec section 3 uniform rule): # the assignment raises, and the error names the replacement, # Policy(strip_emoji=False). constants = Constants() with pytest.raises(TypeError, match="strip_emoji"): constants.regexes.emoji = False # type: ignore[assignment] def test_remove_bidi_control_chars(self) -> None: # LRM/RLM and friends ride along with copy-pasted names and stick to # the parts they surround. Covers every character in the bidi set. for mark in ("\u200e", "\u200f", "\u061c", "\u202a", "\u202b", "\u202c", "\u202d", "\u202e", "\u2066", "\u2067", "\u2068", "\u2069"): hn = HumanName(mark + "John" + mark + " Smith") self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) def test_bidi_stripped_name_compares_equal(self) -> None: # The reported symptom: an invisible RLM around an RTL name makes the # parsed part fail equality against the clean string (issue #266). hn = HumanName("\u200fمحمد بن سلمان\u200f") self.assertEqual(hn.first, "محمد") def test_keep_bidi_opt_out_moved_to_policy(self) -> None: # v1's regexes.bidi = False opt-out is not supported in 2.0 # (deliberate divergence, migration spec section 3 uniform rule): # the assignment raises, and the error names the replacement, # Policy(strip_bidi=False). constants = Constants() with pytest.raises(TypeError, match="strip_bidi"): constants.regexes.bidi = False # type: ignore[assignment] python-nameparser-2.1.0/tests/test_prefixes.py000066400000000000000000000403221523530431700215720ustar00rootroot00000000000000import pytest from nameparser import HumanName from nameparser.config import CONSTANTS, Constants from nameparser.config.prefixes import NON_FIRST_NAME_PREFIXES from tests.base import HumanNameTestBase class PrefixesTestCase(HumanNameTestBase): def test_prefix(self) -> None: hn = HumanName("Juan del Sur") self.m(hn.first, "Juan", hn) self.m(hn.last, "del Sur", hn) def test_prefix_with_period(self) -> None: hn = HumanName("Jill St. John") self.m(hn.first, "Jill", hn) self.m(hn.last, "St. John", hn) def test_prefix_before_two_part_last_name(self) -> None: hn = HumanName("pennie von bergen wessels") self.m(hn.first, "pennie", hn) self.m(hn.last, "von bergen wessels", hn) def test_prefix_is_first_name(self) -> None: hn = HumanName("Van Johnson") self.m(hn.first, "Van", hn) self.m(hn.last, "Johnson", hn) def test_prefix_is_first_name_with_middle_name(self) -> None: hn = HumanName("Van Jeremy Johnson") self.m(hn.first, "Van", hn) self.m(hn.middle, "Jeremy", hn) self.m(hn.last, "Johnson", hn) def test_prefix_before_two_part_last_name_with_suffix(self) -> None: hn = HumanName("pennie von bergen wessels III") self.m(hn.first, "pennie", hn) self.m(hn.last, "von bergen wessels", hn) self.m(hn.suffix, "III", hn) def test_prefix_before_two_part_last_name_with_acronym_suffix(self) -> None: hn = HumanName("pennie von bergen wessels M.D.") self.m(hn.first, "pennie", hn) self.m(hn.last, "von bergen wessels", hn) self.m(hn.suffix, "M.D.", hn) def test_title_before_and_after_prefixed_last_name(self) -> None: # Issue #100: a repeated title/suffix token ("dr") before AND after a # prefixed last name used to corrupt the middle name into # " dr Vincent van" because the suffix-boundary lookup matched the # LEADING "dr" instead of the trailing one. hn = HumanName("dr Vincent van Gogh dr") self.m(hn.title, "dr", hn) self.m(hn.first, "Vincent", hn) self.m(hn.middle, "", hn) self.m(hn.last, "van Gogh", hn) self.m(hn.suffix, "dr", hn) def test_suffix_token_collision_with_two_word_prefix(self) -> None: # Same fix as #100 but with a two-word prefix ("van der"). Exercises a # different iteration count through the prefix-joining loop. hn = HumanName("dr Vincent van der Gogh dr") self.m(hn.title, "dr", hn) self.m(hn.first, "Vincent", hn) self.m(hn.middle, "", hn) self.m(hn.last, "van der Gogh", hn) self.m(hn.suffix, "dr", hn) def test_title_before_and_after_prefixed_last_name_with_middle(self) -> None: # The pre-fix bug corrupted the middle field; verify it is not disturbed # when a genuine middle name is present alongside the repeated token. hn = HumanName("dr Vincent James van Gogh dr") self.m(hn.title, "dr", hn) self.m(hn.first, "Vincent", hn) self.m(hn.middle, "James", hn) self.m(hn.last, "van Gogh", hn) self.m(hn.suffix, "dr", hn) @pytest.mark.timeout(2) def test_many_repeated_prefixes_does_not_blow_up(self) -> None: # Issue #108: a name with a long run of repeated prefixes used to grow # the pieces list exponentially and exhaust memory. The 2-second timeout # enforces this locally and in CI — if the test hangs, an exponential # regression has been reintroduced. name = "Jan " + "van der " * 30 + "Berg" hn = HumanName(name) self.m(hn.first, "Jan", hn) self.assertIn("Berg", hn.last) def test_two_part_last_name_with_suffix_comma(self) -> None: hn = HumanName("pennie von bergen wessels, III") self.m(hn.first, "pennie", hn) self.m(hn.last, "von bergen wessels", hn) self.m(hn.suffix, "III", hn) def test_two_part_last_name_with_suffix(self) -> None: hn = HumanName("von bergen wessels, pennie III") self.m(hn.first, "pennie", hn) self.m(hn.last, "von bergen wessels", hn) self.m(hn.suffix, "III", hn) def test_last_name_two_part_last_name_with_two_suffixes(self) -> None: hn = HumanName("von bergen wessels MD, pennie III") self.m(hn.first, "pennie", hn) self.m(hn.last, "von bergen wessels", hn) self.m(hn.suffix, "MD, III", hn) def test_comma_two_part_last_name_with_acronym_suffix(self) -> None: hn = HumanName("von bergen wessels, pennie MD") self.m(hn.first, "pennie", hn) self.m(hn.last, "von bergen wessels", hn) self.m(hn.suffix, "MD", hn) def test_comma_two_part_last_name_with_suffix_in_first_part(self) -> None: # I'm kinda surprised this works, not really sure if this is a # realistic place for a suffix to be. hn = HumanName("von bergen wessels MD, pennie") self.m(hn.first, "pennie", hn) self.m(hn.last, "von bergen wessels", hn) self.m(hn.suffix, "MD", hn) def test_title_two_part_last_name_with_suffix_in_first_part(self) -> None: hn = HumanName("pennie von bergen wessels MD, III") self.m(hn.first, "pennie", hn) self.m(hn.last, "von bergen wessels", hn) self.m(hn.suffix, "MD, III", hn) def test_portuguese_dos(self) -> None: hn = HumanName("Rafael Sousa dos Anjos") self.m(hn.first, "Rafael", hn) self.m(hn.middle, "Sousa", hn) self.m(hn.last, "dos Anjos", hn) def test_portuguese_prefixes(self) -> None: hn = HumanName("Joao da Silva do Amaral de Souza") self.m(hn.first, "Joao", hn) self.m(hn.middle, "da Silva do Amaral", hn) self.m(hn.last, "de Souza", hn) def test_three_conjunctions(self) -> None: hn = HumanName("Dr. Juan Q. Xavier de la dos Vega III") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la dos Vega", hn) self.m(hn.title, "Dr.", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.suffix, "III", hn) def test_lastname_three_conjunctions(self) -> None: hn = HumanName("de la dos Vega, Dr. Juan Q. Xavier III") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la dos Vega", hn) self.m(hn.title, "Dr.", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.suffix, "III", hn) def test_comma_three_conjunctions(self) -> None: hn = HumanName("Dr. Juan Q. Xavier de la dos Vega, III") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la dos Vega", hn) self.m(hn.title, "Dr.", hn) self.m(hn.middle, "Q. Xavier", hn) self.m(hn.suffix, "III", hn) # The subset-of-PREFIXES and disjoint-from-BOUND_FIRST_NAMES invariants # are enforced by import-time asserts in nameparser/config/prefixes.py, # so they are not repeated as tests here. def test_non_first_name_prefixes_expected_members(self) -> None: # 'abu' is in PREFIXES but excluded (it is a bound_first_name); # 'von'/'van'/'della'/'di'/'del' are excluded (they can be first names). self.assertIn('de', NON_FIRST_NAME_PREFIXES) self.assertIn('dos', NON_FIRST_NAME_PREFIXES) self.assertNotIn('abu', NON_FIRST_NAME_PREFIXES) self.assertNotIn('von', NON_FIRST_NAME_PREFIXES) self.assertNotIn('van', NON_FIRST_NAME_PREFIXES) self.assertNotIn('della', NON_FIRST_NAME_PREFIXES) def test_constants_exposes_non_first_name_prefixes(self) -> None: self.assertEqual(set(CONSTANTS.non_first_name_prefixes), NON_FIRST_NAME_PREFIXES) def test_non_first_name_prefixes_disjoint_from_titles(self) -> None: # A member that is also a title is consumed as a title before the fold # would run, making it inert (the st/ste footgun). Pin the invariant on # the live default sets. self.assertEqual( set(CONSTANTS.non_first_name_prefixes) & set(CONSTANTS.titles), set(), ) def test_non_first_name_prefixes_constructor_arg(self) -> None: c = Constants(non_first_name_prefixes={'zzz'}) self.assertEqual(set(c.non_first_name_prefixes), {'zzz'}) class LastNamePrefixSplitTestCase(HumanNameTestBase): def test_van_gogh_last_base(self) -> None: hn = HumanName("Vincent van Gogh") self.m(hn.last_base, "Gogh", hn) def test_van_gogh_last_prefixes(self) -> None: hn = HumanName("Vincent van Gogh") self.m(hn.last_prefixes, "van", hn) def test_van_gogh_last_base_list(self) -> None: hn = HumanName("Vincent van Gogh") self.m(hn.last_base_list, ["Gogh"], hn) def test_van_gogh_last_prefixes_list(self) -> None: hn = HumanName("Vincent van Gogh") self.m(hn.last_prefixes_list, ["van"], hn) def test_von_bergen_wessels(self) -> None: hn = HumanName("pennie von bergen wessels") self.m(hn.last_base, "bergen wessels", hn) self.m(hn.last_prefixes, "von", hn) self.m(hn.last_base_list, ["bergen", "wessels"], hn) self.m(hn.last_prefixes_list, ["von"], hn) def test_de_la_vega_multiword_prefix(self) -> None: hn = HumanName("Juan de la Vega") self.m(hn.last_base, "Vega", hn) self.m(hn.last_prefixes, "de la", hn) self.m(hn.last_prefixes_list, ["de", "la"], hn) def test_no_prefix(self) -> None: hn = HumanName("John Smith") self.m(hn.last_base, "Smith", hn) self.m(hn.last_prefixes, "", hn) # self.m() coerces [] via `expected or empty_attribute_default`; use assertEqual for empty lists self.assertEqual(hn.last_prefixes_list, []) def test_do_guard_surname_equals_prefix_word(self) -> None: # "Do" is in PREFIXES; without the guard last_base would be empty hn = HumanName("Anh Do") self.m(hn.last_base, "Do", hn) self.m(hn.last_prefixes, "", hn) def test_all_particles_guard(self) -> None: # Artificial case: last name whose every word is a prefix — must not strip hn = HumanName("Smith van der") # last="van der"; both words are prefixes — guard fires, base = full last self.m(hn.last_base, hn.last, hn) self.m(hn.last_prefixes, "", hn) def test_da_silva_title_plus_prefix(self) -> None: hn = HumanName("Dra. Andréia da Silva") self.m(hn.last_base, "Silva", hn) self.m(hn.last_prefixes, "da", hn) def test_empty_name(self) -> None: hn = HumanName() self.m(hn.last_base, "", hn) self.m(hn.last_prefixes, "", hn) # self.m() coerces [] via `expected or empty_attribute_default`; use assertEqual for empty lists self.assertEqual(hn.last_base_list, []) self.assertEqual(hn.last_prefixes_list, []) def test_case_insensitive_prefix_detection(self) -> None: hn = HumanName("VINCENT VAN GOGH") self.m(hn.last_prefixes, "VAN", hn) self.m(hn.last_base, "GOGH", hn) # --- targets: leading non-first-name prefix becomes the surname --- def test_leading_non_first_name_prefix_de(self) -> None: hn = HumanName("de Mesnil") self.m(hn.first, "", hn) self.m(hn.middle, "", hn) self.m(hn.last, "de Mesnil", hn) def test_leading_non_first_name_prefix_dos(self) -> None: hn = HumanName("dos Santos") self.m(hn.first, "", hn) self.m(hn.last, "dos Santos", hn) def test_leading_non_first_name_prefix_chain(self) -> None: hn = HumanName("de la Vega") self.m(hn.first, "", hn) self.m(hn.last, "de la Vega", hn) def test_leading_non_first_name_prefix_derived_props(self) -> None: hn = HumanName("de Mesnil") self.m(hn.last_prefixes, "de", hn) self.m(hn.last_base, "Mesnil", hn) def test_non_first_name_prefix_with_custom_title(self) -> None: # 'Gunny' is NOT a default title -> genuinely exercises the custom-title # path. Title is consumed first (first_list == ['']), so the fold does # not fire and the surname is already correct; asserts we don't corrupt # the title case. Runs on a private Constants (2.0 deprecates shared- # CONSTANTS mutation; the custom-title path under test is identical). c = Constants() c.titles.add('gunny') hn = HumanName("Gunny de Mesnil", constants=c) self.m(hn.title, "Gunny", hn) self.m(hn.first, "", hn) self.m(hn.last, "de Mesnil", hn) def test_repeated_prefix_chain_de_la(self) -> None: hn = HumanName("Juan de la de la Vega") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la de la Vega", hn) def test_repeated_prefix_chain_van_der(self) -> None: hn = HumanName("Charles van der van der Berg") self.m(hn.first, "Charles", hn) self.m(hn.last, "van der van der Berg", hn) def test_triple_repeated_prefix_chain(self) -> None: # a stronger regression guard than the 2-repeat cases above: the # contiguous-prefix absorption loop should chain any number of # repeats, not just handle exactly two hn = HumanName("Juan de la de la de la Vega") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la de la de la Vega", hn) def test_repeated_prefix_chain_followed_by_suffix(self) -> None: # the prefix-run absorption loop and the suffix-boundary loop share # the same index variable, so a repeated chain immediately # followed by a suffix is worth pinning down explicitly hn = HumanName("Juan de la de la Vega Jr.") self.m(hn.first, "Juan", hn) self.m(hn.last, "de la de la Vega", hn) self.m(hn.suffix, "Jr.", hn) # --- safety: excluded / ambiguous particles are unchanged --- def test_leading_von_is_unchanged(self) -> None: hn = HumanName("von Braun") self.m(hn.first, "von", hn) self.m(hn.last, "Braun", hn) def test_leading_van_is_unchanged(self) -> None: hn = HumanName("Van Johnson") self.m(hn.first, "Van", hn) self.m(hn.last, "Johnson", hn) def test_leading_della_is_unchanged(self) -> None: hn = HumanName("Della Reese") self.m(hn.first, "Della", hn) self.m(hn.last, "Reese", hn) def test_leading_di_is_unchanged(self) -> None: hn = HumanName("Di Caprio") self.m(hn.first, "Di", hn) self.m(hn.last, "Caprio", hn) def test_leading_del_is_unchanged(self) -> None: hn = HumanName("Del Toro") self.m(hn.first, "Del", hn) self.m(hn.last, "Toro", hn) def test_non_leading_prefix_is_unchanged(self) -> None: hn = HumanName("Jean de Mesnil") self.m(hn.first, "Jean", hn) self.m(hn.last, "de Mesnil", hn) # --- guard: bare particle with nothing to attach to --- def test_bare_non_first_name_prefix_guard(self) -> None: hn = HumanName("de") self.m(hn.first, "de", hn) self.m(hn.last, "", hn) # --- interactions with opt-in handlers that also run in post_process --- def test_leading_non_first_name_prefix_case_insensitive(self) -> None: hn = HumanName("DE MESNIL") self.m(hn.first, "", hn) self.m(hn.last, "DE MESNIL", hn) def test_leading_non_first_name_prefix_with_suffix(self) -> None: hn = HumanName("de Mesnil Jr.") self.m(hn.first, "", hn) self.m(hn.last, "de Mesnil", hn) self.m(hn.suffix, "Jr.", hn) def test_leading_non_first_name_prefix_with_patronymic_name_order(self) -> None: # The fold requires a single-token first_list; patronymic_name_order # only matters once first_list has more than one piece, so the fold # and this opt-in handler don't fight over the same input shape here. constants = Constants(patronymic_name_order=True) hn = HumanName("de Mesnil", constants=constants) self.m(hn.first, "", hn) self.m(hn.last, "de Mesnil", hn) def test_leading_non_first_name_prefix_with_middle_name_as_last(self) -> None: # handle_non_first_name_prefix runs first and empties middle_list, so # the later opt-in handle_middle_name_as_last has nothing left to do. constants = Constants(middle_name_as_last=True) hn = HumanName("de Mesnil Garcia", constants=constants) self.m(hn.first, "", hn) self.m(hn.last, "de Mesnil Garcia", hn) python-nameparser-2.1.0/tests/test_python_api.py000066400000000000000000000625731523530431700221330ustar00rootroot00000000000000import copy import pickle import re import warnings import pytest try: import dill except ImportError: dill = False # type: ignore[assignment] from nameparser import HumanName from nameparser.config import CONSTANTS, Constants, TupleManager from tests.base import HumanNameTestBase class HumanNamePythonTests(HumanNameTestBase): def test_utf8(self) -> None: hn = HumanName("de la Véña, Jüan") self.m(hn.first, "Jüan", hn) self.m(hn.last, "de la Véña", hn) def test_string_output(self) -> None: hn = HumanName("de la Véña, Jüan") self.m(str(hn), "Jüan de la Véña", hn) def test_escaped_utf8_bytes(self) -> None: # bytes input was removed in 2.0 (#245, warned since 1.3.0): # decode first with pytest.raises(TypeError, match="decode"): HumanName(b'B\xc3\xb6ck, Gerald') # type: ignore[arg-type] hn = HumanName(b'B\xc3\xb6ck, Gerald'.decode('utf-8')) self.m(hn.first, "Gerald", hn) self.m(hn.last, "Böck", hn) def test_bytes_full_name_raises_with_decode_hint(self) -> None: # bytes input was removed in 2.0 (#245); both entry points raise with pytest.raises(TypeError, match="decode"): HumanName(b'John Smith') # type: ignore[arg-type] hn = HumanName("Jane Doe") with pytest.raises(TypeError, match="decode"): hn.full_name = b'John Smith' # type: ignore[assignment] # the failed assignment must not have corrupted the parse self.m(hn.first, "Jane", hn) def test_str_full_name_does_not_warn(self) -> None: with warnings.catch_warnings(): warnings.simplefilter("error") hn = HumanName("John Smith") hn.full_name = "Jane Doe" self.m(hn.first, "Jane", hn) def test_len(self) -> None: hn = HumanName("Doe-Ray, Dr. John P., CLU, CFP, LUTC") self.m(len(hn), 5, hn) hn = HumanName("John Doe") self.m(len(hn), 2, hn) # empty input parses to an all-empty name; len == 0 is the # documented emptiness check (see usage.rst) self.assertEqual(len(HumanName("")), 0) def test_iteration_restarts_after_break(self) -> None: hn = HumanName("John Doe") for _ in hn: break # a plain loop, not list(hn): list() presizes via __len__, which # under the old shared-cursor implementation reset the cursor and # masked this bug collected = [] for part in hn: collected.append(part) self.assertEqual(collected, ["John", "Doe"]) def test_iterators_are_independent(self) -> None: hn = HumanName("John Doe") it1 = iter(hn) it2 = iter(hn) self.assertEqual(next(it1), "John") self.assertEqual(next(it2), "John") self.assertEqual(next(it1), "Doe") self.assertEqual(next(it2), "Doe") def test_len_during_iteration(self) -> None: hn = HumanName("John Doe") it = iter(hn) self.assertEqual(next(it), "John") # len() must count all members and leave the live iterator intact self.assertEqual(len(hn), 2) self.assertEqual(next(it), "Doe") def test_instance_is_not_its_own_iterator(self) -> None: # iterator state must never live on the instance; see release log # for the next(name) -> next(iter(name)) migration hn = HumanName("John Doe") with pytest.raises(TypeError): next(hn) # type: ignore[call-overload] def test_iterating_empty_name_yields_nothing(self) -> None: collected = [] for part in HumanName(""): collected.append(part) self.assertEqual(collected, []) @pytest.mark.skipif(not dill, reason="requires python-dill module to test pickling") def test_config_pickle(self) -> None: constants = Constants() self.assertTrue(dill.pickles(constants)) @pytest.mark.skipif(not dill, reason="requires python-dill module to test pickling") def test_name_instance_pickle(self) -> None: hn = HumanName("Title First Middle Middle Last, Jr.") self.assertTrue(dill.pickles(hn)) def test_name_instance_pickle_preserves_instance_config(self) -> None: """A HumanName carrying its own config must parse identically after a pickle round-trip. HumanName pickles its instance Constants (``.C``) through the default __dict__ path, so a broken Constants round-trip silently produced a differently-configured parser on the other side. """ # Passing None as the second argument gives this name its own Constants. hn = HumanName("Smith, Dr. John", Constants()) hn.C.titles.add('chancellor') hn.parse_full_name() # Safe: round-tripping a HumanName the test just built, not untrusted data. restored = pickle.loads(pickle.dumps(hn)) self.assertIn('chancellor', restored.C.titles) restored.full_name = "Chancellor Jane Smith" self.assertEqual(restored.title, "Chancellor") def test_name_instance_deepcopy(self) -> None: """copy.deepcopy of a HumanName must round-trip. HumanName has no custom copy hooks, so deepcopy recurses into its Constants (`.C`), and previously into `.C.regexes`, whose __getattr__ answered copy's __deepcopy__ probe with a re.Pattern — making deepcopy of *any* HumanName raise TypeError. """ hn = HumanName("Dr. John P. Doe-Ray, CLU") dup = copy.deepcopy(hn) self.assertEqual(str(dup), str(hn)) def test_name_instance_deepcopy_isolates_instance_config(self) -> None: """A deep-copied HumanName with its own config must be independent.""" hn = HumanName("Smith, Dr. John", Constants()) hn.C.titles.add('chancellor') dup = copy.deepcopy(hn) dup.C.titles.add('marker') self.assertIn('chancellor', dup.C.titles) self.assertNotIn('marker', hn.C.titles) def test_pickle_default_name_preserves_singleton_identity(self) -> None: """A default HumanName must re-attach to CONSTANTS after a pickle round-trip. Without __getstate__/__setstate__, pickle serializes .C by value, so the restored name gets a detached copy — has_own_config flips to True and every pickled default name carries a full Constants copy. """ hn = HumanName("John Doe") self.assertFalse(hn.has_own_config) self.assertIs(hn.C, CONSTANTS) # Safe: round-tripping an object we just built, not untrusted data. restored = pickle.loads(pickle.dumps(hn)) self.assertIs(restored.C, CONSTANTS) self.assertFalse(restored.has_own_config) self.assertEqual(str(restored), str(hn)) self.assertEqual(restored.first, hn.first) self.assertEqual(restored.last, hn.last) def test_pickle_instance_config_name_preserves_own_config(self) -> None: """A HumanName with its own Constants must not be collapsed onto CONSTANTS after pickle.""" hn = HumanName("Smith, Dr. John", Constants()) hn.C.titles.add('chancellor') hn.parse_full_name() self.assertTrue(hn.has_own_config) self.assertIsNot(hn.C, CONSTANTS) # Safe: round-tripping a HumanName the test just built, not untrusted data. restored = pickle.loads(pickle.dumps(hn)) self.assertTrue(restored.has_own_config) self.assertIsNot(restored.C, CONSTANTS) self.assertIn('chancellor', restored.C.titles) def test_shallow_copy_default_name_preserves_singleton_identity(self) -> None: """copy.copy of a default HumanName shares the CONSTANTS reference without hooks.""" hn = HumanName("John Doe") sc = copy.copy(hn) self.assertIs(sc.C, CONSTANTS) self.assertFalse(sc.has_own_config) def test_deepcopy_default_name_preserves_singleton_identity(self) -> None: """copy.deepcopy of a default HumanName must re-attach to CONSTANTS.""" hn = HumanName("John Doe") dup = copy.deepcopy(hn) self.assertIs(dup.C, CONSTANTS) self.assertFalse(dup.has_own_config) self.assertEqual(str(dup), str(hn)) self.assertEqual(dup.first, hn.first) self.assertEqual(dup.last, hn.last) def test_comparison(self) -> None: # == and hash reverted to object identity in 2.0 (#223, warned since # 1.3.0): distinct instances never compare equal, however they parse, # and strings never match. matches()/comparison_key() are the # replacements (covered below). hn1 = HumanName("Doe-Ray, Dr. John P., CLU, CFP, LUTC") hn2 = HumanName("Dr. John P. Doe-Ray, CLU, CFP, LUTC") self.assertTrue(hn1 == hn1) self.assertFalse(hn1 == hn2) self.assertIsNot(hn1, hn2) self.assertFalse(hn1 == "Dr. John P. Doe-Ray CLU, CFP, LUTC") self.assertTrue(hn1.matches(hn2)) # the 2.0 spelling of the old == hn1 = HumanName("Doe, Dr. John P., CLU, CFP, LUTC") hn2 = HumanName("Dr. John P. Doe-Ray, CLU, CFP, LUTC") self.assertTrue(not hn1 == hn2) self.assertTrue(not hn1 == 0) self.assertTrue(not hn1 == "test") self.assertTrue(not hn1 == ["test"]) self.assertTrue(not hn1 == {"test": hn2}) def test_assignment_to_full_name(self) -> None: hn = HumanName("John A. Kenneth Doe, Jr.") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A. Kenneth", hn) self.m(hn.suffix, "Jr.", hn) hn.full_name = "Juan Velasquez y Garcia III" self.m(hn.first, "Juan", hn) self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "III", hn) def test_assignment_to_attribute(self) -> None: hn = HumanName("John A. Kenneth Doe, Jr.") hn.last = "de la Vega" self.m(hn.last, "de la Vega", hn) hn.title = "test" self.m(hn.title, "test", hn) hn.first = "test" self.m(hn.first, "test", hn) hn.middle = "test" self.m(hn.middle, "test", hn) hn.suffix = "test" self.m(hn.suffix, "test", hn) with pytest.raises(TypeError): hn.suffix = [['test']] # type: ignore[list-item] with pytest.raises(TypeError): hn.suffix = {"test": "test"} # type: ignore[assignment] def test_assign_list_to_attribute(self) -> None: hn = HumanName("John A. Kenneth Doe, Jr.") hn.title = ["test1", "test2"] self.m(hn.title, "test1 test2", hn) hn.first = ["test3", "test4"] self.m(hn.first, "test3 test4", hn) hn.middle = ["test5", "test6", "test7"] self.m(hn.middle, "test5 test6 test7", hn) hn.last = ["test8", "test9", "test10"] self.m(hn.last, "test8 test9 test10", hn) hn.suffix = ['test'] self.m(hn.suffix, "test", hn) def test_comparison_case_insensitive(self) -> None: # 2.0 identity semantics (#223): equivalently-parsed instances no # longer compare ==; matches() carries the case-insensitive # component comparison instead hn1 = HumanName("Doe-Ray, Dr. John P., CLU, CFP, LUTC") hn2 = HumanName("dr. john p. doe-Ray, CLU, CFP, LUTC") self.assertFalse(hn1 == hn2) self.assertIsNot(hn1, hn2) self.assertFalse(hn1 == "Dr. John P. Doe-ray clu, CFP, LUTC") self.assertTrue(hn1.matches(hn2)) self.assertTrue(hn1.matches("Dr. John P. Doe-ray clu, CFP, LUTC")) def test_hash_is_identity_based(self) -> None: # 2.0 identity semantics (#223): hash reverted to the object default, # consistent with identity ==; equal-parsing instances stay distinct # in sets/dicts. comparison_key() is the dedup replacement. hn1 = HumanName("Doe-Ray, Dr. John P., CLU, CFP, LUTC") hn2 = HumanName("dr. john p. doe-Ray, CLU, CFP, LUTC") self.assertEqual(hash(hn1), object.__hash__(hn1)) self.assertEqual(len({hn1, hn2}), 2) # strings no longer interoperate with HumanName in sets/dicts hn = HumanName("John Smith") self.assertNotEqual(hash(hn), hash("john smith")) self.assertNotIn("john smith", {hn}) self.assertEqual(hn1.comparison_key(), hn2.comparison_key()) def test_not_equal_operator(self) -> None: # != routes through the 2.0 identity __eq__ (#223): any two distinct # instances are unequal, however similar their parse self.assertTrue(HumanName("John Smith") != HumanName("Jane Smith")) self.assertTrue(HumanName("John Smith") != HumanName("john smith")) hn = HumanName("John Smith") self.assertFalse(hn != hn) def test_comparison_key_components(self) -> None: hn = HumanName("Dr. Juan Q. Xavier de la Vega III") self.assertEqual( hn.comparison_key(), ('dr.', 'juan', 'q. xavier', 'de la vega', 'iii', '', '')) def test_comparison_key_case_insensitive_across_formats(self) -> None: hn1 = HumanName("Dr. Juan Q. Xavier de la Vega III") hn2 = HumanName("de la vega, dr. juan Q. xavier III") self.assertEqual(hn1.comparison_key(), hn2.comparison_key()) def test_comparison_key_independent_of_string_format(self) -> None: # unlike ==, which compares str(self) and so changes meaning when # display config changes, the key is built from the parsed lists hn1 = HumanName("John Smith") hn2 = HumanName("John Smith", string_format="{last}") self.assertEqual(hn1.comparison_key(), hn2.comparison_key()) def test_comparison_key_includes_maiden(self) -> None: # maiden isn't in the default string_format, so == can't see it; # the key includes all seven members hn1 = HumanName(first="Jenny", last="Baker", maiden="Johnson") hn2 = HumanName(first="Jenny", last="Baker") self.assertNotEqual(hn1.comparison_key(), hn2.comparison_key()) def test_comparison_key_usable_for_dedup(self) -> None: names = [HumanName("John Smith"), HumanName("Smith, John"), HumanName("JOHN SMITH"), HumanName("Jane Smith")] unique = {n.comparison_key(): n for n in names} self.assertEqual(len(unique), 2) def test_matches_str_is_semantic_not_textual(self) -> None: # any written form of the same name matches, unlike == which only # matches strings that render exactly like str(self) hn = HumanName("Dr. Juan Q. Xavier de la Vega III") self.assertTrue(hn.matches("de la vega, dr. juan Q. xavier III")) self.assertTrue(hn.matches("Dr. Juan Q. Xavier de la Vega III")) self.assertFalse(hn.matches("Juan de la Vega")) def test_matches_humanname_operand(self) -> None: hn = HumanName("John Smith") self.assertTrue(hn.matches(HumanName("JOHN SMITH"))) self.assertFalse(hn.matches(HumanName("Jane Smith"))) def test_matches_parses_str_with_instance_constants(self) -> None: # the custom title must not be a default one ('chancellor' is!), or # this passes without the self.C parse path ever mattering c = Constants() c.titles.add('zephyrmark') self.assertNotIn('zephyrmark', CONSTANTS.titles) hn = HumanName("Zephyrmark Jane Smith", constants=c) # the str operand is parsed with self.C, so the custom title is # recognized in the comma form; parsed with the shared CONSTANTS, # 'zephyrmark' would land in first/middle and the keys would differ self.assertTrue(hn.matches("smith, zephyrmark jane")) def test_matches_humanname_operand_keeps_its_own_parse(self) -> None: # asymmetry, pinned deliberately: a str operand is reparsed with # self.C, but a HumanName operand is compared as already parsed -- # its own constants determined its components c = Constants() c.titles.add('zephyrmark') with_title = HumanName("Zephyrmark Jane Smith", constants=c) default_parse = HumanName("Zephyrmark Jane Smith") self.assertTrue(with_title.matches("Zephyrmark Jane Smith")) self.assertFalse(with_title.matches(default_parse)) def test_empty_parses_share_a_comparison_key(self) -> None: # documented caveat: empty/unparsable input collapses to the # all-empty key, so such names match each other and collide in # dedup; screen with len(name) == 0 first self.assertTrue(HumanName("").matches(HumanName(","))) self.assertEqual(HumanName("()").comparison_key(), HumanName("").comparison_key()) def test_matches_non_ascii_case_insensitive(self) -> None: hn = HumanName("JOSÉ GARCÍA") self.assertTrue(hn.matches("José García")) def test_matches_rejects_other_types(self) -> None: hn = HumanName("John Smith") for bad in (None, 42, b"John Smith", ["John Smith"]): with pytest.raises(TypeError, match="str or HumanName"): hn.matches(bad) # type: ignore[arg-type] def test_eq_and_hash_are_silent_in_2_0(self) -> None: # the 1.3.0/1.4 DeprecationWarnings on __eq__/__hash__ ended with the # 2.0 switch to identity semantics (#223): both are now warning-free hn1 = HumanName("John Smith") hn2 = HumanName("john smith") with warnings.catch_warnings(): warnings.simplefilter("error") self.assertFalse(hn1 == hn2) hash(hn1) def test_new_comparison_api_does_not_warn(self) -> None: # the replacements must be adoptable before 2.0 without tripping # -W error test suites; matches(str) parses, so this also covers # the parse path hn = HumanName("John Smith") with warnings.catch_warnings(): warnings.simplefilter("error") hn.comparison_key() hn.matches("Smith, John") hn.matches(HumanName("John Smith")) def test_unparsable_attribute_removed(self) -> None: # Removed in 1.3.0: the guard that reported unparsable names was # unreachable, so the attribute was always False after any parse. self.assertFalse(hasattr(HumanName("John Smith"), "unparsable")) self.assertFalse(hasattr(HumanName(first="John"), "unparsable")) def test_str_fallback_without_string_format(self) -> None: # string_format=None falls back to joining the non-empty attributes hn = HumanName("Dr. John A. Doe, Jr.") hn.string_format = None self.assertEqual(str(hn), "Dr. John A. Doe Jr.") def test_repr_blank_name(self) -> None: hn = HumanName() self.assertIn("first: ''", repr(hn)) self.assertIn(hn.__class__.__name__, repr(hn)) def test_slice_removed(self) -> None: # slice access was removed in 2.0 (#258, warned since 1.4); iteration # and string-key access are the remaining spellings hn = HumanName("Doe-Ray, Dr. John P., CLU, CFP, LUTC") self.m(list(hn), ['Dr.', 'John', 'P.', 'Doe-Ray', 'CLU, CFP, LUTC'], hn) with pytest.raises(TypeError, match="258"): hn[1:] # type: ignore[index] with pytest.raises(TypeError, match="258"): hn[1:-3] # type: ignore[index] # string-key access is unaffected and stays silent with warnings.catch_warnings(): warnings.simplefilter("error") hn['first'] def test_getitem(self) -> None: hn = HumanName("Dr. John A. Kenneth Doe, Jr.") self.m(hn['title'], "Dr.", hn) self.m(hn['first'], "John", hn) self.m(hn['last'], "Doe", hn) self.m(hn['middle'], "A. Kenneth", hn) self.m(hn['suffix'], "Jr.", hn) def test_setitem_removed(self) -> None: # __setitem__ was removed in 2.0 (#258, warned since 1.4): item # assignment raises TypeError for every key, valid or not; plain # attribute assignment is the replacement hn = HumanName("Dr. John A. Kenneth Doe, Jr.") with pytest.raises(TypeError, match="item assignment"): hn['title'] = 'test' # type: ignore[index] with pytest.raises(TypeError, match="item assignment"): hn['bogus'] = 'value' # type: ignore[index] hn.first = 'Jane' # the replacement spelling self.m(hn.first, "Jane", hn) def test_conjunction_names(self) -> None: hn = HumanName("johnny y") self.m(hn.first, "johnny", hn) self.m(hn.last, "y", hn) def test_prefix_names(self) -> None: hn = HumanName("vai la") self.m(hn.first, "vai", hn) self.m(hn.last, "la", hn) def test_degenerate_comma_input_leaves_no_empty_pieces(self) -> None: # Regression: HumanName(',') (no-comma path after whitespace collapse) # and HumanName('Doe,, Jr.') (lastname-comma path) appended '' to # first_list — a silent empty member in the public *_list attributes. hn = HumanName(",") self.assertEqual(hn.first_list, []) self.assertEqual(hn.middle_list, []) self.assertEqual(hn.last_list, []) self.assertEqual(len(hn), 0) hn = HumanName("Doe,, Jr.") self.assertEqual(hn.first_list, []) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "Jr.", hn) # empty parts[0] exercises the lastname_pieces call site: the empty # last-name segment must not become a member of last_list hn = HumanName(", John") self.assertEqual(hn.last_list, []) self.m(hn.first, "John", hn) def test_assignment_filters_empty_tokens(self) -> None: # parse_pieces() drops tokens that strip to nothing at every entry # point, including the setters: whitespace-only strings and empty # list members never become *_list members (they previously survived, # e.g. hn.first = ' ' left first == ' ' and middle 'a b' gained an # empty member from ['a', '', 'b']). hn = HumanName("John Doe") hn.first = " " self.assertEqual(hn.first_list, []) self.m(hn.first, "", hn) hn.middle = ["a", "", "b"] self.assertEqual(hn.middle_list, ["a", "b"]) self.m(hn.middle, "a b", hn) hn.last = ["", "Smith"] self.assertEqual(hn.last_list, ["Smith"]) def test_blank_name(self) -> None: hn = HumanName() self.m(hn.first, "", hn) self.m(hn.last, "", hn) def test_surnames_list_attribute(self) -> None: hn = HumanName("John Edgar Casey Williams III") self.m(hn.surnames_list, ["Edgar", "Casey", "Williams"], hn) def test_surnames_attribute(self) -> None: hn = HumanName("John Edgar Casey Williams III") self.m(hn.surnames, "Edgar Casey Williams", hn) def test_given_names_list_attribute(self) -> None: hn = HumanName("John Edgar Casey Williams III") self.m(hn.given_names_list, ["John", "Edgar", "Casey"], hn) def test_given_names_attribute(self) -> None: hn = HumanName("John Edgar Casey Williams III") self.m(hn.given_names, "John Edgar Casey", hn) def test_given_names_attribute_first_only(self) -> None: hn = HumanName("John Williams") self.m(hn.given_names_list, ["John"], hn) self.m(hn.given_names, "John", hn) def test_given_names_attribute_empty(self) -> None: # empty attributes are always '' in 2.0 (#255) hn = HumanName("Dr. Williams") self.m(hn.given_names, "", hn) def test_override_constants(self) -> None: C = Constants() hn = HumanName(constants=C) self.assertIs(hn.C, C) def test_override_regex_raises(self) -> None: # Custom regexes are not supported in 2.0 (deliberate divergence, # migration spec §3 uniform rule): the constructor kwarg raises the # same TypeError as attribute assignment, pointing at named Policy # flags. Reads of the built-in patterns stay available. var = TupleManager([("spaces", re.compile(r"\s+")),]) with pytest.raises(TypeError, match="Policy"): Constants(regexes=var) # type: ignore[call-arg] def test_override_titles(self) -> None: var = ["abc","def"] C = Constants(titles=var) hn = HumanName(constants=C) self.assertTrue(sorted(hn.C.titles) == sorted(var)) def test_override_first_name_titles(self) -> None: var = ["abc","def"] C = Constants(first_name_titles=var) hn = HumanName(constants=C) self.assertTrue(sorted(hn.C.first_name_titles) == sorted(var)) def test_override_prefixes(self) -> None: var = ["abc","def"] C = Constants(prefixes=var) hn = HumanName(constants=C) self.assertTrue(sorted(hn.C.prefixes) == sorted(var)) def test_override_suffix_acronyms(self) -> None: var = ["abc","def"] C = Constants(suffix_acronyms=var) hn = HumanName(constants=C) self.assertTrue(sorted(hn.C.suffix_acronyms) == sorted(var)) def test_override_suffix_not_acronyms(self) -> None: var = ["abc","def"] C = Constants(suffix_not_acronyms=var) hn = HumanName(constants=C) self.assertTrue(sorted(hn.C.suffix_not_acronyms) == sorted(var)) def test_override_conjunctions(self) -> None: var = ["abc","def"] C = Constants(conjunctions=var) hn = HumanName(constants=C) self.assertTrue(sorted(hn.C.conjunctions) == sorted(var)) def test_override_capitalization_exceptions(self) -> None: var = TupleManager([("abc", "ABC")]) C = Constants(capitalization_exceptions=var) hn = HumanName(constants=C) self.assertTrue(hn.C.capitalization_exceptions == var) python-nameparser-2.1.0/tests/test_suffixes.py000066400000000000000000000501031523530431700215770ustar00rootroot00000000000000import pytest from nameparser import HumanName from tests.base import HumanNameTestBase class SuffixesTestCase(HumanNameTestBase): def test_suffix(self) -> None: hn = HumanName("Joe Franklin Jr") self.m(hn.first, "Joe", hn) self.m(hn.last, "Franklin", hn) self.m(hn.suffix, "Jr", hn) def test_suffix_with_periods(self) -> None: hn = HumanName("Joe Dentist D.D.S.") self.m(hn.first, "Joe", hn) self.m(hn.last, "Dentist", hn) self.m(hn.suffix, "D.D.S.", hn) def test_two_suffixes(self) -> None: hn = HumanName("Kenneth Clarke QC MP") self.m(hn.first, "Kenneth", hn) self.m(hn.last, "Clarke", hn) # NOTE: this adds a comma when the original format did not have one. # not ideal but at least its in the right bucket self.m(hn.suffix, "QC, MP", hn) def test_two_suffixes_lastname_comma_format(self) -> None: hn = HumanName("Washington Jr. MD, Franklin") self.m(hn.first, "Franklin", hn) self.m(hn.last, "Washington", hn) # NOTE: this adds a comma when the original format did not have one. self.m(hn.suffix, "Jr., MD", hn) def test_roman_numeral_v_suffix_comma_format(self) -> None: # suffix-comma position is unambiguous: 'V' must be a suffix, not a single-letter initial hn = HumanName("John W. Ingram, V") self.m(hn.first, "John", hn) self.m(hn.middle, "W.", hn) self.m(hn.last, "Ingram", hn) self.m(hn.suffix, "V", hn) def test_roman_numeral_i_suffix_comma_format(self) -> None: # 'I' has the same single-letter ambiguity as 'V' hn = HumanName("John W. Smith, I") self.m(hn.first, "John", hn) self.m(hn.middle, "W.", hn) self.m(hn.last, "Smith", hn) self.m(hn.suffix, "I", hn) def test_suffix_not_acronym_then_acronym_suffix_comma_format(self) -> None: # single-letter suffix_not_acronyms entry followed by an acronym suffix hn = HumanName("John Smith, V MD") self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) self.m(hn.suffix, "V MD", hn) def test_two_suffix_not_acronyms_suffix_comma_format(self) -> None: hn = HumanName("John Smith, V Jr.") self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) self.m(hn.suffix, "V Jr.", hn) def test_two_suffixes_suffix_comma_format(self) -> None: hn = HumanName("Franklin Washington, Jr. MD") self.m(hn.first, "Franklin", hn) self.m(hn.last, "Washington", hn) self.m(hn.suffix, "Jr. MD", hn) def test_suffix_containing_periods(self) -> None: hn = HumanName("Kenneth Clarke Q.C.") self.m(hn.first, "Kenneth", hn) self.m(hn.last, "Clarke", hn) self.m(hn.suffix, "Q.C.", hn) def test_suffix_containing_periods_lastname_comma_format(self) -> None: hn = HumanName("Clarke, Kenneth, Q.C. M.P.") self.m(hn.first, "Kenneth", hn) self.m(hn.last, "Clarke", hn) self.m(hn.suffix, "Q.C. M.P.", hn) def test_suffix_containing_periods_suffix_comma_format(self) -> None: hn = HumanName("Kenneth Clarke Q.C., M.P.") self.m(hn.first, "Kenneth", hn) self.m(hn.last, "Clarke", hn) self.m(hn.suffix, "Q.C., M.P.", hn) def test_suffix_with_single_comma_format(self) -> None: hn = HumanName("John Doe jr., MD") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "jr., MD", hn) def test_suffix_with_double_comma_format(self) -> None: hn = HumanName("Doe, John jr., MD") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "jr., MD", hn) def test_phd_with_erroneous_space(self) -> None: hn = HumanName("John Smith, Ph. D.") self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) self.m(hn.suffix, "Ph. D.", hn) def test_phd_extracted_without_comma(self) -> None: hn = HumanName("John Smith Ph. D.") self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) self.m(hn.suffix, "Ph. D.", hn) def test_phd_conflict(self) -> None: hn = HumanName("Adolph D") self.m(hn.first, "Adolph", hn) self.m(hn.last, "D", hn) # http://en.wikipedia.org/wiki/Ma_(surname) def test_potential_suffix_that_is_also_last_name(self) -> None: hn = HumanName("Jack Ma") self.m(hn.first, "Jack", hn) self.m(hn.last, "Ma", hn) def test_potential_suffix_that_is_also_last_name_comma(self) -> None: hn = HumanName("Ma, Jack") self.m(hn.first, "Jack", hn) self.m(hn.last, "Ma", hn) def test_potential_suffix_that_is_also_last_name_with_suffix(self) -> None: hn = HumanName("Jack Ma Jr") self.m(hn.first, "Jack", hn) self.m(hn.last, "Ma", hn) self.m(hn.suffix, "Jr", hn) def test_potential_suffix_that_is_also_last_name_with_suffix_comma(self) -> None: hn = HumanName("Ma III, Jack Jr") self.m(hn.first, "Jack", hn) self.m(hn.last, "Ma", hn) self.m(hn.suffix, "III, Jr", hn) # https://github.com/derek73/python-nameparser/issues/27 @pytest.mark.xfail def test_king(self) -> None: hn = HumanName("Dr King Jr") self.m(hn.title, "Dr", hn) self.m(hn.last, "King", hn) self.m(hn.suffix, "Jr", hn) def test_multiple_letter_suffix_with_periods(self) -> None: hn = HumanName("John Doe Msc.Ed.") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "Msc.Ed.", hn) def test_suffix_with_periods_with_comma(self) -> None: hn = HumanName("John Doe, Msc.Ed.") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "Msc.Ed.", hn) def test_suffix_with_periods_with_lastname_comma(self) -> None: hn = HumanName("Doe, John Msc.Ed.") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "Msc.Ed.", hn) def test_roman_numeral_i_lastname_comma_format(self) -> None: # 'I' is in suffix_not_acronyms; trailing suffix_not_acronyms members in # "Lastname, First Middle Suffix" format (no comma-suffix segment) must # parse as suffix, not middle initial (issue #144) hn = HumanName("Maier, Amy Lauren I") self.m(hn.first, "Amy", hn) self.m(hn.middle, "Lauren", hn) self.m(hn.last, "Maier", hn) self.m(hn.suffix, "I", hn) def test_roman_numeral_v_lastname_comma_format(self) -> None: # 'V' shares the same suffix_not_acronyms ambiguity as 'I' (issue #144) hn = HumanName("Smith, John V") self.m(hn.first, "John", hn) self.m(hn.middle, "", hn) self.m(hn.last, "Smith", hn) self.m(hn.suffix, "V", hn) def test_roman_numeral_i_no_middle_lastname_comma_format(self) -> None: # no middle name: trailing 'I' must still be a suffix (issue #144) hn = HumanName("Smith, John I") self.m(hn.first, "John", hn) self.m(hn.middle, "", hn) self.m(hn.last, "Smith", hn) self.m(hn.suffix, "I", hn) def test_roman_numeral_i_after_single_initial_lastname_comma_format(self) -> None: # single-letter middle initial followed by trailing 'I' (reporter pattern, issue #144) hn = HumanName("Chang, Andy C I") self.m(hn.first, "Andy", hn) self.m(hn.middle, "C", hn) self.m(hn.last, "Chang", hn) self.m(hn.suffix, "I", hn) @pytest.mark.xfail def test_roman_numeral_i_with_explicit_suffix_comma_known_limitation(self) -> None: # When an explicit suffix comma is present (len(parts)==3), the trailing 'I' # is conservatively left in middle to avoid misclassifying true initials. # This is a known limitation of the lastname-comma lenient-suffix # guard in parse_full_name (issue #144). hn = HumanName("Maier, Amy I, Jr.") self.m(hn.suffix, "I, Jr.", hn) def test_suffix_delimiter_default_on_constants(self) -> None: from nameparser.config import CONSTANTS self.assertIs(CONSTANTS.suffix_delimiter, None) def test_suffix_delimiter_kwarg_accepted(self) -> None: hn = HumanName("Steven Hardman, RN - CRNA", suffix_delimiter=" - ") self.assertEqual(hn.suffix_delimiter, " - ") def test_suffix_delimiter_basic(self) -> None: hn = HumanName("Steven Hardman, RN - CRNA", suffix_delimiter=" - ") self.m(hn.first, "Steven", hn) self.m(hn.last, "Hardman", hn) self.m(hn.suffix, "RN, CRNA", hn) def test_suffix_delimiter_multiple(self) -> None: hn = HumanName("John Doe, MD - PhD - FACS", suffix_delimiter=" - ") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "MD, PhD, FACS", hn) def test_suffix_delimiter_no_effect_without_comma(self) -> None: # suffix_delimiter only applies after the comma split; space-separated # suffixes already work via the no-comma parse path hn = HumanName("John Doe MD PhD", suffix_delimiter=" - ") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "MD, PhD", hn) def test_suffix_delimiter_constants_level(self) -> None: # ran on the shared CONSTANTS in v1; 2.0 deprecates shared mutation, # so the config-level (non-kwarg) path is exercised on a private # Constants passed as constants= from nameparser.config import Constants c = Constants() c.suffix_delimiter = " - " hn = HumanName("Steven Hardman, RN - CRNA", constants=c) self.m(hn.first, "Steven", hn) self.m(hn.last, "Hardman", hn) self.m(hn.suffix, "RN, CRNA", hn) def test_suffix_delimiter_none_by_default_known_limitation(self) -> None: # Without suffix_delimiter set, " - " between suffixes breaks parsing. # This test documents the known limitation — do not "fix" it. hn = HumanName("Steven Hardman, RN - CRNA") self.m(hn.first, "RN", hn) self.m(hn.last, "Steven Hardman", hn) self.m(hn.suffix, "CRNA", hn) def test_suffix_delimiter_trailing_delimiter_ignored(self) -> None: # Trailing delimiter must not defeat suffix detection. Using a # non-whitespace-terminated delimiter so stripping doesn't consume it. hn = HumanName("John Doe, MD-PhD-", suffix_delimiter="-") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) # 2.0: fix(suffix-delimiter-rendering) -- v1 split the token and # rendered 'MD, PhD'; v2 keeps a no-space delimiter-core token whole # (anti-#100), same rule as the pinned 'RN/CRNA' case row. self.m(hn.suffix, "MD-PhD-", hn) def test_suffix_delimiter_comma_space_is_noop(self) -> None: hn = HumanName("John Doe, MD, PhD", suffix_delimiter=", ") self.m(hn.suffix, "MD, PhD", hn) def test_suffix_delimiter_inverted_format_not_misparsed(self) -> None: # The delimiter only expands parts once they're identified as a # suffix group, so a hyphenated given name in inverted format isn't # mistaken for a suffix split. hn = HumanName("Doe, Mary - Kate, RN", suffix_delimiter=" - ") self.m(hn.first, "Mary", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "RN", hn) # "Kate" stays in the given-name segment rather than being pulled # into the suffix, since it's separated from "RN" by its own comma. # The bare "-" landing in middle is a pre-existing, delimiter- # independent quirk of tokenizing a lone hyphen (reproducible with # suffix_delimiter unset), not something this fix is responsible for. self.m(hn.middle, "- Kate", hn) def test_suffix_delimiter_expands_each_comma_segment(self) -> None: # parts[1:] holds two separate comma segments here ("MD - PhD" and # "FACS"); each must be expanded on its own, not just the first. hn = HumanName("John Doe, MD - PhD, FACS", suffix_delimiter=" - ") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "MD, PhD, FACS", hn) def test_suffix_delimiter_detection_with_multi_word_side(self) -> None: # The suffix-comma detection check flattens on spaces after # expanding on the delimiter, so a multi-word token on one side of # the delimiter is still tokenized correctly. hn = HumanName("Doe, John, MD PhD - FACS Fellow", suffix_delimiter=" - ") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "MD PhD, FACS Fellow", hn) def test_suffix_delimiter_no_effect_when_not_suffix_comma(self) -> None: # When the comma format isn't recognized as suffix-comma (here the # last-name part is a single word), the delimiter must not affect # parsing at all: output should match the no-delimiter baseline. with_delim = HumanName("Smith, MD - PhD - FACS", suffix_delimiter=" - ") without_delim = HumanName("Smith, MD - PhD - FACS") self.assertEqual( (with_delim.first, with_delim.middle, with_delim.last, with_delim.suffix), (without_delim.first, without_delim.middle, without_delim.last, without_delim.suffix), ) def test_suffix_acronyms_ambiguous_is_customizable(self) -> None: from nameparser.config import Constants custom = Constants(suffix_acronyms_ambiguous=['xyz']) self.assertEqual(set(custom.suffix_acronyms_ambiguous), {'xyz'}) # Constructing without the kwarg still works and uses the module default. default = Constants() self.assertIn('jd', default.suffix_acronyms_ambiguous) def test_suffix_in_parenthesis_with_other_suffixes(self) -> None: hn = HumanName("Andrew Perkins, Jr., Col. (Ret)") self.m(hn.first, "Andrew", hn) self.m(hn.last, "Perkins", hn) self.assertIn("Ret", hn.suffix) self.m(hn.nickname, "", hn) def test_suffix_in_parenthesis_mid_name(self) -> None: # "Jr." is suffix-shaped, so parse_nicknames() no longer treats it as # a nickname. But it isn't in trailing position, and parse_full_name's # suffix detection only recognizes a trailing run of suffix-shaped # pieces -- so it lands wherever normal parsing would put a bare # mid-name "Jr." token, exactly as if the parens were never there # (verified: HumanName("Lon Jr. Williams") parses identically). # Known limitation: making this land in `suffix` would require # changing parse_full_name's suffix detection, out of scope here -- # issue #111 is specifically about the nickname misclassification. hn = HumanName("Lon (Jr.) Williams") self.m(hn.first, "Lon", hn) self.m(hn.middle, "Jr.", hn) self.m(hn.last, "Williams", hn) self.m(hn.suffix, "", hn) self.m(hn.nickname, "", hn) def test_suffix_in_parenthesis_with_period(self) -> None: # Same known limitation as above: "Ret." is mid-name (no comma), so # it's outside the trailing run parse_full_name's suffix detection # requires. It parses exactly as bare "Col. Ret. Smith" would: since # "Ret." is an unrecognized period-abbreviation appearing before the # first name is set, is_leading_title() treats it as a second title # token (see #109), joining "Col." into a single title string. hn = HumanName("Col. (Ret.) Smith") self.m(hn.title, "Col. Ret.", hn) self.m(hn.first, "", hn) self.m(hn.last, "Smith", hn) self.m(hn.suffix, "", hn) self.m(hn.nickname, "", hn) def test_acronym_suffix_in_parenthesis(self) -> None: hn = HumanName("Andrew Perkins (MBA)") self.m(hn.first, "Andrew", hn) self.m(hn.last, "Perkins", hn) self.m(hn.suffix, "MBA", hn) self.m(hn.nickname, "", hn) def test_acronym_suffix_with_internal_periods_in_parenthesis(self) -> None: # "M.D" has a non-trailing period between every letter -- unlike # is_suffix(), handle_match()'s suffix_acronyms check must also strip # internal periods (not just rely on the trailing content.endswith('.') # heuristic, which doesn't fire here since "M.D" has no trailing period). hn = HumanName("Andrew Perkins (M.D)") self.m(hn.first, "Andrew", hn) self.m(hn.last, "Perkins", hn) self.m(hn.suffix, "M.D", hn) self.m(hn.nickname, "", hn) def test_period_terminated_content_in_parenthesis_not_forced_either_way(self) -> None: # "Mgr." isn't in any suffix list, but it ends in a period, so the # period heuristic (rule 2) excludes it from nickname_list. It flows # into normal parsing instead of being force-classified as a suffix. hn = HumanName("Andrew Perkins (Mgr.)") self.m(hn.nickname, "", hn) self.m(hn.suffix, "", hn) def test_suffix_in_single_quotes(self) -> None: # handle_match() is shared across all three delimiter regexes, not # just parenthesis -- confirm suffix-shaped single-quoted content # routes the same way. hn = HumanName("Andrew Perkins 'MBA'") self.m(hn.first, "Andrew", hn) self.m(hn.last, "Perkins", hn) self.m(hn.suffix, "MBA", hn) self.m(hn.nickname, "", hn) def test_suffix_in_double_quotes(self) -> None: hn = HumanName('Andrew Perkins "MBA"') self.m(hn.first, "Andrew", hn) self.m(hn.last, "Perkins", hn) self.m(hn.suffix, "MBA", hn) self.m(hn.nickname, "", hn) def test_suffix_acronyms_ambiguous_custom_entry_stays_nickname(self) -> None: # A custom suffix_acronyms_ambiguous entry keeps a suffix_acronyms # member classified as a nickname instead of a suffix, confirming # the exception list -- not a hardcoded check -- drives the behavior. from nameparser.config import Constants C = Constants( suffix_acronyms=['xyz'], suffix_acronyms_ambiguous=['xyz'], ) hn = HumanName("Andrew Perkins (XYZ)", constants=C) self.m(hn.nickname, "XYZ", hn) self.m(hn.suffix, "", hn) def test_suffix_acronyms_ambiguous_removal_routes_to_suffix(self) -> None: # Removing 'jd' from a custom suffix_acronyms_ambiguous flips JD # from nickname to suffix. Uses a trailing-position name (unlike the # JEFFREY (JD) BRICKEN regression guard in test_nicknames.py) so # parse_full_name's trailing-run suffix detection actually picks it # up -- see the known mid-name limitation noted on the tests above. from nameparser.config import Constants C = Constants(suffix_acronyms_ambiguous=[]) hn = HumanName("Andrew Perkins (JD)", constants=C) self.m(hn.nickname, "", hn) self.m(hn.suffix, "JD", hn) def test_empty_comma_segment_does_not_drop_following_suffix(self) -> None: # Regression: "Doe, John,, Jr." produced an empty third segment, and # the old `if parts[2]:` guard skipped every remaining segment -- # silently dropping the trailing suffix. Empty segments should be # skipped individually. hn = HumanName("Doe, John,, Jr.") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.suffix, "Jr.", hn) # each empty segment is skipped individually; segments after a later # empty must survive too (guards against a break-instead-of-continue # regression that the single-empty case above would not catch) hn = HumanName("Doe, John,, Jr.,, III") self.m(hn.suffix, "Jr., III", hn) def test_suffix_comma_empty_segment_not_added_to_suffix_list(self) -> None: # Regression: in suffix-comma format ("John Doe, Jr.,," -- one # trailing comma survives collapse_whitespace), the unguarded # parts[1:] loop leaked '' into suffix_list via # expand_suffix_delimiter('') returning ['']. hn = HumanName("John Doe, Jr.,,") self.assertEqual(hn.suffix_list, ["Jr."]) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) python-nameparser-2.1.0/tests/test_titles.py000066400000000000000000000352501523530431700212550ustar00rootroot00000000000000import pytest from nameparser import HumanName from tests.base import HumanNameTestBase class TitleTestCase(HumanNameTestBase): def test_last_name_is_also_title(self) -> None: hn = HumanName("Amy E Maid") self.m(hn.first, "Amy", hn) self.m(hn.middle, "E", hn) self.m(hn.last, "Maid", hn) def test_last_name_is_also_title_no_comma(self) -> None: hn = HumanName("Dr. Martin Luther King Jr.") self.m(hn.title, "Dr.", hn) self.m(hn.first, "Martin", hn) self.m(hn.middle, "Luther", hn) self.m(hn.last, "King", hn) self.m(hn.suffix, "Jr.", hn) def test_last_name_is_also_title_with_comma(self) -> None: hn = HumanName("Dr Martin Luther King, Jr.") self.m(hn.title, "Dr", hn) self.m(hn.first, "Martin", hn) self.m(hn.middle, "Luther", hn) self.m(hn.last, "King", hn) self.m(hn.suffix, "Jr.", hn) def test_leading_period_abbreviation_is_title(self) -> None: hn = HumanName("Major. Dona Smith") self.m(hn.title, "Major.", hn) self.m(hn.first, "Dona", hn) self.m(hn.last, "Smith", hn) def test_last_name_is_also_title3(self) -> None: hn = HumanName("John King") self.m(hn.first, "John", hn) self.m(hn.last, "King", hn) def test_title_with_conjunction(self) -> None: hn = HumanName("Secretary of State Hillary Clinton") self.m(hn.title, "Secretary of State", hn) self.m(hn.first, "Hillary", hn) self.m(hn.last, "Clinton", hn) def test_compound_title_with_conjunction(self) -> None: hn = HumanName("Cardinal Secretary of State Hillary Clinton") self.m(hn.title, "Cardinal Secretary of State", hn) self.m(hn.first, "Hillary", hn) self.m(hn.last, "Clinton", hn) def test_title_is_title(self) -> None: hn = HumanName("Coach") self.m(hn.title, "Coach", hn) # TODO: fix handling of U.S. -- 2.0 matches v1 here: lc()-style # normalization keeps 'u.s' out of the titles vocabulary, so the # chain never starts (an interim 2.0 build that stripped interior # periods made this pass, but that normalization wrongly turned # 'J.R.' into the title 'jr'; v1 parity won) @pytest.mark.xfail def test_chained_title_first_name_title_is_initials(self) -> None: hn = HumanName("U.S. District Judge Marc Thomas Treadwell") self.m(hn.title, "U.S. District Judge", hn) self.m(hn.first, "Marc", hn) self.m(hn.middle, "Thomas", hn) self.m(hn.last, "Treadwell", hn) def test_conflict_with_chained_title_first_name_initial(self) -> None: hn = HumanName("U. S. Grant") self.m(hn.first, "U.", hn) self.m(hn.middle, "S.", hn) self.m(hn.last, "Grant", hn) def test_chained_title_first_name_initial_with_no_period(self) -> None: hn = HumanName("US Magistrate Judge T Michael Putnam") self.m(hn.title, "US Magistrate Judge", hn) self.m(hn.first, "T", hn) self.m(hn.middle, "Michael", hn) self.m(hn.last, "Putnam", hn) def test_chained_hyphenated_title(self) -> None: hn = HumanName("US Magistrate-Judge Elizabeth E Campbell") self.m(hn.title, "US Magistrate-Judge", hn) self.m(hn.first, "Elizabeth", hn) self.m(hn.middle, "E", hn) self.m(hn.last, "Campbell", hn) def test_chained_hyphenated_title_with_comma_suffix(self) -> None: hn = HumanName("Mag-Judge Harwell G Davis, III") self.m(hn.title, "Mag-Judge", hn) self.m(hn.first, "Harwell", hn) self.m(hn.middle, "G", hn) self.m(hn.last, "Davis", hn) self.m(hn.suffix, "III", hn) @pytest.mark.xfail def test_title_multiple_titles_with_apostrophe_s(self) -> None: hn = HumanName("The Right Hon. the President of the Queen's Bench Division") self.m(hn.title, "The Right Hon. the President of the Queen's Bench Division", hn) def test_title_starts_with_conjunction(self) -> None: hn = HumanName("The Rt Hon John Jones") self.m(hn.title, "The Rt Hon", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Jones", hn) def test_conjunction_before_title(self) -> None: hn = HumanName('The Lord of the Universe') self.m(hn.title, "The Lord of the Universe", hn) def test_double_conjunction_on_title(self) -> None: hn = HumanName('Lord of the Universe') self.m(hn.title, "Lord of the Universe", hn) def test_triple_conjunction_on_title(self) -> None: hn = HumanName('Lord and of the Universe') self.m(hn.title, "Lord and of the Universe", hn) def test_multiple_conjunctions_on_multiple_titles(self) -> None: hn = HumanName('Lord of the Universe and Associate Supreme Queen of the World Lisa Simpson') self.m(hn.title, "Lord of the Universe and Associate Supreme Queen of the World", hn) self.m(hn.first, "Lisa", hn) self.m(hn.last, "Simpson", hn) def test_title_with_last_initial_is_suffix(self) -> None: hn = HumanName("King John V.") self.m(hn.title, "King", hn) self.m(hn.first, "John", hn) self.m(hn.last, "V.", hn) def test_initials_also_suffix(self) -> None: hn = HumanName("Smith, J.R.") self.m(hn.first, "J.R.", hn) # self.m(hn.middle, "R.", hn) self.m(hn.last, "Smith", hn) def test_two_title_parts_separated_by_periods(self) -> None: hn = HumanName("Lt.Gen. John A. Kenneth Doe IV") self.m(hn.title, "Lt.Gen.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A. Kenneth", hn) self.m(hn.suffix, "IV", hn) def test_two_part_title(self) -> None: hn = HumanName("Lt. Gen. John A. Kenneth Doe IV") self.m(hn.title, "Lt. Gen.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A. Kenneth", hn) self.m(hn.suffix, "IV", hn) def test_two_part_title_with_lastname_comma(self) -> None: hn = HumanName("Doe, Lt. Gen. John A. Kenneth IV") self.m(hn.title, "Lt. Gen.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A. Kenneth", hn) self.m(hn.suffix, "IV", hn) def test_two_part_title_with_suffix_comma(self) -> None: hn = HumanName("Lt. Gen. John A. Kenneth Doe, Jr.") self.m(hn.title, "Lt. Gen.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A. Kenneth", hn) self.m(hn.suffix, "Jr.", hn) def test_possible_conflict_with_middle_initial_that_could_be_suffix(self) -> None: hn = HumanName("Doe, Rev. John V, Jr.") self.m(hn.title, "Rev.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "V", hn) self.m(hn.suffix, "Jr.", hn) def test_possible_conflict_with_suffix_that_could_be_initial(self) -> None: hn = HumanName("Doe, Rev. John A., V, Jr.") self.m(hn.title, "Rev.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) self.m(hn.middle, "A.", hn) self.m(hn.suffix, "V, Jr.", hn) # 'ben' is removed from PREFIXES in v0.2.5 # this test could re-enable this test if we decide to support 'ben' as a prefix @pytest.mark.xfail def test_ben_as_conjunction(self) -> None: hn = HumanName("Ahmad ben Husain") self.m(hn.first, "Ahmad", hn) self.m(hn.last, "ben Husain", hn) def test_ben_as_first_name(self) -> None: hn = HumanName("Ben Johnson") self.m(hn.first, "Ben", hn) self.m(hn.last, "Johnson", hn) def test_ben_as_first_name_with_middle_name(self) -> None: hn = HumanName("Ben Alex Johnson") self.m(hn.first, "Ben", hn) self.m(hn.middle, "Alex", hn) self.m(hn.last, "Johnson", hn) def test_ben_as_middle_name(self) -> None: hn = HumanName("Alex Ben Johnson") self.m(hn.first, "Alex", hn) self.m(hn.middle, "Ben", hn) self.m(hn.last, "Johnson", hn) # http://code.google.com/p/python-nameparser/issues/detail?id=13 def test_last_name_also_prefix(self) -> None: hn = HumanName("Jane Doctor") self.m(hn.first, "Jane", hn) self.m(hn.last, "Doctor", hn) def test_title_with_periods(self) -> None: hn = HumanName("Lt.Gov. John Doe") self.m(hn.title, "Lt.Gov.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) def test_title_with_periods_lastname_comma(self) -> None: hn = HumanName("Doe, Lt.Gov. John") self.m(hn.title, "Lt.Gov.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) def test_title_with_periods_and_single_letter_middle_name(self) -> None: # A derived title ("Lt.Gov.") must be excluded from the rootname # count that join_on_conjunctions() uses for its single-letter # conjunction heuristic. If is_rootname() misses the derived titles, # the count reaches 4 and "e" is treated as a conjunction, joining # "juan e garcia" into a single last-name piece with no first name. hn = HumanName("Lt.Gov. juan e garcia") self.m(hn.title, "Lt.Gov.", hn) self.m(hn.first, "juan", hn) self.m(hn.middle, "e", hn) self.m(hn.last, "garcia", hn) def test_mac_with_spaces(self) -> None: hn = HumanName("Jane Mac Beth") self.m(hn.first, "Jane", hn) self.m(hn.last, "Mac Beth", hn) def test_mac_as_first_name(self) -> None: hn = HumanName("Mac Miller") self.m(hn.first, "Mac", hn) self.m(hn.last, "Miller", hn) def test_multiple_prefixes(self) -> None: hn = HumanName("Mike van der Velt") self.m(hn.first, "Mike", hn) self.m(hn.last, "van der Velt", hn) def test_2_same_prefixes_in_the_name(self) -> None: hh = HumanName("Vincent van Gogh van Beethoven") self.m(hh.first, "Vincent", hh) self.m(hh.middle, "van Gogh", hh) self.m(hh.last, "van Beethoven", hh) # Non-ASCII title normalization — confirm diacritic titles survive # the lowercase lookup path end-to-end. def test_señora_non_ascii_title(self) -> None: hn = HumanName("Señora María García") self.m(hn.title, "Señora", hn) self.m(hn.first, "María", hn) self.m(hn.last, "García", hn) def test_señora_lowercase_non_ascii_title(self) -> None: hn = HumanName("señora María García") self.m(hn.title, "señora", hn) self.m(hn.first, "María", hn) self.m(hn.last, "García", hn) def test_frøken_non_ascii_title(self) -> None: hn = HumanName("Frøken Jensen") self.m(hn.title, "Frøken", hn) self.m(hn.first, "", hn) self.m(hn.last, "Jensen", hn) def test_herr_title_not_first_name(self) -> None: hn = HumanName("Herr Schmidt") self.m(hn.title, "Herr", hn) self.m(hn.first, "", hn) self.m(hn.last, "Schmidt", hn) def test_herr_title_with_first_name(self) -> None: hn = HumanName("Herr Klaus Schmidt") self.m(hn.title, "Herr", hn) self.m(hn.first, "Klaus", hn) self.m(hn.last, "Schmidt", hn) def test_leading_period_abbreviation_suffix_comma(self) -> None: hn = HumanName("Major. John Smith, Jr.") self.m(hn.title, "Major.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) self.m(hn.suffix, "Jr.", hn) def test_leading_period_abbreviation_lastname_comma(self) -> None: hn = HumanName("Smith, Major. John") self.m(hn.title, "Major.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) def test_leading_period_abbreviation_unknown_word(self) -> None: hn = HumanName("Foo. John Smith") self.m(hn.title, "Foo.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) def test_leading_period_abbreviation_chained(self) -> None: hn = HumanName("Foo. Xyz. John Smith") self.m(hn.title, "Foo. Xyz.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) def test_leading_single_letter_initial_excluded(self) -> None: hn = HumanName("J. Smith") self.m(hn.first, "J.", hn) self.m(hn.last, "Smith", hn) self.m(hn.title, "", hn) def test_leading_internal_period_abbreviation_excluded(self) -> None: hn = HumanName("E.T. Smith") self.m(hn.first, "E.T.", hn) self.m(hn.last, "Smith", hn) self.m(hn.title, "", hn) def test_period_abbreviation_after_first_name_stays_middle(self) -> None: hn = HumanName("John Major. Smith") self.m(hn.first, "John", hn) self.m(hn.middle, "Major.", hn) self.m(hn.last, "Smith", hn) self.m(hn.title, "", hn) def test_known_title_with_period_still_a_title(self) -> None: hn = HumanName("Dr. John Smith") self.m(hn.title, "Dr.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) def test_middle_initial_with_period_unaffected(self) -> None: hn = HumanName("John Q. Smith") self.m(hn.first, "John", hn) self.m(hn.middle, "Q.", hn) self.m(hn.last, "Smith", hn) def test_leading_period_abbreviation_excludes_digits(self) -> None: hn = HumanName("No1. John Smith") self.m(hn.title, "", hn) self.m(hn.first, "No1.", hn) self.m(hn.last, "Smith", hn) def test_leading_period_abbreviation_excludes_apostrophe(self) -> None: hn = HumanName("O'B. John Smith") self.m(hn.title, "", hn) self.m(hn.first, "O'B.", hn) self.m(hn.last, "Smith", hn) def test_leading_period_abbreviation_case_insensitive(self) -> None: hn = HumanName("xyz. John Smith") self.m(hn.title, "xyz.", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) def test_leading_period_abbreviation_with_nickname(self) -> None: hn = HumanName("Xyz. (Bud) Smith") self.m(hn.title, "Xyz.", hn) self.m(hn.first, "Smith", hn) self.m(hn.nickname, "Bud", hn) def test_charge_daffaires_chains_as_title(self) -> None: hn = HumanName("Chargé d'Affaires John Smith") self.m(hn.title, "Chargé d'Affaires", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) def test_unaccented_charge_daffaires_chains_as_title(self) -> None: # both spellings ship, like attaché/attache hn = HumanName("Charge d'Affaires John Smith") self.m(hn.title, "Charge d'Affaires", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) python-nameparser-2.1.0/tests/test_turkic_patronymic_order.py000066400000000000000000000303061523530431700247070ustar00rootroot00000000000000import re from typing import cast from nameparser import HumanName from nameparser.config import Constants from tests.base import FlaggedConstantsTestBase, HumanNameTestBase # The 2.0 regexes proxy types its attributes as ``object`` (reads are # informational); the pattern-inspection tests below cast once here. _R = Constants().regexes _TURKIC = cast('re.Pattern[str]', _R.turkic_patronymic_marker) _TURKIC_CYR = cast('re.Pattern[str]', _R.turkic_patronymic_marker_cyrillic) _EAST_SLAVIC = cast('re.Pattern[str]', _R.east_slavic_patronymic) _EAST_SLAVIC_CYR = cast('re.Pattern[str]', _R.east_slavic_patronymic_cyrillic) def test_marker_is_whole_word_not_substring() -> None: # is_turkic_patronymic_marker() deliberately uses whole-word .match(), # not suffix .search() (unlike is_east_slavic_patronymic()) — pin this # so a future .match()->.search() slip doesn't silently start matching # surnames/given-names that merely contain a marker as a substring. assert not _TURKIC.match("ogluu") assert not _TURKIC.match("Bogluchik") assert not _TURKIC_CYR.match("оглуш") assert not _TURKIC_CYR.match("Оглуев") class TurkicPatronymicNameOrderReorderTests(FlaggedConstantsTestBase): """Names that SHOULD be rotated when the flag is on.""" constants_kwargs = {"patronymic_name_order": True} def test_oglu(self) -> None: n = self.hn("Aliyev Vusal Said oglu") assert n.first == "Vusal" assert n.middle == "Said oglu" assert n.last == "Aliyev" def test_oglu_native_orthography(self) -> None: n = self.hn("Aliyev Vusal Said oğlu") assert n.first == "Vusal" assert n.middle == "Said oğlu" assert n.last == "Aliyev" def test_ogly(self) -> None: n = self.hn("Aliyev Vusal Said ogly") assert n.first == "Vusal" assert n.middle == "Said ogly" assert n.last == "Aliyev" def test_ogli(self) -> None: n = self.hn("Aliyev Vusal Said ogli") assert n.first == "Vusal" assert n.middle == "Said ogli" assert n.last == "Aliyev" def test_ogli_uzbek_modifier_apostrophe(self) -> None: # U+02BB modifier letter turned comma, the official Uzbek orthography. n = self.hn("Yusupov Aziz Karim oʻgʻli") assert n.first == "Aziz" assert n.middle == "Karim oʻgʻli" assert n.last == "Yusupov" def test_ogli_straight_apostrophe(self) -> None: n = self.hn("Yusupov Aziz Karim o'g'li") assert n.first == "Aziz" assert n.middle == "Karim o'g'li" assert n.last == "Yusupov" def test_ogli_right_single_quote(self) -> None: # U+2019 right single quotation mark, common in informal data. n = self.hn("Yusupov Aziz Karim o’g’li") assert n.first == "Aziz" assert n.middle == "Karim o’g’li" assert n.last == "Yusupov" def test_qizi(self) -> None: n = self.hn("Aliyeva Mehriban Arif qizi") assert n.first == "Mehriban" assert n.middle == "Arif qizi" assert n.last == "Aliyeva" def test_qizi_native_orthography(self) -> None: n = self.hn("Aliyeva Mehriban Arif qızı") assert n.first == "Mehriban" assert n.middle == "Arif qızı" assert n.last == "Aliyeva" def test_kizi(self) -> None: n = self.hn("Yusupova Nodira Karim kizi") assert n.first == "Nodira" assert n.middle == "Karim kizi" assert n.last == "Yusupova" def test_kyzy(self) -> None: n = self.hn("Bekova Aigul Nurlan kyzy") assert n.first == "Aigul" assert n.middle == "Nurlan kyzy" assert n.last == "Bekova" def test_gyzy(self) -> None: n = self.hn("Annayeva Gozel Merdan gyzy") assert n.first == "Gozel" assert n.middle == "Merdan gyzy" assert n.last == "Annayeva" def test_uly(self) -> None: n = self.hn("Nazarbayev Nursultan Abish uly") assert n.first == "Nursultan" assert n.middle == "Abish uly" assert n.last == "Nazarbayev" def test_uulu(self) -> None: n = self.hn("Atambayev Almazbek Sharshenovich uulu") assert n.first == "Almazbek" assert n.middle == "Sharshenovich uulu" assert n.last == "Atambayev" def test_cyrillic_oglu(self) -> None: n = self.hn("Алиев Вусал Саид оглу") assert n.first == "Вусал" assert n.middle == "Саид оглу" assert n.last == "Алиев" def test_cyrillic_ogly(self) -> None: n = self.hn("Алиев Вусал Саид оглы") assert n.first == "Вусал" assert n.middle == "Саид оглы" assert n.last == "Алиев" def test_cyrillic_oglu_azerbaijani_native(self) -> None: n = self.hn("Алиев Вусал Саид оғлу") assert n.first == "Вусал" assert n.middle == "Саид оғлу" assert n.last == "Алиев" def test_cyrillic_ugli_uzbek_native(self) -> None: n = self.hn("Юсупов Азиз Карим ўғли") assert n.first == "Азиз" assert n.middle == "Карим ўғли" assert n.last == "Юсупов" def test_cyrillic_ugli_russian_rendering(self) -> None: n = self.hn("Юсупов Азиз Карим угли") assert n.first == "Азиз" assert n.middle == "Карим угли" assert n.last == "Юсупов" def test_cyrillic_kyzy(self) -> None: n = self.hn("Бекова Айгуль Нурлан кызы") assert n.first == "Айгуль" assert n.middle == "Нурлан кызы" assert n.last == "Бекова" def test_cyrillic_gyzy(self) -> None: n = self.hn("Аннаева Гозель Мердан гызы") assert n.first == "Гозель" assert n.middle == "Мердан гызы" assert n.last == "Аннаева" def test_cyrillic_qizi_uzbek(self) -> None: n = self.hn("Юсупова Нодира Карим қизи") assert n.first == "Нодира" assert n.middle == "Карим қизи" assert n.last == "Юсупова" def test_cyrillic_qyzy_kazakh_native(self) -> None: n = self.hn("Назарбаева Дана Абиш қызы") assert n.first == "Дана" assert n.middle == "Абиш қызы" assert n.last == "Назарбаева" def test_cyrillic_uly_russian_rendering(self) -> None: n = self.hn("Назарбаев Нурсултан Абиш улы") assert n.first == "Нурсултан" assert n.middle == "Абиш улы" assert n.last == "Назарбаев" def test_cyrillic_uly_kazakh_native(self) -> None: n = self.hn("Назарбаев Нурсултан Абиш ұлы") assert n.first == "Нурсултан" assert n.middle == "Абиш ұлы" assert n.last == "Назарбаев" def test_cyrillic_uulu(self) -> None: n = self.hn("Атамбаев Алмазбек Шаршенович уулу") assert n.first == "Алмазбек" assert n.middle == "Шаршенович уулу" assert n.last == "Атамбаев" def test_all_caps_latin(self) -> None: n = self.hn("ALIYEV VUSAL SAID OGLU") assert n.first == "VUSAL" assert n.middle == "SAID OGLU" assert n.last == "ALIYEV" def test_all_caps_cyrillic(self) -> None: n = self.hn("АЛИЕВ ВУСАЛ САИД ОГЛЫ") assert n.first == "ВУСАЛ" assert n.middle == "САИД ОГЛЫ" assert n.last == "АЛИЕВ" class TurkicPatronymicNameOrderGuardsTests(FlaggedConstantsTestBase): """Names that must NOT be reordered even when the flag is on.""" constants_kwargs = {"patronymic_name_order": True} def test_already_correct_natural_order(self) -> None: n = self.hn("Vusal Said oglu Aliyev") assert n.first == "Vusal" assert n.middle == "Said oglu" assert n.last == "Aliyev" def test_comma_guard(self) -> None: n = self.hn("Aliyev, Vusal Said oglu") assert n.first == "Vusal" assert n.middle == "Said oglu" assert n.last == "Aliyev" def test_three_token_no_surname(self) -> None: # No surname to rotate into place — parses as last=oglu, unchanged. n = self.hn("Vusal Said oglu") assert n.first == "Vusal" assert n.middle == "Said" assert n.last == "oglu" def test_five_token_extra_given_name(self) -> None: # Extra token before the patronymic phrase breaks the strict 4-token shape. n = self.hn("Aliyev Vusal Rza Said oglu") assert n.first == "Aliyev" assert n.middle == "Vusal Rza Said" assert n.last == "oglu" def test_ordinary_western_four_token_name(self) -> None: # Last word isn't a recognised marker → no rotation. n = self.hn("Smith John Michael Anderson") assert n.first == "Smith" assert n.middle == "John Michael" assert n.last == "Anderson" class TurkicPatronymicNameOrderFlagOffTests(HumanNameTestBase): """With default Constants (flag=False) nothing changes.""" def test_reversed_order_unchanged(self) -> None: n = HumanName("Aliyev Vusal Said oglu") assert n.first == "Aliyev" assert n.middle == "Vusal Said" assert n.last == "oglu" class PatronymicHandlerInteractionTests(FlaggedConstantsTestBase): """Both handlers run in sequence under the same flag; confirm no interference.""" constants_kwargs = {"patronymic_name_order": True} def test_east_slavic_shape_unaffected_by_turkic_handler(self) -> None: # If handle_turkic_patronymic_name_order() ran BEFORE the East-Slavic # handler could fire, middle_list would already be length 1 (not 2), # so the Turkic guard is a no-op here regardless of call order. n = self.hn("Ivanov Ivan Ivanovich") assert n.first == "Ivan" assert n.middle == "Ivanovich" assert n.last == "Ivanov" def test_turkic_shape_unaffected_by_east_slavic_handler(self) -> None: # East-Slavic's own guard requires middle_list of length 1; this # input has middle_list of length 2, so it's a genuine no-op rather # than a partial mutation before the Turkic handler runs. n = self.hn("Aliyev Vusal Said oglu") assert n.first == "Vusal" assert n.middle == "Said oglu" assert n.last == "Aliyev" def test_no_regex_collision_latin(self) -> None: east_slavic_examples = [ "Ivanovich", "Ivanovna", "Sergeevich", "Sergeevna", "Nikitichna", "Ilyich", "Kuzmich", "Lukich", "Fomich", "Fokich", ] turkic_examples = [ "oglu", "oğlu", "ogly", "ogli", "o'g'li", "qizi", "qızı", "kizi", "kyzy", "gyzy", "uly", "uulu", ] for word in east_slavic_examples: # Sanity check: each word actually matches its own family's # regex, so the non-collision assertion below is non-vacuous. assert _EAST_SLAVIC.search(word), word assert not _TURKIC.match(word), word for word in turkic_examples: assert _TURKIC.match(word), word assert not _EAST_SLAVIC.search(word), word def test_no_regex_collision_cyrillic(self) -> None: east_slavic_examples = [ "Иванович", "Ивановна", "Сергеевич", "Сергеевна", "Никитична", "Ильич", "Кузьмич", "Лукич", "Фомич", "Фокич", ] turkic_examples = [ "оглу", "оглы", "оғлу", "ўғли", "угли", "кызы", "гызы", "қызы", "қизи", "улы", "ұлы", "уулу", ] for word in east_slavic_examples: # Sanity check: each word actually matches its own family's # regex, so the non-collision assertion below is non-vacuous. assert _EAST_SLAVIC_CYR.search(word), word assert not _TURKIC_CYR.match(word), word for word in turkic_examples: assert _TURKIC_CYR.match(word), word assert not _EAST_SLAVIC_CYR.search(word), word python-nameparser-2.1.0/tests/test_variations.py000066400000000000000000000175501523530431700221330ustar00rootroot00000000000000from nameparser import HumanName from tests.base import HumanNameTestBase from types import SimpleNamespace TEST_NAMES = ( "John Doe", "John Doe, Jr.", "John Doe III", "Doe, John", "Doe, John, Jr.", "Doe, John III", "John A. Doe", "John A. Doe, Jr.", "John A. Doe III", "Doe, John A.", "Doe, John A., Jr.", "Doe, John A. III", "John A. Kenneth Doe", "John A. Kenneth Doe, Jr.", "John A. Kenneth Doe III", "Doe, John A. Kenneth", "Doe, John A. Kenneth, Jr.", "Doe, John A. Kenneth III", "Dr. John Doe", "Dr. John Doe, Jr.", "Dr. John Doe III", "Doe, Dr. John", "Doe, Dr. John, Jr.", "Doe, Dr. John III", "Dr. John A. Doe", "Dr. John A. Doe, Jr.", "Dr. John A. Doe III", "Doe, Dr. John A.", "Doe, Dr. John A. Jr.", "Doe, Dr. John A. III", "Dr. John A. Kenneth Doe", "Dr. John A. Kenneth Doe, Jr.", "Dr. John A. Kenneth Doe III", "Doe, Dr. John A. Kenneth", "Doe, Dr. John A. Kenneth Jr.", "Doe, Dr. John A. Kenneth III", "Juan de la Vega", "Juan de la Vega, Jr.", "Juan de la Vega III", "de la Vega, Juan", "de la Vega, Juan, Jr.", "de la Vega, Juan III", "Juan Velasquez y Garcia", "Juan Velasquez y Garcia, Jr.", "Juan Velasquez y Garcia III", "Velasquez y Garcia, Juan", "Velasquez y Garcia, Juan, Jr.", "Velasquez y Garcia, Juan III", "Dr. Juan de la Vega", "Dr. Juan de la Vega, Jr.", "Dr. Juan de la Vega III", "de la Vega, Dr. Juan", "de la Vega, Dr. Juan, Jr.", "de la Vega, Dr. Juan III", "Dr. Juan Velasquez y Garcia", "Dr. Juan Velasquez y Garcia, Jr.", "Dr. Juan Velasquez y Garcia III", "Velasquez y Garcia, Dr. Juan", "Velasquez y Garcia, Dr. Juan, Jr.", "Velasquez y Garcia, Dr. Juan III", "Juan Q. de la Vega", "Juan Q. de la Vega, Jr.", "Juan Q. de la Vega III", "de la Vega, Juan Q.", "de la Vega, Juan Q., Jr.", "de la Vega, Juan Q. III", "Juan Q. Velasquez y Garcia", "Juan Q. Velasquez y Garcia, Jr.", "Juan Q. Velasquez y Garcia III", "Velasquez y Garcia, Juan Q.", "Velasquez y Garcia, Juan Q., Jr.", "Velasquez y Garcia, Juan Q. III", "Dr. Juan Q. de la Vega", "Dr. Juan Q. de la Vega, Jr.", "Dr. Juan Q. de la Vega III", "de la Vega, Dr. Juan Q.", "de la Vega, Dr. Juan Q., Jr.", "de la Vega, Dr. Juan Q. III", "Dr. Juan Q. Velasquez y Garcia", "Dr. Juan Q. Velasquez y Garcia, Jr.", "Dr. Juan Q. Velasquez y Garcia III", "Velasquez y Garcia, Dr. Juan Q.", "Velasquez y Garcia, Dr. Juan Q., Jr.", "Velasquez y Garcia, Dr. Juan Q. III", "Juan Q. Xavier de la Vega", "Juan Q. Xavier de la Vega, Jr.", "Juan Q. Xavier de la Vega III", "de la Vega, Juan Q. Xavier", "de la Vega, Juan Q. Xavier, Jr.", "de la Vega, Juan Q. Xavier III", "Juan Q. Xavier Velasquez y Garcia", "Juan Q. Xavier Velasquez y Garcia, Jr.", "Juan Q. Xavier Velasquez y Garcia III", "Velasquez y Garcia, Juan Q. Xavier", "Velasquez y Garcia, Juan Q. Xavier, Jr.", "Velasquez y Garcia, Juan Q. Xavier III", "Dr. Juan Q. Xavier de la Vega", "Dr. Juan Q. Xavier de la Vega, Jr.", "Dr. Juan Q. Xavier de la Vega III", "de la Vega, Dr. Juan Q. Xavier", "de la Vega, Dr. Juan Q. Xavier, Jr.", "de la Vega, Dr. Juan Q. Xavier III", "Dr. Juan Q. Xavier Velasquez y Garcia", "Dr. Juan Q. Xavier Velasquez y Garcia, Jr.", "Dr. Juan Q. Xavier Velasquez y Garcia III", "Velasquez y Garcia, Dr. Juan Q. Xavier", "Velasquez y Garcia, Dr. Juan Q. Xavier, Jr.", "Velasquez y Garcia, Dr. Juan Q. Xavier III", "John Doe, CLU, CFP, LUTC", "John P. Doe, CLU, CFP, LUTC", "Dr. John P. Doe-Ray, CLU, CFP, LUTC", "Doe-Ray, Dr. John P., CLU, CFP, LUTC", "Hon. Barrington P. Doe-Ray, Jr.", "Doe-Ray, Hon. Barrington P. Jr.", "Doe-Ray, Hon. Barrington P. Jr., CFP, LUTC", "Jose Aznar y Lopez", "John E Smith", "John e Smith", "John and Jane Smith", "Rev. John A. Kenneth Doe", "Donovan McNabb-Smith", "Rev John A. Kenneth Doe", "Doe, Rev. John A. Jr.", "Buca di Beppo", "Lt. Gen. John A. Kenneth Doe, Jr.", "Doe, Lt. Gen. John A. Kenneth IV", "Lt. Gen. John A. Kenneth Doe IV", 'Mr. and Mrs. John Smith', 'John Jones (Google Docs)', 'john e jones', 'john e jones, III', 'jones, john e', 'E.T. Smith', 'E.T. Smith, II', 'Smith, E.T., Jr.', 'A.B. Vajpayee', 'Rt. Hon. Paul E. Mary', 'Maid Marion', 'Amy E. Maid', 'Jane Doctor', 'Doctor, Jane E.', 'dr. ben alex johnson III', 'Lord of the Universe and Supreme King of the World Lisa Simpson', 'Benjamin (Ben) Franklin', 'Benjamin "Ben" Franklin', "Brian O'connor", "Sir Gerald", "Magistrate Judge John F. Forster, Jr", # "Magistrate Judge Joaquin V.E. Manibusan, Jr", Intials seem to mess this up "Magistrate-Judge Elizabeth Todd Campbell", "Mag-Judge Harwell G Davis, III", "Mag. Judge Byron G. Cudmore", "Chief Judge J. Leon Holmes", "Chief Judge Sharon Lovelace Blackburn", "Judge James M. Moody", "Judge G. Thomas Eisele", # "Judge Callie V. S. Granade", "Judge C Lynwood Smith, Jr", "Senior Judge Charles R. Butler, Jr", "Senior Judge Harold D. Vietor", "Senior Judge Virgil Pittman", "Honorable Terry F. Moorer", "Honorable W. Harold Albritton, III", "Honorable Judge W. Harold Albritton, III", "Honorable Judge Terry F. Moorer", "Honorable Judge Susan Russ Walker", "Hon. Marian W. Payson", "Hon. Charles J. Siragusa", "US Magistrate Judge T Michael Putnam", "Designated Judge David A. Ezra", "Sr US District Judge Richard G Kopf", "U.S. District Judge Marc Thomas Treadwell", "Dra. Andréia da Silva", "Srta. Andréia da Silva", "Herr Klaus Schmidt", "Frau Anna Müller", "Monsieur Jean Dupont", "Señor Carlos García", "Señora María García", "Signor Marco Rossi", "Mevrouw Anna de Vries", ) class HumanNameVariationTests(HumanNameTestBase): # test automated variations of names in TEST_NAMES. # Helps test that the 3 code trees work the same TEST_NAMES = TEST_NAMES def test_variations_of_TEST_NAMES(self) -> None: for name in self.TEST_NAMES: hn = HumanName(name) if len(hn.suffix_list) > 1: d = SimpleNamespace(**hn.as_dict()) hn = HumanName(f"{d.title} {d.first} {d.middle} {d.last} {d.suffix}".split(',')[0]) # v1 forced empty_attribute_default='' here so the format # strings below got '' for empty parts; 2.0 removed the setting # (#255) and empty attributes are always '' -- nothing to force. d = SimpleNamespace(**hn.as_dict()) nocomma = HumanName(f"{d.title} {d.first} {d.middle} {d.last} {d.suffix}") lastnamecomma = HumanName(f"{d.last}, {d.title} {d.first} {d.middle} {d.suffix}") if d.suffix: suffixcomma = HumanName(f"{d.title} {d.first} {d.middle} {d.last}, {d.suffix}") if d.nickname: nocomma = HumanName(f"{d.title} {d.first} {d.middle} {d.last} {d.suffix} ({d.nickname})") lastnamecomma = HumanName(f"{d.last}, {d.title} {d.first} {d.middle} {d.suffix} ({d.nickname})") if d.suffix: suffixcomma = HumanName(f"{d.title} {d.first} {d.middle} {d.last}, {d.suffix} ({d.nickname})") # v1 iterated the private hn._members; 2.0's public equivalent # is the as_dict() key set (same seven fields) for attr in hn.as_dict(): self.m(getattr(hn, attr), getattr(nocomma, attr), hn) self.m(getattr(hn, attr), getattr(lastnamecomma, attr), hn) if hn.suffix: self.m(getattr(hn, attr), getattr(suffixcomma, attr), hn) python-nameparser-2.1.0/tests/v2/000077500000000000000000000000001523530431700166625ustar00rootroot00000000000000python-nameparser-2.1.0/tests/v2/__init__.py000066400000000000000000000000001523530431700207610ustar00rootroot00000000000000python-nameparser-2.1.0/tests/v2/cases.py000066400000000000000000003261731523530431700203460ustar00rootroot00000000000000"""THE shared behavior case table (core spec §7.2). Format is fixed here, in the first pipeline PR, and never per-PR: one Case per input, expected values for exactly the non-empty fields, optional Policy/Locale context, and a mandatory classification -- "parity" (matches v1.4.0, pinned live 2026-07-12) or "fix(#N)" / "fix()" (an intentional 2.0 behavior change, annotated with its issue or a design-decision slug). No silent expectation edits: changing a row means changing its classification. "UNDETERMINED" is a fourth value and a TEMPORARY one: a row added before anyone has run its input against 1.4.0 carries it until the comparison is made and it resolves to one of the three above. It is not a standing category, and nothing enforces its removal -- a row still wearing it is a row whose parity is simply unknown. The v1 suite's full corpus is extracted into this table by the migration plan (facade runner consumes the same rows); this file seeds it with the pinned battery. """ from __future__ import annotations from dataclasses import dataclass from nameparser import Policy from nameparser._policy import PatronymicRule @dataclass(frozen=True) class Case: id: str text: str expect: dict[str, str] # field -> value; absent fields == "" policy: Policy | None = None locale: str | None = None # locale CODE (keeps this table # import-light); mutually exclusive # with policy classification: str = "parity" ambiguities: tuple[str, ...] = () # expected AmbiguityKind values notes: str = "" def __post_init__(self) -> None: if self.policy is not None and self.locale is not None: raise ValueError( f"{self.id}: policy and locale are mutually exclusive") _ES = Policy(patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})) _TK = Policy(patronymic_rules=frozenset({PatronymicRule.TURKIC})) _SD = Policy(extra_suffix_delimiters=frozenset({" - "})) CASES: tuple[Case, ...] = ( Case("plain", "John Smith", {"given": "John", "family": "Smith"}), Case("family_comma", "Smith, John", {"given": "John", "family": "Smith"}), Case("suffix_comma", "John Smith, PhD", {"given": "John", "family": "Smith", "suffix": "PhD"}), Case("bound_given_pairwise_only", "Salem, Abdul Rahman Ahmed", {"given": "Abdul Rahman", "middle": "Ahmed", "family": "Salem"}, notes="the bound-given join is PAIRWISE (one merge, v1 " "parity): the third piece stays a middle name"), Case("family_comma_three_part_trailing_strict", "Smith, John V, Jr.", {"given": "John", "middle": "V", "family": "Smith", "suffix": "Jr."}, notes="the lenient trailing test applies only to TWO-part " "names; a third comma part makes the trailing token a " "middle initial (v1 parity, pinned live 2026-07-17)"), Case("triple_trailing_commas", "Doe,,,", {"family": "Doe"}, notes="one trailing comma is cosmetic; the rest are " "structural empties (pinned live 2026-07-17)"), Case("paren_suffix_word_escapes_nickname", "John Smith (Esq)", {"given": "John", "family": "Smith", "suffix": "Esq"}, notes="the suffix_words branch of the delimited-content " "escape (v1 parity, pinned live 2026-07-17)"), Case("suffix_acronym_multidot_spelling", "John Smith E.S.Q.", {"given": "John", "family": "Smith", "suffix": "E.S.Q."}, notes="'esq' is in BOTH suffix_acronyms and suffix_words on " "purpose, and the two are not redundant: the word test " "strips only EDGE periods, the acronym test strips all " "of them, so only the acronym membership matches the " "multi-dot spelling (v1 parity, pinned live 2026-07-19)"), Case("bound_given_whole_segment", "salem, abdul salam", {"given": "abdul salam", "family": "salem"}, notes="v1 joins bound given names freely in the post-comma " "segment (reserve_last=False, parser.py:1366) -- even " "when the join consumes the whole segment"), Case("middle_as_family_fold_order", "Hassan, Mohamad Ahmad Ali", {"given": "Mohamad", "family": "Ahmad Ali Hassan"}, policy=Policy(middle_as_family=True), notes="v1 PREPENDED middle_list to last_list; folded tokens " "carry vocab:folded-middle and the family view orders " "them first (spans cannot reorder)"), Case("ambiguous_surname_acronyms", "Jack MA", {"given": "Jack", "family": "MA"}, ambiguities=("suffix-or-name",), notes="'ma'/'do' joined suffix_acronyms_ambiguous: with only " "two pieces, 'one of them is a credential' is the less " "likely reading, so the ambiguous acronym stays the " "family name (v1 parity via its reserve_last)"), Case("ambiguous_surname_acronym_with_suffix", "Jack MA Jr", {"given": "Jack", "family": "MA", "suffix": "Jr"}, ambiguities=("suffix-or-name",), notes="'Jr' peels first; 'MA' would then be the only piece " "left beside the given name, so it stays family"), Case("ambiguous_acronym_is_a_suffix_when_a_family_name_remains", "John Smith MA", {"given": "John", "family": "Smith", "suffix": "MA"}, ambiguities=("suffix-or-name",), notes="the other half of the same rule: three pieces means " "peeling 'MA' still leaves given+family, so the " "credential reading wins (v1 parity)"), Case("ambiguous_acronym_suffix_with_middle", "John Q Smith MA", {"given": "John", "middle": "Q", "family": "Smith", "suffix": "MA"}, ambiguities=("suffix-or-name",)), Case("titled_ambiguous_particle_chains", "Dr. Van Johnson", {"title": "Dr.", "family": "Van Johnson"}, ambiguities=("particle-or-given",), notes="the other branch of 'Van Johnson': a leading title " "shifts Van off the given position, the prefix chain " "fires, and the fork is reported from group rather than " "assign (v1 parity on the fields)"), Case("titled_ambiguous_particle_no_op_chain", "Dr. Van Jr.", {"title": "Dr.", "given": "Van", "suffix": "Jr."}, notes="the piece after the particle is a suffix, so the chain " "scan never advances and the merge is a no-op -- nothing " "was chained, so there is no fork to report (the emitter " "fired here for all 39 ambiguous particles, and _assign " "double-reported the same token)"), Case("initial_shaped_not_conjunction", "john e. smith", {"given": "john", "middle": "e.", "family": "smith"}, notes="v1 is_conjunction excludes initials at classify too"), Case("family_comma_lenient_trailing", "Smith, John V", {"given": "John", "family": "Smith", "suffix": "V"}, notes="v1 #144: the trailing piece of a two-part comma name " "takes the lenient suffix test"), Case("family_comma_first_piece_is_given", "Steven Hardman, RN - CRNA", {"given": "RN", "middle": "-", "family": "Steven Hardman", "suffix": "CRNA"}, notes="v1 walk order: the first post-comma piece is the given " "before any suffix check (the delimiter is UNSET here " "-- v1's documented limitation, kept)"), Case("family_comma_lone_suffix_piece", "Andrews, M.D.", {"family": "Andrews", "suffix": "M.D."}, classification="fix(comma-family)", notes="v1 made the lone post-comma strict-suffix piece the " "given; 2.0 routes it to suffix (same family as the " "'Smith, Dr.' row)"), Case("period_joined_ambiguous_chunk", "John Doe, Msc.Ed.", {"given": "John", "family": "Doe", "suffix": "Msc.Ed."}, notes="chunk-level suffix membership is v1's is_suffix: bare " "ambiguous acronyms count within period-joined tokens"), Case("suffix_comma_split_phd", "John Smith, Ph. D.", {"given": "John", "family": "Smith", "suffix": "Ph. D."}, notes="the adjacent Ph./D. pair counts as one suffix unit in " "the suffix-comma detection (v1 fix_phd parity). The " "pair leads the run here; the row below is where it does " "not"), Case("suffix_comma_split_phd_after_another_suffix", "John Smith, Jr. Ph. D.", {"given": "John", "family": "Smith", "suffix": "Jr. Ph. D."}, classification="fix(credential-pair-order)", notes="the pair is merged wherever it sits in the run, not " "only at its head, and the row above cannot say so -- " "there the pair IS the head. Restricted to position 0 " "the merge never fires, 'D.' is suffix vocabulary in no " "lexicon, and the run stops being wholly suffix: this " "reads as a FAMILY comma instead, family 'John Smith' / " "title 'Jr.' / suffix 'Ph. D.'. Pinned at the segment " "stage too (test_segment.test_the_credential_pair_" "merges_anywhere_in_the_run), because the neighbouring " "input 'John Smith, MD, Jr. Ph. D.' keeps every field " "under that break and gains only a comma-structure " "ambiguity, so fields alone do not catch it. Measured: " "1.4.0 gave suffix 'Ph. D., Jr.' -- fix_phd EXTRACTED " "the pair pre-parse and re-appended it, reordering the " "tail, where 2.0 renders it as written. Same words, " "same roles, different order, and the harness cannot " "currently see it: expected_since_1.4.0.toml states that a " "diffing trailing 'Ph. D.' must fail the run, but this " "input is absorbed by fix(comma-family), whose " "name_regex is a bare comma -- measured on a probe " "corpus, unexplained 0. See the note there"), Case("tail_segment_entry_space_joined", "John Smith, V MD", {"given": "John", "family": "Smith", "suffix": "V MD"}, notes="v1 renders each tail comma segment as ONE suffix " "entry; words within an entry space-join via the " "'joined' tag"), Case("maiden_delimiters_win_when_shared", 'Baker (Johnson), Jenny', {"given": "Jenny", "family": "Baker", "maiden": "Johnson"}, policy=Policy(maiden_delimiters=frozenset({("(", ")")})), notes="listing a pair in maiden_delimiters drops it from the " "effective nickname set (maiden wins, 2026-07-19) -- the " "one-liner replaces the bucket-move idiom. The facade " "runner reaches this row by making that same move, so " "it agrees. What v1 does NOT agree on is a pair left in " "BOTH buckets, which it gives to nickname; that spelling " "is a different config, not this row, and is pinned in " "test_config_shim"), # #269 follow-up: Arabic-script bound given names, mirroring the # Latin transliterations' behavior (probed live 2026-07-19: bound # join fires only with 3+ tokens, eats the NEXT token into given; # two-token kunya stays split -- 'أبو مازن' pinned in test_locales; # non-leading أبو still prefix-chains onto family). Case("arabic_bound_given_abd", 'عبد الرحمن محمد', {"given": "عبد الرحمن", "family": "محمد"}, classification="feat(#269)", notes="script twin of 'Abdul Rahman Mohammed'; عبد was the " "missing bound entry -- 1.x split it into given عبد + " "middle الرحمن"), Case("arabic_bound_given_kunya", 'أبو بكر أحمد', {"given": "أبو بكر", "family": "أحمد"}, classification="feat(#269)"), Case("arabic_bound_given_kunya_hamzaless", 'ابو بكر احمد', {"given": "ابو بكر", "family": "احمد"}, classification="feat(#269)", notes="both kunya spellings ship, like the أبو/ابو prefix pair"), Case("arabic_bound_given_umm", 'أم كلثوم إبراهيم', {"given": "أم كلثوم", "family": "إبراهيم"}, classification="feat(#269)"), # #273: typographic nickname delimiters ship as defaults -- one row # per new pair; expectations verified live 2026-07-19 before the # pairs were added to DEFAULT_NICKNAME_DELIMITERS. Case("smart_double_quotes_nickname", 'John “Jack” Kennedy', {"given": "John", "family": "Kennedy", "nickname": "Jack"}, classification="feat(#273)"), Case("low_high_quotes_nickname", 'Hans „Hansi“ Müller', {"given": "Hans", "family": "Müller", "nickname": "Hansi"}, classification="feat(#273)", notes="the closing '“' doubles as the English pair's opener; " "no spurious unbalanced-delimiter ambiguity (pinned in " "test_extract)"), Case("guillemets_nickname_inner_spaces", 'Jean « Petit » Dupont', {"given": "Jean", "family": "Dupont", "nickname": "Petit"}, classification="feat(#273)", notes="French spacing: inner padding is trimmed from the " "extracted nickname"), Case("reversed_guillemets_nickname", 'Hans »Hansi« Müller', {"given": "Hans", "family": "Müller", "nickname": "Hansi"}, classification="feat(#273)"), Case("swedish_right_quotes_nickname", 'Anna ”Ann” Larsson', {"given": "Anna", "family": "Larsson", "nickname": "Ann"}, classification="feat(#273)"), Case("cjk_corner_bracket_nickname", '山田「タロ」太郎', {"family": "山田", "given": "太郎", "nickname": "タロ"}, classification="feat(#273) + fix(#271)", notes="extraction also splits the unspaced remainder -- the " "masked region acts as a token boundary. Both name " "pieces are then wholly Han, so script_orders reads " "them family-first; the nickname's kana is outside the " "name pieces and does not enter that test"), Case("cjk_white_corner_bracket_nickname", '田中『ハナ』花子', {"family": "田中", "given": "花子", "nickname": "ハナ"}, classification="feat(#273) + fix(#271)"), Case("fullwidth_paren_nickname", 'John （Jack） Kennedy', {"given": "John", "family": "Kennedy", "nickname": "Jack"}, classification="feat(#273)"), Case("curly_apostrophe_stays_literal", 'Sean O’Connor', {"given": "Sean", "family": "O’Connor"}, notes="U+2019 is the typographic apostrophe; curly single " "quotes are deliberately NOT delimiters (#273 excludes " "them)"), Case("family_segment_trailing_suffix", "Smith Jr., John", {"given": "John", "family": "Smith", "suffix": "Jr."}, notes="v1: the family part may have suffixes in it " "(parser.py:1368); the first piece is always the family " "(pinned live 2026-07-17)"), Case("family_segment_multiple_suffixes", "Smith Jr. MD, John", {"given": "John", "family": "Smith", "suffix": "Jr., MD"}), Case("family_segment_particle_chain_suffix", "de la Vega III, Juan", {"given": "Juan", "family": "de la Vega", "suffix": "III"}), Case("interior_periods_block_vocab", "Smith, J.R.", {"given": "J.R.", "family": "Smith"}, notes="v1's lc() keeps interior periods: 'J.R.' is not the " "title 'jr' (pinned live 2026-07-17)"), Case("dotted_acronym_suffix", "John Smith M.D.", {"given": "John", "family": "Smith", "suffix": "M.D."}, notes="suffix-ACRONYM membership alone strips periods (v1 " "is_suffix parity)"), Case("nickname_rule_counts_whole_segment", "Xyz. (Bud) Smith", {"title": "Xyz.", "given": "Smith", "nickname": "Bud"}, notes="v1's lone-piece nickname rule counts the segment " "BEFORE title peeling (parser.py:1285, pinned live " "2026-07-17)"), Case("suffix_comma_decided_by_first_segment", "Dr. John P. Doe-Ray, CLU, CFP, LUTC", {"title": "Dr.", "given": "John", "middle": "P.", "family": "Doe-Ray", "suffix": "CLU, CFP, LUTC"}, ambiguities=("comma-structure",), notes="only parts[1] decides the suffix-comma structure " "(v1 parser.py:1318); 'lutc' is not in the vocabulary " "but rides along (v1 parity, pinned live 2026-07-16)"), Case("suffix_comma_nonsuffix_tail_flagged", "John Smith, MD, Xyzzy", {"given": "John", "family": "Smith", "suffix": "MD, Xyzzy"}, ambiguities=("comma-structure",), notes="the unrecognized tail is consumed best-effort with the " "flag (v1 consumed it silently)"), Case("period_joined_titles", "Lt.Gov. John Doe", {"title": "Lt.Gov.", "given": "John", "family": "Doe"}, notes="v1 derived-title rule: ANY period chunk being a title " "makes the token a title (pinned live 2026-07-16)"), Case("period_joined_suffixes", "John Doe JD.CPA", {"given": "John", "family": "Doe", "suffix": "JD.CPA"}), Case("period_joined_any_rule", "Mr.Smith", {"title": "Mr.Smith"}, notes="the ANY rule is deliberate v1 parity: one title chunk " "claims the whole token"), Case("doubled_comma_suffix", "Doe,, Jr.", {"family": "Doe", "suffix": "Jr."}, notes="the EMPTY given segment keeps its position: 'Jr.' is a " "tail suffix, not a lone post-comma title (v1 parity, " "pinned live 2026-07-16)"), Case("doubled_comma_given_kept", "Doe, John,, Jr.", {"given": "John", "family": "Doe", "suffix": "Jr."}), Case("single_trailing_comma_cosmetic", "John,", {"given": "John"}, notes="v1 collapse_whitespace strips exactly ONE trailing " "comma before parsing"), Case("double_trailing_comma_structural", "Doe,,", {"family": "Doe"}, notes="one trailing comma is cosmetic, the second is " "structural: an empty given segment makes this a " "family-comma parse (v1 parity, pinned live 2026-07-16)"), Case("doubled_comma_blocks_suffix_comma", "John Smith,, MD", {"family": "John Smith", "suffix": "MD"}, notes="an empty segment 1 fails the suffix-comma detection " "(v1 parity)"), Case("suffix_delimiter_tail_segment", "Doe, John, RN - CRNA", {"given": "John", "family": "Doe", "suffix": "RN, CRNA"}, policy=_SD, notes="v1 suffix_delimiter parity (#191): the delimiter token " "is dropped from consumed tail segments (pinned live " "2026-07-16)"), Case("suffix_delimiter_detection", "Doe, John RN - CRNA", {"given": "John", "middle": "-", "family": "Doe", "suffix": "RN, CRNA"}, policy=_SD, notes="the delimiter fires only at suffix sites; the stray " "token keeps its per-piece walk role (v1 parity, pinned " "live 2026-07-16)"), Case("suffix_delimiter_suffix_comma", "John Smith, RN - CRNA", {"given": "John", "family": "Smith", "suffix": "RN, CRNA"}, policy=_SD, notes="delimiter transparency in the SUFFIX_COMMA " "determination: the post-comma segment counts as " "all-suffix (v1 parity, pinned live 2026-07-16)"), Case("suffix_delimiter_no_space_core", "John Smith, RN/CRNA", {"given": "John", "family": "Smith", "suffix": "RN/CRNA"}, policy=Policy(extra_suffix_delimiters=frozenset({"/"})), classification="fix(suffix-delimiter-rendering)", notes="v1 split 'RN/CRNA' and rendered 'RN, CRNA'; v2 keeps " "the token whole (anti-#100) with Role.SUFFIX -- role " "assignment matches, rendering differs (migration plan " "deviation 5, release-log classified)"), Case("suffix_delimiter_name_segment_untouched", "Doe, Mary - Kate, RN", {"given": "Mary", "middle": "- Kate", "family": "Doe", "suffix": "RN"}, policy=_SD, notes="a delimiter token in a NAME segment is kept (v1 parity, " "pinned live 2026-07-16)"), Case("comma_extras_become_suffixes", "Smith, John, Extra, Jr.", {"given": "John", "family": "Smith", "suffix": "Extra, Jr."}, ambiguities=("comma-structure",), notes="post-comma segments land in suffix even when not " "suffix-shaped; the ambiguity flags the guess (v1 " "parity, pinned live 2026-07-13)"), Case("delavega", "Dr. Juan de la Vega III", {"title": "Dr.", "given": "Juan", "family": "de la Vega", "suffix": "III"}), Case("prefix_chain_to_end", "Juan de la Vega Martinez", {"given": "Juan", "family": "de la Vega Martinez"}), Case("van_johnson", "Van Johnson", {"given": "Van", "family": "Johnson"}, ambiguities=("particle-or-given",), notes="v2 surfaces #121's irreducible ambiguity"), Case("family_comma_particles", "de la Vega, Juan", {"given": "Juan", "family": "de la Vega"}), Case("paren_suffix_escapes_nickname", "Andrew Perkins (MBA)", {"given": "Andrew", "family": "Perkins", "suffix": "MBA"}, notes="v1 parse_nicknames: suffix-shaped delimited content is " "left in place for normal parsing (pinned live " "2026-07-17)"), Case("paren_period_escapes_nickname", "Andrew Perkins (Ret.)", {"given": "Andrew", "family": "Perkins", "suffix": "Ret."}), Case("nickname_quotes", 'John "Jack" Kennedy', {"given": "John", "family": "Kennedy", "nickname": "Jack"}), Case("nickname_parens", "John (Jack) Kennedy", {"given": "John", "family": "Kennedy", "nickname": "Jack"}), Case("sir_bob", "Sir Bob Andrew Dole", {"title": "Sir", "given": "Bob", "middle": "Andrew", "family": "Dole"}), Case("long_title", "President of the United States Barack Obama", {"title": "President of the United States", "given": "Barack", "family": "Obama"}), Case("secretary", "The Secretary of State Hillary Clinton", {"title": "The Secretary of State", "given": "Hillary", "family": "Clinton"}), Case("comma_middle_initial", "Doe, John A.", {"given": "John", "middle": "A.", "family": "Doe"}), Case("single", "John", {"given": "John"}), Case("title_only", "Dr.", {"title": "Dr."}), Case("double_comma_suffix", "Smith, John, Jr.", {"given": "John", "family": "Smith", "suffix": "Jr."}), Case("bound_given_two", "abdul rahman", {"given": "abdul", "family": "rahman"}), Case("bound_given_three", "abdul rahman al-said", {"given": "abdul rahman", "family": "al-said"}), Case("mr_and_mrs", "Mr. and Mrs. John Smith", {"title": "Mr. and Mrs.", "given": "John", "family": "Smith"}), Case("roman_suffix", "John Smith V", {"given": "John", "family": "Smith", "suffix": "V"}, ambiguities=("suffix-or-name",), notes="a trailing single letter is a name part unless it is a " "roman numeral; V/X/I are also ordinary middle initials, " "so the reading is reported"), Case("initial_not_suffix", "John V. Smith", {"given": "John", "middle": "V.", "family": "Smith"}), Case("lenient_after_comma", "John Ingram, V", {"given": "John", "family": "Ingram", "suffix": "V"}), Case("comma_then_title", "Smith, Dr. John", {"title": "Dr.", "given": "John", "family": "Smith"}), Case("nickname_single_name", "John (Jack)", {"family": "John", "nickname": "Jack"}), Case("nickname_only", "(Jack)", {"nickname": "Jack"}), Case("suffix_run", "John Jack Kennedy PhD MD", {"given": "John", "middle": "Jack", "family": "Kennedy", "suffix": "PhD, MD"}), Case("maiden_marker", "Jane Smith née Jones", {"given": "Jane", "family": "Smith", "maiden": "Jones"}, classification="fix(#274)", notes="v1 mangles to middle='Smith née'"), Case("maiden_marker_kyusei", "山田花子 旧姓 佐藤", {"family": "山田花子", "maiden": "佐藤"}, classification="fix(#309)", notes="旧姓 is default vocabulary, not pack data: a " "native-script marker cannot collide with a Latin-script " "name and matching is whole-token, the same rule that " "admitted урожд. Reaches a marker that is its own TOKEN. " "Japanese more often brackets the marker and writes a " "fullwidth colon after it, and '山田（旧姓：佐藤）' under " "maiden_delimiters still gives maiden '旧姓：佐藤', marker " "and colon attached -- not because the marker escapes " "tagging (classify tags it fine wherever it is a token; " "what #329 fixed was the CONSUMING, since group's #274 " "rule walks pieces and a role-bearing token is not in " "pieces) but because ：glues marker to name into a " "single token, leaving nothing to drop. The spaced " "bracketed form is pinned by " "maiden_marker_kyusei_delimited below; the glued one " "wants the head-peel #317 tracks. 1.4.0 read this " "first 山田花子 / middle 旧姓 / last 佐藤 -- the marker " "sat in the name"), Case("maiden_marker_kyusei_segmented", "山田 花子 旧姓 佐藤", {"given": "花子", "family": "山田", "maiden": "佐藤"}, classification="fix(#309)", notes="the row above with the family name spaced, so the " "marker is consumed from a name that already has a " "given side -- pinned because #274's consuming rule " "takes the marker plus the piece after it, and the " "pieces before it are what could have gone wrong. " "1.4.0 read this first 山田 / middle '花子 旧姓' / " "last 佐藤"), Case("maiden_marker_delimited", "Jane Smith (née Jones)", {"given": "Jane", "family": "Smith", "maiden": "Jones"}, policy=Policy(maiden_delimiters=frozenset({("(", ")")})), classification="fix(#329)", notes="#329: the bracketed form now agrees with the bare " "maiden_marker row above -- both give maiden 'Jones'. " "Before, the marker rode along inside the value: " "extract records the clause as a span (it makes no " "tokens at all), tokenize gives the tokens it cuts " "there Role.MAIDEN, and group's #274 consuming rule " "walks pieces, which hold no role-bearing token; the " "fix drops the marker inside the CLAUSE instead. The " "facade runner reaches this row by performing the " "bucket move itself, so it is exercised twice. 1.4.0 " "expresses the policy the same way: " "measured 2026-08-02 through the bucket-move idiom " "maiden_delimiters['parenthesis'] = " "nickname_delimiters.pop('parenthesis'), it gave first " "Jane / last Smith / maiden 'née Jones' -- same name " "fields, marker still inside the value, which is the " "single field this change moves"), Case("maiden_marker_delimited_unaccented", "Jane Smith (nee Jones)", {"given": "Jane", "family": "Smith", "maiden": "Jones"}, policy=Policy(maiden_delimiters=frozenset({("(", ")")})), classification="fix(#329)", notes="the row above with the marker spelled unaccented, and " "it is the ONLY row in the suite whose value depends " "on 'nee' being in the default MAIDEN_MARKERS: " "everything else reaches the marker branch through " "'née', 'geb' or '旧姓', or through a stage lexicon of " "its own. Removing the entry now fails exactly this " "row's two tests, one per runner (measured 2026-08-03); " "before this row existed it left the whole suite green, " "so the shipped spelling English writes most often was " "one vocabulary edit from silence. 1.4.0 under the " "bucket-move idiom gave first Jane / last Smith / " "maiden 'nee Jones' (2026-08-03) -- the same diff the " "accented row records, which is the point: the two " "spellings behave alike on both sides"), Case("maiden_marker_delimited_unmarked_content", "Jane Smith (Mary Jones)", {"given": "Jane", "family": "Smith", "maiden": "Mary Jones"}, policy=Policy(maiden_delimiters=frozenset({("(", ")")})), classification="parity", notes="the row the rest of the #329 battery leaves out: a " "multi-token clause whose first token is NOT a marker, " "which keeps every one of its tokens. The clause-size " "test and the marker-tag test are separate conditions, " "and this shape is one of the two that separates them " "-- with the tag test removed the pass eats the opening " "word of every delimited maiden name ('Jones' here), " "and five tests go red across this row and " "maiden_marker_delimited_trailing_marker (measured " "2026-08-03). What is this row's alone is that NO token " "in its clause is a marker; the trailing-marker row has " "one, just not first. Measured against " "1.4.0 2026-08-03 through the bucket-move idiom " "maiden_delimiters['parenthesis'] = " "nickname_delimiters.pop('parenthesis'): first Jane / " "last Smith / maiden 'Mary Jones', so #329 leaves this " "input exactly where v1 had it"), Case("maiden_marker_delimited_three_token_clause", "Jane Smith (née Mary Jones)", {"given": "Jane", "family": "Smith", "maiden": "Mary Jones"}, policy=Policy(maiden_delimiters=frozenset({("(", ")")})), classification="fix(#329)", notes="the only clause in the battery holding THREE tokens, " "which is what bounds the drop in both directions: it " "takes the marker and stops. Every other delimited row " "has a two-token clause, where 'the first token' and " "'all but the last token' agree, so two opposite " "mistakes both survive them -- restricting the drop to " "a clause of exactly two tokens gives maiden 'née Mary " "Jones' here (marker never dropped), and letting it eat " "the token after the marker gives maiden 'Jones' " "(a name eaten). Both measured 2026-08-03. 1.4.0 under " "the bucket-move idiom " "maiden_delimiters['parenthesis'] = " "nickname_delimiters.pop('parenthesis') gave first Jane " "/ last Smith / maiden 'née Mary Jones' (2026-08-03) -- " "marker inside the value, the single field #329 moves"), Case("maiden_marker_delimited_trailing_marker", "Jane Smith (Jones née)", {"given": "Jane", "family": "Smith", "maiden": "Jones née"}, policy=Policy(maiden_delimiters=frozenset({("(", ")")})), classification="parity", notes="the drop takes the clause's FIRST token or nothing: a " "marker anywhere else in the clause is content. Pinned " "because the cheap generalization -- drop every marker " "in the clause -- passes the whole battery above and " "gives maiden 'Jones' here, and because no marker the " "shipped vocabulary carries is written after the name " "it marks. Measured against 1.4.0 2026-08-03 " "through the bucket-move idiom: first Jane / last " "Smith / maiden 'Jones née'"), Case("maiden_marker_delimited_beside_a_nickname_clause", 'Jane "née Janie" Smith {née Jones}', {"given": "Jane", "family": "Smith", "nickname": "née Janie", "maiden": "Jones"}, policy=Policy(maiden_delimiters=frozenset({("{", "}")})), classification="fix(#329)", notes="the pass is scoped to MAIDEN clauses, and this is the " "row that says so: two extracted clauses, both opening " "with a marker word, and only the maiden one loses it. " "Braces route to maiden here precisely so the default " "nickname set survives untouched -- the parenthesis " "rows above cannot show this, since Policy's " "maiden-wins canonicalization would take ( ) away from " "nickname. Without the role filter the nickname reads " "'Janie'. 1.4.0 cannot express a brace delimiter at " "all (its buckets hold the NAMES of compiled regexes " "and there is no brace one; measured 2026-08-03, " "maiden_delimiters['brace'] = ('{', '}') is accepted " "and then raises ValueError('references unknown " "regexes key') at parse time), so the " "classification compares against its single reading, " "first Jane / middle 'Smith {née' / last 'Jones}' / " "nickname 'née Janie' -- braces as name text, the same " "convention maiden_marker_kyusei_delimited uses for a " "knob with no v1 spelling. The nickname agreed even " "there"), Case("maiden_marker_delimited_two_clauses", "Jane Smith (Nee) (Jones)", {"given": "Jane", "family": "Smith", "maiden": "Nee Jones"}, policy=Policy(maiden_delimiters=frozenset({("(", ")")})), classification="parity", notes="the scoping pin for #329, and the row a simplification " "would break: the drop is CLAUSE-scoped, so a one-token " "clause keeps its token even when the next clause could " "read as the name it marks. A neighbour-scoped rule -- " "drop a marker whose successor is also maiden -- gives " "'Jones' here, eating a real surname (Irish Ní/Nee, and " "a Chinese romanization). Unaccented 'nee' is in the " "default MAIDEN_MARKERS, so the 'Nee' token really is " "tagged and the CLAUSE bound is the only thing keeping " "it -- which is what makes this row kill a rule that " "drops the bound. The VALUE does not depend on that " "vocabulary entry, though: the clause test is checked " "before the tag test, so 'nee' leaving MAIDEN_MARKERS " "would leave this expectation green. " "maiden_marker_delimited_unaccented above is the row " "that fails when it goes. " "Parity is measured, not inferred from the row being " "untouched: 1.4.0 under the same bucket move gave first " "Jane / last Smith / maiden 'Nee Jones' (2026-08-02), " "so the two clauses joined with a space on that side " "too -- the classification the facade runner checks " "against, since it expresses this policy through the " "same bucket move"), Case("maiden_marker_delimited_content_free", "(née —)", {}, policy=Policy(maiden_delimiters=frozenset({("(", ")")})), classification="fix(#329)", notes="the drop can empty the WHOLE parse, and that is a " "decision rather than fallout. assemble's content test " "runs over the SURVIVING tokens, so once the marker " "goes structural the em dash is the only one left, no " "alnum character remains and every field clears -- " "bool() False. Reachable only where a maiden clause is " "the entire input and its non-marker tokens are pure " "punctuation -- the same clause inside a name is " "maiden_marker_delimited_content_free_in_a_name below. " "Coherent with the model 2.0 " "already had: a dropped marker is structural like a " "delimiter character, and '(-)' empties on both sides " "of this change. Structurally unreachable on the bare " "#274 path, whose scan starts at piece 1, so a token " "always survives ahead of the marker ('née —' gives " "given 'née', family '—'). Do NOT restore the old value " "with a guard on what else the clause holds: that is a " "different rule, and it would leave maiden holding " "marker-plus-punctuation. fix rather than parity " "because 1.4.0 CAN express this policy and disagrees: " "measured 2026-08-03 through the bucket-move idiom " "maiden_delimiters['parenthesis'] = " "nickname_delimiters.pop('parenthesis'), it gave maiden " "'née —' and a truthy name, as pre-#329 did. The 2.0 " "content rule already deviated from 1.4.0 here ('(-)' " "is maiden '-' in 1.4.0); this change moves one more " "input into its reach"), Case("maiden_marker_delimited_content_free_in_a_name", "Jane Smith (née —)", {"given": "Jane", "family": "Smith", "maiden": "—"}, policy=Policy(maiden_delimiters=frozenset({("(", ")")})), classification="fix(#329)", notes="the row above with a name in front of the clause, " "which is what bounds the emptying: assemble's content " "test is about the WHOLE parse, so a clause of " "marker-plus-punctuation empties only a name that is " "nothing else. Here Jane Smith carries the alnum " "content and maiden keeps the em dash. Pinned because " "the drop could plausibly have been widened to take the " "clause's punctuation with the marker -- that mutation " "gives maiden '' here (measured 2026-08-03) and leaves " "the row above green, since both readings empty a parse " "that is only the clause. 1.4.0 under the bucket-move " "idiom gave first Jane / last Smith / maiden 'née —' " "(2026-08-03)"), Case("maiden_marker_kyusei_delimited", "山田 花子（旧姓 佐藤）", {"given": "花子", "family": "山田", "maiden": "佐藤"}, policy=Policy( maiden_delimiters=frozenset({("(", ")"), ("（", "）")})), classification="fix(#329)", notes="the Japanese bracketed form that #329 reaches: the " "marker is spaced off inside fullwidth brackets, so it " "is a token of its own and the clause-scoped drop " "applies. The form Japanese more often writes puts a " "fullwidth colon after the marker instead, and " "'山田（旧姓：佐藤）' is ONE token -- nothing reaches it, " "and it wants the head-peel #317 tracks (see " "maiden_marker_kyusei above). Unlike its two Latin " "siblings, 1.4.0 cannot express this policy at all: v1's " "delimiter buckets hold the NAMES of compiled regexes, " "and no fullwidth pair is among them (#273 added it), so " "maiden_delimiters['fullwidth_parenthesis'] = ('（', '）') " "raises ValueError('references unknown regexes key') at " "parse time. The classification therefore compares " "against 1.4.0's single reading, first 山田 / middle " "'花子（旧姓' / last '佐藤）' -- the brackets were name " "text -- the same convention " "ko_honorific_period_under_strict_comma_suffixes uses " "for a knob with no v1 spelling. That reading is also " "what the differential harness sees, since it runs the " "corpus under the DEFAULT policy where （） is a #273 " "NICKNAME delimiter and nothing in #329 is reachable; " "the diff is classified there under " "fix(cjk-fullwidth-paren-nickname)"), Case("east_slavic", "Сидоров Иван Петрович", {"given": "Иван", "middle": "Петрович", "family": "Сидоров"}, policy=_ES), Case("turkic", "Mammadova Aygun Ali kizi", {"given": "Aygun", "middle": "Ali kizi", "family": "Mammadova"}, policy=_TK), Case("ru_pack_formal_rotation", "Сидоров Иван Петрович", {"given": "Иван", "middle": "Петрович", "family": "Сидоров"}, locale="ru", notes="the RU pack end-to-end: same expectation as the " "east_slavic synthetic row, through parser_for"), Case("ru_pack_transliterated", "Petrov Ivan Sergeyevich", {"given": "Ivan", "middle": "Sergeyevich", "family": "Petrov"}, locale="ru"), Case("ru_pack_comma_untouched", "Петров, Иван", {"given": "Иван", "family": "Петров"}, locale="ru", notes="a comma is an explicit signal that suppresses the " "rotation (spec §1)"), Case("tr_az_pack_marker", "Mammadova Aygun Ali kizi", {"given": "Aygun", "middle": "Ali kizi", "family": "Mammadova"}, locale="tr_az", notes="the TR_AZ pack end-to-end: same expectation as the " "turkic synthetic row (pinned live during Plan 3), " "through parser_for"), Case("empty", "", {}), Case("whitespace", " ", {}), Case("bare_ambiguous_acronym", "John Ed", {"given": "John", "family": "Ed"}, ambiguities=("suffix-or-name",), notes="'ed' is an ambiguous acronym; bare form is a name (C1), " "and the parse reports which reading it took"), Case("comma_ambiguous_acronym", "Smith, Ed", {"given": "Ed", "family": "Smith"}), Case("ambiguous_acronym_with_suffix", "John Ed III", {"given": "John", "family": "Ed", "suffix": "III"}, ambiguities=("suffix-or-name",)), Case("phd_split", "John Ph. D.", {"given": "John", "suffix": "Ph. D."}, notes="v1 fix_phd; healed via the stable 'joined' tag"), Case("phd_split_mid_name", "Dr. John Ph. D. Smith", {"title": "Dr.", "given": "John", "family": "Smith", "suffix": "Ph. D."}), Case("phd_split_leading", "Ph. D. John Smith", {"given": "John", "family": "Smith", "suffix": "Ph. D."}, classification="fix", notes="v1 healed 'Ph.'+'D.' only when trailing; leading it " "split them (title 'Ph.', given 'D.', real given name " "pushed to middle). Surfaced by the issue-tracker " "corpus, which is where this shape existed at all."), Case("leading_never_given_particle", "de la Vega", {"family": "de la Vega"}, notes="v1 handle_non_first_name_prefix: never-given leading " "particle folds the whole name into family"), Case("unbalanced_quote", 'Jon "Nick Smith', {"given": "Jon", "middle": '"Nick', "family": "Smith"}, ambiguities=("unbalanced-delimiter",), notes="quote char stays literal (spec §5a)"), Case("suffix_stays_suffix", "Johnson PhD", {"given": "Johnson", "suffix": "PhD"}, classification="fix(suffix-routing)", notes="v1 routes a lone trailing suffix to family " "(first=Johnson last=PhD); v2 keeps recognized " "suffixes in suffix"), Case("suffix_stays_suffix_title", "Mr. Johnson PhD", {"title": "Mr.", "given": "Johnson", "suffix": "PhD"}, classification="fix(suffix-routing)", notes="v1 routes a lone trailing suffix to family " "(title=Mr. first=Johnson last=PhD); v2 keeps " "recognized suffixes in suffix"), Case("family_comma_lone_title", "Smith, Dr.", {"title": "Dr.", "family": "Smith"}, classification="fix(comma-family)", notes="pre-comma is definitionally family; v1 put it in first"), # -- #271: script-scoped order + segmentation (amendment 2026-07-27) Case("ko_unspaced_default", "김민준", {"family": "김", "given": "민준"}, classification="fix(#271)", notes="hangul is unambiguously Korean: census surnames ship " "as default vocabulary and HANGUL segmentation is " "default-on"), Case("ko_two_syllable_surname_default", "남궁민수", {"family": "남궁", "given": "민수"}, classification="fix(#271)", ambiguities=("segmentation",), notes="남 is itself a shipped surname; longest-first takes " "남궁 and records the decided fork"), Case("ko_bare_two_syllable_surname", "남궁", {"family": "남궁"}, classification="fix(#271)", notes="a token that IS a surname never splits by its " "shorter prefix (남+궁); whole token takes the script " "order's first role; nothing was split, so no fork is " "recorded"), Case("ko_family_comma_stays_whole", "남궁민수, 지훈", {"family": "남궁민수", "given": "지훈"}, classification="fix(#271)", notes="the comma already decided the family: segmentation " "is inert under FAMILY_COMMA (comma doctrine -- see " "the script_segment stage docstring, which uses this " "exact example)"), Case("ko_suffix_comma_name_part_splits", "Dr 김민준, Jr.", {"title": "Dr", "family": "김", "given": "민준", "suffix": "Jr."}, classification="fix(#271)", notes="the one comma structure where segmentation still " "fires: a second word before the comma makes it " "SUFFIX_COMMA, and the name part is a full positional " "name"), Case("ko_spaced_family_first_default", "김 민준", {"family": "김", "given": "민준"}, classification="fix(#271)", notes="script_orders, no segmentation involved"), Case("han_spaced_family_first_default", "毛 泽东", {"family": "毛", "given": "泽东"}, classification="fix(#271)", notes="Han ORDER is default-safe without knowing zh from ja " "(both write family-first natively); only " "SEGMENTATION needs the zh opt-in"), Case("han_unspaced_unsegmented_default", "毛泽东", {"family": "毛泽东"}, classification="fix(#271)", notes="no default Han segmentation: one token, and a lone " "wholly-Han token takes the script order's first " "role = family"), Case("mixed_script_untouched_by_script_orders", "John 王", {"given": "John", "family": "王"}, notes="effective_script is None for a mixed name: script_orders " "declines and the positional default governs"), Case("two_han_scripts_untouched_by_script_orders", "毛 김", {"given": "毛", "family": "김"}, notes="two scripts also decline -- the rule is one script, or " "the Han/kana repertoire the #272 license covers"), Case("zh_unspaced", "毛泽东", {"family": "毛", "given": "泽东"}, locale="zh", classification="fix(#271)"), Case("zh_unspaced_two_char", "张伟", {"family": "张", "given": "伟"}, locale="zh", classification="fix(#271)"), Case("zh_compound_tie_break", "夏侯惇", {"family": "夏侯", "given": "惇"}, locale="zh", classification="fix(#271)", ambiguities=("segmentation",), notes="夏 (rank 65) is itself listed, so longest-first " "decides a real fork here and records it; most " "compounds' first chars are NOT listed"), Case("zh_compound_two_char_given", "司马相如", {"family": "司马", "given": "相如"}, locale="zh", classification="fix(#271)"), Case("zh_traditional_compound", "諸葛亮", {"family": "諸葛", "given": "亮"}, locale="zh", classification="fix(#271)"), Case("zh_no_surname_match", "阿明", {"family": "阿明"}, locale="zh", classification="fix(#271)", notes="no surname prefix in the vocabulary: the token stays " "whole and takes the script order's first role"), Case("zh_japanese_kanji_tradeoff", "高橋一郎", {"family": "高", "given": "橋一郎"}, locale="zh", classification="fix(#271)", notes="the RECORDED tradeoff, not a bug: applying the zh " "pack declares the data Chinese, so Japanese kanji " "names mis-split (高橋 is the real surname). This is " "why Han segmentation is opt-in and why the gate " "cannot guard it (DEVIATES declares all Han); " "Japanese data belongs under locales.JA and its " "segmenter instead"), # -- #272: the kana license + nakaguro (amendment 2026-07-29). # Segmenter-dependent divisions cannot live here (Case has no # segmenter field); they are pinned in test_locales.py's # integration tests instead. Case("ja_kana_spaced_family_first", "高橋 みなみ", {"family": "高橋", "given": "みなみ"}, classification="fix(#272)", notes="hiragana identifies Japanese as certainly as hangul " "identifies Korean; kana-licensed names read " "family-first by default"), Case("ja_kanji_katakana_pieces", "山田 エミ", {"family": "山田", "given": "エミ"}, classification="fix(#272)", notes="a kanji piece + a katakana piece cannot be a foreign " "transcription (those are katakana-only): native, " "licensed"), Case("ja_unspaced_unsegmented_default", "高橋みなみ", {"family": "高橋みなみ"}, classification="fix(#272)", notes="no segmenter by default: one token, family role via " "the kana license"), Case("ja_pure_katakana_positional", "マイケル ジャクソン", {"given": "マイケル", "family": "ジャクソン"}, notes="parity row guarding the license's boundary: " "pure-katakana is predominantly transcribed foreign " "names in original order -- never licensed"), Case("ja_nakaguro_divides_the_transcription", "マイケル・ジャクソン", {"given": "マイケル", "family": "ジャクソン"}, classification="fix(#272)", notes="the katakana middle dot is the transcription's own " "part divider: it separates like whitespace, the " "license declines each katakana token, and the " "positional default keeps the source-language order"), Case("ja_nakaguro_han_takes_the_han_order", "高橋・一郎", {"family": "高橋", "given": "一郎"}, classification="fix(#272)", notes="the dot splits; both tokens are pure Han, so the HAN " "family-first entry fires -- the katakana row's sibling " "through the other outcome"), Case("ja_lone_hiragana_takes_family", "みなみ", {"family": "みなみ"}, classification="fix(#272)", notes="hiragana earns a script_orders entry in its own right: " "a lone token takes the entry's first role"), Case("ja_lone_katakana_stays_given", "マイケル", {"given": "マイケル"}, notes="parity: katakana deliberately has no entry, so the " "positional default holds -- transcribed foreign names " "keep source order"), Case("ja_iteration_mark_is_han", "佐々木 太郎", {"family": "佐々木", "given": "太郎"}, classification="fix(#272)", notes="々 (U+3005, the ideographic iteration mark) repeats " "the preceding kanji and is Script=Han under UAX #24, " "but sits outside every CJK ideograph BLOCK -- and the " "classifier is a block table, so it needs its own " "entry to count as Han; without one 佐々木 -- a top-20 " "Japanese surname -- would be a mixed-script token and " "reverse"), Case("ja_shime_mark_is_han", "〆木 太郎", {"family": "〆木", "given": "太郎"}, classification="fix(shime-mark)", notes="〆 (U+3006, the shime mark) opens real Japanese " "surnames -- 〆木 Shimeki, 〆谷 Shimetani, 〆野 -- but " "carries Script=Common under UAX #24; the table counts " "it as Han anyway (a deliberate step PAST the Script " "property, unlike 々's), else 〆木 is a mixed-script " "token and the name reverses"), Case("ja_shime_lone_token_takes_family", "〆木太郎", {"family": "〆木太郎"}, classification="fix(shime-mark)", notes="wholly-Han lone token under the family-first entry: " "the whole token lands in family, unsegmented by " "default"), Case("ja_shime_with_kana_given", "〆木 ひろ", {"family": "〆木", "given": "ひろ"}, classification="fix(shime-mark)", notes="the kana license composes with the widened Han span: " "〆木 reads as kanji beside a kana given name"), Case("ja_nakaguro_inside_a_nickname", "山田 太郎 (マイケル・ジャクソン)", {"family": "山田", "given": "太郎", "nickname": "マイケル ジャクソン"}, classification="fix(#272)", notes="delimited content tokenizes under the same separator " "rules: the dot renders back as a space in the nickname " "join -- a decision, not an accident"), Case("zh_interpunct_transcription_source_order", "威廉·莎士比亚", {"given": "威廉", "family": "莎士比亚"}, classification="fix(#298)", notes="间隔号-divided Han is a transcribed foreign name and " "keeps source order -- the B7 is the transcription " "marker, playing the role pure katakana plays in the " "kana license; it divides only between classified " "characters"), Case("zh_interpunct_nakaguro_typed_stays_roster", "威廉・莎士比亚", {"given": "莎士比亚", "family": "威廉"}, classification="fix(#272)", notes="the SAME transcription typed with the Japanese " "nakaguro reads as the dot's own typography says -- a " "姓・名 roster pair, family-first (#272) -- because the " "nakaguro records nothing (spec 2026-07-30 decision 5: " "codepoint-scoped; only the Chinese B7 marks a " "transcription). A limitation row: chosen, not " "accidental -- cross-convention input reads by the " "convention of the codepoint it was typed with"), Case("ja_interpunct_b7_katakana", "マイケル·ジャクソン", {"given": "マイケル", "family": "ジャクソン"}, classification="fix(#298)", notes="sloppy-IME B7 between katakana divides like the " "nakaguro; katakana was never licensed, so order was " "already positional -- the split is the fix"), Case("latin_punt_volat_is_name_interior", "Gal·la Marcet", {"given": "Gal·la", "family": "Marcet"}, notes="the Catalan punt volat: U+00B7 with Latin neighbors " "is interior to the name, never a divider -- the flank " "guard's whole reason"), Case("zh_interpunct_suppresses_segmentation", "马丁·路德·金", {"given": "马丁", "middle": "路德", "family": "金"}, locale="zh", classification="fix(#298)", notes="马 is a listed zh surname, but a 间隔号-divided name " "is a transcription: the dot gates segmentation off, " "so 马丁 stays whole and the positional read stands"), Case("ko_interpunct_transcription_source_order", "마이클·잭슨", {"given": "마이클", "family": "잭슨"}, classification="fix(#298)", notes="Korean writes transcribed foreign names with the " "interpunct too: the dot suppresses the hangul " "family-first entry AND the default segmentation -- 마 " "is a listed census surname, and the spaced form " "마이클 잭슨 really does mis-split 마|이클 today, so " "the dot is what rescues the ko transcription"), Case("zh_interpunct_with_suffix_comma", "威廉·莎士比亚, PhD", {"given": "威廉", "family": "莎士比亚", "suffix": "PhD"}, classification="fix(#298)", notes="the transcription reading composes with a suffix " "comma: the marker is structure-independent"), Case("zh_interpunct_half_flanked_stays", "王·Smith", {"given": "王·Smith"}, notes="one classified neighbor is not enough: the guard " "requires both, so the undivided dot remains part of " "the word -- declining, not deciding"), Case("zh_honorific_suffix_spaced", "王小明 先生", {"family": "王小明", "suffix": "先生"}, classification="fix(#307)", notes="CJK honorifics FOLLOW the name; a spaced 先生 (Mr.) is " "a suffix, and recognizing it must come before the " "family-first order hands it a role -- unrecognized it " "read as the GIVEN name under the 2.1 defaults"), Case("ko_honorific_ssi", "김민준 씨", {"family": "김", "given": "민준", "suffix": "씨"}, classification="fix(#307)", notes="Korean orthography standardly SPACES 씨, so the " "whole-token suffix machinery reaches it; the name " "still segments (suffix classification runs after the " "script_segment stage, which only ever saw 김민준)"), Case("ko_degree_baksa", "김민준 박사", {"family": "김", "given": "민준", "suffix": "박사"}, classification="fix(#307)", notes="박사 (doctorate) is the ko analogue of a trailing " "PhD: fix(suffix-routing)'s two-token shape, one " "script over"), Case("ja_sama_spaced", "田中 太郎 様", {"family": "田中", "given": "太郎", "suffix": "様"}, classification="fix(#307)", notes="the spaced 様 of forms and databases, which whole-token " "matching reaches on its own; the glued " "mail-addressing form is ja_sama_glued below, reached " "by #308's peel instead"), Case("ja_san_spaced", "田中 さん", {"family": "田中", "suffix": "さん"}, classification="fix(#308)", notes="the kana honorifics ship as suffix vocabulary so the " "glued peel has somewhere to hand its tail; spaced " "recognition falls out of the same entry -- until this " "change さん read as the given name under the " "family-first default"), Case("ja_san_glued", "田中さん", {"family": "田中", "suffix": "さん"}, classification="fix(#308)", notes="the everyday glued form, and the one that also " "corrupted classification: 田中さん is Han plus " "hiragana, so the kana license read the whole string " "as a Japanese name. The peel runs before the license " "is consulted, so it now sees 田中 alone"), Case("ja_honorific_glued_before_a_roman_suffix", "田中さん II", {"family": "田中", "suffix": "さん, II"}, classification="fix(#308)", notes="an unrelated trailing suffix does not hide the peel " "site: the scan-back steps over II and peels さん off " "the token behind it. Half of the pair that pins " "_is_post_nominal's use of is_suffix_STRICT -- the " "other half is the row below, and swapping in " "is_suffix_lenient changes that one and not this one"), Case("ja_honorific_glued_before_an_initial", "田中さん V.", {"given": "田中さん", "family": "V."}, notes="the strict/lenient discriminator, and the reason " "_is_post_nominal reads is_suffix_strict. Bare 'v' is " "a suffix word, but 'V.' is an initial and the strict " "test vetoes it -- so the scan-back stops HERE rather " "than stepping over it, finds no tail on 'V.', and " "does not peel. Under is_suffix_lenient it would step " "over and give given 田中, middle さん, family 'V.'. " "Parity besides: 1.4.0 read this first 田中さん / last " "'V.', which is these fields under the 2.0 names. " "Classification agrees with what classify does with " "the same token downstream -- 'V.' is a middle " "initial, not a post-nominal"), Case("ja_honorific_with_a_period_no_comma", "田中さん 様.", {"family": "田中", "suffix": "さん, 様."}, classification="fix(#320)", notes="the SPACED form, and the example _vocab.is_initial's " "own docstring cites as what #320 cost. Same fields as " "the comma row below, reached without a comma: the peel " "scans segment 0 either way, and with '様.' no longer " "vetoed the scan-back steps over it onto 田中さん and " "peels さん. Worth its own row because it is the only " "one of the pair where the SCAN-BACK is what #320 " "fixes, and since #319 the only one where the PEEL " "depends on #320 at all: here '様.' shares the run with " "田中さん and has to be stepped over, while the comma " "form declines its post-comma run and never looks at " "'様.' in the peel. That decline does not carry the " "veto over -- is_wholly_suffix asks is_suffix_lenient " "under the default policy, which takes '様.' whether " "the veto is in place or not (see the row below) -- so " "restoring the veto strands さん HERE and nowhere else " "in the pair, giving given '田中さん', family '様.'. " "(Before #319 the distinction was structural -- " "FAMILY_COMMA flattened TWO runs before scanning where " "this has one -- which is no longer what separates " "them.) 1.4.0 read this first " "田中さん / last '様.', which is what 2.0 produced until " "#320: parity before, a classified change after"), Case("ja_honorific_period_does_not_stop_the_peel", "田中さん, 様.", {"family": "田中", "suffix": "さん, 様."}, classification="fix(#320)", notes="the comma spelling of the row above, and since #319 a " "DIFFERENT dependence on #320. At #320 this row was " "about the peel: the veto made _is_suffix_strict say " "no of '様.', so the #312 scan-back stopped AT it as " "the site, found no listed tail there and abandoned, " "leaving さん glued to 田中 over one period. #319 then " "declined the post-comma run outright, and what is " "left here for #320 to fix is the ASSIGNMENT -- with " "the veto restored the peel still fires (family 田中, " "suffix さん) and only '様.' moves, to given. The " "fields and the classification are unchanged either " "way. CORRECTION, recorded here because commit " "4aff219's message claims otherwise and cannot be " "amended: the veto does NOT reach is_wholly_suffix " "under this row's DEFAULT policy, so it is not true " "that #319 merely handed the same veto to a different " "predicate. is_wholly_suffix selects is_suffix_lenient " "there, whose contract is suffix_words accepted " "unconditionally, BYPASSING the initial veto, and " "_normalize('様.') is '様', a suffix word -- so " "is_wholly_suffix(['様.']) is True veto or no veto, " "the run is declined either way, and simulating the " "veto PEELS さん here rather than stranding it. The " "is_wholly_suffix route to an abandoned peel exists " "only under Policy(lenient_comma_suffixes=False), " "which drops the call to is_suffix_strict and gives " "family '田中さん', given '様.'; this row does not set " "that knob, and " "ko_honorific_period_under_strict_comma_suffixes is " "the table's row for it. Was the " "shape of " "ja_honorific_glued_before_an_initial above, except " "that here the token stopping the scan is a real " "honorific rather than an initial, which is what makes " "it a bug rather than the intended veto. The structure " "is FAMILY_COMMA throughout -- SUFFIX_COMMA needs more " "than one word ahead of the comma and 田中さん is one -- " "so at #320 both runs were in the peel's reach and " "only the strict test's answer moved. " "1.4.0 read " "this first '様.' / last 田中さん, which is exactly what " "2.0 produced before this change -- the row sat at " "parity until #320 moved it"), Case("ja_sama_glued", "山田太郎様", {"family": "山田太郎", "suffix": "様"}, classification="fix(#308)", notes="the mail-addressing form. Undivided without a " "segmenter -- no surname list divides a kanji name -- " "so the family name is the whole 山田太郎; " "tests/v2/test_locales.py pins the divided twin under " "locales.JA"), Case("ko_honorific_nim_glued", "김민준님", {"family": "김", "given": "민준", "suffix": "님"}, classification="fix(#308)", notes="the online/formal glued address form, 씨's twin"), Case("ko_honorific_written_with_a_period", "김민준, 씨.", {"family": "김민준", "suffix": "씨."}, classification="fix(#320)", notes="the period-written form of ko_honorific_after_comma " "('김민준, 씨'), whose field ASSIGNMENT it must match " "and before #320 did not -- same roles, the suffix " "VALUE differing by the period it was written with. " "_normalize strips the trailing period, " "so the vocabulary sees 씨 either way -- the initial " "veto was the only thing rejecting the written form, " "and literally the veto: _is_suffix_piece is " "'vocab:suffix' in tags and 'initial' not in tags, and " "'씨.' carried both, so the suffix-shaped piece went to " "the given. 1.4.0 read this first '씨.' / last 김민준 -- " "the same fields 2.0 gave before this change, so the row " "was at parity and #320 is what moves it"), Case("ko_honorific_period_under_strict_comma_suffixes", "김민준, 씨.", {"family": "김민준", "suffix": "씨."}, policy=Policy(lenient_comma_suffixes=False), classification="fix(#320)", notes="the row above under the knob that governs exactly this " "shape, and one of the table's three exercises of it " "(ja_honorific_glued_family_comma_strict_knob is the " "second, on the initial-shaped side of the same gap; " "ja_honorific_glued_family_comma_credential_pair_strict_" "knob is the third, showing where the knob changes " "nothing). " "lenient_comma_suffixes=False drops segment's post-comma " "test to the strict one, so a 'Family, Suffix' input " "whose suffix is INITIAL-SHAPED reads as a given-name " "initial instead ('John Smith, V' -> given 'V'). '씨.' " "is a single character plus a period, so it was in that " "class by shape, and before #320 the knob decided it: " "given '씨.' / family 김민준. It is out of the class now, " "and the honorific parses identically under both " "settings -- which is the claim this row exists to " "hold, since the knob's own documentation scopes it to " "the initial-shaped suffix words ('John Smith, V') and " "_script_segment names it as the setting that keeps a " "glued honorific inside the name in a neighbouring " "shape ('田中さん, V.' gives family '田中さん' under " "it). No v1 spelling exists for the knob " "(the facade runner skips this row), so the " "classification compares against 1.4.0's single " "reading, first '씨.' / last 김민준 -- the same fields " "2.0 gave under EITHER setting before this change"), Case("ko_honorific_with_a_period_no_comma", "김민준 씨.", {"given": "민준", "family": "김", "suffix": "씨."}, classification="fix(#320)", notes="the period-written ko_honorific_ssi ('김민준 씨'), and " "#320 by a different route than the row above: the veto " "left '씨.' a NAME piece, and effective_script('씨.') is " "None because the trailing period defeats the " "wholly-one-script test, so script_orders declined for " "the whole name and the three pieces fell back to " "name_order -- given 김, middle 민준, family '씨.'. " "Hangul segmentation is NOT what moves: it divides " "김민준 identically either way, and only the order the " "pieces are read in changes. 1.4.0 read this first " "김민준 / last '씨.' -- undivided, no suffix. The only " "row of the three that already differed from 1.4.0 " "before this change, but it differed as given 김 / " "middle 민준 / family '씨.'; the fields above are #320's, " "not the segmenter's, so the row is classified to it -- " "as ko_honorific_ssi is classified to #307 without " "naming the same segmentation it also depends on"), Case("ko_honorific_glued_teacher", "김선생님", {"family": "김", "suffix": "선생님"}, classification="fix(#307)", notes="longest-first, end to end: 선생님 peels whole where " "님 alone would have left 김선생 to segment into a " "family 김 and a given 선생. Classified to #307 " "because the fields do not move in this change -- " "segmentation already delivered this shape (김 is a " "listed surname, so the split reached it); #308 " "changes which mechanism gets there first"), Case("latin_stem_glued_kana_honorific", "Andersonさん", {"given": "Anderson", "suffix": "さん"}, classification="fix(#308)", notes="no script precondition on the remainder -- the tail " "is the license. Japanese text about a foreigner, and " "the Latin remainder keeps the positional default"), Case("latin_stem_glued_hangul_honorific", "Anderson선생님", {"given": "Anderson", "suffix": "선생님"}, classification="fix(#308)", notes="the hangul twin of latin_stem_glued_kana_honorific, " "and the one that shows why a post-nominal is not a " "surname site: 선 is a listed census surname, so the " "peeled 선생님 would otherwise be split into 선 + 생님 " "-- the stage dissecting the honorific it had just " "manufactured"), Case("ko_honorific_glued_doctor", "김민준박사님", {"family": "김", "given": "민준", "suffix": "박사님"}, classification="fix(#308)", notes="박사님 is one honorific, not 박사 plus 님, and ships " "as one entry: 선생님, 교수님 and 박사님 are the three " "standard -님 professional honorifics and the first two " "shipped without it, which stranded 박사 in the given " "name here and rendered the spaced 김민준 박사님 as two " "suffixes. This row USED to pin the one-peel rule " "(named ko_glued_stack_peels_once) on the reading that " "left 박사 behind; no shipped vocabulary now has that " "shape, so the pin lives at stage level with a " "synthetic lexicon -- test_script_segment.py's " "test_one_peel_never_a_stack"), Case("ko_honorific_glued_doctor_spaced", "김민준 박사님", {"family": "김", "given": "민준", "suffix": "박사님"}, classification="fix(#308)", notes="the spaced twin, and the second half of the same gap: " "without a 박사님 entry the peel cut this token too -- " "it is not a whole-token suffix word, so 박사 + 님 came " "back as two suffixes for one honorific"), Case("ko_glued_tail_alone_never_peels", "씨", {"family": "씨"}, classification="fix(#271)", notes="a token that IS a tail is not a peel site at all -- " "every tail is a suffix word, so the site scan steps " "past it and finds nothing else. NOT the length cap, " "which is never reached here. Classified to #271 " "because that is what moves the lone token from first " "to family; the row exists for the guard"), Case("ko_honorific_token_alone_stays_whole", "선생님", {"family": "선생님"}, classification="fix(#308)", notes="a lone honorific is not a name to be taken apart: it " "is not a peel site (every tail is a suffix word) and " "not a surname site, though 선 is listed and the " "default segmentation split it 선 + 생님 before this " "change"), Case("ja_glued_tail_alone_never_peels", "さん", {"family": "さん"}, classification="fix(#272)", notes="the kana twin of ko_glued_tail_alone_never_peels, and " "of that row only: a lone さん carries none of " "ko_honorific_token_alone_stays_whole's risk, since no " "surname vocabulary is written in kana. The field " "placement is the kana-licensed order default, not the " "peel"), Case("ja_glued_degree_stays", "田中博士", {"family": "田中博士"}, classification="fix(#271)", notes="the exclusion pinned: glued 田中博士 IS Tanaka " "Hiroshi, an attested given name, so 博士 never peels " "-- it stays recognized in the SPACED position only, " "where the writer's own token boundary settles it"), Case("zh_glued_jun_stays", "王君", {"family": "王君"}, classification="fix(#271)", notes="likewise unpeeled: 君 is a common Chinese given-name " "final, so 王君 is a complete name, not Mr. Wang. " "Unlike its neighbours here, 君 is in NEITHER set -- " "there is no spaced entry to fall back on either, and " "only its kana spelling くん peels"), Case("zh_glued_shi_stays", "王氏", {"family": "王氏"}, classification="fix(#271)", notes="likewise: 王氏 is a historical name form ('the Wang " "woman'). The spaced 田中氏 keeps its entry"), Case("ja_glued_dono_stays", "鵜殿", {"family": "鵜殿"}, classification="fix(#271)", notes="the exclusion with the longest argument behind it and, " "until this row, the only one nothing held: adding 殿 " "back to GLUED_HONORIFICS passed the whole suite. 鵜殿 " "(Udono) is a real surname, one of roughly ninety " "Japanese surnames ending in 殿 (真殿, 大殿, ...), and a " "peeled 殿 would give family 鵜 with 殿 in suffix. It " "has to be the BARE surname: in a two-token 真殿 太郎 " "the site scan lands on 太郎 and the exclusion is never " "consulted, so only a lone surname discriminates. " "Classified to #271 like its neighbours, not parity: " "1.4 gave first 鵜殿 and no last, and it is the CJK " "order flip that makes the one token a family name"), Case("ja_dono_spaced", "田中 殿", {"family": "田中", "suffix": "殿"}, classification="fix(#308)", notes="殿 waited on an argument in #307 and gets one here: " "spaced it is safe for the reason 양/군 are -- a " "殿-surnamed person's name LEADS, and the suffix gate " "is trailing-only -- while glued it would cut 鵜殿 and " "真殿 in two, so it ships spaced only"), Case("ko_honorific_nim_spaced", "김민준 님", {"family": "김", "given": "민준", "suffix": "님"}, classification="fix(#308)", notes="님 is new in both sets -- #307 shipped only the -님 " "compounds 선생님/교수님. Standardly glued in online " "address, spaced too, and never the end of a Korean " "given name, which is what qualifies it for the " "harsher glued vetting as well"), Case("ko_honorific_glued_via_segmentation", "김씨", {"family": "김", "suffix": "씨"}, classification="fix(#307)", notes="the one glued shape that was already reachable before " "#308, which is why this row stays fix(#307) where its " "neighbours are fix(#308): stage order alone delivered " "it, since segmentation split 김 off the front and the " "honorific was whatever remained. The peel now takes 씨 " "off the END before segmentation is consulted, so the " "route changed and these fields did not. The glued " "shapes that needed the peel to be reached at all are " "the rows around it (김민준씨 below)"), Case("ko_honorific_after_comma", "김민준, 씨", {"family": "김민준", "suffix": "씨"}, classification="fix(#307)", notes="the post-comma run is normally the given name -- " "'김민준, 태호' gives given 태호 -- and group's " "_is_suffix_piece diverts this one because 씨 is a " "single-token piece carrying vocab:suffix. NOT the " "lenient comma gate, which an earlier note named: " "measured, lenient_comma_suffixes=False leaves this " "row unchanged. The comma disables segmentation per " "the comma doctrine, so 김민준 stays whole"), Case("ko_honorific_glued_given", "김민준씨", {"family": "김", "given": "민준", "suffix": "씨"}, classification="fix(#308)", notes="the common full-name glued shape, and the row this " "replaces (ko_honorific_glued_given_stays) pinned the " "old boundary: 씨 peels off the last token first, and " "the remainder 김민준 then segments as usual -- peel " "and split compose, in that order"), Case("ko_honorific_glued_given_trailing_suffix", "김민준씨 Jr.", {"family": "김", "given": "민준", "suffix": "씨, Jr."}, classification="fix(#308)", notes="the peel site is the last token that is not itself a " "post-nominal, so an unrelated trailing suffix cannot " "hide it -- this now agrees with the comma-written " "'Dr 김민준씨, Jr.', where the suffix comma had " "already put 씨 within reach"), Case("ko_honorific_glued_given_suffix_comma", "Dr 김민준씨, Jr.", {"title": "Dr", "family": "김", "given": "민준", "suffix": "씨, Jr."}, classification="fix(#308)", notes="the peel scans the NAME's runs, not the token stream: " "under a suffix comma that is segments[0] alone, a " "strict subset, and the peel site is found within it " "(a FAMILY comma is the one structure that splits the " "name across two runs, #312). Pairs with " "ko_honorific_glued_given_trailing_suffix, whose " "comma-less spelling of the same name reaches the same " "answer by the scan-back instead"), Case("ko_honorific_glued_given_nickname", "김민준씨 (Jimmy)", {"family": "김", "given": "민준", "suffix": "씨", "nickname": "Jimmy"}, classification="fix(#308)", notes="the other half of scanning the NAME's runs: extracted " "content is still in the token stream at this stage but " "in NO segment, so the scan-back never reaches " "Jimmy. Scanning the tokens instead would take Jimmy as " "the site -- it is no post-nominal -- and lose the peel " "entirely, with 씨 back in the given name. Nothing else " "pins that choice: under NO_COMMA the two are otherwise " "the same run"), Case("ko_honorific_glued_given_nickname_family_comma", "김, 민준씨 (Jimmy)", {"family": "김", "given": "민준", "suffix": "씨", "nickname": "Jimmy"}, classification="fix(#312)", notes="the family-comma half of the same argument, which the " "row above cannot make: there the two runs are one, so " "'flatten the name's segments' and 'take segments[0]' " "agree. Here the peel has to cross the comma AND still " "not reach Jimmy, so declining to cross whenever " "extract_delimited claimed something passes the row " "above and fails only here"), Case("ja_honorific_glued_family_comma", "田中さん, PhD", {"title": "PhD", "family": "田中", "suffix": "さん"}, classification="fix(#312)", notes="the peel reaches 田中さん across the comma, though no " "longer by crossing it: since #319 is_wholly_suffix " "declines the post-comma run outright (PhD is suffix " "vocabulary and it is the whole run), so the scan never " "leaves segments[0] and never examines PhD at all. It " "did cross before, stepping OVER PhD because that " "spelling satisfies _is_post_nominal's strict test -- " "the same fields by the older route. The spaced " "田中さん PhD still peels that way, its single run " "holding both tokens, so the two spellings now agree on " "the outcome through DIFFERENT mechanisms; " "ja_honorific_glued_family_comma_suffixy_second_run " "cites this row for the outcome, not the route. Where " "PhD itself lands still differs " "between the two spellings -- suffix spaced, title " "post-comma -- and that is fix(comma-family)'s, not " "the peel's. The expectation bakes in TWO deviations " "from 1.4.0 and only one of them is #312's: the peel " "is, while first -> family is comma-family's, witnessed " "on pure Latin by family_comma_lone_title (1.4.0 first " "Smith, here family Smith) -- so no script-conditional " "rule is reaching that half"), Case("ja_honorific_glued_family_comma_suffixy_second_run", "田中さん, V.", {"given": "V.", "family": "田中", "suffix": "さん"}, classification="fix(#319)", notes="under a family comma the peel scanned both runs on " "the premise that segments[1] is name text, and here it " "is not: segment picks FAMILY_COMMA when the pre-comma " "part is a single word, even where the post-comma part " "is entirely suffix-shaped, so the scan reached 'V.' -- " "which is_suffix_strict rejects as an initial where " "segment admitted the run on is_suffix_lenient. 'V.' " "was therefore the site, ended in no tail, and the peel " "was abandoned with さん still in the family name. #319 " "asks segment's own predicate (_vocab.is_wholly_suffix) " "instead of inferring name text from the structure: the " "run is declined, the scan stays inside segments[0], " "and さん peels off 田中さん as it always did without a " "second run. Same credential, three spellings, ONE " "answer FROM THE PEEL now -- this, '田中さん, PhD' " "above and '田中さん, Ph. D.' in " "ja_honorific_glued_family_comma_credential_pair below " "all reach 田中さん. Where the credential itself LANDS " "still differs by spelling (title 'PhD', given 'V.', " "suffix 'さん, Ph. D.'); that is assign's question, not the " "peel's, and this row is not a claim about it. Nor does " "the comma form now agree with its SPACED twin: " "ja_honorific_glued_before_an_initial ('田中さん V.') " "still does not peel, because under NO_COMMA there is " "one run and 'V.' is inside the name's own tokens where " "the scan-back legitimately stops -- declining a " "post-comma run does not reach it. Parity with 1.4.0 " "(first V., last 田中さん) until this change, which is " "what moves it; the 1.4.0 fields are still reachable " "through Policy(lenient_comma_suffixes=False), pinned " "by ja_honorific_glued_family_comma_strict_knob below"), Case("ja_honorific_glued_family_comma_credential_pair", "田中さん, Ph. D.", {"family": "田中", "suffix": "さん, Ph. D."}, classification="fix(#319)", notes="the third of the three spellings #319 named, and the " "only one the Ph./D. merge reaches: is_wholly_suffix " "folds the adjacent pair into the single unit 'phd' " "(v1's fix_phd extracted the credential pre-parse), so " "the run counts as wholly suffix, the peel declines it, " "and さん comes off 田中さん exactly as in " "ja_honorific_glued_family_comma_suffixy_second_run " "above. The merge is also why this spelling is NOT the " "one to reach for when exercising " "Policy(lenient_comma_suffixes=False): 'phd' satisfies " "is_suffix_strict as readily as is_suffix_lenient, so " "the run is declined under EITHER setting and this row " "is identical under the knob, which " "ja_honorific_glued_family_comma_credential_pair_strict_" "knob below holds rather than leaving to the claim -- " "ja_honorific_glued_family_comma_strict_knob below uses " "'V.' because the initial-shaped words are where the " "two predicates actually part. Where the credential " "lands is a separate question and answers differently " "again: suffix here, title for 'PhD', given for 'V.' " "Measured: 1.4.0 gave first 田中さん / suffix 'Ph. D.' " "(fix_phd lifted the credential pre-parse, leaving a " "lone pre-comma word), so like " "ja_honorific_glued_family_comma above the expectation " "carries TWO deviations -- the peel is #319's, first -> " "family is comma-family's, which 2.0 already had before " "this change (family 田中さん / suffix 'Ph. D.')"), Case("ja_honorific_glued_family_comma_strict_knob", "田中さん, V.", {"family": "田中さん", "given": "V."}, policy=Policy(lenient_comma_suffixes=False), classification="parity", notes="the same input as " "ja_honorific_glued_family_comma_suffixy_second_run " "above under the knob that keeps the pre-#319 answer, " "and one of the table's three exercises of it " "(ko_honorific_period_under_strict_comma_suffixes and " "ja_honorific_glued_family_comma_credential_pair_strict_" "knob are the others). The knob drops is_wholly_suffix " "to the " "strict predicate, which rejects 'V.' as an initial, so " "the post-comma run reads as name text after all, the " "scan crosses into it, 'V.' is the site, it ends in no " "listed tail and the peel is abandoned -- さん stays in " "the family name. Not a blanket freeze of pre-#319 " "behavior, and the row must not be read as one: it " "holds for the INITIAL-shaped suffix words ('V.', 'V', " "'I'), which is the whole of where the strict/lenient " "gap lives. The counterexample is the row above, " "ja_honorific_glued_family_comma_credential_pair: its " "Ph./D. pair merges to a form strict accepts too, so " "that run is " "declined and the peel fires under this setting as " "well -- pinned, not merely stated, by " "ja_honorific_glued_family_comma_credential_pair_strict_" "knob above. No v1 spelling exists for the knob, so the " "facade runner skips this row and the classification " "compares against 1.4.0's single reading of the same " "text, as the other exercise of the knob named above " "does: measured, 1.4.0 gave first 'V.' / last " "田中さん, which is field for field what the knob holds " "here -- parity, and the point of the knob"), Case("ja_honorific_glued_family_comma_credential_pair_strict_knob", "田中さん, Ph. D.", {"family": "田中", "suffix": "さん, Ph. D."}, policy=Policy(lenient_comma_suffixes=False), classification="fix(#319)", notes="the counterexample the two rows above assert and " "neither measured: the knob does NOT freeze the " "pre-#319 reading in general, only for the " "initial-shaped suffix words. Here the Ph./D. merge " "folds the run into 'phd', which is_suffix_strict " "accepts as readily as is_suffix_lenient, so the run is " "declined and さん peels under this setting exactly as " "under the default -- field for field the same " "expectation as " "ja_honorific_glued_family_comma_credential_pair, which " "is the whole claim. Cheap to hold and worth holding " "separately, because the two rows differ only in the " "policy and a knob that started gating the decline " "wholesale would move this one alone. Same " "classification and the same two deviations as its " "default-policy twin (1.4.0: first 田中さん / suffix " "'Ph. D.'), which is also why the knob cannot be judged " "against a v1 spelling here -- there is none, so the " "facade runner skips this row as it does the other two " "knob rows"), Case("ko_honorific_glued_family_comma_suffixy_second_run", "김민준씨, V.", {"given": "V.", "family": "김민준", "suffix": "씨"}, classification="fix(#319)", notes="ja_honorific_glued_family_comma_suffixy_second_run in " "hangul, and pinned because the decline reads no script " "at all: it asks is_wholly_suffix about the post-comma " "run and the peel's own scan-back about segments[0], " "both vocabulary questions, so a script-conditional " "regression would be invisible in a table whose every " "other witness to #319 is written in kana. The family " "stays 김민준 undivided -- the FAMILY comma gates the " "surname split off, so hangul segmentation never runs " "here and only the peel acts. 1.4.0 gave first 'V.' / " "last 김민준씨, peeling nothing"), Case("zh_honorific_glued_family_comma_suffixy_second_run", "王先生, V.", {"given": "V.", "family": "王", "suffix": "先生"}, classification="fix(#319)", notes="the Han spelling of the row above, and the third " "script. 先生 is a shipped tail, so the peel fires with " "no locale opted in -- honorific_tails is licensed by " "the entries themselves rather than by " "Policy.segment_scripts, and HAN is not activated here " "(the family is what the peel left behind, not a " "vocabulary split). 1.4.0 gave first 'V.' / last " "王先生"), Case("ko_honorific_glued_family_comma_site_only_beyond_the_comma", "이, J.씨", {"given": "J.", "family": "이", "suffix": "씨"}, classification="fix(#312)", notes="the limit of #319's decline, and the row that says why " "it carries a second condition. is_wholly_suffix reaches " "period_joined_vocab, which calls an interior-period " "token a suffix when ANY chunk is suffix vocabulary -- " "and every honorific tail is a suffix WORD by the " "Lexicon invariant, so 'J.씨' reads as suffix-shaped " "BECAUSE of the 씨 the peel exists to remove. Declining " "on that evidence does not move the peel elsewhere the " "way 田中さん, V. does: segments[0] is the lone 이, which " "ends in no tail, so the scan finds no site at all, " "nothing peels, and the given name goes to suffix " "glued to its honorific ('J.씨'). So the gate declines " "only where segments[0] holds a peel site of its own, " "and this input reaches the pre-#319 fields by having " "none. Not rescued by " "Policy(lenient_comma_suffixes=False) either, unlike the " "strict-knob row above: the knob picks between " "is_suffix_lenient and is_suffix_strict per token and " "period_joined_vocab is downstream of neither, so both " "settings call this run wholly suffix. 1.4.0 gave first " "'J.씨' / last 이 -- it peels nothing, so the deviation " "here is #312's crossing, which is what puts the site on " "'J.씨' in the first place"), Case("ko_honorific_glued_family_comma_site_in_both_runs", "김민준씨, J.씨", {"family": "김민준", "suffix": "씨, J.씨"}, classification="fix(#319)", notes="the two-honorific input, where both runs hold a site " "and the decline therefore stands -- a deliberate choice " "between two readings rather than a fallout. Pre-#319 " "the scan crossed and took the LAST site, peeling the 씨 " "off the junk 'J.씨' and leaving the person's own " "honorific glued in the family name (given 'J.', family " "김민준씨, suffix 씨); now 씨 comes off 김민준씨 and " "'J.씨' is consumed whole as a suffix. It is also the " "row that rules out the narrower gate: declining only " "where the SECOND run has no peel site fixes " "ko_honorific_glued_family_comma_site_only_beyond_the_" "comma above and reverts this input to the pre-#319 " "reading, which is the worse of the two -- the same " "junk-tail reach " "ko_honorific_glued_given_suffix_comma_initial's note " "names under a suffix comma. 1.4.0 gave first 'J.씨' / " "last 김민준씨, peeling neither"), Case("ko_honorific_glued_family_comma_lone_post_nominal_before_it", "선생님, J.씨", {"given": "J.", "family": "선생님", "suffix": "씨"}, classification="fix(#312)", notes="segments[0] is a single token that IS a listed tail " "entire, which the site scan skips as a post-nominal " "(the same guard that keeps 선생님 from being peeled to " "선생 + 님) -- so there is no site before the comma, the " "run beyond it is scanned after all, and the fields are " "the pre-#319 ones as in the 이 row above. Pinned " "because it separates asking for the site with the " "peel's own scan-back from asking the cheaper question, " "whether any token ENDS in a tail: the cheaper one " "counts 선생님, declines, and then finds nothing to cut " "-- 씨 lost into suffix 'J.씨'. Only the lone-token " "shape of that divergence is reachable from the gate, " "which is why this row is one token before the comma; " "_peel_site's docstring derives the bound. " "1.4.0 gave first 'J.씨' / last 선생님"), Case("ko_honorific_glued_given_after_family_comma", "김, 민준씨", {"family": "김", "given": "민준", "suffix": "씨"}, classification="fix(#312)", notes="under a family comma the name spans both segments and " "the honorific is on the GIVEN side, where the peel " "never looked before #312. Agrees with the spaced " "김 민준씨"), Case("ja_honorific_glued_given_after_family_comma", "田中, 太郎さん", {"family": "田中", "given": "太郎", "suffix": "さん"}, classification="fix(#312)", notes="the Han twin of the row above"), Case("zh_interpunct_transcription_glued_honorific", "威廉·莎士比亚さん", {"given": "威廉", "family": "莎士比亚", "suffix": "さん"}, classification="fix(#312)", notes="the dot still gates the surname split -- a " "transcription's pieces are syllable groups -- but an " "honorific glued to a transcribed name is still an " "honorific, so the peel crosses it. Agrees with the " "spaced 威廉·莎士比亚 さん"), Case("ja_honorific_glued_family_comma_no_site", "田中さん, 太郎", {"family": "田中さん", "given": "太郎"}, notes="unchanged, and pinned because a naive fix breaks it: " "the site is the last NON-POST-NOMINAL token, which is " "太郎, so nothing peels -- exactly as in the spaced " "田中さん 太郎. #312 was originally filed naming this " "pair as a disagreement; it never was one"), Case("ko_honorific_glued_given_suffix_comma_initial", "Dr 김민준씨, V.", {"title": "Dr", "family": "김", "given": "민준", "suffix": "씨, V."}, classification="fix(#308)", notes="fix(#308) rather than fix(#312) because the fields do " "not move in this change -- a SUFFIX comma keeps the " "whole name in segments[0], which is what the peel " "already scanned. The row is here as the end-to-end " "guard for the scoping #312 introduced: widen the scan " "to every segment and 'V.' becomes the site, since " "segment admits a post-comma run on is_suffix_lenient " "while the site scan asks is_suffix_strict and an " "initial fails it. 'V.' then ends in no tail, the peel " "is abandoned, and 씨 is back in the given name -- the " "original bug. It was the table's only witness to that " "widening until #319; ja_honorific_glued_family_comma_" "suffixy_second_run and " "ja_honorific_glued_family_comma_credential_pair notice " "it as well now, but from the FAMILY_COMMA side, where " "the guard is is_wholly_suffix rather than this " "structural scoping. Their strict-knob sibling does NOT " "join them: under the knob is_wholly_suffix is False on " "the same run, so the scan was already crossing into " "segments[1] and widening past it changes nothing. This " "row is still the only SUFFIX comma among the three. " "Its comma-less twin ja_honorific_glued_before_an_initial " "shows the same veto from the other side, where 'V.' is " "in the name's own run and so IS the site"), Case("zh_honorific_glued_surname", "王先生", {"family": "王", "suffix": "先生"}, locale="zh", classification="fix(#307)", notes="the Han twin of 김씨: the zh pack's segmentation " "splits off the surname and the remaining 先生 is the " "honorific token"), Case("zh_honorific_glued_given", "王小明先生", {"family": "王", "given": "小明", "suffix": "先生"}, locale="zh", classification="fix(#308)", notes="the Han twin, replacing zh_honorific_glued_given_stays: " "先生 peels, and the zh pack's surname vocabulary then " "divides the remainder 王小明"), Case("zh_honorific_glued_given_default", "王小明先生", {"family": "王小明", "suffix": "先生"}, classification="fix(#308)", notes="the same input WITHOUT the pack: the peel is default-on " "and script-independent, so the honorific still routes " "to suffix -- only the surname split needs the opt-in, " "so the undivided 王小明 stays one family name"), Case("ko_suffix_matching_is_whole_token", "김지양", {"family": "김", "given": "지양"}, classification="fix(#271)", notes="지양 ENDS with the honorific 양 but is a given name: " "suffix matching is whole-token, never endswith -- the " "pin the differential rule's anchor mirrors at the " "name-string level. #308 leaves it alone too: 양 is " "excluded from the glued tail set for exactly this " "name. Classified to #271, not parity: 1.4 gave first " "김지양 and no last, and it is #271's hangul " "segmentation plus the CJK order flip that produces " "these two fields"), Case("ko_surname_yang_leads", "양 미선", {"family": "양", "given": "미선"}, classification="fix(#271)", notes="양 is both a top-tier surname (Yang) and a shipped " "honorific: position decides, and a surname LEADS -- " "the trailing-only suffix gate never sees it here. " "Classified to #271, not parity: 1.4 gave first 양, " "last 미선, and it is the CJK order flip that swaps " "them"), Case("ko_honorific_yang_trails", "김민준 양", {"family": "김", "given": "민준", "suffix": "양"}, classification="fix(#307)", notes="the other side of ko_surname_yang_leads: the same " "token trailing a name is 'Miss', and that is the whole " "argument shipping it -- suffixes.py singles 양 out " "(with 군) as the risk class it takes, a top-tier " "surname admitted to the vocabulary on the strength of " "position alone. Nothing pinned the trailing reading " "before this row, so the leading rows carried the pair " "by themselves. Classified to #307 like ko_honorific_ssi " "(1.4 gave first 김민준, last 양; the recognition and " "the order flip both move it) -- the point of the row " "is the twin below"), Case("ko_honorific_yang_written_with_a_period", "김민준 양.", {"family": "김", "given": "민준", "suffix": "양."}, classification="fix(#320)", notes="the period-written twin, whose fields must equal the " "row above and before #320 did not (given 김, middle " "민준, family '양.' -- the veto kept '양.' a name piece, " "exactly ko_honorific_with_a_period_no_comma's route). " "The pair is the point: 양 is the shipped vocabulary's " "acknowledged risk, so if the 양/군 policy is ever " "tightened or withdrawn, both spellings have to move " "together and neither row can be adjusted alone. 군 " "gets no pair of its own -- it parses identically and " "is the SAFER half (no surname reading), so it would " "pin nothing these two do not. Classified to #320 like " "its 씨 counterpart: 1.4.0 read this first 김민준 / last " "'양.', and the fields above are the ones this change " "produced, not the segmenter's"), Case("ko_surname_yang_leads_a_segmentable_given", "양 지훈", {"family": "양", "given": "지훈"}, classification="fix(#271)", notes="양 is a surname AND a shipped honorific, and 지 is a " "listed surname too -- so this is the name that " "catches a segmentation site scanning PAST the " "honorific reading of 양 into the given name. The " "first script-written token decides, and deciding " "includes deciding there is no surname site. " "Classified to #271, not parity: 1.4 gave first 양, " "last 지훈, and it is the CJK order flip that puts 양 " "in family -- nothing in #308 moves these fields"), Case("ko_honorific_stack", "김민준 박사 씨", {"family": "김", "given": "민준", "suffix": "박사, 씨"}, classification="fix(#307)", notes="a trailing RUN of honorifics peels whole, like " "'Smith PhD MD' -- the multi-suffix loop the peel " "shares with Latin suffixes"), ) python-nameparser-2.1.0/tests/v2/conftest.py000066400000000000000000000030761523530431700210670ustar00rootroot00000000000000"""Shared helpers for the tests/v2 package. The parent suite's autouse `_isolate_constants` fixture (tests/conftest.py) applies here ON PURPOSE: the v2 facade/shim tests mutate the shared CONSTANTS singleton (tests/v2/test_config_shim.py), so the snapshot/restore is wanted protection, not overhead. This conftest once overrode the parent's dual-run fixture by name to skip it -- that override went dead when the parent was renamed for #255, and it was deleted rather than reconnected because its premise ("v2 code never reads shared CONSTANTS") had stopped being true. """ import json from pathlib import Path def differential_corpus() -> list[str]: """Every name the differential harness compares, from all of its corpus files. Globs `corpus*.jsonl` for the reason compare.py does: a corpus you have to ask for by name is one that stops being run. Two test modules had grown their own hard-coded `corpus.jsonl` loader, so both silently missed `corpus_issues.jsonl` -- 198 reported names, 166 of them found nowhere else. tools/ is still not imported: it is a standalone uv-run script tree rather than a package, and the one-JSON-string-per-line format is cheaper to read than to make importable. """ corpus_dir = Path(__file__).parents[2] / "tools" / "differential" paths = sorted(corpus_dir.glob("corpus*.jsonl")) assert paths, f"no corpus*.jsonl in {corpus_dir}" names = [json.loads(line) for path in paths for line in path.read_text().splitlines() if line.strip()] return list(dict.fromkeys(names)) python-nameparser-2.1.0/tests/v2/data/000077500000000000000000000000001523530431700175735ustar00rootroot00000000000000python-nameparser-2.1.0/tests/v2/data/constants_v14.pickle000066400000000000000000000323721523530431700235010ustar00rootroot000000000000004nameparser.config Constants)}(prefixesh SetManager)}(elements(vel'tdíaufibndelliheerdeglivdvomdellaopabudelaanbintderdeidellevondialledellomcdenstavaenthoafdeladaldodebarfreiherrhetdasantamacsanbatduzudemthoe freiherrinbonvandevanderde'dosteterbinlavanste _on_changebuiltinsgetattrh_invalidate_pstRubsuffix_acronymsh)}(h (lvofacepcipmisocgasgmscmpcbecgmacwepsiasccgmemdiaccpcplpbclfacofpcisspdomairaaslacpcmcpfogisprcpsccpccmtcpofhameschtcprecapprmsksaspqpmcsslpkcmglitlcscpasid nbcdch-psrdmscfedsbampcdnprtrpphrpgmpapssnp[18]cbtcphtdcvoemt-bocpfac-ccgcmchesdiplaccrttcirocchpcpgchbaaasfwsaprsphrcwnagrpmsmcpscnctoccsplactpcprcedfmvafalarvtcplmswamndtrissp-cspipepthmcwapstmieeealcnbc-hisclacdssiiecmcdepcapspebncbchseeaomdfcmcsditillpcmdivabpplgqgmbpicmfolcmtbpecxadabfmcpfasasmissp-sacfcmfeckciecnadveprdncfceddgaee[14]kbecismrairvmcluchrmgridrphmscchpnmdchicgfmemt-i/85cydsocakgciapmpidsmcspcsrebls-icarochescdallsidmdfacsoscpcaeccnacaplitpqfsmafasmacccacgapmscmsmcamscapmgcmgmdhbrbaaqpphlmtelsceausarpmccigfacemlvtabrusacpwaabcfacphphdpmi-acpcsbecprpccimpharmdmemmmasmtcrpstbccprpsm iimfacelafsaamdpccpa-crdcdfmapncva[22]gbemtsphcctfaaedfmcfiusafcnpncsocipsaphrprmcgspcfrccesecbkcsicwpcpcuvdalpcfpsfnssdcfaafpmdbbchse-aptardcssedd kchs/dchsleed apcma nicet iiichapsmafcmcpdccisoceasncidqcrmendriduxcncpsccnplcpcaemcgrcipmmaddtrcvocrisccspocwbrphrrt-accsfaslacbntfacogfaicpkpllmarrcachausnrcsacedscp-cfieeeeitcppmjpodlpngchsesqbtcsdrbceraameaicprrtdptabamirmmdbfrmcemfasidmaaachmmamieeecltdcrmahrsddscrnacplnicet ivdedmcsacfesfpcbvocmppsccierhcambsfmpbemmltfcacwspchpsefpcdvmawbfaspencpacccmslitkchdmemt-prfpmieeecfadnpsracppdbeenpuscgmlelaccsinbcch-psqamfasmagcrlscpmmengchpalpcvrscrtchfcfaapqcswjdresfachaobimdlcmspkcvocmgcsidmaacpbcalpssusnmlscstctocdmpcbmfacdcpacronrpsscpkcbaceshrm-scpcehatcficifgictnremterdbtsameacadabvlmiombptcysaiccm-dcgbpspkhs/dhsipmcwnepsyddminfp&acuspphrcaacasceppsm iabdeicmtcvpalagitgmruxmccisaktccacscumicpdjurbdsdrmprncbrtecnmcaiachmcnccprcidacvimciecphlvnagsfcasgmchmnaatdcicsancarbmnaecebsmbaafmclsdqsdpgccmcapaccpnbcfchcsrtenbcdchusmccrdecbnecccmfaanqcppmdepcgfocbsthdqspmvoiccm-ffashpchsc8-vsbcactsficfmphdscplsevpcpsifaccchplnudccufcvcpcpbegbarpdcbnraemtbscctsfsdpdsmmctcmgptfaiaarmdiplommcdbaemt-i/99lsitaiarrccpegcbchsoscmasfacpcbpcetmpadpmshrm-cpcpssgsmieeecahalmnbctlpampsefcasvccssmmcmmbenremrcbspcccgphrmcpcsmobemlsecmpfp-cmmchpecvaaprnnbcchiaeedsocbteasarpapfmpnicet iictbsbcsschssccp-c nbcfch-pscuadcmcsterplcaspcwdprdhvrddsscpimcfptbr-ctcgpcfccfcelamsamcsechesmieeeltcsepmramdamsdcmgcdtcptmcbtsnicet ilcswmacfcegcvocfmgcierrt-npshccpotcfrecscmdgmbamspcimcctrrt-sdsfdhGhJhhKRubsuffix_not_acronymsh)}(h (iretesqjunioresquiresnrdriiiii2srjrvvetjnrivhGhJhhKRubtitlesh)}(h (staffukmystery bodhisattvamarcher marketingpaleontologistchiefasstsgm1lt evangelistlawplayerrangerdrummercomediansingerstates assistantsrtascomicdo1stlt co-founder politicianrasaintecpo gentiluomo biologistmajgeneditorgyani fräulein architect monseigneurpresssg technicalcustomsmrsbishopliterarybailiffexecqueen'smagtimi abolitionist printmakerfederal saoshyantdancershaykhattache co-chairsco-chairdptysultana swordbearer psychiatristcountessflagproceordmlteacheroperalcpl signorina archbishopmissessergeant historicusmlle playwrightfirst venerablevicar señoritaladysailorchef photographerlinguistprofmayor barristerroyalsgt ambassadorrevenue industrialistcmsgt bandleadermadammsgrep designatedpriestmanagerneuroscientistsr philosopherpiramnprinmusician broadcastermostsaint treasurersinger-songwritermcponadvocate großfürstfürsterzbischof knowledgequeen1stsgtbgenproducerenglishguru pursuivantforeignking'sblessedtenorjuniorviscount subalternabbot analyticswo5expertinventormarquiscomposerservantmcpocsnreverendsubedarsres honourable associatesrtaprovostus2ndkingshaikhmarchessstrategyhighcdrdivisionhajji professorsheriffpettysfcsultanpharaohwpdutchessmobsterherzogwoodmangroupimam councillor mathematicsvrouwe clergymanpianist marchionesspvtmajesty erzbischof mag-judgecwo-4poetforestercardinalsheik counselorcritic corporatetaxdeaconfraupolice celebrityadmiral controllerrepresentative naturalistbaronfriar ornithologist instructorbgdistrictanthropologistfru obstetritianphdpilotbpsenhorasurgeon senhoritaritterbusinesswo-1sheikh historiansecurity presbyter researchersistertravel mathematicianacademic burlesquefrjrsgtmajmcdesignerfilmalmonersoftwareminister economist mademoisellemahdi biochemistprivatepv2 operatingwo2vice appellate developer rangatiravardapetdocket keyboardist excellent schoolmasterpremierdamecfouncleprimatefamilyspcbaba physicianmgysgtbriggen technologistwardenkingdomlegalattachérevsecwo5superiorpriorassochermilitaryakhoonddirectorjudge tirthankar guitaristmistercwo-3pioneer ayatollahbaseballnavybluesgysgtcwo-5lord biographer historien excellencyfreifrauprefectdra9thmothercolburgessseñoraeffendiholinessdelegateseniorbarddcnemperor principalarranger dipl.-inglamaleadermsgtwing maharajahresidentpslc conductorhostsenator archdeaconherrenshaykacolytesignoractor activist compositeurarhatrttsgtfadmrearlecturermesignore presidingdocentmaharanichairlawyer apprentice politicalvadmcpl entertainerrockengineer theologian comptrollercriminalgräfinmgrmodellyricistcheikhchairsltgenrebbeprime priestess dramatist detectiveprinceenssupremeattorney award-winning discovery entrepreneurcmsafmonsieurspeakermerchantpo2adjutantsrasoccernannybrothermevrouwe behavioral chieftainbearerauntpfcadviser classicalscholar archdruidcomtessepastorcwo-2 librarianclerk matriarchmpco-cgltcolshekhcwo3eminence cartoonistuniteddiplomat2ltbloggersoldieradeptcoachmuftistatecanonhon missionarydrsmsgt arbitratorgrand archeologist correspondent monsignorherrntheatrecredit ecologista1cnationalmet intelligencedeputygovernoredohen comediennehismagnatemevrcolonelarchdukeairmurshid sociologist1stjudicial lieutenant pediatriciansopranoempressgoodwifeeducatorgeneralchargé d'affairesssgtdhr businesswomanenvoybuddhachaplainstgraf hereditarymullah10thballetcyclist journalist cartographer magistrate registrarradmseñorcorporal informationfatherflyingfh-profviziersa physicistcaptainhighnessfrk3rdprinterrisk personalityshehuabbess sysselmannsiddhaverdererltg zoologist television reichsgraf solicitorabbenchdoyenvoicegianiedmicomputerbaronessactress dissidentarmy elerunwonedler bookseller metropolitanseñoreswo3commander-in-chief commandermaidmx executivegafmonkscpo scientistekegbianprelateheir bibliographerpo1wo4lcdrinvestor philantropistfield municipalsmaherrprinzwmspecialpope mountaineerchemist anarchistheren memoirist2ndlt7thprincessbwanamadamesatiristswamiwriterdircwo4 patriarchwarrantwo1 generalissimolamidopd mag/judgejuristmarquessgen criminologist financialtsar musicologistvcgoodmanheiress6th businessmanpo3ccmsgtsgtmajcsm psychologistshaiknovelist publisher secretarycwo2frøken chancellor consultanttheatersocialbiblicalbankerofficerltjg archduchess investigatormrpaintervocalist astronomer8thhrcount senior-judge priv.-doz advertisingmevrouw brigadiercurator suffragist illustratoressayist choreographer1sgt collector intendant honorable president univ.profmcpomgcapt catholicos screenwritermajheadmanltmagistrate-judgebotanistfoundernarratormembercourtierduchessesirsenhormarquisealderman commodore journeymanmsmme señoritascptsignoratheoristauthor5thmasteradmbannerseigneurrabbifreiherrfürstradio4thmdanimatortsarinadoctorauntie translatormisscivilbritishdruidnurserightltchGhJhhKRubfirst_name_titlesh)}(h (sheikshaykmastershaykhbrotherdamemotheruncleshaikshekhsheikhmaidfathersisterauntqueenpopekingshaikhsirauntiecheikhhGNub conjunctionsh)}(h (undyofand&etheethGNubbound_first_namesh)}(h (abdulabdalabuummabdelabouhGNubnon_first_name_prefixesh)}(h (af'tdeaufibndellideglivdvomophetbintderdeidellezudemde'dosdelloteravhGNubsuffix_acronyms_ambiguoush)}(h (jdedhGNubcapitalization_exceptionsh TupleManager)R}(iiIIiiiIIIivIVmdM.D.phdPh.D.ubregexeshRegexTupleManager)R}(spacesre_compile\s+K Rwordj(\w|\.)+K Rmacj^(ma?c)(\w{2,})K"Rinitialj^(\w\.|[A-Z])?$K R quoted_wordj(? ParsedName: return assemble(run(ParseState(original=text, lexicon=_LEX, policy=Policy()))) def test_assemble_produces_validated_parsedname() -> None: pn = _parse("Dr. Juan de la Vega III") assert pn.title == "Dr." assert pn.given == "Juan" assert pn.family == "de la Vega" assert pn.family_base == "Vega" # particle tags carried over assert pn.family_particles == "de la" assert pn.suffix == "III" assert pn.original == "Dr. Juan de la Vega III" for t in pn.tokens: assert t.span is not None assert t.text == pn.original[t.span.start:t.span.end] def test_assemble_materializes_ambiguities_on_final_tokens() -> None: pn = _parse("Van Johnson") assert len(pn.ambiguities) == 1 amb = pn.ambiguities[0] assert amb.kind is AmbiguityKind.PARTICLE_OR_GIVEN assert amb.tokens[0] is pn.tokens[0] def test_assemble_drops_structural_marker_tokens() -> None: pn = _parse("Jane Smith née Jones") assert [t.text for t in pn.tokens] == ["Jane", "Smith", "Jones"] assert pn.maiden == "Jones" def test_empty_parse_is_falsy() -> None: assert not _parse("") assert not _parse(" ") def test_content_free_input_parses_to_empty() -> None: # An input with no alphanumeric character is not a name. v1 kept # it (parse('.') -> first='.'); 2.0 empties it so bool() stays an # honest "did I get a name?" check. isalnum is Unicode-aware, so # this only fires on pure punctuation/symbols. The test is over the # SURVIVING tokens, which since #329 is not the same as over the # input -- see cases.py's maiden_marker_delimited_content_free, # where dropping a marker is what leaves nothing behind. for junk in [".", ".,", "- -", ". .", "'", "∫≜⩕", "()", "-"]: pn = _parse(junk) assert not pn, f"{junk!r} should be falsy" assert pn.tokens == (), f"{junk!r} should have no tokens" assert pn.original == junk # the raw input is still remembered def test_a_single_letter_of_content_survives() -> None: # the guard keys on content, not length: one alnum char is a name for real in ["a.", ".a", "a", "李", "О"]: assert _parse(real), f"{real!r} should be truthy" def test_a_content_free_parse_keeps_its_diagnostics() -> None: # Emptying the name must not take the reports with it. "Was this # input malformed?" is the one question still worth answering when # there is no parse left to infer it from, and every junk row in # the test above is balanced or delimiter-free, so none of them # would notice if the ambiguities were dropped again. pn = _parse("(") assert not pn and pn.tokens == () assert [a.kind for a in pn.ambiguities] == [ AmbiguityKind.UNBALANCED_DELIMITER] # points at no token, because none survived -- the load-bearing # assertion: a fix that materialized a phantom token would # otherwise pass, and ParsedName would reject it anyway assert pn.ambiguities[0].tokens == () assert len(_parse("((").ambiguities) == 2 # one report each # a delimiter that IS balanced still reports nothing assert _parse("()").ambiguities == () def test_assemble_falls_back_to_given_for_unassigned_role() -> None: # This should never happen through the real pipeline -- assign/group # always set a role on every main-stream token before assemble runs. # This test constructs the invariant violation directly to pin the # documented last-resort behavior: assemble() must stay total over # its input and never raise just because upstream left a role unset. state = ParseState( original="Jane", lexicon=Lexicon.default(), policy=Policy(), tokens=(WorkToken("Jane", Span(0, 4), role=None),), ) pn = assemble(state) assert pn.tokens[0].role is Role.GIVEN assert pn.given == "Jane" def test_ambiguity_with_all_indices_dropped_is_omitted() -> None: # an ambiguity whose referent tokens were ALL dropped describes # nothing; emitting it hollow would mislead consumers. Born-empty # ambiguities (unbalanced delimiters) are kept -- they are # token-independent by design. from nameparser._pipeline._state import PendingAmbiguity from nameparser._types import AmbiguityKind as AK import dataclasses state = run(ParseState(original="Jane Smith née Jones", lexicon=_LEX, policy=Policy())) née_idx = next(i for i, t in enumerate(state.tokens) if t.text == "née") poisoned = dataclasses.replace( state, ambiguities=state.ambiguities + ( PendingAmbiguity(AK.ORDER, "refers only to the marker", (née_idx,)), PendingAmbiguity(AK.UNBALANCED_DELIMITER, "born empty", ()), )) pn = assemble(poisoned) kinds = [a.kind for a in pn.ambiguities] assert AK.ORDER not in kinds # fully dangled: omitted assert AK.UNBALANCED_DELIMITER in kinds # born empty: kept python-nameparser-2.1.0/tests/v2/pipeline/test_assign.py000066400000000000000000000227301523530431700233700ustar00rootroot00000000000000# tests/v2/pipeline/test_assign.py from nameparser._lexicon import Lexicon from nameparser._pipeline._assign import assign from nameparser._pipeline._classify import classify from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._group import group from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState from nameparser._pipeline._tokenize import tokenize from nameparser._policy import ( FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, Policy, Script, ) from nameparser._types import AmbiguityKind, Role _LEX = Lexicon( titles=frozenset({"dr", "mr", "mrs", "sir"}), given_name_titles=frozenset({"sir"}), suffix_acronyms=frozenset({"phd", "md"}), suffix_words=frozenset({"jr", "iii", "v"}), particles=frozenset({"de", "la", "van"}), particles_ambiguous=frozenset({"van"}), conjunctions=frozenset({"and"}), maiden_markers=frozenset({"née"}), ) def _assigned(text: str, policy: Policy | None = None) -> ParseState: state = ParseState(original=text, lexicon=_LEX, policy=policy or Policy()) return assign(group(classify(segment(tokenize( extract_delimited(state)))))) def _by_role(state: ParseState, role: Role) -> str: return " ".join(t.text for t in state.tokens if t.role is role) def test_given_first_positional() -> None: out = _assigned("Dr. Juan de la Vega III") assert _by_role(out, Role.TITLE) == "Dr." assert _by_role(out, Role.GIVEN) == "Juan" assert _by_role(out, Role.FAMILY) == "de la Vega" assert _by_role(out, Role.SUFFIX) == "III" def test_middles() -> None: out = _assigned("John Quincy Adams Smith") assert _by_role(out, Role.GIVEN) == "John" assert _by_role(out, Role.MIDDLE) == "Quincy Adams" assert _by_role(out, Role.FAMILY) == "Smith" def test_single_token_takes_name_order_head() -> None: assert _by_role(_assigned("John"), Role.GIVEN) == "John" out = _assigned("John", Policy(name_order=FAMILY_FIRST)) assert _by_role(out, Role.FAMILY) == "John" def test_title_only() -> None: out = _assigned("Dr.") assert _by_role(out, Role.TITLE) == "Dr." assert not _by_role(out, Role.GIVEN) def test_leading_ambiguous_particle_reads_as_given_with_ambiguity() -> None: out = _assigned("Van Johnson") assert _by_role(out, Role.GIVEN) == "Van" assert _by_role(out, Role.FAMILY) == "Johnson" assert any(a.kind is AmbiguityKind.PARTICLE_OR_GIVEN for a in out.ambiguities) # unambiguous input: no ambiguity recorded assert not _assigned("John Smith").ambiguities def test_family_comma() -> None: out = _assigned("de la Vega, Juan") assert _by_role(out, Role.FAMILY) == "de la Vega" assert _by_role(out, Role.GIVEN) == "Juan" def test_family_comma_with_title_and_middle() -> None: out = _assigned("Smith, Dr. John A.") assert _by_role(out, Role.TITLE) == "Dr." assert _by_role(out, Role.GIVEN) == "John" assert _by_role(out, Role.MIDDLE) == "A." assert _by_role(out, Role.FAMILY) == "Smith" def test_family_comma_lone_post_comma_title() -> None: # v1 parity: a lone post-comma title is a TITLE ('Smith, Dr.'), # not a given name or suffix; post_rules later applies the # title+lone-family handling out = _assigned("Smith, Dr.") assert _by_role(out, Role.TITLE) == "Dr." assert _by_role(out, Role.FAMILY) == "Smith" assert not _by_role(out, Role.GIVEN) def test_suffix_comma_and_extra_segments() -> None: out = _assigned("Smith, John, Jr.") assert _by_role(out, Role.GIVEN) == "John" assert _by_role(out, Role.FAMILY) == "Smith" assert _by_role(out, Role.SUFFIX) == "Jr." def test_suffix_comma_structure() -> None: # Structure.SUFFIX_COMMA (segment.py): >1 word before the first # comma AND every post-first segment is entirely lenient-suffix. # ('Smith, John, Jr.' above only has ONE word before its first # comma, so it is FAMILY_COMMA with a trailing suffix segment, not # this branch.) Previously this assign() branch (tail=1) was only # reached via the full-corpus test, not its own isolated case. out = _assigned("John Smith, PhD") assert _by_role(out, Role.GIVEN) == "John" assert _by_role(out, Role.FAMILY) == "Smith" assert _by_role(out, Role.SUFFIX) == "PhD" def test_trailing_suffix_run_no_comma() -> None: out = _assigned("John Jack Kennedy PhD MD") assert _by_role(out, Role.MIDDLE) == "Jack" assert _by_role(out, Role.FAMILY) == "Kennedy" assert _by_role(out, Role.SUFFIX) == "PhD MD" def test_initial_veto_keeps_v_in_middle() -> None: out = _assigned("John V. Smith") assert _by_role(out, Role.MIDDLE) == "V." assert _by_role(out, Role.FAMILY) == "Smith" def test_nickname_only_leaves_name_fields_empty() -> None: out = _assigned("(Jack)") assert _by_role(out, Role.NICKNAME) == "Jack" assert not _by_role(out, Role.GIVEN) and not _by_role(out, Role.FAMILY) def test_single_name_with_nickname_goes_to_family() -> None: out = _assigned("John (Jack)") assert _by_role(out, Role.FAMILY) == "John" assert not _by_role(out, Role.GIVEN) def test_family_first_order() -> None: out = _assigned("Yamada Taro", Policy(name_order=FAMILY_FIRST)) assert _by_role(out, Role.FAMILY) == "Yamada" assert _by_role(out, Role.GIVEN) == "Taro" out2 = _assigned("Yamada Hanako Taro", Policy(name_order=FAMILY_FIRST)) assert _by_role(out2, Role.FAMILY) == "Yamada" assert _by_role(out2, Role.GIVEN) == "Hanako" assert _by_role(out2, Role.MIDDLE) == "Taro" def test_family_first_given_last_order() -> None: # Pinned against the actual built pipeline (2026-07-12), not guessed: # 'Van' is particles_ambiguous and non-leading, so group's prefix # chain (which is name_order-agnostic -- it lands upstream of assign # and doesn't consult Policy) absorbs 'Van Anh Thu' into ONE piece # before assign ever sees it. With only two name pieces left, # FAMILY_FIRST_GIVEN_LAST's positional rule (family, then given for # count==2) correctly assigns Nguyen -> FAMILY and the whole merged # piece -> GIVEN; there is no MIDDLE piece to assign. This is a # sensible consequence of the current, order-agnostic group stage -- # not a contract violation of assign -- and is the exact kind of gap # the Vietnamese-aware locale pack (#270/#272 follow-ups) is meant to # close by teaching group to suppress the particle chain under this # name_order. Do not "fix" this by changing assign. out = _assigned("Nguyen Van Anh Thu", Policy(name_order=FAMILY_FIRST_GIVEN_LAST)) assert _by_role(out, Role.FAMILY) == "Nguyen" assert _by_role(out, Role.GIVEN) == "Van Anh Thu" assert _by_role(out, Role.MIDDLE) == "" def test_script_order_applies_when_every_piece_is_one_script() -> None: out = _assigned("毛 泽东") assert _by_role(out, Role.FAMILY) == "毛" assert _by_role(out, Role.GIVEN) == "泽东" def test_script_order_declines_on_a_mixed_piece_set() -> None: # {None, HAN}: one Latin piece is enough to put the name back on # the positional default, even though the other piece is Han. out = _assigned("毛 Smith") assert _by_role(out, Role.GIVEN) == "毛" assert _by_role(out, Role.FAMILY) == "Smith" def test_script_order_declines_when_no_piece_has_a_script() -> None: # all-None: the ordinary Latin path, unreachable by the table. out = _assigned("John Smith") assert _by_role(out, Role.GIVEN) == "John" assert _by_role(out, Role.FAMILY) == "Smith" def test_kana_licensed_piece_resolves_via_hiragana_entry() -> None: # a kanji+hiragana mixed piece set takes the license (#272) and # resolves through the HIRAGANA table entry, family-first out = _assigned("高橋 みなみ") assert _by_role(out, Role.FAMILY) == "高橋" assert _by_role(out, Role.GIVEN) == "みなみ" def test_pure_katakana_piece_falls_back_to_name_order() -> None: # katakana alone is not in the table (transcription ambiguity), so # a pure-katakana piece set falls back to the positional default out = _assigned("マイケル ジャクソン") assert _by_role(out, Role.GIVEN) == "マイケル" assert _by_role(out, Role.FAMILY) == "ジャクソン" def test_interpunct_divided_name_reads_positionally() -> None: # 威廉·莎士比亚 is William Shakespeare: a 间隔号-divided name is a # transcription and keeps source order (spec 2026-07-30) -- the # B7 is the marker, playing the role pure katakana plays in the # kana license out = _assigned("威廉·莎士比亚") assert _by_role(out, Role.GIVEN) == "威廉" assert _by_role(out, Role.FAMILY) == "莎士比亚" def test_interpunct_suppression_yields_to_explicit_name_order() -> None: # the suppression falls back to name_order, it does not force # given-first: an explicit FAMILY_FIRST governs the transcription out = _assigned("威廉·莎士比亚", Policy(name_order=FAMILY_FIRST)) assert _by_role(out, Role.FAMILY) == "威廉" assert _by_role(out, Role.GIVEN) == "莎士比亚" def test_script_with_no_table_entry_falls_back() -> None: # A single, well-defined script the table simply does not list: # resolution must fall through to name_order rather than pick an # arbitrary entry. hangul_only = Policy(script_orders={Script.HANGUL: FAMILY_FIRST}) # type: ignore[arg-type] out = _assigned("毛 泽东", hangul_only) assert _by_role(out, Role.GIVEN) == "毛" assert _by_role(out, Role.FAMILY) == "泽东" python-nameparser-2.1.0/tests/v2/pipeline/test_classify.py000066400000000000000000000101261523530431700237150ustar00rootroot00000000000000from nameparser._lexicon import Lexicon from nameparser._pipeline._classify import classify from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState from nameparser._pipeline._tokenize import tokenize from nameparser._policy import Policy _LEX = Lexicon( titles=frozenset({"dr", "sir"}), given_name_titles=frozenset({"sir"}), suffix_acronyms=frozenset({"phd", "ma"}), suffix_words=frozenset({"jr", "v"}), suffix_acronyms_ambiguous=frozenset({"ma"}), particles=frozenset({"de", "la", "van"}), particles_ambiguous=frozenset({"van"}), # й is COPIED from the shipped conjunctions (Ukrainian, #267) so the # collision test_cyrillic_initial_outranks_the_conjunction pins is # one that really ships and the reader can check against the # defaults. The copy is local: this file never reads the shipped set conjunctions=frozenset({"and", "y", "й"}), bound_given_names=frozenset({"abdul"}), maiden_markers=frozenset({"née"}), ) def _classified(text: str) -> ParseState: state = ParseState(original=text, lexicon=_LEX, policy=Policy()) return classify(segment(tokenize(extract_delimited(state)))) def _tags(state: ParseState, text: str) -> frozenset[str]: return next(t.tags for t in state.tokens if t.text == text) def test_vocabulary_tags() -> None: out = _classified("Dr. van de la Smith and abdul née PhD Jr") assert "vocab:title" in _tags(out, "Dr.") assert {"particle", "vocab:particle-ambiguous"} <= _tags(out, "van") assert "particle" in _tags(out, "de") assert "vocab:particle-ambiguous" not in _tags(out, "de") assert "conjunction" in _tags(out, "and") assert "vocab:bound-given" in _tags(out, "abdul") assert "vocab:maiden-marker" in _tags(out, "née") assert "vocab:suffix" in _tags(out, "PhD") assert {"vocab:suffix", "vocab:suffix-word"} <= _tags(out, "Jr") assert _tags(out, "Smith") == frozenset() def test_given_title_tag() -> None: out = _classified("Sir John") assert {"vocab:title", "vocab:given-title"} <= _tags(out, "Sir") def test_initial_tag() -> None: out = _classified("John A. B Smith") assert "initial" in _tags(out, "A.") assert "initial" in _tags(out, "B") assert "initial" not in _tags(out, "John") def test_cyrillic_initial_outranks_the_conjunction() -> None: """#320 regression. 'й' is the Ukrainian conjunction (#267); 'Й.' is an initial and must not be read as it. Narrowing is_initial to [A-Za-z] -- the fix #320 originally proposed -- flips this token to 'conjunction' and strips 'initial' off every Cyrillic, Greek, Arabic, Hebrew, Devanagari and Armenian initial (the six test_vocab.test_is_initial_script_repertoire asserts). Neither moves field output on a short name, so this asserts the TAG. _LEX copies й from the SHIPPED conjunctions so the collision is a real one -- but the copy is local, so this test would keep passing if й were dropped from the defaults.""" out = _classified("Й. Сліпий") assert "initial" in _tags(out, "Й.") assert "conjunction" not in _tags(out, "Й.") def test_ambiguous_suffix_acronym_needs_periods() -> None: out = _classified("M.A. Ma") assert "vocab:suffix" in _tags(out, "M.A.") assert "vocab:suffix" not in _tags(out, "Ma") assert "vocab:suffix-ambiguous" in _tags(out, "Ma") def test_v_is_suffix_word_and_initial() -> None: # both tags present; assign applies the veto, not classify out = _classified("John V Smith") assert {"vocab:suffix", "vocab:suffix-word", "initial"} <= _tags(out, "V") def test_bare_ambiguous_acronym_in_acronyms_is_not_suffix() -> None: # the default lexicon has suffix_acronyms_ambiguous SUBSET OF # suffix_acronyms (v1 data shape); the plain membership test must # exclude the ambiguous members or the period gate is dead code # and bare 'Ed'/'Jd' silently become suffixes (PR review C1) out = _classified("Ma M.A.") assert "vocab:suffix" not in _tags(out, "Ma") assert "vocab:suffix" in _tags(out, "M.A.") python-nameparser-2.1.0/tests/v2/pipeline/test_extract.py000066400000000000000000000252171523530431700235610ustar00rootroot00000000000000import dataclasses import pytest from nameparser import parse from nameparser._lexicon import Lexicon from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._state import ParseState from nameparser._policy import Policy from nameparser._types import AmbiguityKind, Role, Span def _state(text: str, policy: Policy | None = None) -> ParseState: return ParseState(original=text, lexicon=Lexicon.empty(), policy=policy or Policy()) def test_double_quoted_nickname_extracted() -> None: # 0123456789012345678 out = extract_delimited(_state('John "Jack" Kennedy')) assert out.extracted == ((Role.NICKNAME, Span(6, 10)),) assert out.masked == (Span(5, 11),) def test_parenthesized_nickname_extracted() -> None: out = extract_delimited(_state("John (Jack) Kennedy")) assert out.extracted == ((Role.NICKNAME, Span(6, 10)),) def test_same_char_delimiter_needs_boundaries() -> None: # the apostrophe in O'Connor is not an opening quote out = extract_delimited(_state("Sean O'Connor")) assert out.extracted == () and out.masked == () def test_single_quoted_nickname_at_boundaries() -> None: # 01234567890123456789 out = extract_delimited(_state("John 'Jack' Kennedy")) assert out.extracted == ((Role.NICKNAME, Span(6, 10)),) def test_unbalanced_delimiter_left_literal_with_ambiguity() -> None: out = extract_delimited(_state('Jon "Nick Smith')) assert out.extracted == () and out.masked == () assert len(out.ambiguities) == 1 assert out.ambiguities[0].kind is AmbiguityKind.UNBALANCED_DELIMITER def test_no_spurious_unbalanced_from_role_overlapping_pairs() -> None: # #273: '“' closes the German „…“ pair but OPENS the English “…” # pair, and '»' closes «…» but opens the reversed »…« pair. A # delimiter character consumed by another pair's successful # extraction is literal for every other pair -- it must not # surface as an unbalanced-delimiter ambiguity. policy = Policy(nickname_delimiters=frozenset( {("“", "”"), ("„", "“"), ("«", "»"), ("»", "«")})) for text, inner in (("Hans „Hansi“ Müller", "Hansi"), ("Jean «Petit» Dupont", "Petit")): out = extract_delimited(_state(text, policy)) assert [text[s.start:s.end] for _, s in out.extracted] == [inner] assert out.ambiguities == () def test_mixed_conventions_both_extract_leftmost_wins() -> None: # Review finding (2026-07-19, three agents independently): with the # default #273 pairs, a string containing TWO delimited asides in # different conventions mis-extracted -- the earlier-sorted pair # claimed the other's close character ('“' closes „…“ but opens # “…”) as its own open and swallowed a bogus span across the real # boundary, silently dropping the legitimate match. The scan must # be position-driven: the leftmost genuine opener wins. text = "Hans „Erster“ und “Zweiter” Müller" out = extract_delimited(_state(text)) assert [text[s.start:s.end] for _, s in out.extracted] == [ "Erster", "Zweiter"] assert out.ambiguities == () def test_mixed_guillemet_directions_both_extract() -> None: text = "Jean «Petit» Dupont »Rex« Martin" out = extract_delimited(_state(text)) assert [text[s.start:s.end] for _, s in out.extracted] == [ "Petit", "Rex"] assert out.ambiguities == () def test_mixed_conventions_across_maiden_and_nickname_buckets() -> None: # same shape across ROLES: guillemets routed to maiden must not let # the reversed nickname pair steal the maiden close policy = Policy(maiden_delimiters=frozenset({("«", "»")})) text = "Jean «Dupont» Martin »Rex« Smith" out = extract_delimited(_state(text, policy)) got = {(role, text[s.start:s.end]) for role, s in out.extracted} assert got == {(Role.MAIDEN, "Dupont"), (Role.NICKNAME, "Rex")} assert out.ambiguities == () def test_extraction_and_genuine_unbalanced_coexist() -> None: # the kept path: a successful extraction must not suppress a REAL # dangling open elsewhere in the same string text = 'John (Jack) Smith "Extra' out = extract_delimited(_state(text)) assert [text[s.start:s.end] for _, s in out.extracted] == ["Jack"] assert [a.kind for a in out.ambiguities] == [ AmbiguityKind.UNBALANCED_DELIMITER] def test_same_char_bulk_unbalanced_filtered_by_other_pairs_mask() -> None: # the same-char forward-scan records EVERY dangling quote; one that # sits inside a region another pair successfully extracts is # literal content, not a dangling delimiter -- only the truly # dangling ones surface. All three quotes here fail the close-walk # (no quote is followed by a boundary), so the bulk pass records # all three; the middle one lands inside the paren match. # 01234567890123456789 text = 'Jon "A (x "y) Bob "C' out = extract_delimited(_state(text)) assert [text[s.start:s.end] for _, s in out.extracted] == ['x "y'] # quotes at 4 and 18 are genuinely dangling; the one at 10 was # consumed by the parenthesized extraction assert sorted(a.detail for a in out.ambiguities) == [ "unmatched '\"' at offset 18; treated as literal text", "unmatched '\"' at offset 4; treated as literal text", ] assert all(a.kind is AmbiguityKind.UNBALANCED_DELIMITER for a in out.ambiguities) def test_genuine_unbalanced_still_flagged_alongside_overlapping_pairs() -> None: # the suppression must not swallow REAL unbalanced opens: here the # German open has no close anywhere, and no other pair extracts policy = Policy(nickname_delimiters=frozenset( {("“", "”"), ("„", "“")})) out = extract_delimited(_state("Hans „Hansi Müller", policy)) assert out.extracted == () assert [a.kind for a in out.ambiguities] == [ AmbiguityKind.UNBALANCED_DELIMITER] def test_maiden_delimiters_route_to_maiden() -> None: policy = dataclasses.replace( Policy(), nickname_delimiters=frozenset({("'", "'"), ('"', '"')}), maiden_delimiters=frozenset({("(", ")")}), ) out = extract_delimited(_state("Jane Smith (Jones)", policy)) assert out.extracted == ((Role.MAIDEN, Span(12, 17)),) def test_empty_enclosure_is_masked_but_not_extracted() -> None: out = extract_delimited(_state("John () Kennedy")) assert out.extracted == () assert out.masked == (Span(5, 7),) def test_multiple_extracts_and_no_overlap() -> None: # 0123456789012345678901234 out = extract_delimited(_state('"Jack" John (Jonny) Kim')) roles = {span: role for role, span in out.extracted} assert roles == {Span(1, 5): Role.NICKNAME, Span(13, 18): Role.NICKNAME} def test_adjacent_pairs_extract_separately() -> None: out = extract_delimited(_state('John "A" "B" Kim')) assert len(out.extracted) == 2 def test_close_at_end_of_string() -> None: out = extract_delimited(_state('John "Jack"')) assert out.extracted == ((Role.NICKNAME, Span(6, 10)),) def test_unmatched_open_at_last_char() -> None: out = extract_delimited(_state('John "')) assert out.extracted == () assert len(out.ambiguities) == 1 def test_nested_delimiters_inner_scan_order_pins() -> None: # parens win over quotes when the quote chars sit flush against the # parens (their word-boundary test fails); the inner quote chars # flow into the extracted span verbatim -- v1 parity, deliberate out = extract_delimited(_state('John ("Jack") Kim')) assert out.extracted == ((Role.NICKNAME, Span(6, 12)),) @pytest.mark.parametrize(("text", "char"), [ ("John Smith)", ")"), ("John Jack) Smith", ")"), ('John Smith"', '"'), ("John Smith」", "」"), # a #273 typographic pair ]) def test_unmatched_close_is_reported(text: str, char: str) -> None: # the scan is opener-driven, so a close with no open to its left was # never in its search space and went unreported -- even though the # AmbiguityKind docstring promises "or closed without opening" ambiguities = parse(text).ambiguities assert [a.kind for a in ambiguities] == [ AmbiguityKind.UNBALANCED_DELIMITER] assert char in ambiguities[0].detail def test_word_final_apostrophe_is_not_an_unbalanced_delimiter() -> None: # ambiguity means the part could REASONABLY read two ways where it # sits. A "'" after a word character is an apostrophe -- it is the # one delimiter character that occurs inside and at the end of real # name parts, and the least likely to mark a nickname. The same # position with a quote IS ambiguous, because quotes do not appear # inside names. (#273 excluded the curly apostrophe from the # delimiters entirely for this reason; the straight one is a # delimiter, so it needs the carve-out here instead.) assert parse("Mari' Aube'").ambiguities == () assert parse("Ali Baba'").ambiguities == () quoted = parse('Mari" Aube"') assert [a.kind for a in quoted.ambiguities] == \ [AmbiguityKind.UNBALANCED_DELIMITER] * 2 def test_apostrophe_after_a_space_is_still_reported() -> None: # the carve-out is about what PRECEDES the character: an apostrophe # opening a quote is still an unmatched open when nothing closes it assert [a.kind for a in parse("John 'Jack Smith").ambiguities] == \ [AmbiguityKind.UNBALANCED_DELIMITER] @pytest.mark.parametrize("text", [ "Brian O'connor", # apostrophe mid-word, not a delimiter "O'B. John Smith", "The Queen's Bench", "John (Jack) Smith", # matched pair 'John "Jack" Smith', ]) def test_no_spurious_unmatched_close(text: str) -> None: assert [a for a in parse(text).ambiguities if a.kind is AmbiguityKind.UNBALANCED_DELIMITER] == [] def test_unmatched_open_is_not_double_reported_as_a_close() -> None: # a symmetric delimiter can satisfy both boundary tests; it must # still produce exactly one ambiguity assert len(parse('John " Smith').ambiguities) == 1 assert len(parse('Jon "Nick Smith').ambiguities) == 1 @pytest.mark.parametrize(("text", "expected"), [ ('Jon "Nick Smith', '"Nick'), ("John (Jack Smith", "(Jack"), ("John Smith)", "Smith)"), ("John Jack) Smith", "Jack)"), ]) def test_unbalanced_delimiter_points_at_its_token( text: str, expected: str ) -> None: # the kind was documented as possibly carrying no tokens, but the # stray character does survive into one -- without it the ambiguity # is locatable only by parsing an offset back out of the detail # string, which is not an API (amb,) = parse(text).ambiguities assert [t.text for t in amb.tokens] == [expected] python-nameparser-2.1.0/tests/v2/pipeline/test_group.py000066400000000000000000000263211523530431700232400ustar00rootroot00000000000000from nameparser._lexicon import Lexicon from nameparser._pipeline._classify import classify from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._group import group from nameparser._pipeline._script_segment import script_segment from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState from nameparser._pipeline._tokenize import tokenize from nameparser._policy import Policy, Script from nameparser._types import Role _LEX = Lexicon( titles=frozenset({"mr", "mrs", "secretary", "the", "state"}), suffix_acronyms=frozenset({"phd"}), suffix_words=frozenset({"jr"}), particles=frozenset({"de", "la", "van", "von", "und", "zu"}), particles_ambiguous=frozenset({"van"}), conjunctions=frozenset({"and", "of", "the", "y"}), bound_given_names=frozenset({"abdul"}), maiden_markers=frozenset({"née", "geb"}), ) # NOTE: "the" sits in both titles and conjunctions here, mirroring v1's # real vocabulary overlap; the subset rule only constrains particles. def _grouped(text: str, policy: Policy | None = None, lexicon: Lexicon | None = None) -> ParseState: state = ParseState(original=text, lexicon=lexicon or _LEX, policy=policy or Policy()) return group(classify(segment(tokenize(extract_delimited(state))))) def _piece_texts(state: ParseState) -> list[list[str]]: return [[" ".join(state.tokens[i].text for i in piece) for piece in seg] for seg in state.pieces] def test_no_joins_pass_through() -> None: out = _grouped("John Smith") assert _piece_texts(out) == [["John", "Smith"]] def test_conjunction_joins_neighbors() -> None: out = _grouped("Mr. and Mrs. John Smith") assert _piece_texts(out) == [["Mr. and Mrs.", "John", "Smith"]] # joined-to-a-title piece is flagged a derived title assert "title" in out.piece_tags[0][0] def test_contiguous_conjunctions_join_first() -> None: out = _grouped("The Secretary of State Hillary Clinton") assert _piece_texts(out) == [["The Secretary of State", "Hillary", "Clinton"]] assert "title" in out.piece_tags[0][0] def test_single_letter_conjunction_prefers_initial_when_short() -> None: # v1 Google Code issue 11 ("john e smith"): 3 rootname parts, # single-letter conjunction "y" out = _grouped("John y Smith") assert _piece_texts(out) == [["John", "y", "Smith"]] def test_prefix_chain_joins_to_following() -> None: out = _grouped("Juan de la Vega") assert _piece_texts(out) == [["Juan", "de la Vega"]] def test_prefix_chain_absorbs_through_to_next_suffix() -> None: out = _grouped("Juan de la Vega Martinez PhD") assert _piece_texts(out) == [["Juan", "de la Vega Martinez", "PhD"]] def test_leading_prefix_is_never_chained() -> None: # "Van Johnson": the leading piece is a first name, not a particle out = _grouped("Van Johnson") assert _piece_texts(out) == [["Van", "Johnson"]] def test_von_und_zu_bridges() -> None: # conjunction "und" joins two prefixes; the joined piece is a derived # prefix and still chains onto the following name (v1 PR #191) out = _grouped("Otto von und zu Habsburg") assert _piece_texts(out) == [["Otto", "von und zu Habsburg"]] def test_bound_given_joins_with_three_rootnames() -> None: assert _piece_texts(_grouped("abdul rahman al-said")) == \ [["abdul rahman", "al-said"]] # only two rootname pieces: no join (v1 reserve_last) assert _piece_texts(_grouped("abdul rahman")) == [["abdul", "rahman"]] def test_phd_split_across_tokens_merges_as_suffix() -> None: out = _grouped("John Smith Ph. D.") assert _piece_texts(out) == [["John", "Smith", "Ph. D."]] assert "suffix" in out.piece_tags[0][2] # continuation tokens of the merged piece carry the stable "joined" # tag so the suffix string view can heal the split (", " join would # otherwise render 'Ph., D.') d_tok = next(t for t in out.tokens if t.text == "D.") assert "joined" in d_tok.tags def test_maiden_marker_consumes_tail() -> None: out = _grouped("Jane Smith née Jones") assert _piece_texts(out) == [["Jane", "Smith"]] maiden = [t.text for t in out.tokens if t.role is Role.MAIDEN] assert maiden == ["Jones"] # the marker token itself is structural: dropped from assembly née_idx = next(i for i, t in enumerate(out.tokens) if t.text == "née") assert née_idx in out.dropped def test_maiden_marker_stops_at_suffix() -> None: out = _grouped("Jane Smith née Jones PhD") maiden = [t.text for t in out.tokens if t.role is Role.MAIDEN] assert maiden == ["Jones"] assert _piece_texts(out)[0][-1] == "PhD" def test_leading_marker_is_not_consumed() -> None: # "née Jones" alone: marker at piece 0 has no name before it out = _grouped("née Jones") assert _piece_texts(out) == [["née", "Jones"]] _MAIDEN_PARENS = Policy(maiden_delimiters=frozenset({("(", ")")})) #: `nee` a marker AND `Nee` a surname in one vocabulary -- the collision #: the clause-size guard exists for. _LEX alone leaves `Nee` untagged, #: which would let the guard tests pass under every mutant. _NEE_LEX = _LEX.add(maiden_markers=frozenset({"nee"})) def test_delimited_marker_is_dropped() -> None: """#329: the marker IS tagged -- classify reaches it like any other token. What it never enters is `pieces`: extract claims the clause and its tokens carry Role.MAIDEN from tokenize, so segment keeps them out of the main stream. The #274 rule above walks pieces, so the tag alone does it no good.""" out = _grouped("Jane Smith (née Jones)", policy=_MAIDEN_PARENS) maiden = [t.text for i, t in enumerate(out.tokens) if t.role is Role.MAIDEN and i not in out.dropped] assert maiden == ["Jones"] née_idx = next(i for i, t in enumerate(out.tokens) if t.text == "née") assert née_idx in out.dropped def test_lone_delimited_marker_is_kept_when_a_clause_follows() -> None: """The clause-size guard, and it is load-bearing rather than defensive: `Nee` is a real surname (Irish Ni/Nee, and a Chinese romanization), so a one-token clause is a maiden NAME, not a marker. The trailing "(Jones)" is what makes this pin the guard. With "(Nee)" alone the marker is also the last token in the string, so a rule that merely checked for a following token would keep it too and the mutant would live. Here a token does follow -- only the CLAUSE bound distinguishes them.""" out = _grouped("Jane Smith (Nee) (Jones)", policy=_MAIDEN_PARENS, lexicon=_NEE_LEX) kept = [t.text for i, t in enumerate(out.tokens) if t.role is Role.MAIDEN and i not in out.dropped] assert kept == ["Nee", "Jones"] def test_marker_in_the_bare_form_is_left_to_the_piece_rule() -> None: """#274 sets Role.MAIDEN on consumed tokens itself, so a maiden role is NOT proof that extract produced it. Keying this pass on state.extracted spans is what keeps it off the bare path -- a neighbour test would eat the `Nee` here, the very surname the guard above exists to protect. Reads inert and is not: deleting the #329 pass outright leaves this green, because the value comes from #274's piece rule and the pass never touches the bare path. What it kills is a SPELLING of the pass -- drop a maiden marker whose next token is also maiden, the form this fix originally took -- and nothing else here covers the bare path against it. Checked both ways 2026-08-03.""" out = _grouped("Jane Smith nee Nee Jones", lexicon=_NEE_LEX) kept = [t.text for i, t in enumerate(out.tokens) if t.role is Role.MAIDEN and i not in out.dropped] assert kept == ["Nee", "Jones"] def test_every_delimited_marker_is_dropped_not_only_the_first() -> None: """Two maiden clauses land as ONE contiguous run of maiden-role tokens, so a rule keyed on the run would strip the first marker and keep the second. Each clause is scoped separately, so each loses its own leading marker.""" out = _grouped("Jane Smith (née Jones) (geb Braun)", policy=_MAIDEN_PARENS) kept = [t.text for i, t in enumerate(out.tokens) if t.role is Role.MAIDEN and i not in out.dropped] assert kept == ["Jones", "Braun"] def test_clause_containment_survives_script_segmentation() -> None: """The #329 pass finds the clause's first token by SPAN, and the comment on it rests that on script_segment only ever cutting a token into sub-slices. _grouped omits that stage, so this is the one place the two meet: 王小明 becomes 王 + 小明 before group runs, which shifts every token index after it while the spans stay exact, and the marker is still the token the clause drops.""" lex = Lexicon(surnames=frozenset({"王"}), maiden_markers=frozenset({"旧姓"})) policy = Policy(segment_scripts=frozenset({Script.HAN}), maiden_delimiters=frozenset({("（", "）")})) state = ParseState(original="王小明（旧姓 李四）", lexicon=lex, policy=policy) out = group(classify(script_segment(segment( tokenize(extract_delimited(state)))))) assert [t.text for t in out.tokens] == ["王", "小明", "旧姓", "李四"] kept = [t.text for i, t in enumerate(out.tokens) if t.role is Role.MAIDEN and i not in out.dropped] assert kept == ["李四"] def test_initials_do_not_count_as_rootnames_for_conjunction_carveout() -> None: # v1 parity: 'J.' is an initial, so total rootnames stay under 4 and # the single-letter conjunction 'y' is treated as an initial, not joined out = _grouped("J. Ruiz y Gomez") assert _piece_texts(out) == [["J.", "Ruiz", "y", "Gomez"]] def test_extra_suffix_delimiter_splits_tail_entries() -> None: # v1 expand_suffix_delimiter parity (#191): a configured delimiter # is transparent in a FAMILY_COMMA tail segment -- it separates # suffix ENTRIES (each rendered independently) and is itself # dropped, rather than becoming a suffix token or fusing the # entries on either side into one "joined" run. lex = Lexicon(suffix_acronyms=frozenset({"phd"}), suffix_words=frozenset({"jr", "v", "md"})) out = _grouped("Smith, John, V MD / PhD", Policy(extra_suffix_delimiters=frozenset({"/"})), lexicon=lex) # one tail SEGMENT, split into three suffix pieces: "/" is dropped # rather than surviving as its own piece assert _piece_texts(out) == [["Smith"], ["John"], ["V", "MD", "PhD"]] slash_idx = next(i for i, t in enumerate(out.tokens) if t.text == "/") assert slash_idx in out.dropped # "MD" continues the "V MD" entry (joined); "PhD" starts a fresh # entry across the dropped delimiter, so it does NOT get "joined" md_tok = next(t for t in out.tokens if t.text == "MD") phd_tok = next(t for t in out.tokens if t.text == "PhD") assert "joined" in md_tok.tags assert "joined" not in phd_tok.tags def test_suffix_comma_name_segment_gets_no_additional_count() -> None: # v1 parity: additional_parts_count applies to FAMILY_COMMA parts only; # ', PhD' must not tip the single-letter-conjunction carve-out out = _grouped("John y Smith, PhD") assert _piece_texts(out) == [["John", "y", "Smith"], ["PhD"]] python-nameparser-2.1.0/tests/v2/pipeline/test_post_rules.py000066400000000000000000000104271523530431700243030ustar00rootroot00000000000000from nameparser._lexicon import Lexicon from nameparser._pipeline import run from nameparser._pipeline._state import ParseState from nameparser._policy import PatronymicRule, Policy from nameparser._types import Role _LEX = Lexicon( titles=frozenset({"mr", "sir"}), given_name_titles=frozenset({"sir"}), particles=frozenset({"de", "la", "van"}), particles_ambiguous=frozenset({"van"}), ) def _parsed(text: str, policy: Policy | None = None) -> ParseState: return run(ParseState(original=text, lexicon=_LEX, policy=policy or Policy())) def _by_role(state: ParseState, role: Role) -> str: return " ".join(t.text for t in state.tokens if t.role is role) def test_plain_title_with_single_name_swaps_to_family() -> None: out = _parsed("Mr. Johnson") assert _by_role(out, Role.FAMILY) == "Johnson" assert not _by_role(out, Role.GIVEN) def test_given_name_title_keeps_given() -> None: out = _parsed("Sir Bob") assert _by_role(out, Role.GIVEN) == "Bob" assert not _by_role(out, Role.FAMILY) def test_no_swap_when_more_fields_present() -> None: out = _parsed("Mr. John Johnson") assert _by_role(out, Role.GIVEN) == "John" assert _by_role(out, Role.FAMILY) == "Johnson" _ES = Policy(patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})) _TK = Policy(patronymic_rules=frozenset({PatronymicRule.TURKIC})) def test_east_slavic_rotation() -> None: out = _parsed("Сидоров Иван Петрович", _ES) assert _by_role(out, Role.GIVEN) == "Иван" assert _by_role(out, Role.MIDDLE) == "Петрович" assert _by_role(out, Role.FAMILY) == "Сидоров" def test_east_slavic_needs_one_one_one() -> None: # four tokens: left unchanged (v1 parity) out = _parsed("Anna Maria Petrova Ivanovna", _ES) assert _by_role(out, Role.GIVEN) == "Anna" def test_east_slavic_skips_comma_forms() -> None: # v1 parity: patronymic reorder never fires on comma input -- # the comma already established the family out = _parsed("Abramovich, Roman Petrovich", _ES) assert _by_role(out, Role.FAMILY) == "Abramovich" assert _by_role(out, Role.GIVEN) == "Roman" def test_east_slavic_skips_when_middle_is_also_patronymic() -> None: # v1 parity: given + patronymic + patronymic-derived surname # (Abramovich) must not rotate out = _parsed("Roman Petrovich Abramovich", _ES) assert _by_role(out, Role.GIVEN) == "Roman" assert _by_role(out, Role.MIDDLE) == "Petrovich" assert _by_role(out, Role.FAMILY) == "Abramovich" def test_east_slavic_off_by_default() -> None: out = _parsed("Сидоров Иван Петрович") assert _by_role(out, Role.GIVEN) == "Сидоров" def test_turkic_rotation() -> None: out = _parsed("Mammadova Aygun Ali kizi", _TK) assert _by_role(out, Role.GIVEN) == "Aygun" assert _by_role(out, Role.MIDDLE) == "Ali kizi" assert _by_role(out, Role.FAMILY) == "Mammadova" def test_leading_never_given_particle_folds_into_family() -> None: # v1 handle_non_first_name_prefix: a leading particle that is never # a given name ('de') means the whole name is a surname out = _parsed("de la Vega") assert _by_role(out, Role.FAMILY) == "de la Vega" assert not _by_role(out, Role.GIVEN) def test_leading_ambiguous_particle_stays_given() -> None: # 'van' is particles_ambiguous: the given reading stands (v1 parity) out = _parsed("van Gogh") assert _by_role(out, Role.GIVEN) == "van" assert _by_role(out, Role.FAMILY) == "Gogh" def test_degenerate_bare_particle_stays_given() -> None: # v1's guard: with no middle or family, a bare 'de' keeps given='de' # rather than inventing a surname out = _parsed("de") assert _by_role(out, Role.GIVEN) == "de" assert not _by_role(out, Role.FAMILY) def test_middle_as_family_folds_middles() -> None: # v1 handle_middle_name_as_last, opt-in: middles prepend to family out = _parsed("John Quincy Adams Smith", Policy(middle_as_family=True)) assert _by_role(out, Role.GIVEN) == "John" assert not _by_role(out, Role.MIDDLE) assert _by_role(out, Role.FAMILY) == "Quincy Adams Smith" def test_middle_as_family_off_by_default() -> None: out = _parsed("John Quincy Adams Smith") assert _by_role(out, Role.MIDDLE) == "Quincy Adams" python-nameparser-2.1.0/tests/v2/pipeline/test_script_segment.py000066400000000000000000001143701523530431700251340ustar00rootroot00000000000000"""Stage: script_segment (#271, #272) -- unspaced CJK splitting, by vocabulary and by the pluggable segmenter behind it.""" import dataclasses import pytest from nameparser import Parser from nameparser._lexicon import Lexicon from nameparser._pipeline._script_segment import script_segment from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ( ParseState, PendingAmbiguity, Structure, ) from nameparser._pipeline._tokenize import tokenize from nameparser._policy import Policy, Script from nameparser._types import AmbiguityKind, Segmentation, Segmenter _HAN = Policy(segment_scripts=frozenset({Script.HAN})) _HANGUL = Policy() # HANGUL is the default activation # the JA pack's own activation (amendment 2026-07-29): HIRAGANA is the # kana license's carrier key, so a kanji+kana composite gates in # through it; KATAKANA is deliberately absent from every set. _JA = Policy(segment_scripts=frozenset({Script.HAN, Script.HIRAGANA})) # 欧 AND 欧阳 both present on purpose: compound-before-single must # pick 欧阳 (the issue's own acceptance example); same for 남/남궁. # "jr" so the comma cases can reach SUFFIX_COMMA. _LEX = Lexicon(surnames=frozenset({"毛", "欧", "欧阳", "김", "남", "남궁"}), suffix_words=frozenset({"jr"})) # The peel's own vocabulary (#308). Tails are also suffix words here, # as the shipped config asserts they are: the neighbour rule in the # segmenter tests below reads that membership, so a stage lexicon that # omitted it would pin a shape no real configuration can have. _TAILS = frozenset({"씨", "님", "선생님", "박사", "さん", "先生", "様"}) _LEX_TAILS = _LEX.add(suffix_words=_TAILS, honorific_tails=_TAILS) def _run(text: str, policy: Policy = _HAN, lexicon: Lexicon = _LEX, segmenter: Segmenter | None = None) -> ParseState: state = ParseState(original=text, lexicon=lexicon, policy=policy, segmenter=segmenter) return script_segment(segment(tokenize(state))) def _texts(state: ParseState) -> list[str]: return [t.text for t in state.tokens] def _fake(splits: tuple[int, ...], confidence: float | None = None) -> Segmenter: """A segmenter that gives the same answer whatever it is handed: every case below pins what the STAGE does with an answer, not how a real segmenter arrives at one (no external dependency here).""" def segmenter(text: str) -> Segmentation | None: return Segmentation(splits, confidence=confidence) return segmenter def _capture( splits: tuple[int, ...] = (2,)) -> tuple[list[str], Segmenter]: """_fake, plus the list of texts it was handed: the two cases below that pin WHETHER and WITH WHAT the stage consults a segmenter need the same closure.""" asked: list[str] = [] def segmenter(text: str) -> Segmentation | None: asked.append(text) return Segmentation(splits) return asked, segmenter def _declines(text: str) -> Segmentation | None: return None def test_splits_leading_surname_and_spans_index_the_original() -> None: state = _run("毛泽东") assert _texts(state) == ["毛", "泽东"] assert [(t.span.start, t.span.end) for t in state.tokens] == [ (0, 1), (1, 3)] assert all(state.original[t.span.start:t.span.end] == t.text for t in state.tokens) def test_compound_before_single() -> None: assert _texts(_run("欧阳明")) == ["欧阳", "明"] assert _texts(_run("남궁민수", policy=_HANGUL)) == ["남궁", "민수"] def test_hangul_splits_under_the_default_policy() -> None: assert _texts(_run("김민준", policy=_HANGUL)) == ["김", "민준"] def test_han_needs_the_opt_in() -> None: # default policy activates HANGUL only: Han tokens stay whole assert _texts(_run("毛泽东", policy=_HANGUL)) == ["毛泽东"] def test_inert_without_surname_vocabulary() -> None: assert _texts(_run("毛泽东", lexicon=Lexicon.empty())) == ["毛泽东"] def test_inert_with_segmentation_disabled() -> None: off = Policy(segment_scripts=()) # type: ignore[arg-type] assert _texts(_run("김민준", policy=off)) == ["김민준"] def test_whole_token_surname_does_not_split() -> None: # a bare surname has nothing to split off; a lone token's role is # the order resolution's call, not this stage's. The trap: 欧 and # 남 are ALSO listed, so without the whole-token guard these # would split 欧+阳 / 남+궁 by the shorter prefix assert _texts(_run("欧阳")) == ["欧阳"] assert _texts(_run("남궁", policy=_HANGUL)) == ["남궁"] def test_no_match_leaves_the_token_alone() -> None: assert _texts(_run("阿明")) == ["阿明"] def test_only_the_first_script_token_is_considered() -> None: assert _texts(_run("毛泽东 毛泽东")) == ["毛", "泽东", "毛泽东"] def test_leading_non_script_token_is_skipped_over() -> None: assert _texts(_run("Dr 毛泽东")) == ["Dr", "毛", "泽东"] def test_mixed_script_token_is_not_split() -> None: assert _texts(_run("毛zedong")) == ["毛zedong"] def test_first_script_token_decides_even_on_no_match() -> None: # no fallthrough: the front of the name is the only surname site assert _texts(_run("阿明 毛泽东")) == ["阿明", "毛泽东"] def test_compound_fork_emits_segmentation_ambiguity() -> None: # both 欧阳+明 and 欧+阳明 are vocabulary-supported: the stage # decided a fork, and the pipeline contract is that deciding # stages record it (indices = the two result tokens) out = _run("欧阳明") assert [a.kind for a in out.ambiguities] == [ AmbiguityKind.SEGMENTATION] assert out.ambiguities[0].indices == (0, 1) # the documented promise: detail names BOTH readings detail = out.ambiguities[0].detail assert "欧阳" in detail and "明" in detail assert "阳明" in detail ko = _run("남궁민수", policy=_HANGUL) assert [a.kind for a in ko.ambiguities] == [ AmbiguityKind.SEGMENTATION] def test_single_possible_split_emits_no_ambiguity() -> None: # only 毛 matches: no fork, no noise assert _run("毛泽东").ambiguities == () assert _run("김민준", policy=_HANGUL).ambiguities == () def test_a_family_comma_stands_the_surname_split_down() -> None: # the comma declared the family: splitting it would invent a # boundary the writer did not draw ("남궁 민수" with a space). # The SPLIT only -- the stage as a whole is not inert under a # family comma since #312, and would not be on a lexicon carrying # honorific_tails (_LEX does not; see the peel's own cases below). out = _run("남궁민수, 지훈", policy=_HANGUL) assert out.structure is Structure.FAMILY_COMMA assert _texts(out) == ["남궁민수", "지훈"] assert out.ambiguities == () def test_family_comma_given_side_untouched() -> None: assert _texts(_run("박, 남궁민수", policy=_HANGUL)) == [ "박", "남궁민수"] def test_one_word_before_the_comma_is_never_suffix_comma() -> None: # an unspaced CJK name is ONE word, and segment's suffix-comma # rule needs >1 word before the comma -- so this reads as # FAMILY_COMMA and the opt-out above covers it too out = _run("김민준, Jr.", policy=_HANGUL) assert out.structure is Structure.FAMILY_COMMA assert _texts(out) == ["김민준", "Jr."] def test_suffix_comma_name_part_still_splits() -> None: out = _run("Dr 김민준, Jr.", policy=_HANGUL) assert out.structure is Structure.SUFFIX_COMMA assert _texts(out) == ["Dr", "김", "민준", "Jr."] def test_segments_remap_after_the_split() -> None: out = _run("Dr 김민준, Jr.", policy=_HANGUL) # name segment gains the second half in place; the suffix # segment's index follows the token it names assert out.segments == ((0, 1, 2), (3,)) assert [[out.tokens[i].text for i in run] for run in out.segments] \ == [["Dr", "김", "민준"], ["Jr."]] def test_ambiguity_indices_shift_past_the_split() -> None: state = segment(tokenize(ParseState( original="毛泽东 X", lexicon=_LEX, policy=_HAN))) state = dataclasses.replace(state, ambiguities=( PendingAmbiguity(AmbiguityKind.UNBALANCED_DELIMITER, "synthetic", indices=(0,)), PendingAmbiguity(AmbiguityKind.UNBALANCED_DELIMITER, "synthetic", indices=(1,)), )) out = script_segment(state) # index 0 (the split token itself) stays; index 1 moves right assert out.ambiguities[0].indices == (0,) assert out.ambiguities[1].indices == (2,) def test_the_real_fold_splits_and_keeps_the_stage_after_segment() -> None: # the stage-order contract end to end: a set-comparison ownership # test cannot catch a reorder, but this can -- placed before # segment, the split would feed segment two words where the writer # wrote one and turn the family-comma parse below into a # suffix-comma one p = Parser(lexicon=_LEX, policy=_HANGUL) plain = p.parse("김민준") assert (plain.family, plain.given) == ("김", "민준") declared = p.parse("남궁민수, 지훈") assert (declared.family, declared.given) == ("남궁민수", "지훈") def test_delimited_only_input_reaches_the_empty_segments_guard() -> None: # extraction masks the whole string, so segment produces no runs # at all -- the guard the stage-level helper above cannot reach nick = Parser(lexicon=_LEX, policy=_HANGUL).parse("(민준)") assert nick.nickname == "민준" # -- the pluggable segmenter, consulted on decline (#272) --------------- def test_segmenter_splits_on_vocabulary_decline() -> None: # the baseline: with no segmenter configured a declined token is # simply left whole, so every split below is the hook's doing assert _texts(_run("山田太郎", policy=_JA)) == ["山田太郎"] # 山田太郎 has no vocabulary surname prefix at all, so the # vocabulary declines and the token reaches the segmenter state = _run("山田太郎", policy=_JA, segmenter=_fake((2,))) assert _texts(state) == ["山田", "太郎"] assert [(t.span.start, t.span.end) for t in state.tokens] == [ (0, 2), (2, 4)] assert all(state.original[t.span.start:t.span.end] == t.text for t in state.tokens) def test_vocabulary_wins_over_the_segmenter() -> None: # precedence, and the reason parser_for(ZH, JA, segmenter=...) # composes: a matched surname is a dictionary certainty, so the # segmenter is not even asked asked, seg = _capture() state = _run("毛泽东", policy=_JA, segmenter=seg) assert _texts(state) == ["毛", "泽东"] assert asked == [] def test_segmenter_decline_and_confident_whole_are_distinct() -> None: # two different ANSWERS -- None is "I don't know", () is # "confidently one token" -- with the same effect on the tokens: # the difference is the protocol's to carry, not this stage's for seg in (_declines, _fake(())): out = _run("山田太郎", policy=_JA, segmenter=seg) assert _texts(out) == ["山田太郎"] assert out.ambiguities == () # nothing decided, nothing to report def test_kana_licensed_token_gates_in_for_the_segmenter() -> None: # the gate reads effective_script, not single_script: 高橋みなみ is # mixed Han+hiragana, which the kana license resolves to the # HIRAGANA carrier entry -- and HIRAGANA is in the JA activation state = _run("高橋みなみ", policy=_JA, segmenter=_fake((2,))) assert _texts(state) == ["高橋", "みなみ"] def test_pure_katakana_never_gates_in() -> None: # the license's other half: a lone katakana token is predominantly # a transcribed foreign name, so KATAKANA sits in no activation set # and no segmenter is consulted for one out = _run("マイケル", policy=_JA, segmenter=_fake((2,))) assert _texts(out) == ["マイケル"] assert out.ambiguities == () def test_kana_licensed_token_stays_whole_under_default_policy() -> None: # the pack IS the language declaration: without it the default # activation is HANGUL alone, so nothing gates in and the # configured segmenter is inert out = _run("高橋みなみ", segmenter=_fake((2,))) assert _texts(out) == ["高橋みなみ"] assert out.ambiguities == () def test_segmenter_multi_split_produces_n_plus_one_tokens() -> None: # n cuts make n+1 sub-slices, and every index the earlier stages # recorded shifts by n rather than by one state = segment(tokenize(ParseState( original="山田太 X", lexicon=_LEX, policy=_JA, segmenter=_fake((1, 2))))) state = dataclasses.replace(state, ambiguities=( PendingAmbiguity(AmbiguityKind.UNBALANCED_DELIMITER, "synthetic", indices=(1,)),)) out = script_segment(state) assert _texts(out) == ["山", "田", "太", "X"] assert [(t.span.start, t.span.end) for t in out.tokens] == [ (0, 1), (1, 2), (2, 3), (4, 5)] assert all(out.original[t.span.start:t.span.end] == t.text for t in out.tokens) assert out.segments == ((0, 1, 2, 3),) assert out.ambiguities[0].indices == (3,) def test_low_confidence_emits_segmentation_ambiguity() -> None: # a statistical guess is a fork the stage decided, so a score under # _SEGMENTER_CONFIDENCE_FLOOR is reported -- unlike a vocabulary # split, which reports only when a SECOND surname also matched state = _run("山田太郎", policy=_JA, segmenter=_fake((2,), confidence=0.42)) assert [a.kind for a in state.ambiguities] == [ AmbiguityKind.SEGMENTATION] assert "0.42" in state.ambiguities[0].detail assert state.ambiguities[0].indices == (0, 1) def test_high_confidence_and_no_confidence_emit_nothing() -> None: # a confident answer is not a fork, and an opinionless one gives # the stage nothing to judge -- neither is worth a report # 0.9 is the floor itself: the comparison is strict, so an answer # AT the floor is not under it for conf in (0.99, 0.9, None): state = _run("山田太郎", policy=_JA, segmenter=_fake((2,), confidence=conf)) assert _texts(state) == ["山田", "太郎"] assert state.ambiguities == () def test_out_of_bounds_split_raises() -> None: # Segmentation cannot check the upper bound (it never sees the # text), so the stage does -- and it RAISES, the same call the # wrong-answer-type check beside it makes: both are protocol # violations by the segmenter's author, inside the declared # totality exception. Declining silently instead (as this did # before the review) made an off-by-one segmenter undebuggable -- # every answer vanished and the name merely looked undivided. with pytest.raises(ValueError, match=r"segmenter returned splits beyond the " r"token: last offset 5, token length 2"): _run("山田", policy=_JA, segmenter=_fake((5,))) # 2 on a two-character token is the boundary itself: the check is # >=, not >, since a cut at len(text) would leave an empty piece with pytest.raises(ValueError, match="last offset 2, token length 2"): _run("山田", policy=_JA, segmenter=_fake((2,))) def test_family_comma_gates_the_segmenter_too() -> None: # the comma doctrine is about the input's structure, not the # split's source: the structural opt-out runs before any consult out = _run("山田太郎, 花子", policy=_JA, segmenter=_fake((2,))) assert out.structure is Structure.FAMILY_COMMA assert _texts(out) == ["山田太郎", "花子"] assert out.ambiguities == () def test_empty_vocabulary_still_consults_the_segmenter() -> None: # "consult on vocabulary decline" has no unstated exception for a # lexicon that declines EVERYTHING: a surname-less vocabulary is # the JA pack's own shape (the segmenter is its whole mechanism), # so an emptiness bail here would make it silently inert state = _run("山田太郎", policy=_JA, lexicon=Lexicon.empty(), segmenter=_fake((2,))) assert _texts(state) == ["山田", "太郎"] def test_a_wrong_answer_type_raises_a_curated_error() -> None: # a duck-typed lookalike would otherwise wander into the split # path and surface as a ValueError naming Token -- the reader's # bug is in their segmenter, and the message must say so class NotASegmentation: splits = (2,) confidence = None def seg(text: str) -> Segmentation | None: return NotASegmentation() # type: ignore[return-value] with pytest.raises(TypeError, match="segmenter must return Segmentation or None, " "got NotASegmentation"): _run("山田太郎", policy=_JA, segmenter=seg) def test_a_neighbour_in_an_UNACTIVATED_script_still_blocks_the_consult() -> None: # The precondition counts a second script-written token whatever # script it is written in -- effective_script merely non-None, NOT # membership in the activation set. A katakana or hangul neighbour # is a boundary its writer drew just as deliberately as a Han one, # so the name is already divided and the segmenter must not be # asked. Narrowing the check to `in scripts` would make both of # these split 山 + 田太郎 while the writer's own boundary sat one # token to the right. for name in ("山田太郎 マイケル", "山田太郎 김민준"): out = _run(name, policy=_JA, segmenter=_fake((1,))) assert _texts(out) == name.split(), name assert out.ambiguities == (), name def test_only_a_manufactured_tail_does_not_block_the_consult() -> None: # The exemption is PROVENANCE, not vocabulary. A tail the peel # manufactured is a boundary nobody drew -- glued 山田太郎様 was # written as one token -- so the segmenter is asked, and asked # about the name alone. The SPACED spelling of the same honorific # is the opposite case: its writer drew that boundary and wrote # 山田太郎 as a unit beside it, so the consult is declined and the # unit stays whole. Asking the suffix VOCABULARY instead cannot # tell these apart -- same word, same place -- and answering both # the first way divided 佐藤 氏 into 佐 + 藤. asked, seg = _capture() out = _run("山田太郎様", policy=_JA, lexicon=_LEX_TAILS, segmenter=seg) assert asked == ["山田太郎"] assert _texts(out) == ["山田", "太郎", "様"] asked, seg = _capture() out = _run("山田太郎 様", policy=_JA, lexicon=_LEX_TAILS, segmenter=seg) assert asked == [] assert _texts(out) == ["山田太郎", "様"] def test_a_manufactured_tail_does_not_excuse_a_real_boundary() -> None: # The peeled tail stops counting, and nothing else does: 太郎 is # still a boundary its writer drew, so the name is already divided # and the segmenter must not be asked. Without the narrowing the # rule reads as "an exempt token anywhere disables the neighbour # test", which passes every other test in this file and divides # 山田 into 山 + 田. asked, seg = _capture() out = _run("山田 太郎様", policy=_JA, lexicon=_LEX_TAILS, segmenter=seg) assert asked == [] assert _texts(out) == ["山田", "太郎", "様"] # -- the glued honorific peel (#308) ----------------------------------- def test_peels_a_listed_tail_without_any_activation() -> None: # The peel is licensed by the TAIL, not by the script, so it runs # under a policy that activates nothing -- 田中さん must peel under # the DEFAULT policy, where HAN is in no activation set. Spans stay # sub-slices of the original (anti-#100), like every split here. off = Policy(segment_scripts=()) # type: ignore[arg-type] out = _run("田中さん", policy=off, lexicon=_LEX_TAILS) assert _texts(out) == ["田中", "さん"] assert [(t.span.start, t.span.end) for t in out.tokens] == [ (0, 2), (2, 4)] assert all(out.original[t.span.start:t.span.end] == t.text for t in out.tokens) assert out.ambiguities == () # decisive vocabulary, no fork def test_peel_then_segmentation_compose() -> None: # the peel runs first, so the segmentation half sees the REMAINDER assert _texts(_run("김민준씨", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["김", "민준", "씨"] def test_a_token_that_is_a_tail_never_peels() -> None: # nothing to split off, and the guard is the peel site's scan past # post-nominals, not the length cap: 선생님 ENDS in the shorter # tail 님, so a cap-based guard would peel it to 선생 + 님 assert _texts(_run("씨", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["씨"] assert _texts(_run("さん", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["さん"] assert _texts(_run("선생님", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["선생님"] def test_longest_tail_wins() -> None: # 님 and 선생님 both match by endswith, and only the longer one # leaves a remainder that is a name (박선생 is not one) assert _texts(_run("박선생님", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["박", "선생님"] def test_one_peel_never_a_stack() -> None: # The remainder here ENDS in another listed tail (박사) and is # still not peeled again: one peel, then the stage moves on to # segmentation, which splits the surname off what is left. # The SOLE home of this pin, and it needs the synthetic lexicon: # _TAILS deliberately omits 박사님, which the shipped vocabulary # carries, so 김민준박사님 gives up the whole honorific there (the # ko_honorific_glued_doctor case row). No shipped entry pair has # the shape this needs -- a tail whose remainder ends in a # DIFFERENT tail, the two not forming a listed entry themselves -- # so no realistic name can carry the pin end to end. assert _texts(_run("김민준박사님", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["김", "민준박사", "님"] def test_peel_needs_no_script_on_the_remainder() -> None: # the tail is the license: Japanese text about a foreigner assert _texts(_run("Andersonさん", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["Anderson", "さん"] def test_an_unlisted_tail_never_peels() -> None: # endswith against the vocabulary, never against a shape: 지양 ends # in a shipped SPACED honorific that is deliberately not a tail assert _texts(_run("김지양", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["김", "지양"] def test_an_interpunct_gates_the_split_but_not_the_peel() -> None: # The dot marks a transcription, which is a claim about where the # name divides into syllable groups -- so it still gates the # surname split. An honorific glued to a transcribed name is # still an honorific, so the peel crosses it (#312). 威 is a # surname in this lexicon precisely so the first clause bites: # without the gate 威廉 would divide 威 + 廉. lex = _LEX_TAILS.add(surnames={"威"}) state = segment(tokenize(ParseState( original="威廉·莎士比亚先生", lexicon=lex, policy=_HAN))) assert state.interpunct_offsets out = script_segment(state) assert _texts(out) == ["威廉", "莎士比亚", "先生"] def test_a_peeled_tail_is_not_a_surname_site() -> None: # The peel manufactures a token, and the site scan below must not # then treat it as a name: 남궁 opens with the listed surname 남, # so without the post-nominal guard the stage would split the # honorific it had just created. lex = Lexicon(surnames=frozenset({"남"}), suffix_words=frozenset({"남궁"}), honorific_tails=frozenset({"남궁"})) assert _texts(_run("Anderson남궁", policy=_HANGUL, lexicon=lex)) == ["Anderson", "남궁"] def test_a_leading_post_nominal_is_not_a_surname_site() -> None: # A surname LEADS, so an honorific in the surname's own position # means there is no surname to find -- decline rather than scan # on. Scanning on would reach the given name, which is what the # first-token rule above exists to prevent: 지 is a listed # surname, so 양 지훈 would split its own given name in half. assert _texts(_run("씨 김민준", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["씨", "김민준"] def test_a_trailing_post_nominal_does_not_hide_the_peel_site() -> None: # the site is the last token that is not ITSELF a post-nominal, so # an unrelated trailing suffix cannot put the glued one out of # reach -- "김민준씨 jr" must answer as "Dr 김민준씨, Jr." does assert _texts(_run("김민준씨 jr", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["김", "민준", "씨", "jr"] def test_the_peel_only_looks_at_the_last_non_post_nominal_token() -> None: # a tail anywhere but the end is somebody's name, not an # honorific: only the trailing position is an honorific site, the # same reasoning the trailing-only suffix gate follows assert _texts(_run("田中さん 太郎", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["田中さん", "太郎"] def test_the_peel_reaches_past_a_family_comma() -> None: # A comma says where the FAMILY name ends. It says nothing about # whether a trailing honorific is part of the name, so the peel # does not inherit the gate that stops the surname split -- and # under FAMILY_COMMA the name spans both segments, so the site # scan has to cross the boundary to find 민준씨 at all. assert _texts(_run("김, 민준씨", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["김", "민준", "씨"] def test_the_peel_scans_the_name_runs_and_no_further() -> None: # Crossing a FAMILY comma is not crossing every comma: past the # name's own runs lie post-nominals, and taking one as the site # abandons the peel. Two shapes, both silently broken by a scan # over ALL segments. segment admits a post-comma run on # is_suffix_lenient while the site scan asks is_suffix_strict, so # an initial-shaped suffix word is a site rather than a token to # step over -- "V." ends in no tail, and 씨 would stay glued. And # a later junk segment would take the peel off the person's own # name onto 박씨, peeling the wrong 씨 of two. lex = _LEX_TAILS.add(suffix_words={"v"}) assert _texts(_run("Dr 김민준씨, V.", policy=_HANGUL, lexicon=lex)) == ["Dr", "김", "민준", "씨", "V."] assert _texts(_run("Dr 김민준씨, Jr., 박씨", policy=_HANGUL, lexicon=lex)) == [ "Dr", "김", "민준", "씨", "Jr.", "박씨"] def test_a_suffix_comma_keeps_the_peel_inside_the_name() -> None: # The test above shows a suffix comma not being crossed while the # name HOLDS a site, which is the easy half: segments[0] answers # the scan either way, so a gate that had stopped keying on # FAMILY_COMMA -- on "there is a second run at all", say -- would # still land the peel on 민준씨 and pass. This is the twin that can # see it: two words before the comma so the structure is # SUFFIX_COMMA, NO site among them, and a token beyond the comma # that ends in a listed tail. The peel must answer None, not reach # past the name for the junk one -- "J.씨" is a stray post-nominal, # and its 씨 is nobody's honorific. assert _texts(_run("John Smith, J.씨", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["John", "Smith", "J.씨"] def test_the_peel_crosses_a_family_comma_and_stops_there() -> None: # The family-comma mirror of the test above, and the only input # that pins the SECOND half of segments[:2] -- that it is two runs # and not three. Every other family-comma case here has exactly # two segments, so the slice is otherwise only ever exercised as # "more than one run": widening it to three passes them all. # Both shapes above, re-spelled with the name split across a family # comma -- but only the SECOND of the two pins the slice, and the # measurement says so: widen this scan to three runs and 박씨 is # the junk tail whose 씨 gets peeled instead of the person's own, # while "Jr." is a post-nominal by the STRICT test as well, so a # wider scan steps over it onto 민준씨 and peels correctly anyway. # It is the initial-shaped words ("V." in the test above) that fall # in the strict/lenient gap, not "Jr."; this input keeps 씨's # placement pinned against a third run of ordinary post-nominals. assert _texts(_run("김, 민준씨, Jr.", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["김", "민준", "씨", "Jr."] assert _texts(_run("김, 민준씨, 박씨", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["김", "민준", "씨", "박씨"] def test_a_wholly_suffix_run_after_a_family_comma_is_declined() -> None: # The two tests above pin HOW FAR the crossing reaches; this pins # WHETHER it happens, which the structure alone does not decide. # segment answers FAMILY_COMMA whenever a single word precedes the # comma, even where the part after it is entirely suffix-shaped, so # "the second run is name text" is an inference and a wrong one # here. Scanning it lands the site on "V." -- admitted to the run by # segment's is_suffix_lenient, rejected as an initial by the scan's # is_suffix_strict -- which ends in no listed tail, so the peel is # abandoned and さん stays glued to 田中 (#319). The peel asks # is_wholly_suffix, segment's own predicate, and declines the run. lex = _LEX_TAILS.add(suffix_words={"v"}) assert _texts(_run("田中さん, V.", policy=_HANGUL, lexicon=lex)) == ["田中", "さん", "V."] def test_a_long_wholly_suffix_post_comma_run_is_declined_too() -> None: # The decline is a question about the run's VOCABULARY and not its # LENGTH, and no other input here says so: every case pinning the # decline has a one- or two-token run, so a gate that declined only # short runs passes all of them and this one alone catches it. # Three post-nominals, with the initial-shaped one LEFTMOST on # purpose -- a scan that reached this run would step over PhD and # MD (both post-nominals by the strict test) and stop on "V.", # which ends in no tail, abandoning the peel exactly as above. lex = _LEX_TAILS.add(suffix_words={"v"}, suffix_acronyms={"md", "phd"}) assert _texts(_run("田中さん, V. MD PhD", policy=_HANGUL, lexicon=lex)) == ["田中", "さん", "V.", "MD", "PhD"] def test_a_third_segment_does_not_reopen_the_declined_run() -> None: # WHICH run the decline reads, and how many there may be, are two # more axes no other input here moves: every case pinning the # decline has exactly two segments, so reading segments[-1] instead # of segments[1], or requiring exactly two segments before # declining at all, passes all of them. Both mistakes give the same # visible loss on this input -- the run consulted is 太郎, or none # is, so the scan crosses, "V." is the site, it ends in no tail and # the peel is abandoned. The third segment is junk beyond the name # either way (segment flags it, and the peel already refuses to # scan it -- test_the_peel_crosses_a_family_comma_and_stops_there # above); the question here is only whether its presence changes # the answer about the SECOND. lex = _LEX_TAILS.add(suffix_words={"v"}) assert _texts(_run("田中さん, V., 太郎", policy=_HANGUL, lexicon=lex)) == ["田中", "さん", "V.", "太郎"] def test_a_configured_suffix_delimiter_reaches_the_peel() -> None: # is_wholly_suffix is handed the CALLER's policy, so # extra_suffix_delimiters decides what counts as a suffix run here # exactly as it does inside segment -- a peel that passed a default # policy instead would answer differently from the stage whose # question it is borrowing. Two routes into that answer, neither # reachable from the other's input, so both are pinned: a # delimiter WITHOUT whitespace leaves one token no core equals # ("PhD/MD"), admitted by splits_into_suffixes, while a # whitespace-padded one surfaces as a standalone token ("-"), # admitted by plain core membership. Each peels only with its # delimiter configured; the bare-policy line under each is the # baseline that says so. lex = _LEX_TAILS.add(suffix_words={"v"}, suffix_acronyms={"md", "phd"}) slash = Policy(extra_suffix_delimiters=frozenset({"/"})) assert _texts(_run("田中さん, PhD/MD", policy=slash, lexicon=lex)) == ["田中", "さん", "PhD/MD"] assert _texts(_run("田中さん, PhD/MD", policy=_HANGUL, lexicon=lex)) == ["田中さん", "PhD/MD"] dash = Policy(extra_suffix_delimiters=frozenset({" - "})) assert _texts(_run("田中さん, Jr. - V.", policy=dash, lexicon=lex)) == ["田中", "さん", "Jr.", "-", "V."] assert _texts(_run("田中さん, Jr. - V.", policy=_HANGUL, lexicon=lex)) == ["田中さん", "Jr.", "-", "V."] def test_an_empty_first_run_still_reaches_the_second() -> None: # A leading comma empties segments[0] without emptying the name: # the scan flattens both runs, so an empty first one is nothing to # stop at. Guarding the crossing on a non-empty segments[0] passes # the ", , 씨" test below, where BOTH runs are empty, and fails # only here. assert _texts(_run(", 민준씨", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["민준", "씨"] def test_the_peel_crosses_a_comma_and_a_dot_at_once() -> None: # The two gates #312 made siblings, composed. Both stand over the # surname split here -- 威 is a listed surname in this lexicon, so # bare 威廉 would divide 威 + 廉, and either gate alone stops that # (the family comma reaches its return first) -- while the peel # crosses both and reaches 太郎さん on the far side of the comma. # Neither gate alone pins this: keying the cross-comma reach on # interpunct-free input passes every other case in this file. lex = _LEX_TAILS.add(surnames={"威"}) state = segment(tokenize(ParseState( original="威廉·莎士比亚, 太郎さん", lexicon=lex, policy=_HAN))) assert state.interpunct_offsets assert state.structure is Structure.FAMILY_COMMA assert _texts(script_segment(state)) == [ "威廉", "莎士比亚", "太郎", "さん"] def test_the_peel_survives_a_name_with_no_name_tokens() -> None: # The name's runs are EMPTY -- a leading comma yields an empty one # per comma -- so the scan has nothing to look at and answers None # rather than indexing. Reachable, so taking the last token # outright would raise here. The scan's other way to answer None, # a name that is nothing but post-nominals, is not what this # exercises: next() short-circuits on the empty runs and never # asks. test_a_token_that_is_a_tail_never_peels pins that one. assert _texts(_run(", , 씨", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["씨"] def test_a_comma_does_not_by_itself_change_the_peel() -> None: # The counterpart, and the shape #312 was originally filed about: # the site is the last NON-POST-NOMINAL token, which is 太郎 here # whichever spelling is used, so neither peels. Crossing the comma # must not turn this into a peel. for name in ("田中さん 太郎", "田中さん, 太郎"): expected = name.replace(",", "").split() assert _texts(_run(name, policy=_HANGUL, lexicon=_LEX_TAILS)) == expected, name # -- the 间隔号 gate: a divided name is a transcription (#298) ---------- def test_interpunct_divided_name_never_segments() -> None: # 马丁·路德·金 (Martin Luther King): 马 is a listed surname, but a # 间隔号-divided name is a transcription -- its pieces are syllable # groups, not surname+given, so the dot gates the stage off # entirely, vocabulary AND segmenter (spec 2026-07-30). Built # through the real tokenize so interpunct_offsets is the field's # own producer's, not hand-set. state = segment(tokenize(ParseState( original="马丁·路德·金", lexicon=Lexicon(surnames=frozenset({"马"})), policy=_HAN))) assert state.interpunct_offsets # tokenize recorded the dots out = script_segment(state) assert out.tokens == state.tokens # nothing split: no 马 + 丁 assert out.ambiguities == () def test_interpunct_divided_name_never_consults_the_segmenter() -> None: # the gate's other consumer, pinned separately: today the # second-script-token precondition would decline this consult # anyway, but the transcription doctrine must not depend on it -- # the dot gates BEFORE either mechanism is reached asked, seg = _capture() state = segment(tokenize(ParseState( original="马丁·路德·金", lexicon=Lexicon.empty(), policy=_HAN, segmenter=seg))) out = script_segment(state) assert out.tokens == state.tokens assert asked == [] def test_interpunct_gates_the_whole_name_not_just_dotted_tokens() -> None: # state-global on purpose: the marker reads the WHOLE name as a # transcription listing, so the un-dotted hangul token stays whole # too -- the baseline shows it segments without the B7 elsewhere assert _texts(_run("김민준 马丁 路德", policy=_HANGUL)) == [ "김", "민준", "马丁", "路德"] state = segment(tokenize(ParseState( original="김민준 马丁·路德", lexicon=_LEX, policy=_HANGUL))) assert state.interpunct_offsets out = script_segment(state) assert out.tokens == state.tokens def test_the_segmenter_is_handed_the_gated_token_only() -> None: # It is asked where ONE token divides, so it gets that token's text # and nothing else -- not the original, not the name part joined # back together. A leading Latin title is the case that can tell # the difference: it is skipped by the gate (no script) and is not # a neighbour for the precondition either, so the consult happens # and the argument is observable. asked, seg = _capture() out = _run("Dr 山田太郎", policy=_JA, segmenter=seg) assert asked == ["山田太郎"] assert _texts(out) == ["Dr", "山田", "太郎"] python-nameparser-2.1.0/tests/v2/pipeline/test_segment.py000066400000000000000000000114211523530431700235410ustar00rootroot00000000000000from nameparser._lexicon import Lexicon from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState, Structure from nameparser._pipeline._tokenize import tokenize import dataclasses from nameparser._policy import Policy from nameparser._types import AmbiguityKind # synthetic vocabulary: behavior given a lexicon, never default() contents _LEX = Lexicon( suffix_acronyms=frozenset({"phd"}), suffix_words=frozenset({"jr", "v"}), ) def _segmented(text: str) -> ParseState: state = ParseState(original=text, lexicon=_LEX, policy=Policy()) return segment(tokenize(extract_delimited(state))) def _texts(state: ParseState, seg: tuple[int, ...]) -> list[str]: return [state.tokens[i].text for i in seg] def test_no_comma() -> None: out = _segmented("John Smith") assert out.structure is Structure.NO_COMMA assert [_texts(out, s) for s in out.segments] == [["John", "Smith"]] def test_family_comma() -> None: out = _segmented("Smith, John") assert out.structure is Structure.FAMILY_COMMA assert [_texts(out, s) for s in out.segments] == [["Smith"], ["John"]] def test_suffix_comma_when_all_rest_groups_are_suffixes() -> None: out = _segmented("John Smith, PhD") assert out.structure is Structure.SUFFIX_COMMA assert [_texts(out, s) for s in out.segments] == [["John", "Smith"], ["PhD"]] def test_suffix_comma_lenient_accepts_initial_shaped_suffix_word() -> None: # "V" is initial-shaped; the strict test vetoes it, the post-comma # lenient test accepts suffix_words unconditionally (v1 parity) out = _segmented("John Ingram, V") assert out.structure is Structure.SUFFIX_COMMA def test_the_credential_pair_merges_anywhere_in_the_run() -> None: # v1's fix_phd healed a split 'Ph. D.' wherever it fell, and the # suffix-comma test inherits that: the pair counts as ONE unit at # any position, not just at the head of the run. Nothing else # reaches past position 0 -- 'John Smith, Ph. D.' above puts the # pair first -- and the flip is silent, because 'D.' alone is # suffix vocabulary in no lexicon, so the run stops being wholly # suffix and this reads as a FAMILY comma instead. out = _segmented("John Smith, Jr. Ph. D.") assert out.structure is Structure.SUFFIX_COMMA assert [_texts(out, s) for s in out.segments] == [ ["John", "Smith"], ["Jr.", "Ph.", "D."]] def test_family_comma_with_trailing_suffix_segment() -> None: out = _segmented("Smith, John, Jr.") assert out.structure is Structure.FAMILY_COMMA assert [_texts(out, s) for s in out.segments] == [["Smith"], ["John"], ["Jr."]] def test_single_pre_comma_word_never_suffix_comma() -> None: # v1: suffix-comma requires >1 word before the comma out = _segmented("Johnson, Jr.") assert out.structure is Structure.FAMILY_COMMA def test_excess_non_suffix_segment_flags_comma_structure() -> None: out = _segmented("Smith, John, Extra, Jr.") assert out.structure is Structure.FAMILY_COMMA kinds = [a.kind for a in out.ambiguities] assert AmbiguityKind.COMMA_STRUCTURE in kinds def test_empty_input_yields_no_segments() -> None: assert _segmented("").segments == () def test_comma_only_input_is_no_comma_structure() -> None: out = _segmented(",,,") assert out.structure is Structure.NO_COMMA assert out.segments == () def test_single_trailing_comma_is_cosmetic_and_dropped() -> None: # v1 parity: exactly ONE trailing comma is cosmetic and stripped # (segment.py's "pop the trailing empty bucket" step), so # 'Smith, John,' segments identically to 'Smith, John' -- unlike a # genuinely empty INTERIOR bucket ('Doe,, Jr.'), which stays # structural and keeps its position. out = _segmented("Smith, John,") assert out.structure is Structure.FAMILY_COMMA assert [_texts(out, s) for s in out.segments] == [["Smith"], ["John"]] def test_leading_comma_yields_empty_first_segment() -> None: # A leading comma produces a non-trailing empty bucket, which is # structural (not cosmetic) and keeps its position as segment 0. out = _segmented(",John Smith") assert out.structure is Structure.FAMILY_COMMA assert [_texts(out, s) for s in out.segments] == [[], ["John", "Smith"]] def test_strict_comma_suffixes_veto_lenient_only_members() -> None: # lenient_comma_suffixes=False: the post-comma test drops back to # the strict predicate, so initial-shaped suffix words no longer # qualify and the structure reads FAMILY_COMMA state = ParseState( original="John Ingram, V", lexicon=_LEX, policy=dataclasses.replace(Policy(), lenient_comma_suffixes=False)) out = segment(tokenize(extract_delimited(state))) assert out.structure is Structure.FAMILY_COMMA python-nameparser-2.1.0/tests/v2/pipeline/test_state.py000066400000000000000000000132671523530431700232310ustar00rootroot00000000000000import dataclasses from nameparser._lexicon import Lexicon from nameparser._pipeline._state import ParseState, Structure, WorkToken from nameparser._policy import Policy from nameparser._types import Role, Span def _state(text: str) -> ParseState: return ParseState(original=text, lexicon=Lexicon.empty(), policy=Policy()) def test_state_defaults_are_empty() -> None: s = _state("John Smith") assert s.tokens == () and s.segments == () and s.pieces == () assert s.structure is Structure.NO_COMMA assert s.ambiguities == () and s.extracted == () and s.masked == () assert s.comma_offsets == () and s.interpunct_offsets == () assert s.dropped == () and s.piece_tags == () assert s.segmenter is None def test_state_is_frozen_and_replace_works() -> None: s = _state("x") tok = WorkToken("x", Span(0, 1)) s2 = dataclasses.replace(s, tokens=(tok,)) assert s.tokens == () and s2.tokens == (tok,) assert s2.tokens[0].role is None and s2.tokens[0].tags == frozenset() def test_worktoken_carries_optional_role() -> None: t = WorkToken("Jack", Span(6, 10), role=Role.NICKNAME) assert t.role is Role.NICKNAME def test_stage_field_ownership() -> None: # The ParseState docstring's ownership map, pinned mechanically: run # the whole case corpus through the fold stage by stage and assert # each stage only changes the fields it owns. Converts the prose # contract into a test (a future stage clobbering another stage's # field fails here, not in a distant assertion). import dataclasses as _dc from nameparser import Lexicon as _Lexicon from nameparser._pipeline import STAGES from ..cases import CASES ownership = { "extract_delimited": {"extracted", "masked", "ambiguities"}, # tokenize also rewrites ambiguities: extract_delimited runs # before tokens exist, so its UNBALANCED_DELIMITER entries carry # a character offset that tokenize resolves to a token index "tokenize": {"tokens", "comma_offsets", "interpunct_offsets", "ambiguities"}, "segment": {"segments", "structure", "ambiguities"}, # script_segment splits one unspaced CJK token into n+1 pieces # (n = 1 from the vocabulary, any n from a segmenter), so it # rewrites tokens and shifts every later index the earlier # stages recorded -- the segment runs included. It is # deliberately absent from token_ownership below, whose # token-count assert is the one contract this stage is exempt # from. structure and segmenter it only READS (the # FAMILY_COMMA opt-out, and the hook it consults on a # vocabulary decline). "script_segment": {"tokens", "segments", "ambiguities"}, # classify also emits SUFFIX_OR_NICKNAME: the delimiter escape # that decides it lives in extract_delimited, which has no token # index to point at, so the report is raised here instead "classify": {"tokens", "ambiguities"}, # group also emits PARTICLE_OR_GIVEN: the prefix chain takes # the particle branch of a fork whose given branch _assign # takes, so each stage reports the side it decides "group": {"tokens", "pieces", "piece_tags", "dropped", "ambiguities"}, "assign": {"tokens", "ambiguities"}, "post_rules": {"tokens"}, } assert {s.__name__ for s in STAGES} == set(ownership) # Within the tokens themselves the contract is finer: texts and # spans are fixed at tokenize (the anti-#100 invariant -- tokens # are never re-created), classify touches only tags, and the # role-assigning stages touch only roles (group also tags, for the # ph-d "joined" marker). token_ownership = { "classify": {"tags"}, "group": {"tags", "role"}, "assign": {"role"}, # post_rules also tags: the middle_as_family fold marks folded # tokens vocab:folded-middle for the family view's prepend order "post_rules": {"role", "tags"}, } for case in CASES: if case.locale is not None: # locale rows dissolve into a Policy only through parser_for # (nameparser._parser); this test exercises the raw stage # pipeline directly, and the same inputs already run here # under the equivalent synthetic Policy row (_ES/_TK) -- # covering them again through a resolved locale policy would # be redundant, not additive. continue state = ParseState(original=case.text, lexicon=_Lexicon.default(), policy=case.policy or Policy()) for stage in STAGES: before = {f.name: getattr(state, f.name) for f in _dc.fields(state)} state = stage(state) changed = {name for name, value in before.items() if getattr(state, name) != value} assert changed <= ownership[stage.__name__], ( f"{case.id}: {stage.__name__} changed {changed - ownership[stage.__name__]}") if stage.__name__ not in token_ownership: continue allowed = token_ownership[stage.__name__] assert len(state.tokens) == len(before["tokens"]), ( f"{case.id}: {stage.__name__} changed the token count") for old, new in zip(before["tokens"], state.tokens): token_changed = { f.name for f in _dc.fields(old) if getattr(old, f.name) != getattr(new, f.name)} assert token_changed <= allowed, ( f"{case.id}: {stage.__name__} changed token fields " f"{token_changed - allowed} on {old.text!r}") python-nameparser-2.1.0/tests/v2/pipeline/test_tokenize.py000066400000000000000000000225221523530431700237330ustar00rootroot00000000000000import dataclasses from nameparser._lexicon import Lexicon from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._state import ParseState from nameparser._pipeline._tokenize import tokenize from nameparser._policy import Policy from nameparser._types import Role def _tokenized(text: str, policy: Policy | None = None) -> ParseState: state = ParseState(original=text, lexicon=Lexicon.empty(), policy=policy or Policy()) return tokenize(extract_delimited(state)) def test_whitespace_split_with_spans() -> None: out = _tokenized("John Smith") assert [(t.text, tuple(t.span)) for t in out.tokens] == [ ("John", (0, 4)), ("Smith", (6, 11)), ] assert all(t.role is None for t in out.tokens) def test_provenance_text_equals_original_slice() -> None: out = _tokenized(" Dr. Juan de la Vega ") for t in out.tokens: assert t.text == out.original[t.span.start:t.span.end] def test_commas_are_separators_and_recorded() -> None: out = _tokenized("Smith, John") assert [t.text for t in out.tokens] == ["Smith", "John"] assert out.comma_offsets == (5,) def test_fullwidth_and_arabic_commas_segment() -> None: out = _tokenized("سميث، جون") assert out.comma_offsets == (4,) out2 = _tokenized("山田，太郎") assert out2.comma_offsets == (2,) def test_extracted_regions_are_skipped_and_tokenized_with_role() -> None: out = _tokenized('John "Jack Jr" Kennedy') main = [(t.text, t.role) for t in out.tokens if t.role is None] nick = [(t.text, t.role) for t in out.tokens if t.role is Role.NICKNAME] assert main == [("John", None), ("Kennedy", None)] assert nick == [("Jack", Role.NICKNAME), ("Jr", Role.NICKNAME)] # tokens are span-sorted overall starts = [t.span.start for t in out.tokens] assert starts == sorted(starts) def test_two_simultaneous_extracted_regions_interleave_correctly() -> None: # tokenize's masked/extracted loops each run over ALL their regions # before the final span-sort; every other test here exercises at # most one masked region at a time. With two active extraction # types (nickname parens AND a configured maiden bracket) on the # same string, main-stream and both extracted regions must still # interleave into the correct overall span order. pol = Policy(maiden_delimiters=frozenset({("[", "]")})) out = _tokenized("John (Jack) [Doe] Smith", pol) assert [(t.text, t.role) for t in out.tokens] == [ ("John", None), ("Jack", Role.NICKNAME), ("Doe", Role.MAIDEN), ("Smith", None), ] starts = [t.span.start for t in out.tokens] assert starts == sorted(starts) def test_comma_inside_extracted_region_is_not_an_offset() -> None: out = _tokenized('John "Jack, Jr" Kim') assert out.comma_offsets == () nick = [t.text for t in out.tokens if t.role is Role.NICKNAME] assert nick == ["Jack", "Jr"] def test_emoji_and_bidi_are_ignorable_by_policy() -> None: out = _tokenized("John‏ \U0001f600Smith") assert [t.text for t in out.tokens] == ["John", "Smith"] keep = dataclasses.replace(Policy(), strip_emoji=False, strip_bidi=False) out2 = _tokenized("John \U0001f600Smith", keep) assert [t.text for t in out2.tokens] == ["John", "\U0001f600Smith"] def test_empty_and_whitespace_yield_no_tokens() -> None: assert _tokenized("").tokens == () assert _tokenized(" ").tokens == () def test_nakaguro_separates_and_enters_no_token() -> None: # U+30FB divides transcribed foreign names (マイケル・ジャクソン); # native names never contain it -- amendment 2026-07-29 section 1b state = _tokenized("マイケル・ジャクソン") assert [t.text for t in state.tokens] == ["マイケル", "ジャクソン"] assert all(state.original[t.span.start:t.span.end] == t.text for t in state.tokens) # like whitespace, and unlike COMMA_CHARS: it never records an offset assert state.comma_offsets == () # leading/trailing dots are plain separators, not content assert [t.text for t in _tokenized("・ジャクソン").tokens] == ["ジャクソン"] def test_nakaguro_next_to_a_real_comma_still_only_the_comma_offsets() -> None: # the dot and a real comma side by side must not blur together -- # the comma still segments and records; the dot still separates # and stays silent state = _tokenized("マイケル・, X") assert [t.text for t in state.tokens] == ["マイケル", "X"] assert state.comma_offsets == (5,) def test_interpunct_splits_between_classified_characters() -> None: state = _tokenized("威廉·莎士比亚") assert [t.text for t in state.tokens] == ["威廉", "莎士比亚"] assert state.interpunct_offsets == (2,) def test_interpunct_stays_in_latin_names() -> None: # the Catalan punt volat is INTERIOR to names; Latin neighbors # never qualify state = _tokenized("Gal·la Marcet") assert [t.text for t in state.tokens] == ["Gal·la", "Marcet"] assert state.interpunct_offsets == () def test_interpunct_between_kana_splits() -> None: # sloppy-IME katakana transcription state = _tokenized("マイケル·ジャクソン") assert [t.text for t in state.tokens] == ["マイケル", "ジャクソン"] def test_interpunct_at_edges_stays() -> None: # no classified neighbor on one side: not a divider position state = _tokenized("·威廉") assert [t.text for t in state.tokens] == ["·威廉"] def test_stripped_bidi_mark_is_transparent_to_the_flank_guard() -> None: # an RTL document quoting a transcription: U+200F beside the dot, # visually identical to the clean form -- the stripped mark must # not occupy a flank position state = _tokenized("威廉‏·莎士比亚") assert [t.text for t in state.tokens] == ["威廉", "莎士比亚"] assert state.interpunct_offsets != () def test_stripped_emoji_is_transparent_to_the_flank_guard() -> None: state = _tokenized("威廉\U0001f600·莎士比亚") assert [t.text for t in state.tokens] == ["威廉", "莎士比亚"] def test_unstripped_bidi_mark_defeats_the_guard() -> None: # the contrast: with strip_bidi off the mark is real token text, # an unclassified flank -- the dot stays interior state = _tokenized("威廉‏·莎士比亚", policy=Policy(strip_bidi=False)) assert state.interpunct_offsets == () def test_interpunct_trailing_edge_stays() -> None: # the upper bound: a trailing dot has no right flank -- and # dropping the bound is an IndexError, not a wrong answer state = _tokenized("威廉·") assert [t.text for t in state.tokens] == ["威廉·"] def test_interpunct_minimal_flanks_split() -> None: state = _tokenized("威·廉") assert [t.text for t in state.tokens] == ["威", "廉"] assert state.interpunct_offsets == (1,) def test_interpunct_one_sided_flank_stays() -> None: # the AND in the guard: one classified neighbor is not enough, # and no offset records (a spurious record would fake the # transcription marker downstream) state = _tokenized("Anna·威") assert [t.text for t in state.tokens] == ["Anna·威"] assert state.interpunct_offsets == () def test_hangul_flanks_split_and_record() -> None: # Korean writes transcribed foreign names with the interpunct too # (마틴·루터·킹) -- hangul flanks are classified flanks state = _tokenized("김·민준") assert [t.text for t in state.tokens] == ["김", "민준"] assert state.interpunct_offsets == (1,) def test_interpunct_in_delimited_regions_splits_but_does_not_record() -> None: # a B7 inside a nickname must not mark the OUTER name as a # transcription -- same flag that keeps nickname commas out of # comma_offsets state = _tokenized("山田 太郎 (マイケル·ジャクソン)") assert state.interpunct_offsets == () assert "マイケル" in [t.text for t in state.tokens] def test_nakaguro_never_records() -> None: # the Japanese dot is roster/divider typography, not the # transcription marker: it separates (#272) and records nothing -- # 高橋・一郎 keeps its family-first roster reading downstream state = _tokenized("威廉・莎士比亚") assert [t.text for t in state.tokens] == ["威廉", "莎士比亚"] assert state.interpunct_offsets == () def test_halfwidth_nakaguro_separates_too() -> None: # U+FF65 between halfwidth katakana -- build the string from # escapes and VERIFY the codepoints (the U+F900 homoglyph lesson): # ﾏｲｹﾙ = halfwidth マイケル (MA I KE RU), # ･ = the halfwidth dot, ｼﾞｬｸｿﾝ = # halfwidth ジャクソン (SI + voiced-sound-mark, small-YA, KU, SO, N) text = ("\uff8f\uff72\uff79\uff99" # \uff8f\uff72\uff79\uff99 = halfwidth MA I KE RU "\uff65" # \uff65 = halfwidth middle dot "\uff7c\uff9e\uff6c\uff78\uff7f\uff9d") # halfwidth SI+voice, small-YA, KU, SO, N # belt and braces: the escapes above must spell the halfwidth dot, # not the fullwidth U+30FB it could be confused for by eye -- and # this is exactly the check the un-escaped version was missing: # it would still pass with U+30FB substituted for U+FF65 assert "･" in text state = _tokenized(text) assert len(state.tokens) == 2 assert "･" not in "".join(t.text for t in state.tokens) python-nameparser-2.1.0/tests/v2/pipeline/test_vocab.py000066400000000000000000000434141523530431700232000ustar00rootroot00000000000000import unicodedata from nameparser._lexicon import Lexicon from nameparser._pipeline._vocab import ( effective_script, is_initial, is_initial_shaped, is_suffix_lenient, is_suffix_strict, is_wholly_suffix, resolve_script_set, single_script, ) from nameparser._policy import (Policy, Script, _NO_INITIALS, _SCRIPT_RANGES) _LEX = Lexicon( suffix_acronyms=frozenset({"phd", "ma"}), suffix_words=frozenset({"jr", "v"}), suffix_acronyms_ambiguous=frozenset({"ma"}), ) def test_is_initial() -> None: assert is_initial("A.") assert is_initial("j.") assert is_initial("B") assert not is_initial("Jo") assert not is_initial("b") # bare lowercase letter is not an initial def test_is_initial_script_repertoire() -> None: # An initial is a single LETTER standing in for a name. Alphabets # have letters, so these are real initials ("А. С. Пушкин"). assert is_initial("А.") # Cyrillic assert is_initial("Α.") # Greek assert is_initial("م.") # Arabic assert is_initial("ה.") # Hebrew assert is_initial("र.") # Devanagari assert is_initial("Ա.") # Armenian # Han ideographs, hangul syllables and kana are morphemes or # syllables -- a single one never stands in for a name (#320). assert not is_initial("씨.") assert not is_initial("様.") assert not is_initial("김.") assert not is_initial("さ.") assert not is_initial("ラ.") # unchanged: a digit is ONE edge of \w's reach and '_' is another, # and the shape half still owns both -- only the repertoire narrowed assert is_initial("2.") assert is_initial("_.") # unchanged: the SHAPE half still requires a single character assert not is_initial("राम.") def test_is_initial_shaped_keeps_the_shape_half_reachable() -> None: """The two halves are separately askable (#320): assign's roman-numeral fork asks the SHAPE question about the piece before a trailing 'V', and answering it with the narrowed predicate dropped the family name out of 'John 씨. V' entirely.""" for text in ("A.", "j.", "B", "2."): assert is_initial_shaped(text) is is_initial(text) is True for text in ("Jo", "b", "raam."): assert is_initial_shaped(text) is is_initial(text) is False # the whole difference between them, in both directions for text in ("씨.", "様.", "김.", "さ.", "ラ."): assert is_initial_shaped(text) and not is_initial(text) def test_strict_suffix_veto_skips_cjk() -> None: """#320: the initial veto is what stopped a period-written CJK honorific being recognized. _normalize strips the trailing period, so '씨.' reaches the vocabulary as '씨' -- the veto was the only thing rejecting it.""" lex = Lexicon(suffix_words=frozenset({"씨", "様"})) assert is_suffix_strict("씨.", lex) assert is_suffix_strict("様.", lex) def _representative(script: Script) -> str: """The first codepoint of `script`'s _SCRIPT_RANGES spans that the SHAPE half admits as an initial. Shape-admitted, not simply the first codepoint: a range's first codepoint is often unassigned or punctuation (KATAKANA's 0x30A0 is a hyphen, HIRAGANA's 0x3040 is unassigned), which \\w does not match -- and testing is_initial on such a character answers False for the SHAPE's reason, making the repertoire assertion below vacuously green. Raising when no span holds one is the point rather than a corner: a script whose declared ranges contain no initial-shaped character at all has ranges that do not describe it.""" for lo, hi in _SCRIPT_RANGES[script]: for cp in range(lo, hi + 1): if is_initial_shaped(chr(cp) + "."): return chr(cp) raise AssertionError( f"no character in _SCRIPT_RANGES[{script}] is initial-SHAPED, " f"so this script's declaration cannot be tested against " f"is_initial -- check that the ranges are that script's") def test_every_script_is_classified_for_initials() -> None: """A member joining Script must be classified here on purpose; _policy._NO_INITIALS carries the reasoning. The classification lives in this table rather than the assertion being `set(Script) == set(_NO_INITIALS)`: that passes trivially today, since all four current members are CJK, and the only way to green it again after adding a script would be to declare that script initial-less. That prejudges the answer. The point is to force a decision, not a particular one. Three bindings, not two: the table covers Script, the table's False rows are _NO_INITIALS, and -- the one that makes this a behavioral test rather than a comparison of two constants -- each row is checked against is_initial on a character DERIVED from that script's own _SCRIPT_RANGES entry. Without the third, a script declared initial-less under ranges that are not its own passes all the way through while is_initial still says yes to its characters. """ has_initials = { Script.HAN: False, # ideographs are morphemes Script.HANGUL: False, # syllable blocks Script.HIRAGANA: False, # syllables Script.KATAKANA: False, # syllables } assert set(has_initials) == set(Script), ( "a Script member is unclassified for initials: decide whether " "a single character of it can stand in for a name, add the row, " "and put it in _policy._NO_INITIALS if it cannot") assert {s for s, yes in has_initials.items() if not yes} \ == set(_NO_INITIALS), ( "this table and _policy._NO_INITIALS disagree about which " "scripts have initials: a row here saying False is what puts a " "script in the constant, so add the missing member to " "_NO_INITIALS -- or, if the constant is the one that's right, " "flip the row") for script, yes in has_initials.items(): char = _representative(script) assert is_initial(char + ".") is yes, ( f"the declaration for {script} does not reach is_initial: " f"the row says has_initials={yes}, but is_initial(" f"{char + '.'!r}) -- on a character taken from " f"_SCRIPT_RANGES[{script}] -- says {not yes}. Either the " f"ranges are not this script's, or _NO_INITIALS and the " f"repertoire predicate have come apart") def test_strict_suffix_initial_veto() -> None: assert is_suffix_strict("PhD", _LEX) assert not is_suffix_strict("V.", _LEX) # initial veto assert not is_suffix_strict("V", _LEX) # initial veto assert is_suffix_strict("Jr", _LEX) def test_ambiguous_acronym_needs_periods_and_beats_the_veto() -> None: assert is_suffix_strict("M.A.", _LEX) assert not is_suffix_strict("Ma", _LEX) def test_lenient_accepts_suffix_words_unconditionally() -> None: assert is_suffix_lenient("V", _LEX) assert is_suffix_lenient("V.", _LEX) assert not is_suffix_lenient("Ma", _LEX) def test_strict_excludes_bare_ambiguous_even_when_in_acronyms() -> None: # mirrors the real data shape: ambiguous is a SUBSET of acronyms assert not is_suffix_strict("Ma", _LEX) assert is_suffix_strict("M.A.", _LEX) _SUFFIX_LEX = Lexicon( suffix_acronyms=frozenset({"phd", "md"}), suffix_words=frozenset({"jr", "v"}), ) def test_is_wholly_suffix() -> None: lex, pol = _SUFFIX_LEX, Policy() assert is_wholly_suffix(["Jr."], lex, pol) assert is_wholly_suffix(["PhD", "Jr."], lex, pol) assert not is_wholly_suffix(["Smith"], lex, pol) assert not is_wholly_suffix(["PhD", "Smith"], lex, pol) # an adjacent Ph./D. pair is ONE unit. 'D.' is what makes the merge # load-bearing: it is not suffix vocabulary in ANY lexicon, so the # pair fails without the merge. ('Ph.' happens to fail alone here # too, but only because _SUFFIX_LEX is synthetic -- under # Lexicon.default(), 'ph' IS in suffix_acronyms.) assert is_wholly_suffix(["Ph.", "D."], lex, pol) assert not is_wholly_suffix(["Ph."], lex, pol) assert is_wholly_suffix(["Ph.", "D.", "Jr."], lex, pol) # and the pair is found wherever it sits, not only at the head of # the run: the merge walks the whole list. 'John Smith, Jr. Ph. D.' # is the reachable input -- restricted to position 0 the pair never # merges, 'D.' fails alone, and segment reads a FAMILY comma where # it should read a suffix comma. assert is_wholly_suffix(["Jr.", "Ph.", "D."], lex, pol) # the two routes that are neither the policy-selected predicate nor # the merge, each with the input that reaches ONLY it. Both die in # the v1 banks and the case table today, so a break here is # diagnosed a long way from its cause. assert is_wholly_suffix(["Lt.Jr."], lex, pol) # period-joined assert not is_wholly_suffix(["Lt.Smith"], lex, pol) # no suffix chunk delim = Policy(extra_suffix_delimiters=frozenset({"/"})) assert is_wholly_suffix(["PhD/MD"], lex, delim) # delimiter split assert not is_wholly_suffix(["PhD/MD"], lex, pol) def test_is_wholly_suffix_empty_run_is_false() -> None: """NOT vacuous truth, unlike Python's all() and unlike v1's are_suffixes. v1's suffix-comma detection fails on an empty parts[1] -- 'John Smith,, MD' is a family-comma parse -- and the 'wholly' idiom agrees (_script_matcher(whole=True) requires non-empty too).""" assert not is_wholly_suffix([], _SUFFIX_LEX, Policy()) def test_is_wholly_suffix_is_not_the_plural_of_is_post_nominal() -> None: """The #319 bug in one assertion. _script_segment._is_post_nominal asks is_suffix_strict per token; this asks the POLICY-selected predicate over a run. 'V.' is the input that tells them apart: a suffix run, but not a post-nominal.""" lex, pol = _SUFFIX_LEX, Policy() assert is_wholly_suffix(["V."], lex, pol) assert not is_suffix_strict("V.", lex) # and the knob moves it: under strict, 'V.' is name text assert not is_wholly_suffix( ["V."], lex, Policy(lenient_comma_suffixes=False)) def test_single_script_requires_every_char_in_one_script() -> None: assert single_script("毛泽东") is Script.HAN assert single_script("諸葛") is Script.HAN # traditional assert single_script("김민준") is Script.HANGUL assert single_script("Smith") is None assert single_script("毛zedong") is None # mixed assert single_script("毛김") is None # mixed CJK # kana classifies as its own script (#272), not HAN -- was None # before kana had a table entry; single_script's job is telling # kana apart from Han, not lumping the two together assert single_script("イチロー") is Script.KATAKANA def test_single_script_range_edges() -> None: # one char at each declared bound, and a neighbour outside assert single_script("㐀") is Script.HAN # Ext A first assert single_script("䶿") is Script.HAN # Ext A last assert single_script("䷀") is None # hexagram, not Han # U+F900 is a COMPATIBILITY ideograph; it renders identically to # the URO 豈 (U+8C48) it decomposes to, so spell it as an escape # or this assertion silently tests the URO range instead assert single_script("\uf900") is Script.HAN assert single_script("\U00020bb7") is Script.HAN # Ext B (𠮷) assert single_script("가") is Script.HANGUL assert single_script("힣") is Script.HANGUL # last ASSIGNED syllable assert single_script("ힰ") is None # jungseong, not a syllable assert single_script("ㄱ") is None # bare jamo def test_single_script_empty_string_is_no_script() -> None: assert single_script("") is None def test_no_script_range_reaches_ascii() -> None: # single_script short-circuits on an ASCII token instead of # sweeping the ranges (the hot path: every Latin name). The # classifications above stay pinned either way; what the shortcut # rests on is this floor, so a future script entry that dips below # it fails here rather than silently going unclassified. assert all(lo >= 0x80 for ranges in _SCRIPT_RANGES.values() for lo, _ in ranges) def test_kana_singles_classify() -> None: assert single_script("みなみ") is Script.HIRAGANA assert single_script("エミ") is Script.KATAKANA assert single_script("ー") is Script.KATAKANA # prolonged mark, in-block def test_effective_script_kana_license() -> None: # pure single-script tokens pass through unchanged assert effective_script("山田") is Script.HAN assert effective_script("みなみ") is Script.HIRAGANA assert effective_script("マイケル") is Script.KATAKANA # the license: a MIXED token wholly in Han∪kana is Japanese and # resolves to the HIRAGANA carrier entry assert effective_script("高橋みなみ") is Script.HIRAGANA # kanji+hira assert effective_script("山田エミ") is Script.HIRAGANA # kanji+kata assert effective_script("さくらエミ") is Script.HIRAGANA # hira+kata # outside the license: anything beyond the JA repertoire assert effective_script("毛김") is None assert effective_script("山田x") is None # NOT None: U+30FB sits INSIDE the katakana block (verified by # codepoint), so this stays a PURE-katakana token, same as # "マイケル" above -- the license only ever fires on a MIXED # token, and this one isn't mixed. It has no order-default entry # (DEFAULT_SCRIPT_ORDERS carries no KATAKANA key), which is where # "declines the license" actually shows up. tokenize (#272 Task 2b) # now turns U+30FB into a token separator, so this exact string # arrives at effective_script as two tokens during a real parse -- # this unit test calls effective_script directly on the whole # string, bypassing tokenize, so it still sees the dot and must # keep passing unchanged. assert effective_script("マイケル・ジャクソン") is Script.KATAKANA assert effective_script("") is None def test_resolve_script_set_generalizes_the_license_across_pieces() -> None: # a single script passes through as-is, including a script with no # order-default entry (KATAKANA): the caller decides what to do # with that, this function only reports what was found assert resolve_script_set([Script.HAN]) is Script.HAN assert resolve_script_set([Script.HAN, Script.HAN]) is Script.HAN assert resolve_script_set([Script.KATAKANA]) is Script.KATAKANA # the license, generalized: Han + Hiragana across two SEPARATE, # individually single-script pieces (never mixed within one # token) is exactly the repertoire effective_script licenses # inside a single token, so it collapses to the carrier key assert resolve_script_set([Script.HAN, Script.HIRAGANA]) \ is Script.HIRAGANA assert resolve_script_set([Script.HAN, Script.KATAKANA]) \ is Script.HIRAGANA assert resolve_script_set([Script.HIRAGANA, Script.KATAKANA]) \ is Script.HIRAGANA # outside the license: Han+Hangul is two real scripts, neither # kana -- no single tradition, so this declines like the caller's # own None (Latin, mixed, empty) assert resolve_script_set([Script.HAN, Script.HANGUL]) is None assert resolve_script_set([]) is None def test_nfd_katakana_still_classifies_and_declines_the_license() -> None: # Built via normalize(), never pasted as decomposed literals -- # NFD "ガガ" is base katakana カ + COMBINING VOICED SOUND MARK # (U+3099) twice; U+3099 sits in the HIRAGANA block, not # katakana's, so classifying raw NFD text would see one char from # each block and either call it mixed (single_script: None) or # wrongly grant the kana license (effective_script: HIRAGANA) for # what is really one pure-katakana token. NFC-normalizing first # (the #272 amendment's NFC decision) recomposes it back to ガガ, # which reads as ordinary katakana either way -- the license # still correctly declines, because this token isn't mixed. nfd = unicodedata.normalize("NFD", "ガガ") assert nfd != "ガガ" # sanity: confirms the decomposition actually ran assert single_script(nfd) is Script.KATAKANA assert effective_script(nfd) is Script.KATAKANA def test_nfd_hangul_still_classifies() -> None: # NFD decomposes each precomposed syllable onto 2-3 jamo # (U+1100-U+11FF), entirely outside the HANGUL range -- raw NFD # input would silently miss the shipped family-first order rule # rather than merely misclassify. Built via normalize(), not # pasted decomposed literals, same reason as above. nfd = unicodedata.normalize("NFD", "김민준") assert nfd != "김민준" # sanity: confirms the decomposition actually ran assert single_script(nfd) is Script.HANGUL def test_script_ranges_are_pairwise_disjoint() -> None: # The table's own comment states this invariant and nothing # checked it: single_script returns the FIRST covering entry (dict # iteration order), so any overlap would make the answer depend on # declaration order instead of being well-defined. Compared over # every entry of every script rather than script by script, # because the adjacent blocks are exactly where a future edit # would go wrong -- HIRAGANA ends at U+309F and KATAKANA opens at # U+30A0 with no gap at all, and HAN's U+3005-U+3006 span sits just # below both. spans = [(lo, hi, script) for script, ranges in _SCRIPT_RANGES.items() for lo, hi in ranges] assert len(spans) > 1 # never passes vacuously for i, (lo, hi, script) in enumerate(spans): assert lo <= hi, f"{script} range ({lo:#x}, {hi:#x}) is inverted" for other_lo, other_hi, other in spans[i + 1:]: assert hi < other_lo or other_hi < lo, ( f"{script} range ({lo:#x}, {hi:#x}) overlaps {other} " f"range ({other_lo:#x}, {other_hi:#x})") python-nameparser-2.1.0/tests/v2/test_benchmark.py000066400000000000000000000230521523530431700222270ustar00rootroot00000000000000"""Perf smoke (core spec §7 tail): parse cost stays v1-comparable (microseconds per name). Deliberately generous bound -- guards against order-of-magnitude regressions, does not gate normal variance. Two different things are measured here, and the distinction matters. The thousand-names tests below bound ABSOLUTE cost on constant-size input; the scaling test bounds GROWTH in input length. Neither subsumes the other, and the absolute tests are structurally blind to a complexity regression: their workload is a short name with no delimiters, so a stage that goes quadratic in the number of delimiter pairs stays far inside the one-second bound. Three such quadratics (_extract's mask-overlap scan, ParsedName's token-subset check, and _group's #329 clause scan) were caught in review rather than here -- hence the scaling test. The third is why _POLICY_SHAPES exists below: it was quadratic in a shape the scaling test ALREADY had ("(a) "), and still went unseen, because the stage is gated on an opt-in Policy field that bare parse() leaves empty. A shape guards nothing if the default policy cannot reach the code under it. """ import time from collections.abc import Callable import pytest from nameparser import Parser, parse from nameparser._policy import Policy def test_parse_thousand_names_under_a_second() -> None: parse("warm up the default parser cache") start = time.perf_counter() for i in range(1000): parse(f"Dr. Juan{i} de la Vega III") elapsed = time.perf_counter() - start assert elapsed < 1.0, f"1000 parses took {elapsed:.2f}s" def test_facade_thousand_names_under_a_second() -> None: # the legacy-API path (what all existing users call): snapshot # resolution must stay generation-cached, not rebuilt per instance from nameparser import HumanName HumanName("warm up the caches") start = time.perf_counter() for i in range(1000): HumanName(f"Dr. Juan{i} de la Vega III") elapsed = time.perf_counter() - start assert elapsed < 1.0, f"1000 facade parses took {elapsed:.2f}s" # Pathological shapes: each repeats a unit that drives one stage's inner # loop. Cheap per unit, so a superlinear stage shows up as growth rather # than as a slow parse. # # What each DIMENSION is covered by, measured at _BASE. Several shapes # differ only in a column you cannot see from the string, so measure # before pruning one that looks redundant: # # dimension covered by # token count all ten # piece count unmatched_*, plain_tokens, commas, titles # SEGMENT count commas ONLY -- deleting it leaves every # segment-keyed regression unguarded # intra-piece accumulation particles (one 799-token piece), # conjunctions (800) -- the merge() quadratic # masked-span count delimiter_pairs, quote_pairs (0 pieces: # everything is consumed as a delimited run) # NON-ASCII input honorifics ONLY -- every other unit here is # pure ASCII, so script_segment returns at # its isascii() bail and neither the #308 # peel nor the #271 surname site is measured # at all. A repeated 씨 is what walks the # peel's site scan: every token is a # post-nominal, so the scan-back crosses the # whole run before declining _SHAPES = { "delimiter_pairs": "(a) ", # extract: matched pairs -> masked spans "quote_pairs": '"a" ', # extract: the open==close path "unmatched_opens": "( ", # extract: the bulk unmatched-open sweep "unmatched_closes": ") ", # extract: the unmatched-close sweep "plain_tokens": "a ", # tokenize/assign: long token stream "commas": "a, ", # segment: many comma segments "titles": "Dr. ", # group: one long title chain "particles": "van ", # group: the prefix-chain inner loop "conjunctions": "and ", # group: merge() accumulating one piece "honorifics": "씨 ", # script_segment: the peel's site scan } _BASE = 800 _FACTOR = 4 # Calibrated, not guessed, and calibrated against the WEAKEST signal. # # A quadratic here is MIXED -- linear work dominates at small sizes and # the quadratic term takes over slowly -- so the textbook 16x never # appears at a testable size, different quadratics surface at different # strengths, and the operating point matters more than the bound. # Measured against the two real bugs this guards, per shape, ratio for # 4x the input: # # base clean _extract mask-overlap _group merge re-flatten # 200 3.9 - 4.3 7.7 5.6 # 400 4.0 - 4.3 9.2 6.3 - 6.6 # 800 4.0 - 4.4 >9.2 7.9 - 8.1 # # The bound must clear the weakest column, not the first one measured. # An earlier version calibrated 6.5 against _extract alone and then # gained the conjunction shape, whose signal is 6.3-6.6 at base 400 -- # the bound landed on the MEDIAN of the broken distribution and caught # the regression it was added for about half the time. Raising the base # separates the populations again rather than squeezing the bound into # a gap that is not there: at 800 the bound has ~1.4x over the worst # clean run (noise headroom on a shared runner) and ~1.3x under the # weakest quadratic. Re-derive BOTH numbers when adding a shape. # The tenth shape (honorifics) was measured into the clean column when # it arrived: 4.0-4.3 at base 800 across repeated runs, inside the range # the ASCII shapes already occupy, so neither number moved. _MAX_RATIO = 6.0 def _best(text: str, parse_: Callable[[str], object], repeats: int = 7) -> float: """Minimum of several runs: on a shared runner the mean carries the noise of whatever else is running, while the minimum approaches the true cost.""" parse_(text) # warm the parser cache best = float("inf") for _ in range(repeats): start = time.perf_counter() parse_(text) best = min(best, time.perf_counter() - start) return best def _assert_grows_linearly(unit: str, parse_: Callable[[str], object]) -> None: small = _best(unit * _BASE, parse_) large = _best(unit * (_BASE * _FACTOR), parse_) ratio = large / small assert ratio < _MAX_RATIO, ( f"{unit!r} x{_BASE} took {small * 1e3:.2f}ms, " f"x{_BASE * _FACTOR} took {large * 1e3:.2f}ms -- {ratio:.1f}x for " f"{_FACTOR}x the input, which is superlinear (linear is ~{_FACTOR})") @pytest.mark.parametrize("unit", _SHAPES.values(), ids=list(_SHAPES)) def test_parse_cost_grows_no_worse_than_linearly(unit: str) -> None: _assert_grows_linearly(unit, parse) # Shapes that need a NON-DEFAULT POLICY to reach the code they guard. # Every _SHAPES entry runs bare parse(), and a stage gated on an opt-in # Policy field is dead there -- #329's clause loop exits on its first # line when maiden_delimiters is empty, so the quadratic it shipped # with was invisible to all ten shapes above even though "(a) " was # already one of them. The unit is the SAME string as delimiter_pairs; # only the policy differs, which is the whole point. # # Calibrated like the others, per the note above: the #329 clause scan # measured 14.1x at base 800 (against a 4.1x bare-policy control), and # 4.3x once bisected -- inside the clean column, so _MAX_RATIO did not # move. Every ratio in this file is for _FACTOR x the input, never per # doubling; re-measuring the scan on another runner gave 11.2x against # the same 4.2x control, which is the spread to expect here. # # Third element: a REACHABILITY probe, run before the measurement. # "the shape must reach the code" is the whole premise of this table, # and it is a premise about precedence, which moves -- route ( ) back # to nickname and the parse below stops producing a maiden clause, # leaving this test measuring a no-op at a comfortable 4.2x forever. # That is the module docstring's failure mode one level up: a guard # whose subject has quietly left the building. The probe is per shape # because what "reached" means is per shape. _POLICY_SHAPES: dict[str, tuple[str, Parser, Callable[[Parser], bool]]] = { "maiden_pairs": ( "(a) ", Parser(policy=Policy(maiden_delimiters=frozenset({("(", ")")}))), # a maiden clause whose marker the #329 pass consumes: the # parenthesis pair must route to maiden AND the clause loop # must run for this to read "b" rather than "" or "née b" lambda p: p.parse("a (née b)").maiden == "b", ), } def test_shape_tables_are_not_empty() -> None: # pytest turns an EMPTY parametrize into a SKIP, not a failure, so # deleting the last entry of either table would retire its guard # into the skip count with nothing going red. _POLICY_SHAPES is # the nearer risk, holding only shapes whose stage a default parse # cannot reach at all -- so it gains an entry only when an opt-in # Policy field turns out to have a scaling cliff behind it. assert _SHAPES assert _POLICY_SHAPES @pytest.mark.parametrize("unit,parser,reaches", _POLICY_SHAPES.values(), ids=list(_POLICY_SHAPES)) def test_policy_gated_cost_grows_no_worse_than_linearly( unit: str, parser: Parser, reaches: Callable[[Parser], bool]) -> None: assert reaches(parser), "shape no longer reaches the gated stage" _assert_grows_linearly(unit, parser.parse) python-nameparser-2.1.0/tests/v2/test_cases.py000066400000000000000000000016751523530431700214020ustar00rootroot00000000000000"""Core runner over the shared case table (spec §7.2). The facade runner (migration plan) consumes the same CASES.""" import pytest from nameparser import Parser, Role, locales, parser_for from .cases import CASES, Case _FIELDS = tuple(r.value for r in Role) # declaration order is canonical def _parser_for_case(case: Case) -> Parser: if case.locale is not None: return parser_for(locales.get(case.locale)) return Parser(policy=case.policy) if case.policy else Parser() @pytest.mark.parametrize("case", CASES, ids=lambda c: c.id) def test_case(case: Case) -> None: parser = _parser_for_case(case) pn = parser.parse(case.text) actual = {f: getattr(pn, f) for f in _FIELDS if getattr(pn, f)} assert actual == case.expect, f"{case.text!r} ({case.classification})" kinds = sorted(a.kind.value for a in pn.ambiguities) assert kinds == sorted(case.ambiguities), \ f"{case.text!r} ({case.classification})" python-nameparser-2.1.0/tests/v2/test_cli.py000066400000000000000000000053641523530431700210520ustar00rootroot00000000000000import json import subprocess import sys def _run(*args: str) -> subprocess.CompletedProcess: return subprocess.run([sys.executable, "-m", "nameparser", *args], capture_output=True, text=True) def test_cli_prints_repr_capitalized_and_initials() -> None: out = _run("dr. juan de la vega iii").stdout assert "juan" in out # raw repr first assert "Juan" in out # capitalized repr second assert "Initials:" in out def test_cli_labels_each_section() -> None: # two of the three outputs were unlabeled ParsedName reprs, and for # already-cased input they are byte-identical -- the reader cannot # tell which is the parse and which is the capitalized view out = _run("Dr. Juan de la Vega III").stdout assert out.count(" None: proc = _run("John Smith", "--json") data = json.loads(proc.stdout) assert data["given"] == "John" and data["family"] == "Smith" def test_cli_no_args_usage() -> None: proc = _run() assert proc.returncode != 0 def test_cli_locale() -> None: proc = _run("Сидоров Иван Петрович", "--locale", "ru") assert proc.returncode == 0 assert "Иван" in proc.stdout proc = _run("Сидоров Иван Петрович", "--locale", "ru", "--json") data = json.loads(proc.stdout) assert "Иван" in data["given"] def test_cli_locale_unknown_code_lists_available() -> None: proc = _run("John Smith", "--locale", "xx") assert proc.returncode != 0 assert "ru" in proc.stderr and "tr_az" in proc.stderr def test_cli_locale_empty_string_errors_not_silent_default() -> None: # --locale "" is a mistake (empty shell variable, typo), not a # request for the default parser; a truthiness check would swallow # it silently -- it must error like any other unknown code proc = _run("John Smith", "--locale", "") assert proc.returncode != 0 assert "ru" in proc.stderr and "tr_az" in proc.stderr def test_cli_locale_ja_activates_nothing_by_itself() -> None: # usage.rst's claim, pinned at the layer it is about: the CLI takes # the pack but has no way to attach a segmenter, and the ja pack # ships no vocabulary -- so --locale ja is byte-identical to the # default parser on the very name the pack exists to divide. A # future --segmenter flag, or a pack that grew a surname list, # would land here first. packed = _run("山田太郎", "--locale", "ja") default = _run("山田太郎") assert packed.returncode == 0 assert packed.stdout == default.stdout python-nameparser-2.1.0/tests/v2/test_config_shim.py000066400000000000000000001030041523530431700225560ustar00rootroot00000000000000"""Shim Constants/SetManager/TupleManager (migration spec §3).""" import copy import pickle import warnings from pathlib import Path import pytest from nameparser import GIVEN_FIRST, HumanName, Lexicon, PatronymicRule, Policy from nameparser._config_shim import ( CONSTANTS, Constants, SetManager, TupleManager, _DelimiterManager, _RegexesProxy, _cached_parser, ) from nameparser.config.regexes import EMPTY_REGEX, REGEXES _DATA_DIR = Path(__file__).parent / "data" def test_set_manager_normalizes_and_holds_membership() -> None: s = SetManager(["Dr", "MRS."]) assert "dr" in s and "Dr" in s # lc() normalization, both ways assert "mrs" in s assert len(s) == 2 assert sorted(s) == ["dr", "mrs"] def test_set_manager_add_remove_chain_and_keyerror() -> None: s = SetManager() assert s.add("Dame", "Fra") is s # chainable, v1 parity assert "dame" in s assert s.remove("Dame") is s assert "dame" not in s with pytest.raises(KeyError): # 1.3.0 grace period ended (#243) s.remove("never-there") def test_set_manager_call_is_removed() -> None: s = SetManager(["dr"]) with pytest.raises(TypeError): # #243: __call__ removed in 2.0 s() # type: ignore[operator] def test_set_manager_operators_and_equality() -> None: a, b = SetManager(["a", "b"]), SetManager(["b", "c"]) assert a | b == {"a", "b", "c"} assert a & b == {"b"} assert a - b == {"a"} assert a | {"z"} == {"a", "b", "z"} assert {"z"} | a == {"a", "b", "z"} # reflected: set op manager assert {"a", "b", "c"} - a == {"c"} # operand order matters here assert a == {"a", "b"} assert a == SetManager(["a", "b"]) and a != b with pytest.raises(TypeError): # mutable, unhashable (v1 parity) hash(a) def test_set_manager_reports_mutations_to_owner() -> None: bumps = [] s = SetManager(["a"], _on_change=lambda: bumps.append(1)) s.add("b") s.remove("a") assert len(bumps) == 2 s.add("b") # no-op: already present assert len(bumps) == 2 # must not bump (v1 parity) def test_set_manager_partial_remove_still_notifies_owner() -> None: bumps = [] s = SetManager(["a", "b"], _on_change=lambda: bumps.append(1)) with pytest.raises(KeyError): s.remove("a", "missing") # "a" IS removed before the raise assert "a" not in s assert len(bumps) == 1 # the real removal was reported def test_set_manager_accepts_v1_pickle_state() -> None: s = SetManager.__new__(SetManager) s.__setstate__({"elements": {"dr", "mr"}, "_on_change": None}) assert "dr" in s and len(s) == 2 # the shim's own key spelling, with un-normalized elements: loading # must re-normalize rather than trust the blob passed through lc() s2 = SetManager.__new__(SetManager) s2.__setstate__({"_elements": {"Dr", "MRS."}}) assert "dr" in s2 and "mrs" in s2 assert sorted(s2) == ["dr", "mrs"] def test_set_manager_pickle_round_trip() -> None: # in-process round trip of a blob we just built; pickle is not a # security boundary here (same stance as the v2 pickle guards) t = pickle.loads(pickle.dumps(SetManager(["Dr", "Mrs."]))) assert t == SetManager(["dr", "mrs"]) def test_tuple_manager_attribute_access_and_unknown_key() -> None: t = TupleManager({"mcdonald": "McDonald"}) assert t.mcdonald == "McDonald" assert t["mcdonald"] == "McDonald" with pytest.raises(AttributeError, match="mcdonalds"): # #256 t.mcdonalds def test_tuple_manager_mutations_bump_owner() -> None: bumps = [] t = TupleManager({"a": "A"}, _on_change=lambda: bumps.append(1)) t["b"] = "B" del t["a"] t.pop("b") assert len(bumps) == 3 def test_tuple_manager_pop_default_on_missing_key_is_noop() -> None: bumps = [] t = TupleManager({"a": "A"}, _on_change=lambda: bumps.append(1)) assert t.pop("missing", None) is None assert len(bumps) == 0 # no-op: key was never present def test_tuple_manager_bulk_mutations_bump_owner() -> None: # dict's C fast paths (update, setdefault, |=, clear, popitem) must # notify too, or a cached parser built from the owner goes stale bumps = [] t = TupleManager(_on_change=lambda: bumps.append(1)) t.update({"a": "A", "b": "B"}) assert len(bumps) == 2 # per-key, via __setitem__ assert t.setdefault("c", "C") == "C" assert len(bumps) == 3 assert t.setdefault("a", "ignored") == "A" assert len(bumps) == 3 # existing key: read, not write t |= {"d": "D"} assert len(bumps) == 4 assert t.popitem()[0] in "abcd" assert len(bumps) == 5 t.clear() assert len(bumps) == 6 t.clear() assert len(bumps) == 6 # already empty: no-op def test_tuple_manager_pickle_round_trip() -> None: t = pickle.loads(pickle.dumps(TupleManager({"a": "A"}))) assert dict(t) == {"a": "A"} assert t._on_change is None def test_delimiter_manager_sentinels_only() -> None: d = _DelimiterManager({"parenthesis": "parenthesis"}) moved = d.pop("parenthesis") d2 = _DelimiterManager() d2["parenthesis"] = moved # the documented bucket-move idiom assert "parenthesis" in d2 with pytest.raises(TypeError, match="quoted_word"): d2["angle_brackets"] = "custom" # spec §3: custom keys raise def test_delimiter_manager_no_bypass_via_constructor_or_update() -> None: # dict's C fast paths (dict.__init__, dict.update) skip a subclass's # __setitem__ -- the sentinel rule must hold on every mutation path with pytest.raises(TypeError, match="quoted_word"): _DelimiterManager({"angle_brackets": "x"}) with pytest.raises(TypeError, match="quoted_word"): _DelimiterManager(angle_brackets="x") with pytest.raises(TypeError, match="quoted_word"): _DelimiterManager().update({"angle_brackets": "x"}) with pytest.raises(TypeError, match="quoted_word"): _DelimiterManager().setdefault("angle_brackets", "x") with pytest.raises(TypeError, match="quoted_word"): d0 = _DelimiterManager() d0 |= {"angle_brackets": "x"} # update through the validated path still notifies the owner per key bumps = [] d = _DelimiterManager(_on_change=lambda: bumps.append(1)) d.update({"quoted_word": "quoted_word", "parenthesis": "parenthesis"}) assert dict(d) == {"quoted_word": "quoted_word", "parenthesis": "parenthesis"} assert len(bumps) == 2 def test_regexes_reads_ok_assignment_raises() -> None: r = _RegexesProxy() assert r.word.match("Smith") # type: ignore[attr-defined] # reads keep working with pytest.raises(TypeError, match="strip_bidi"): r.bidi = None # slot-aware message with pytest.raises(TypeError, match="Policy"): r.roman_numeral = None # generic message def test_regexes_membership_iteration_and_deepcopy() -> None: r = _RegexesProxy() assert "word" in r # membership is a read assert "nope" not in r assert "word" in sorted(r) # iteration is a read assert "word" in r.keys() # dunder probes must raise AttributeError, not resolve as regex # names -- the classic copy.deepcopy regression (see AGENTS.md) assert isinstance(copy.deepcopy(r), _RegexesProxy) def test_regexes_supports_the_v1_read_only_mapping_surface() -> None: # v1's regexes was a dict subclass. The proxy is not, and its # catch-all __getattr__ claims every unimplemented mapping method # as a regex name -- the same failure that made .get() raise # "no regex named 'get'". Cover the rest of the read surface. r = _RegexesProxy() assert dict(r.items())["word"] is REGEXES["word"] assert REGEXES["word"] in list(r.values()) assert len(r) == len(REGEXES) assert r.copy() == dict(REGEXES) # v1's dict.copy() def test_regexes_unsupported_method_is_not_reported_as_a_missing_regex() -> None: # "no regex named 'pop'" sends the reader looking for a vocabulary # entry when the truth is the proxy does not implement a mutator r = _RegexesProxy() with pytest.raises(AttributeError, match="not supported"): r.pop with pytest.raises(AttributeError, match="no regex named"): r.definitely_not_a_regex def test_regexes_get_is_the_soft_access_escape_hatch() -> None: # 1.4's #256 deprecation told users ".get() remains available for # intentional soft access"; __getattr__ must not swallow `get` # itself. capitalization_exceptions inherits it from dict -- the # regexes proxy is not a dict, so it has to say so explicitly. r = _RegexesProxy() assert r.get("word") is REGEXES["word"] assert r.get("typo") is None assert r.get("typo", EMPTY_REGEX) is EMPTY_REGEX def test_constants_default_fields_present() -> None: c = Constants() assert "dr" in c.titles assert "phd" in c.suffix_acronyms assert "van" in c.prefixes assert c.patronymic_name_order is False assert c.middle_name_as_last is False assert c.capitalize_name is False assert c.force_mixed_case_capitalization is False assert c.string_format == "{title} {first} {middle} {last} {suffix} ({nickname})" assert c.suffix_delimiter is None # every named sentinel is a default nickname bucket (the v1 trio # plus the #273 typographic pairs); derived from the map itself so # a new sentinel can't leave this pin stale from nameparser._config_shim import _SENTINEL_PAIRS assert set(c.nickname_delimiters) == set(_SENTINEL_PAIRS) assert {"quoted_word", "double_quotes", "parenthesis", "smart_double_quotes", "guillemets"} <= set(_SENTINEL_PAIRS) assert len(c.maiden_delimiters) == 0 def test_constants_mutation_bumps_generation() -> None: c = Constants() g0 = c._generation c.titles.add("zqxtitle") # not a default title (real bump) assert c._generation > g0 g1 = c._generation c.patronymic_name_order = True assert c._generation > g1 def test_private_constants_do_not_warn_shared_singleton_does() -> None: c = Constants() with warnings.catch_warnings(): warnings.simplefilter("error") c.titles.add("zqxtitle") # private: silent with pytest.deprecated_call(match="Lexicon"): CONSTANTS.titles.add("zqxtest") with pytest.deprecated_call(): CONSTANTS.titles.remove("zqxtest") # leave the singleton clean def test_scalar_noop_assignment_neither_bumps_nor_warns() -> None: # managers already suppress no-op mutations (no-op add, pop with # default); re-assigning a scalar's current value must match -- # neither a generation bump nor, on the shared singleton, the # deprecation warning c = Constants() g0 = c._generation c.string_format = c.string_format assert c._generation == g0 g_shared = CONSTANTS._generation with warnings.catch_warnings(): warnings.simplefilter("error") CONSTANTS.string_format = CONSTANTS.string_format assert CONSTANTS._generation == g_shared def test_empty_attribute_default_assignment_raises() -> None: c = Constants() with pytest.raises(AttributeError, match="#255"): c.empty_attribute_default = None def test_regexes_attribute_assignment_raises() -> None: with pytest.raises(TypeError, match="Policy"): Constants().regexes = {} # type: ignore[assignment] def test_constants_copy_is_independent() -> None: # #260 c = Constants() d = c.copy() d.titles.add("zqxtitle") # not a default title (real addition) assert "zqxtitle" in d.titles and "zqxtitle" not in c.titles assert d._generation != -1 # a real instance with its own counter def test_constants_pickle_round_trip() -> None: c = Constants() c.titles.add("zqxtitle") loaded = pickle.loads(pickle.dumps(c)) assert "zqxtitle" in loaded.titles loaded.titles.add("fra") # mutations still tracked assert "fra" in loaded.titles def test_pre_1_3_constants_blob_raises() -> None: # #279 # pre-1.3.0 blobs are recognizable by the computed property their # dir()-sweep __getstate__ captured; the 1.4 warning promised # ValueError in 2.0 with pytest.raises(ValueError, match="#279|1.3"): Constants.__new__(Constants).__setstate__( {"prefixes": set(), "suffixes_prefixes_titles": set()}) def test_v14_constants_state_shape_accepted() -> None: # v1.4 __getstate__ emitted public field names -> manager values; # scalars ride along. empty_attribute_default is accepted and # DROPPED (#255: empty is always '' in 2.0). c = Constants.__new__(Constants) c.__setstate__({"prefixes": {"van", "de"}, "string_format": "{first} {last}", "empty_attribute_default": None}) assert "van" in c.prefixes assert c.string_format == "{first} {last}" assert not hasattr(c, "empty_attribute_default") or True # dropped def test_v14_constants_blob_unpickles_into_shim() -> None: # tests/v2/data/constants_v14.pickle was produced by a real 1.4.0 # install (`uv run --no-project --with "nameparser==1.4.*" ...`), # not synthesized here -- it is the actual bytes a caller upgrading # from 1.4 to 2.0 would hand to pickle.load(). Since the M11 swap # replaced nameparser.config.Constants with this shim, the blob's # pickled class reference now resolves here. Its nested `regexes` # field pickles as nameparser.config.RegexTupleManager, reconstructed # by the unpickler before Constants.__setstate__ runs -- see # RegexTupleManager's docstring in _config_shim.py. with open(_DATA_DIR / "constants_v14.pickle", "rb") as f: loaded = pickle.load(f) assert isinstance(loaded, Constants) assert "van" in loaded.prefixes assert "dr" in loaded.titles assert "jr" in loaded.suffix_not_acronyms assert loaded.string_format == "{title} {first} {middle} {last} {suffix} ({nickname})" def test_v14_pickle_restores_and_parses_warning_free() -> None: # A 1.4 pickle froze the then-shipped vocabulary, including the # eight dead multi-word entries 2.0 removed; restoring it must not # spray warnings about entries the user never added. A multi-word # entry the user DID add still warns (second half) -- at PARSE # time, not add() time, since the shim's Lexicon snapshot is built # lazily on the first parse after a mutation. with open(_DATA_DIR / "constants_v14.pickle", "rb") as f: c = pickle.load(f) with warnings.catch_warnings(): warnings.simplefilter("error") HumanName("John Smith", constants=c) c.titles.add("grand moff") with pytest.warns(UserWarning, match="matched one word at a time"): HumanName("Jane Roe", constants=c) def test_snapshot_keeps_a_multi_word_first_name_title() -> None: # v1 looks first_name_titles up on the joined title string, so a # multi-word entry is reachable with only its WORDS in titles. # Verified on a live 1.4.0: "Grand Duke John" -> first='John'. # Intersecting the entry away made the lookup miss and silently # reclassified the given name as a family name. c = Constants() c.titles.add("grand", "duke") c.first_name_titles.add("grand duke") lexicon, _, _ = c._snapshot() assert "grand duke" in lexicon.given_name_titles name = HumanName("Grand Duke John", constants=c) assert (name.title, name.first, name.last) == ("Grand Duke", "John", "") @pytest.mark.parametrize("entry", [".", "..", "grand duke", "grand\tduke"]) def test_snapshot_drops_first_name_titles_v1_could_not_match(entry: str) -> None: # v1's key is lc(" ".join(title_list)) -- always single-spaced, and # never empty -- so an entry with a whitespace run, or one that is # all periods, could never match there. Translating it anyway would # either widen behavior (the double-space entry starts matching) or # raise (the all-period entry normalizes to empty and _normset # rejects it). 1.4.0 accepts all of these and leaves them inert. c = Constants() c.titles.add("grand", "duke") c.first_name_titles.add(entry) lexicon, _, _ = c._snapshot() # must not raise assert "" not in lexicon.given_name_titles name = HumanName("Grand Duke John", constants=c) assert (name.first, name.last) == ("", "John") @pytest.mark.parametrize(("entry", "text"), [ # v1's lookup key is lc(joined-title-run), and lc strips only the # EDGE periods of the whole string, so an interior word keeps its # period. v2 normalizes per token and then joins, so the entry has # to be TRANSLATED between the two spellings, not filtered out. ("lt. col", "Lt. Col. Smith"), # v1 joins titles across conjunctions, and v2 tags the conjunction # Role.TITLE too, so the joined key contains it in both versions ("sir and dame", "Sir and Dame Alex"), ]) def test_snapshot_translates_a_multi_word_first_name_title( entry: str, text: str ) -> None: c = Constants() c.titles.add("lt", "col") c.first_name_titles.add(entry) name = HumanName(text, constants=c) assert name.first and not name.last, f"{entry!r} was not honored" def test_snapshot_ignores_an_ambiguous_acronym_added_as_a_suffix_word() -> None: # Adding an ambiguous acronym to suffix_not_acronyms is INERT in # v1: is_suffix already accepts it through the acronym branch, and # reserve_last keeps it as the surname anyway. 1.4.0 with this # config parses "Jack Ma" exactly as it does without it. # # Lexicon forbids the overlap, since the word branch would bypass # the period gate. Dropping the word from the AMBIGUOUS set instead # (the reverse subtraction) ungates it and loses the family name # entirely -- worse than the raise it was meant to avoid. Drop it # from suffix_words instead, which reproduces v1's inertness. c = Constants() c.suffix_not_acronyms.add("ma") lexicon, _, _ = c._snapshot() # must not raise assert "ma" in lexicon.suffix_acronyms_ambiguous assert "ma" not in lexicon.suffix_words name = HumanName("Jack Ma", constants=c) assert (name.first, name.last, name.suffix) == ("Jack", "Ma", "") def test_bound_never_given_prefix_deviates_on_two_pieces() -> None: # Pinned DEVIATION, not parity. v1's bound-given join has a # reserve_last guard, so with only two pieces it does not fire and # the word stays never-given: 1.4.0 gives first='', last='dos # Santos'. The shim promotes such a word to may-be-given # unconditionally, because v1's rule is piece-count dependent and a # static vocabulary set cannot express that. Three-piece parity is # the case that matters and is covered below; this records the cost. c = Constants() c.bound_first_names.add("dos") name = HumanName("dos Santos", constants=c) assert (name.first, name.last) == ("dos", "Santos") def test_snapshot_keeps_a_bound_never_given_prefix_parseable() -> None: # prefixes.py asserts its own data keeps non_first_name_prefixes # disjoint from bound_first_names, but nothing stops a v1 caller # adding one at runtime, and 1.4 accepts it -- letting the bound # rule win, so "dos Santos Silva" parses first="dos Santos". # Lexicon rejects that combination, so the shim promotes such a word # to may-be-given rather than raising on config v1 allowed. c = Constants() assert "dos" in c.non_first_name_prefixes # baseline: never-given c.bound_first_names.add("dos") lexicon, _, _ = c._snapshot() # must not raise assert "dos" in lexicon.particles_ambiguous name = HumanName("dos Santos Silva", constants=c) assert (name.first, name.last) == ("dos Santos", "Silva") def test_snapshot_removing_a_honorific_word_turns_the_peel_off() -> None: # The deciding case for honorific_tails, which has no v1 manager of # its own: the snapshot intersects GLUED_HONORIFICS with the WORD # set, so deleting 씨 from suffix_not_acronyms -- the only v1 knob # that reaches it -- must make the tail stop mattering rather than # raise the subset error or leave 씨 peeling into a suffix field # that no longer recognizes it. With the default config the # intersection is a no-op, so the equality test above pins nothing # here. c = Constants() assert HumanName("김민준씨", constants=c).suffix == "씨" # baseline c.suffix_not_acronyms.remove("씨") lexicon, _, _ = c._snapshot() # must not raise assert "씨" not in lexicon.honorific_tails # the peel is off: the glued honorific goes back into the name, # which is 2.0's answer for this input name = HumanName("김민준씨", constants=c) assert (name.first, name.last, name.suffix) == ("민준씨", "김", "") def test_snapshot_field_translation() -> None: c = Constants() lexicon, policy, defaults = c._snapshot() # drift guard: a default Constants must resolve to EXACTLY the v2 # defaults -- a field populated in _default_lexicon() but forgotten # in _snapshot() (or vice versa) fails loudly here assert lexicon == Lexicon.default() assert policy == Policy() assert isinstance(lexicon, Lexicon) and isinstance(policy, Policy) assert lexicon.suffix_words == frozenset(c.suffix_not_acronyms) # complement translation: v1 marks never-given, v2 marks may-be-given assert lexicon.particles_ambiguous == \ frozenset(c.prefixes) - frozenset(c.non_first_name_prefixes) assert lexicon.suffix_acronyms_ambiguous <= lexicon.suffix_acronyms assert policy.name_order == GIVEN_FIRST assert policy.patronymic_rules == frozenset() assert defaults.string_format == c.string_format # a snapshot is a pure read: no generation bump, no deprecation # warning even on the shared singleton g0 = CONSTANTS._generation with warnings.catch_warnings(): warnings.simplefilter("error") CONSTANTS._snapshot() assert CONSTANTS._generation == g0 def test_snapshot_patronymic_and_middle_flags() -> None: c = Constants() c.patronymic_name_order = True c.middle_name_as_last = True _, policy, _ = c._snapshot() assert policy.patronymic_rules == frozenset( {PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) assert policy.middle_as_family is True def test_typographic_delimiter_sentinels_present_and_movable() -> None: # #273: the eight typographic pairs surface in the v1 API as new # named sentinels, so the documented keyed idioms (pop/move/del) # work on them exactly like the original trio c = Constants() assert "smart_double_quotes" in c.nickname_delimiters assert "guillemets" in c.nickname_delimiters c.maiden_delimiters["guillemets"] = c.nickname_delimiters.pop("guillemets") _, policy, _ = c._snapshot() assert ("«", "»") in policy.maiden_delimiters assert ("«", "»") not in policy.nickname_delimiters def test_snapshot_delimiter_bucket_move() -> None: c = Constants() c.maiden_delimiters["parenthesis"] = c.nickname_delimiters.pop("parenthesis") _, policy, _ = c._snapshot() assert ("(", ")") in policy.maiden_delimiters assert ("(", ")") not in policy.nickname_delimiters def test_snapshot_overlap_keeps_v1_nickname_precedence() -> None: # v1 semantics: a pair present in BOTH v1 buckets parses as nickname. # 2.0's Policy resolves overlap the other way (maiden wins), so the # snapshot pre-subtracts on the maiden side -- a v1 user who added # parens to maiden_delimiters WITHOUT removing them from # nickname_delimiters keeps their 1.x parse through the facade. c = Constants() c.maiden_delimiters["parenthesis"] = ("(", ")") # nickname still has it _, policy, _ = c._snapshot() assert ("(", ")") in policy.nickname_delimiters assert ("(", ")") not in policy.maiden_delimiters from nameparser import HumanName n = HumanName("Jane (Jones) Smith", constants=c) assert n.nickname == "Jones" and n.maiden == "" def test_snapshot_ambiguous_removed_acronym_intersects() -> None: c = Constants() c.suffix_acronyms.remove("jd") # 'jd' stays in ambiguous lexicon, _, _ = c._snapshot() assert "jd" not in lexicon.suffix_acronyms assert "jd" not in lexicon.suffix_acronyms_ambiguous # subset holds def test_parser_cache_shared_across_equal_snapshots() -> None: a, b = Constants(), Constants() la, pa, _ = a._snapshot() lb, pb, _ = b._snapshot() assert _cached_parser(la, pa) is _cached_parser(lb, pb) # -- Gap 1: Constants() constructor kwargs (v1.4 parity, #238/#242/#244) ---- def test_constants_kwarg_replaces_field_others_keep_defaults() -> None: default = Constants() c = Constants(titles=["Abc"]) assert set(c.titles) == {"abc"} # lc()-normalized assert "abc" not in default.titles # sanity: not a real default # a kwarg REPLACES that field's vocabulary wholesale (v1 parity); the # other eight set fields are untouched for field in ("prefixes", "suffix_acronyms", "suffix_not_acronyms", "suffix_acronyms_ambiguous", "first_name_titles", "conjunctions", "bound_first_names", "non_first_name_prefixes"): assert set(getattr(c, field)) == set(getattr(default, field)) def test_constants_kwarg_bare_string_raises_typeerror() -> None: with pytest.raises(TypeError, match="titles"): Constants(titles="abc") # type: ignore[arg-type] def test_constants_kwarg_non_iterable_raises_typeerror() -> None: with pytest.raises(TypeError): Constants(titles=5) # type: ignore[arg-type] def test_constants_kwarg_capitalization_exceptions_wraps_tuple_manager() -> None: c = Constants(capitalization_exceptions={"mcdonald": "McDonald"}) assert isinstance(c.capitalization_exceptions, TupleManager) assert c.capitalization_exceptions["mcdonald"] == "McDonald" def test_constants_kwarg_regexes_raises_typeerror() -> None: with pytest.raises(TypeError, match="Policy"): Constants(regexes={}) # type: ignore[call-arg] def test_constants_kwarg_unknown_raises_typeerror() -> None: with pytest.raises(TypeError): Constants(bogus=1) # type: ignore[call-arg] def test_constants_kwarg_feeds_facade_parse() -> None: from nameparser._facade import HumanName c = Constants(titles=["zzqtitle"]) n = HumanName("Zzqtitle Judy Dench", constants=c) assert n.title == "Zzqtitle" # -- Restoring v1.3/1.4 manager hardening (#221/#238/#241/#242/#260) ------- def test_set_manager_discard_is_noop_on_missing_and_notifies_on_real_change() -> None: bumps = [] s = SetManager(["a"], _on_change=lambda: bumps.append(1)) assert s.discard("missing") is s # never raises, chainable assert len(bumps) == 0 # no-op: nothing removed assert s.discard("A") is s # normalized match assert "a" not in s assert len(bumps) == 1 def test_set_manager_clear_notifies_only_if_non_empty() -> None: bumps = [] s = SetManager(["a", "b"], _on_change=lambda: bumps.append(1)) assert s.clear() is s assert len(s) == 0 assert len(bumps) == 1 assert s.clear() is s # already empty: no-op assert len(bumps) == 1 def test_set_manager_operators_accept_arbitrary_iterables() -> None: a = SetManager(["a", "b"]) assert a | ["B.", "z"] == {"a", "b", "z"} # list operand, normalized assert a & (x for x in ["A", "q"]) == {"a"} # generator operand assert a - ["a"] == {"b"} assert a ^ ["b", "c"] == {"a", "c"} # symmetric difference assert ["b", "c"] ^ a == {"a", "c"} # reflected def test_set_manager_bare_str_or_bytes_operand_raises_typeerror() -> None: a = SetManager(["a"]) with pytest.raises(TypeError): a | "ab" with pytest.raises(TypeError): a & b"ab" with pytest.raises(TypeError): "ab" | a # reflected form too def test_set_manager_comparison_operators() -> None: a = SetManager(["a", "b"]) assert a <= {"a", "b", "c"} assert a < {"a", "b", "c"} assert not (a < {"a", "b"}) assert {"a", "b", "c"} >= a assert a <= SetManager(["a", "b", "c"]) assert SetManager(["a", "b", "c"]) > a def test_set_manager_constructor_rejects_bare_str() -> None: with pytest.raises(TypeError): # #238/#241: not shredded to chars SetManager("dr") # type: ignore[arg-type] def test_set_manager_constructor_rejects_bare_bytes_with_decode_hint() -> None: with pytest.raises(TypeError, match="decode"): SetManager(b"dr") # type: ignore[arg-type] def test_set_manager_constructor_rejects_non_str_element() -> None: with pytest.raises(TypeError): SetManager([None]) # type: ignore[list-item] def test_set_manager_add_rejects_bytes_with_decode_hint() -> None: s = SetManager() with pytest.raises(TypeError, match="decode"): s.add(b"dr") # type: ignore[arg-type] def test_constants_setattr_rejects_bare_str_like_constructor() -> None: c = Constants() with pytest.raises(TypeError, match="conjunctions"): c.conjunctions = "and" # type: ignore[assignment] def test_tuple_manager_constructor_rejects_bare_str() -> None: with pytest.raises(TypeError): # #242 TupleManager("ab") # type: ignore[arg-type] def test_tuple_manager_constructor_rejects_iterable_of_strings() -> None: with pytest.raises(TypeError): # #242: 2-char strings silently split TupleManager(["ab", "cd"]) # type: ignore[arg-type] def test_tuple_manager_unknown_key_error_names_known_keys() -> None: t = TupleManager({"mcdonald": "McDonald", "obrien": "O'Brien"}) with pytest.raises(AttributeError, match="mcdonald"): t.bogus def test_tuple_manager_attribute_style_set_and_delete() -> None: bumps = [] t = TupleManager({"a": "A"}, _on_change=lambda: bumps.append(1)) t.b = "B" # attribute-style routes to dict assert t["b"] == "B" assert len(bumps) == 1 del t.a assert "a" not in t assert len(bumps) == 2 # the manager's own private attribute is unaffected assert t._on_change is not None def test_constants_copy_preserves_subclass() -> None: # #260 class MyConstants(Constants): pass c = MyConstants() d = c.copy() assert type(d) is MyConstants def test_constants_repr_shows_collection_counts() -> None: # #221 c = Constants() r = repr(c) assert r.startswith(" None: # #221 c = Constants() assert "capitalize_name" not in repr(c) # default: not shown c.capitalize_name = True assert "capitalize_name: True" in repr(c) def test_constants_bool_kwargs() -> None: # v1.4 constructor kwargs used by the customize.rst doctests c = Constants(middle_name_as_last=True) assert c.middle_name_as_last is True assert c.patronymic_name_order is False d = Constants(patronymic_name_order=True) _, policy, _ = d._snapshot() assert len(policy.patronymic_rules) == 2 def test_set_manager_partial_add_still_notifies_owner() -> None: # a TypeError mid-argument-list leaves earlier additions applied; # the owner must hear about them or its cached parser goes stale bumps: list[int] = [] s = SetManager(["a"], _on_change=lambda: bumps.append(1)) with pytest.raises(TypeError, match="decode"): s.add("b", b"bad") # type: ignore[arg-type] assert "b" in s assert len(bumps) == 1 def test_set_manager_remove_discard_bytes_decode_hint() -> None: s = SetManager(["dr"]) with pytest.raises(TypeError, match="decode"): s.remove(b"dr") # type: ignore[arg-type] with pytest.raises(TypeError, match="decode"): s.discard(b"dr") # type: ignore[arg-type] def test_delimiter_manager_constructor_bare_str_gets_242_error() -> None: # the parent's #242 guard, not dict's cryptic ValueError with pytest.raises(TypeError, match="mapping or iterable"): _DelimiterManager("ab") # type: ignore[arg-type] with pytest.raises(TypeError, match="pairs"): _DelimiterManager(["ab", "cd"]) # type: ignore[list-item] def test_constants_shared_flag_is_read_only() -> None: c = Constants() with pytest.raises(AttributeError, match="read-only"): c._shared = True with pytest.raises(AttributeError, match="read-only"): CONSTANTS._shared = False assert CONSTANTS._shared is True and c._shared is False def test_multiword_warning_through_shim_points_at_caller() -> None: c = Constants() c.titles.add("zqx zqy") with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") HumanName("John Smith", constants=c) # snapshot builds lazily here multi = [x for x in w if "matched one word at a time" in str(x.message)] assert multi and all(x.filename == __file__ for x in multi) def test_2x_pickle_roundtrip_keeps_a_readded_dead_entry() -> None: # the all-eight gate: a 2.0 user's deliberate re-add of ONE legacy # string survives a round-trip (only a full pre-2.0 blob, which # froze all eight, is subtracted) c = Constants() with pytest.warns(UserWarning): c.suffix_acronyms.add("leed ap") HumanName("John Smith", constants=c) # snapshot builds lazily here c2 = pickle.loads(pickle.dumps(c)) assert "leed ap" in c2.suffix_acronyms python-nameparser-2.1.0/tests/v2/test_contracts.py000066400000000000000000000102651523530431700222770ustar00rootroot00000000000000"""Stable-string contract tests (core spec §7.4): every enum member and stable tag has a canonical triggering input, parametrized by iterating the registries -- a new member without an entry here fails loudly.""" import pytest from nameparser import Parser, Policy, parse from nameparser._policy import PatronymicRule from nameparser._types import STABLE_TAGS, AmbiguityKind _AMBIGUITY_TRIGGERS: dict[AmbiguityKind, str | None] = { AmbiguityKind.PARTICLE_OR_GIVEN: "Van Johnson", AmbiguityKind.UNBALANCED_DELIMITER: 'Jon "Nick Smith', AmbiguityKind.COMMA_STRUCTURE: "Smith, John, Extra, Jr.", AmbiguityKind.SUFFIX_OR_NICKNAME: "JEFFREY (JD) BRICKEN", AmbiguityKind.SUFFIX_OR_NAME: "John Smith MA", # no emitter yet -- arrives with locale-pack order detection (2.x) AmbiguityKind.ORDER: None, # 남 and 남궁 are both shipped surnames, so longest-first picks # between two vocabulary-supported splits AmbiguityKind.SEGMENTATION: "남궁민수", } @pytest.mark.parametrize("kind", [ # the per-entry comment above says WHY each None is None (no # emitter yet, or an emitter no default parse can reach) pytest.param(k, marks=pytest.mark.xfail( strict=True, reason=f"{k.value}: no trigger registered yet")) if k in _AMBIGUITY_TRIGGERS and _AMBIGUITY_TRIGGERS[k] is None else k for k in AmbiguityKind ]) def test_every_ambiguity_kind_has_a_registered_trigger( kind: AmbiguityKind) -> None: assert kind in _AMBIGUITY_TRIGGERS, ( f"new AmbiguityKind {kind.value!r} needs a canonical trigger " f"(or an explicit None with its planned emitter)") trigger = _AMBIGUITY_TRIGGERS[kind] assert trigger is not None # None triggers are strict-xfail marked kinds = {a.kind for a in parse(trigger).ambiguities} assert kind in kinds _PATRONYMIC_TRIGGERS: dict[PatronymicRule, tuple[str, str]] = { # rule -> (input, expected given) PatronymicRule.EAST_SLAVIC: ("Сидоров Иван Петрович", "Иван"), PatronymicRule.TURKIC: ("Mammadova Aygun Ali kizi", "Aygun"), } @pytest.mark.parametrize("rule", list(PatronymicRule)) def test_every_patronymic_rule_has_a_trigger(rule: PatronymicRule) -> None: assert rule in _PATRONYMIC_TRIGGERS text, expected_given = _PATRONYMIC_TRIGGERS[rule] p = Parser(policy=Policy(patronymic_rules=frozenset({rule}))) assert p.parse(text).given == expected_given _TAG_TRIGGERS: dict[str, tuple[str, str]] = { # tag -> (input, token text carrying the tag) "particle": ("Juan de la Vega", "de"), "conjunction": ("Mr. and Mrs. John Smith", "and"), "initial": ("John A. Smith", "A."), "joined": ("John Ph. D.", "D."), } @pytest.mark.parametrize("tag", sorted(STABLE_TAGS)) def test_every_stable_tag_has_a_trigger(tag: str) -> None: assert tag in _TAG_TRIGGERS text, token_text = _TAG_TRIGGERS[tag] pn = parse(text) tagged = next(t for t in pn.tokens if t.text == token_text) assert tag in tagged.tags def test_every_guarded_config_module_is_imported() -> None: """Import-time asserts only run if the module is imported. ``maiden_markers`` and ``surnames`` are lazy (imported inside ``_snapshot``/``_default_lexicon``), so their guards fire only incidentally, whenever some other test happens to build a default Lexicon. Importing them all here makes every guard unconditional. The roster is DERIVED from the source tree, not listed: a hand-written tuple fails open, silently skipping the next guarded module nobody remembers to add. """ import importlib import pathlib import nameparser.config config_dir = pathlib.Path(nameparser.config.__file__).parent # private modules excluded: _invariants DEFINES the guard (the # substring hits its own def) and holds no vocabulary of its own guarded = sorted( p.stem for p in config_dir.glob("*.py") if not p.stem.startswith("_") and "assert_normalized(" in p.read_text(encoding="utf-8")) assert guarded, ( f"no guarded config module found in {config_dir} -- the " f"derivation broke, and an empty roster asserts nothing") for name in guarded: importlib.import_module(f"nameparser.config.{name}") python-nameparser-2.1.0/tests/v2/test_differential.py000066400000000000000000000741731523530431700227430ustar00rootroot00000000000000"""Unit tests for the differential gate's decision logic. `tools/` is outside `testpaths`, and adding it would run `--doctest-modules` over the corpus builders, so `compare.py` is imported by path here -- the same way `test_regex_sync.py` already imports `build_cjk_corpus`. Only pure logic is covered: nothing here spawns `uv` or the network. What is tested is what produces FALSE CONFIDENCE when it silently misbehaves -- which surfaces get compared, which ledger gets consulted, and above all whether a version tell is believed. """ import importlib.util from pathlib import Path from types import ModuleType import pytest _TOOLS = Path(__file__).parents[2] / "tools" / "differential" def _load_compare() -> ModuleType: spec = importlib.util.spec_from_file_location( "differential_compare", _TOOLS / "compare.py") assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module compare = _load_compare() def test_parse_version_pads_a_short_release_to_three_parts() -> None: """A requested '2.0' and a wheel reporting '2.0.0' are the same release; comparing the raw strings would call them unequal and abort a correct run as a tell mismatch.""" assert compare._parse_version("2.0") == compare._parse_version("2.0.0") def test_parse_version_orders_numerically_not_lexically() -> None: """The bug string comparison would introduce: '10.0.0' sorts BELOW '2.0.0' as text.""" assert compare._parse_version("10.0.0") > compare._parse_version("2.0.0") def test_parse_version_ignores_a_prerelease_segment() -> None: assert compare._parse_version("2.0.0rc1") == (2, 0, 0) def test_parse_version_rejects_a_string_with_no_release_in_it() -> None: with pytest.raises(SystemExit, match="cannot parse a version"): compare._parse_version("not-a-version") @pytest.mark.parametrize("version,expected", [ ("1.4.0", {"facade"}), ("1.4", {"facade"}), ("1.9.9", {"facade"}), ("2.0.0", {"facade", "v2"}), ("2.0", {"facade", "v2"}), ("2.1.0", {"facade", "v2"}), # the row string comparison gets wrong: '10.0.0' < '2.0.0' as text ("10.0.0", {"facade", "v2"}), ]) def test_surfaces_are_derived_from_the_baseline( version: str, expected: set[str]) -> None: assert compare._surfaces_for(version) == expected def test_allowlist_path_is_named_for_its_baseline() -> None: assert compare._allowlist_for("1.4.0").name == "expected_since_1.4.0.toml" def test_allowlist_for_a_baseline_with_no_ledger_is_a_hard_error() -> None: """Not an empty rule set. An empty set classifies nothing, so every diff reports as unexplained -- which reads as a catastrophic regression rather than as a missing file, and sends the reader hunting the parser instead of the ledger.""" with pytest.raises(SystemExit, match="no allowlist for baseline"): compare._allowlist_for("9.9.9") def test_name_regex_rules_sort_ahead_of_fields_only_rules() -> None: """Most-specific-first BETWEEN tiers. Within a tier the stable sort leaves file order deciding, which the 2.0 ledger relies on.""" rules = [{"issue": "broad", "fields": ["first"]}, {"issue": "specific", "name_regex": "Smith"}] assert [r["issue"] for r in compare._sorted_rules(rules)] \ == ["specific", "broad"] def test_rule_sort_is_stable_within_a_tier() -> None: rules = [{"issue": "a", "name_regex": "A"}, {"issue": "b", "name_regex": "B"}] assert [r["issue"] for r in compare._sorted_rules(rules)] == ["a", "b"] def test_classify_returns_none_when_no_rule_matches() -> None: rules = [{"issue": "x", "name_regex": "Zzz"}] assert compare.classify("John Smith", {"first"}, rules) is None def test_classify_takes_the_first_matching_rule() -> None: rules = [{"issue": "specific", "name_regex": "Smith"}, {"issue": "broad", "fields": ["first"]}] assert compare.classify("John Smith", {"first"}, rules) == "specific" def test_worker_source_carries_the_requested_pin() -> None: src = compare._worker_source("2.0.0", want_v2=True) assert 'dependencies = ["nameparser==2.0.0"]' in src def test_worker_source_always_emits_a_version_tell() -> None: """The tell is the whole defence against a worker that silently resolved to the checkout, so it is not conditional on anything.""" for want_v2 in (True, False): src = compare._worker_source("1.4.0", want_v2=want_v2) assert "__version__" in src and "__file__" in src def test_worker_source_gates_the_v2_import_on_the_baseline() -> None: """1.4 has no nameparser.parse to import; asking for it would make the worker die on import rather than report a clean facade diff.""" assert "WANT_V2 = False" in compare._worker_source("1.4.0", want_v2=False) assert "WANT_V2 = True" in compare._worker_source("2.0.0", want_v2=True) _WHEEL = "/Users/x/.cache/uv/environments-v2/w/lib/python3.11/" \ "site-packages/nameparser/__init__.py" def test_tell_accepts_a_matching_wheel() -> None: compare._check_tell({"__version__": "2.0.0", "__file__": _WHEEL}, "2.0.0") def test_tell_accepts_an_equivalent_short_release() -> None: compare._check_tell({"__version__": "2.0.0", "__file__": _WHEEL}, "2.0") def test_tell_rejects_a_version_mismatch() -> None: with pytest.raises(SystemExit, match="not the requested"): compare._check_tell( {"__version__": "2.1.0", "__file__": _WHEEL}, "2.0.0") def test_tell_rejects_a_module_loaded_from_the_checkout() -> None: """The failure the whole design exists to make impossible. An editable install reports the TREE's version, so when the tree and the baseline share a version the version half of the tell agrees and only the path gives it away. """ checkout = _TOOLS.parents[1] / "nameparser" / "__init__.py" with pytest.raises(SystemExit, match="CHECKOUT"): compare._check_tell( {"__version__": "2.0.0", "__file__": str(checkout)}, "2.0.0") def test_tell_rejects_an_empty_tell() -> None: with pytest.raises(SystemExit): compare._check_tell({}, "2.0.0") def test_facade_field_names_canonicalize_to_role_vocabulary() -> None: """Both surfaces name the same seven roles with different words, and Role's names win -- AGENTS.md already makes Role's declaration order canonical "defined once and derived everywhere", and the facade's vocabulary expires at 3.0.""" assert compare._canonical_field("first") == "given" assert compare._canonical_field("last") == "family" assert compare._canonical_field("middle") == "middle" assert compare._canonical_field("_ambiguities") == "_ambiguities" def test_canonical_field_is_idempotent_on_role_names() -> None: """Both surfaces' diffs pass through it, and the v2 surface's names are already canonical, so applying it must be a no-op there.""" for role in compare.V2_FIELDS: assert compare._canonical_field(role) == role def test_every_ledger_rule_names_roles_canonically() -> None: """A rule written in facade vocabulary parses, and validate_rules now rejects it at startup ("not roles"). Before that guard it validated and then silently never matched -- the ledger growing an entry that did nothing. This keeps a sharper message than the generic role check, and sweeps every ledger, so a new baseline's file is covered the day it is added.""" import tomllib ledgers = sorted(_TOOLS.glob("expected_since_*.toml")) assert ledgers, "no ledgers found; this test would pass vacuously" for ledger in ledgers: rules = tomllib.loads( ledger.read_text(encoding="utf-8")).get("change", []) for rule in rules: for field in rule.get("fields", []): assert field == compare._canonical_field(field), ( f"{ledger.name}: rule {rule['issue']!r} names " f"{field!r}; use " f"{compare._canonical_field(field)!r}") @pytest.mark.parametrize("name,latin", [ ("John Smith", True), ("Anna Müller", True), ("Jane Smith (née Jones)", True), ("田中さん", False), ("김민준", False), ("Хосе Сантос", False), ("威廉·莎士比亚", False), ]) def test_latin_only_partition(name: str, latin: bool) -> None: assert compare._is_latin_only(name) is latin def test_malformed_rule_error_names_the_ledger_it_came_from() -> None: """There is one ledger per baseline now, so a hardcoded filename sends the reader to edit a rule that is not the broken one.""" bad = [{"issue": "x"}] # neither name_regex nor fields with pytest.raises(SystemExit, match="expected_since_2.0.0.toml"): compare.validate_rules(bad, "expected_since_2.0.0.toml") with pytest.raises(SystemExit, match="expected_since_1.4.0.toml"): compare.validate_rules([{}], "expected_since_1.4.0.toml") def test_classify_declines_a_diff_touching_a_field_the_rule_omits() -> None: """The subset check is the tightness mechanism of every `fields` rule -- a rule claims a diff only when EVERY changed field is one it listed. Nothing else pinned it: a rule and a diff that name the same single field satisfy `<=`, `>=`, `==` and `&` alike, so the existing tests pass with the comparison flipped, and every deliberate field omission in both ledgers would quietly stop meaning anything.""" rules = [{"issue": "given-only", "fields": ["given"]}] assert compare.classify("x", {"given"}, rules) == "given-only" assert compare.classify("x", {"given", "suffix"}, rules) is None def test_v2_fields_matches_the_Role_enum() -> None: """AGENTS.md: the seven roles are 'defined once and derived everywhere'. compare.py cannot import Role into the WORKER (that runs under the old wheel), but this copy reads the working tree's ParsedName and must track Role. If a role were added and this tuple not updated, getattr never asks for it and every change in that role is invisible on the v2 surface -- silent under-coverage, exit 0.""" from nameparser import Role assert compare.V2_FIELDS == tuple(str(r) for r in Role) # The malformed-rule family. Most rows are a way a rule can silently # match MORE than its author meant, which is how a real regression # becomes a classified diff and a green run. Three rows are the # opposite -- an empty `fields`, a non-role name, a facade name -- and # make a rule that can never match; those fail loudly (the diff # surfaces as UNEXPLAINED) so their rows buy a precise message rather # than safety. Parametrized rather than written one-by-one because a # guard added to one member of this family belongs on all of it. @pytest.mark.parametrize("rule,expect", [ ({}, "no string 'issue'"), ({"issue": ""}, "no string 'issue'"), ({"issue": "x"}, "neither 'name_regex' nor 'fields'"), # a misspelled key is not ignored -- it deletes that half of the # narrowing and the rule matches on the other half alone ({"issue": "x", "name_regex": ",", "field": ["given"]}, "unknown key"), # wrong types: classify skips them, so the rule silently widens ({"issue": "x", "name_regex": ["a"], "fields": ["given"]}, "non-string 'name_regex'"), ({"issue": "x", "name_regex": "a", "fields": "given"}, "not a list of strings"), # an empty pattern matches every name, and name_regex rules sort # FIRST, so it would shadow the whole ledger ({"issue": "x", "name_regex": ""}, "matches every one of"), ({"issue": "x", "name_regex": "(?:)"}, "matches every one of"), # the shapes the empty-string probe let through: each declines "" # and still matches every name in every corpus ({"issue": "x", "name_regex": "."}, "matches every one of"), ({"issue": "x", "name_regex": ".+"}, "matches every one of"), ({"issue": "x", "name_regex": r"\b"}, "matches every one of"), ({"issue": "x", "name_regex": r"[\s\S]"}, "matches every one of"), # seven roles without _ambiguities: below baseline 2.0 that IS the # whole vocabulary, so it claims every diff ({"issue": "x", "fields": ["title", "given", "middle", "family", "suffix", "nickname", "maiden"]}, "all seven roles"), # uncompilable: without this it raises mid-run, after the worker ({"issue": "x", "name_regex": "Smith("}, "invalid 'name_regex'"), ({"issue": "x", "fields": []}, "empty 'fields'"), ({"issue": "x", "fields": ["famly"]}, "not roles"), # facade vocabulary is not role vocabulary; it would never match ({"issue": "x", "fields": ["first"]}, "not roles"), ({"issue": "x", "fields": ["title", "given", "middle", "family", "suffix", "nickname", "maiden", "_ambiguities"]}, "all seven roles"), ]) def test_validate_rules_rejects_a_rule_that_would_silently_widen( rule: dict, expect: str) -> None: with pytest.raises(SystemExit, match=expect): compare.validate_rules([rule], "expected_since_2.0.0.toml") def test_validate_rules_accepts_the_shipped_ledgers() -> None: """The guards above must not be so strict they reject real rules.""" import tomllib ledgers = sorted(_TOOLS.glob("expected_since_*.toml")) assert ledgers, "no ledgers found; this test would pass vacuously" for ledger in ledgers: rules = tomllib.loads( ledger.read_text(encoding="utf-8")).get("change", []) assert rules, f"{ledger.name} has no [[change]] rules" compare.validate_rules(rules, ledger.name) def test_ambiguities_is_a_legal_field_name() -> None: """A SEGMENTATION-only diff is facade-identical by construction, so this pseudo-field is the only name that can classify it -- and the 2.0 ledger's first rule depends on it.""" compare.validate_rules( [{"issue": "x", "fields": ["_ambiguities"]}], "ledger.toml") #: What _run_worker was asked for, so a test can prove main forwarded #: the baseline and the corpus rather than defaults of its own. _WORKER_CALL: dict = {} def _run_main(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ledger_body: str, baseline_facade: dict, baseline: str = "1.4.0", baseline_v2: dict | None = None, floor: int | None = 1) -> tuple[int, str]: """Drive main() end to end with a faked baseline worker. No uv, no network. The helper exists because every unit test above proves a helper WORKS while none proves main() calls it -- and in a gate, the composition is the part that can go silently permissive. `baseline` defaults to 1.4.0 (facade only). Pass 2.0.0 with `baseline_v2` to exercise the v2 surface, including the ambiguity-only diff that is the stated reason to compare it. """ import sys corpus = tmp_path / "corpus_x.jsonl" corpus.write_text('"John Smith"\n', encoding="utf-8") (tmp_path / f"expected_since_{baseline}.toml").write_text( ledger_body, encoding="utf-8") row: dict = {"facade": baseline_facade} if baseline_v2 is not None: row["v2"] = baseline_v2 _WORKER_CALL.clear() def _fake(v: str, w: bool, n: list[str]) -> tuple[dict, list[dict]]: _WORKER_CALL.update(version=v, want_v2=w, names=list(n)) return ({"__version__": v, "__file__": "/wheel/nameparser/__init__.py"}, [row]) # The fixture corpus needs a floor like any other. `floor=None` # leaves it unregistered, for the test that pins what happens when # a corpus arrives without one. if floor is not None: monkeypatch.setitem(compare._CORPUS_FLOORS, corpus.name, floor) monkeypatch.setattr(compare, "HERE", tmp_path) monkeypatch.setattr(compare, "_run_worker", _fake) monkeypatch.setattr(sys, "argv", ["compare.py", "--baseline", baseline, "--corpus", str(corpus)]) import io import contextlib buf = io.StringIO() with contextlib.redirect_stdout(buf): code = compare.main() return code, buf.getvalue() #: 'John Smith' with the family name altered, so the tree disagrees on #: exactly one role. The facade calls it `last`; the report and any rule #: must call it `family`. _DIFFERS = {"title": "", "first": "John", "middle": "", "last": "SMYTHE", "suffix": "", "nickname": "", "maiden": ""} def test_main_exits_1_and_reports_an_unclassified_diff( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """The gate's entire verdict. Nothing else pins it: mutating the return to a bare 0 leaves every other test in this file passing, and the harness would report unexplained diffs on stdout while exiting 0 forever -- read by exit code, that is silence.""" code, out = _run_main( tmp_path, monkeypatch, '[[change]]\nissue = "unrelated"\nname_regex = "ZZZ"\n', _DIFFERS) assert code == 1 assert "UNEXPLAINED 'John Smith'" in out def test_main_reports_the_unexplained_field_under_its_role_name( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """The block exists to be copy-pasted into a ledger rule, so the label it prints must be the label a rule needs. The facade calls this role `last`; a rule saying `last` never matches.""" _, out = _run_main( tmp_path, monkeypatch, '[[change]]\nissue = "unrelated"\nname_regex = "ZZZ"\n', _DIFFERS) assert "family:" in out and "last:" not in out def test_main_exits_0_when_every_diff_is_claimed( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: code, out = _run_main( tmp_path, monkeypatch, '[[change]]\nissue = "claimed"\nfields = ["family"]\n', _DIFFERS) assert code == 0 assert "UNEXPLAINED" not in out assert "## claimed (1)" in out def test_main_validates_the_ledger_before_running_anything( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """validate_rules has its own tests; this pins that main CALLS it. Deleting the call leaves those tests passing while a match-anything rule shadows the ledger.""" with pytest.raises(SystemExit, match="matches every one of"): _run_main(tmp_path, monkeypatch, '[[change]]\nissue = "wide"\nname_regex = ""\n', _DIFFERS) def test_main_sorts_a_name_regex_rule_ahead_of_a_fields_only_one( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """A broad fields-only rule written FIRST must not claim a diff the specific name_regex rule below it owns. Deleting main's _sorted_rules call leaves _sorted_rules' own test passing.""" _, out = _run_main( tmp_path, monkeypatch, '[[change]]\nissue = "broad"\nfields = ["family"]\n' '[[change]]\nissue = "specific"\nname_regex = "Smith"\n', _DIFFERS) assert "## specific (1)" in out and "broad" not in out def test_check_tree_accepts_the_checkout_and_rejects_anything_else( tmp_path: Path) -> None: """The tree side is the half that had no proof at all: the baseline gets a pinned wheel, a temp dir and a version tell, while the tree was a bare import trusted on sight.""" inside = _TOOLS.parents[1] / "nameparser" / "__init__.py" assert compare._check_tree(str(inside)) == inside.resolve() with pytest.raises(SystemExit, match="not from this checkout's source"): compare._check_tree(str(tmp_path / "nameparser" / "__init__.py")) def test_main_aborts_when_the_tree_side_is_not_the_checkout( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Pins that main CALLS the tree check, not merely that the check works. Measured 2026-08-05: with a released 2.0.0 on PYTHONPATH, compare.py imported THAT and reported `intentional diffs: 0`, exit 0 -- both halves of the baseline tell passing. Run as a script, sys.path[0] is tools/differential/, which holds no nameparser, so PYTHONPATH outranks the editable install. REPO_ROOT is moved rather than the module, because relocating the import is what the trap does and this reproduces its EFFECT: the tree's nameparser is no longer under the root it must be under. """ monkeypatch.setattr(compare, "REPO_ROOT", tmp_path) with pytest.raises(SystemExit, match="not from this checkout's source"): _run_main(tmp_path, monkeypatch, '[[change]]\nissue = "x"\nname_regex = "ZZZ"\n', _DIFFERS) def test_worker_env_strips_the_import_path_overrides( monkeypatch: pytest.MonkeyPatch) -> None: """PEP 723 isolation does not survive PYTHONPATH -- it precedes site-packages, so a directory named there shadows the pinned wheel inside uv's own environment.""" monkeypatch.setenv("PYTHONPATH", "/somewhere/else") monkeypatch.setenv("PYTHONHOME", "/elsewhere") monkeypatch.setenv("PATH", "/usr/bin") env = compare._worker_env() assert "PYTHONPATH" not in env and "PYTHONHOME" not in env assert env["PATH"] == "/usr/bin", "the rest of the env must survive" class _FakePopen: """Records how _run_worker spawned the child, and replays a canned stdout. Lets the subprocess-facing guards be tested without uv.""" last: dict = {} out: str = "" rc: int = 0 def __init__(self, argv: list[str], **kw: object) -> None: _FakePopen.last = {"argv": argv, **kw} self.returncode = _FakePopen.rc def communicate(self, payload: str) -> tuple[str, str]: _FakePopen.last["stdin"] = payload return _FakePopen.out, "" def _fake_popen(monkeypatch: pytest.MonkeyPatch, out: str, rc: int = 0) -> type[_FakePopen]: _FakePopen.out, _FakePopen.rc = out, rc monkeypatch.setattr(compare.subprocess, "Popen", _FakePopen) return _FakePopen _TELL = ('{"__version__": "1.4.0", ' '"__file__": "/wheel/nameparser/__init__.py"}') _ROW = '{"facade": {"first": "John"}}' def test_run_worker_strips_the_import_path_overrides_from_the_child( monkeypatch: pytest.MonkeyPatch) -> None: """_worker_env has its own test; this pins that _run_worker USES it. Deleting `env=_worker_env()` left all 61 tests green -- the same shape as the bug the previous review found, a proved helper with an unproved call site.""" monkeypatch.setenv("PYTHONPATH", "/shadow") _fake_popen(monkeypatch, f"{_TELL}\n{_ROW}\n") compare._run_worker("1.4.0", False, ["John Smith"]) env = _FakePopen.last["env"] assert "PYTHONPATH" not in env and "PYTHONHOME" not in env def test_run_worker_aborts_on_a_nonzero_exit( monkeypatch: pytest.MonkeyPatch) -> None: _fake_popen(monkeypatch, "", rc=3) with pytest.raises(SystemExit, match="exited 3"): compare._run_worker("1.4.0", False, ["John Smith"]) def test_run_worker_aborts_on_empty_output( monkeypatch: pytest.MonkeyPatch) -> None: _fake_popen(monkeypatch, "") with pytest.raises(SystemExit, match="not even a version tell"): compare._run_worker("1.4.0", False, ["John Smith"]) def test_run_worker_aborts_when_fewer_results_than_names( monkeypatch: pytest.MonkeyPatch) -> None: """The guard behind main's zip(), which truncates silently. This is the comparing-fewer-names-than-you-think failure.""" _fake_popen(monkeypatch, f"{_TELL}\n{_ROW}\n") with pytest.raises(SystemExit, match="1 results for 2 corpus names"): compare._run_worker("1.4.0", False, ["John Smith", "Jane Doe"]) def test_run_worker_checks_the_tell_before_returning_results( monkeypatch: pytest.MonkeyPatch) -> None: wrong = ('{"__version__": "9.9.9", ' '"__file__": "/wheel/nameparser/__init__.py"}') _fake_popen(monkeypatch, f"{wrong}\n{_ROW}\n") with pytest.raises(SystemExit, match="not the requested"): compare._run_worker("1.4.0", False, ["John Smith"]) @pytest.mark.parametrize("rel", [ ".venv/lib/python3.11/site-packages/nameparser/__init__.py", "build/lib/nameparser/__init__.py", "dist/unpacked/nameparser/__init__.py", ]) def test_check_tree_rejects_a_wheel_sitting_inside_the_checkout( rel: str) -> None: """The hole in the first version of this guard. It asked "is this under the repo", but the repo contains .venv/, build/ and dist/, any of which can hold a released wheel -- so PYTHONPATH=/build/lib was the same trap one directory to the left, and uv never touches build/ to self-heal it.""" with pytest.raises(SystemExit, match="not from this checkout's source"): compare._check_tree(str(compare.REPO_ROOT / rel)) def test_check_tree_resolves_before_comparing() -> None: """Without .resolve(), a path escaping via .. reads as inside.""" escaped = compare.REPO_ROOT / "nameparser" / ".." / ".." / "x" \ / "nameparser" / "__init__.py" with pytest.raises(SystemExit, match="not from this checkout's source"): compare._check_tree(str(escaped)) #: The tree's own reading of the fixture name, on both surfaces. A fake #: baseline row built from these differs from the tree in exactly the #: one field a test chooses to alter. _SAME_FACADE = {"title": "", "first": "John", "middle": "", "last": "Smith", "suffix": "", "nickname": "", "maiden": ""} _SAME_V2 = {"title": "", "given": "John", "middle": "", "family": "Smith", "suffix": "", "nickname": "", "maiden": "", "_ambiguities": []} def test_main_compares_the_v2_surface_from_baseline_2_0( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """A SEGMENTATION-only diff is facade-identical by construction, so it is invisible unless main actually unions the v2 surface into the diff set. That diff shape is the whole stated reason _surfaces_for compares v2 from 2.0 on -- and every mutation that disabled it (want_v2 forced False, the v2 union deleted, `|=` changed to `=`) passed the suite before this test existed. """ v2 = {**_SAME_V2, "_ambiguities": ["SEGMENTATION"]} code, out = _run_main( tmp_path, monkeypatch, '[[change]]\nissue = "unrelated"\nname_regex = "ZZZ"\n', _SAME_FACADE, baseline="2.0.0", baseline_v2=v2) assert code == 1, "an ambiguity-only regression must not exit 0" assert "UNEXPLAINED 'John Smith'" in out assert "_ambiguities:" in out assert "[v2 surface only]" in out, ( "the tag distinguishes an ambiguity-kind change from a field " "change; without it the row reads as a field diff") def test_main_claims_an_ambiguity_only_diff_when_a_rule_names_it( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: v2 = {**_SAME_V2, "_ambiguities": ["SEGMENTATION"]} code, out = _run_main( tmp_path, monkeypatch, '[[change]]\nissue = "seg"\nfields = ["_ambiguities"]\n', _SAME_FACADE, baseline="2.0.0", baseline_v2=v2) assert code == 0 and "## seg (1)" in out def test_main_reports_a_role_once_when_both_surfaces_moved( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """The `seen` set. Both surfaces name the same role, so a family change shows on each; printing it twice would read as two findings.""" _, out = _run_main( tmp_path, monkeypatch, '[[change]]\nissue = "unrelated"\nname_regex = "ZZZ"\n', {**_SAME_FACADE, "last": "SMYTHE"}, baseline="2.0.0", baseline_v2={**_SAME_V2, "family": "SMYTHE"}) assert out.count("family:") == 1 def test_main_forwards_the_baseline_and_corpus_to_the_worker( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Otherwise main could read the 2.0 ledger while comparing against 1.4, or compare a truncated corpus, and every other test would pass.""" _run_main(tmp_path, monkeypatch, '[[change]]\nissue = "x"\nname_regex = "ZZZ"\n', _SAME_FACADE, baseline="2.0.0", baseline_v2=_SAME_V2) assert _WORKER_CALL == {"version": "2.0.0", "want_v2": True, "names": ["John Smith"]} def test_main_asks_for_the_facade_alone_below_2_0( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: _run_main(tmp_path, monkeypatch, '[[change]]\nissue = "x"\nname_regex = "ZZZ"\n', _SAME_FACADE) assert _WORKER_CALL["want_v2"] is False def test_every_shipped_corpus_has_a_floor_and_clears_it() -> None: """Two bindings, so neither half can rot alone: every corpus file on disk must have a floor, and must be at or above it. The floor exists because the empty-file guard only catches a corpus that lost EVERY name. One truncated to a handful passes that guard, and the run exits 0 having compared a fraction of what its summary line reports -- the harness's own stated nightmare, reached by a file that is merely short rather than absent. """ import json corpora = sorted(_TOOLS.glob("corpus*.jsonl")) assert corpora, "no corpora found; this test would pass vacuously" for path in corpora: names = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] floor = compare._CORPUS_FLOORS.get(path.name) assert floor is not None, ( f"{path.name} has no _CORPUS_FLOORS entry; add one a little " f"under its {len(names)} names") assert len(names) >= floor, ( f"{path.name} holds {len(names)}, below its floor {floor}") def test_a_floor_names_a_corpus_that_exists() -> None: """The other direction: a floor for a file nobody ships is a guard that can never fire, and reads as coverage that is not there.""" on_disk = {p.name for p in _TOOLS.glob("corpus*.jsonl")} assert set(compare._CORPUS_FLOORS) <= on_disk def test_main_aborts_on_a_truncated_corpus( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """A corpus below its floor must stop the run, not shrink it.""" with pytest.raises(SystemExit, match="below its floor"): _run_main(tmp_path, monkeypatch, '[[change]]\nissue = "x"\nname_regex = "ZZZ"\n', _SAME_FACADE, floor=50) def test_main_aborts_on_a_corpus_with_no_floor( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Adding a corpus without a floor must be a decision, not a silent default -- the same force-a-decision shape the Script tables use.""" with pytest.raises(SystemExit, match="no entry in _CORPUS_FLOORS"): _run_main(tmp_path, monkeypatch, '[[change]]\nissue = "x"\nname_regex = "ZZZ"\n', _SAME_FACADE, floor=None) python-nameparser-2.1.0/tests/v2/test_facade.py000066400000000000000000000534301523530431700215030ustar00rootroot00000000000000"""The 2.0 HumanName facade (migration spec §2).""" import pickle import warnings from pathlib import Path import pytest from nameparser._config_shim import CONSTANTS, Constants from nameparser._facade import HumanName from nameparser._types import Role _DATA_DIR = Path(__file__).parent / "data" def test_basic_parse_and_v1_spellings() -> None: n = HumanName("Dr. Juan de la Vega III") assert n.title == "Dr." assert n.first == "Juan" # v1 spelling of core 'given' assert n.last == "de la Vega" # v1 spelling of core 'family' assert n.suffix == "III" assert n.original == "Dr. Juan de la Vega III" assert n.full_name == "Dr. Juan de la Vega III" def test_bytes_raises_with_decode_hint() -> None: # #245 with pytest.raises(TypeError, match="decode"): HumanName(b"John Smith") # type: ignore[arg-type] def test_constants_none_raises_migration_hint() -> None: # #261 with pytest.raises(TypeError, match="constants"): HumanName("John Smith", constants=None) def test_full_name_assignment_reparses() -> None: n = HumanName("John Smith") n.full_name = "Jane Doe" assert (n.first, n.last) == ("Jane", "Doe") def test_private_constants_mutation_honored_lazily() -> None: # NB: the task spec's example used "dame" as a marker not in the # default titles, but nameparser/config/titles.py already lists # "dame" -- swapped in a nonsense marker to keep the test's premise # (mutate a private Constants, next parse sees the change) true. c = Constants() n = HumanName("Zzqtitle Judy Dench", constants=c) assert n.title == "" # 'zzqtitle' not a default title c.titles.add("zzqtitle") n.full_name = "Zzqtitle Judy Dench" # next parse sees the change assert n.title == "Zzqtitle" def test_capitalize_name_flag_capitalizes_on_parse() -> None: c = Constants() c.capitalize_name = True n = HumanName("john smith", constants=c) assert (n.first, n.last) == ("John", "Smith") def test_constructor_suffix_delimiter_layers_onto_policy() -> None: n = HumanName("Doe, John, RN - CRNA", constants=Constants(), suffix_delimiter=" - ") assert n.suffix == "RN, CRNA" def test_subclass_overriding_no_hooks_never_warns() -> None: class Plain(HumanName): pass with warnings.catch_warnings(): warnings.simplefilter("error") Plain("John Smith") def test_subclass_hook_override_warns_once_per_class() -> None: # #280 class Custom(HumanName): def parse_pieces(self, parts: object, additional_parts_count: int = 0) -> object: return parts with pytest.deprecated_call(match="parse_pieces"): Custom("John Smith") with warnings.catch_warnings(): warnings.simplefilter("error") Custom("Jane Doe") # second instance: silent def test_c_property_and_has_own_config() -> None: n = HumanName("John Smith") assert n.C is CONSTANTS assert n.has_own_config is False m = HumanName("John Smith", constants=Constants()) assert m.has_own_config is True def test_dirty_tracking_reuses_cached_parser_across_unchanged_generation() -> None: n = HumanName("John Smith") assert n._resolve() is n._resolve() c = Constants() m = HumanName("John Smith", constants=c) parser_before = m._resolve() c.titles.add("zzq") # bumps the generation parser_after = m._resolve() assert parser_before is not parser_after def test_component_kwargs_bypass_parsing() -> None: n = HumanName(first="John", last="de la Vega") assert n.first == "John" and n.last == "de la Vega" assert n.full_name == "" def test_field_assignment_str_list_none() -> None: n = HumanName("John Smith") n.first = "Jane" assert n.first == "Jane" n.middle = ["Q", "Xavier"] # lists join with space assert n.middle == "Q Xavier" n.middle = None # None clears assert n.middle == "" assert n.last == "Smith" # untouched fields survive assert n.full_name == "John Smith" # no re-parse (v1 parity) def test_list_attributes_are_snapshots() -> None: # spec §2 exc. 1 n = HumanName("John Quincy Adams Smith") lst = n.middle_list lst.append("HACKED") assert "HACKED" not in n.middle_list def test_derived_views() -> None: n = HumanName("Juan Q. de la Vega") assert n.surnames == "Q. de la Vega" # middle + last (v1 shape) assert n.given_names == "Juan Q." # first + middle assert n.last_prefixes == "de la" assert n.last_base == "Vega" def test_split_last_all_prefix_guard() -> None: n = HumanName(last="de la") # every word is a particle assert n.last_prefixes == "" # v1 guard: no stripping assert n.last_base == "de la" @pytest.mark.parametrize("member", [ "title", "first", "middle", "last", "suffix", "nickname", "maiden", ]) def test_every_member_set_get_and_list(member: str) -> None: n = HumanName() setattr(n, member, "Alpha Beta") # the suffix string view joins parts with ", " (v1 parity) joined = "Alpha, Beta" if member == "suffix" else "Alpha Beta" assert getattr(n, member) == joined assert getattr(n, f"{member}_list") == ["Alpha", "Beta"] def test_list_assignment_rejects_non_str_elements() -> None: n = HumanName("John Smith") with pytest.raises(TypeError, match="strings"): n.first = [1, 2] # type: ignore[list-item] with pytest.raises(TypeError, match="strings"): n.first = [None] # type: ignore[list-item] def test_list_assignment_multiword_elements_join() -> None: n = HumanName("John Smith") n.middle = ["ab", "cd ef"] assert n.middle == "ab cd ef" def test_suffix_list_heals_joined_continuations() -> None: # v1 fix_phd n = HumanName("John Ph. D.") assert n.suffix == "Ph. D." assert n.suffix_list == ["Ph. D."] # ONE element, v1 parity def test_str_uses_string_format_with_v1_cleanup() -> None: n = HumanName("Dr. Juan de la Vega III") assert str(n) == "Dr. Juan de la Vega III" n2 = HumanName("John Smith") # empty nickname: no ' ()' assert str(n2) == "John Smith" n2.string_format = "{last}, {first}" assert str(n2) == "Smith, John" def test_str_none_format_falls_back_to_space_join() -> None: # v1-identical: the format wraps the nickname in parens, the plain # member join does not n = HumanName('John "Jack" Kennedy') assert str(n) == "John Kennedy (Jack)" n.string_format = None assert str(n) == "John Kennedy Jack" def test_getitem_unknown_key_raises_attribute_error() -> None: n = HumanName("John Smith") with pytest.raises(AttributeError): # v1 behavior, not KeyError n["nope"] def test_repr_v1_shape() -> None: r = repr(HumanName("John Smith")) assert r.startswith(" None: n = HumanName("John Smith") assert list(n) == ["John", "Smith"] assert len(n) == 2 assert len(HumanName("")) == 0 # documented emptiness check def test_getitem_str_ok_slice_raises() -> None: # #258 n = HumanName("John Smith") assert n["first"] == "John" with pytest.raises(TypeError, match="#258"): n[1:-1] # type: ignore[index] with pytest.raises(TypeError): n["first"] = "Jane" # type: ignore[index] # no __setitem__ in 2.0 def test_getitem_accepts_role_members() -> None: # Role is a StrEnum, so Role members reach __getitem__'s getattr; # the v1 spellings (first/last) must resolve for given/family too. hn = HumanName("Dr. John Quincy Smith Jr.") assert hn[Role.TITLE] == hn.title assert hn[Role.GIVEN] == hn.first assert hn[Role.MIDDLE] == hn.middle assert hn[Role.FAMILY] == hn.last assert hn[Role.SUFFIX] == hn.suffix assert hn[Role.NICKNAME] == hn.nickname assert hn[Role.MAIDEN] == hn.maiden # the plain strings work the same way assert hn["given"] == hn.first assert hn["family"] == hn.last def test_assigning_v2_field_spellings_warns_but_still_sets() -> None: # "given"/"family" are 2.0 spellings the facade has no attribute # for, so plain assignment silently forks the name: a stray # instance attribute appears while the parse (and .first/.last) # keeps the old value. Warn -- but still set, so any v1-legal code # that worked keeps working. hn = HumanName("John Smith") with pytest.warns(UserWarning, match=r"use \.first"): hn.given = "Jane" # type: ignore[attr-defined] assert hn.given == "Jane" # type: ignore[attr-defined] # stray attr still set assert hn.first == "John" # the parse did not change with pytest.warns(UserWarning, match=r"use \.last"): setattr(hn, Role.FAMILY, "Doe") # StrEnum member, same trap def test_other_attribute_assignment_stays_silent() -> None: # The five 2.0 field names that ARE facade properties go through # their setters; the v1 spellings work; ad-hoc attribute stashing # is a legal v1 pattern. None of these warn. hn = HumanName("John Smith") with warnings.catch_warnings(): warnings.simplefilter("error") hn.title = "Dr." hn.first = "Jane" hn.my_cache = 42 # type: ignore[attr-defined] assert hn.title == "Dr." and hn.first == "Jane" def test_eq_and_hash_are_object_identity() -> None: # #223 a, b = HumanName("John Smith"), HumanName("John Smith") assert a != b and a == a assert hash(a) != hash(b) or a is b # default object hash def test_as_dict_v1_keys() -> None: n = HumanName("Dr. John Smith") d = n.as_dict() assert d["title"] == "Dr." and d["first"] == "John" and d["last"] == "Smith" assert set(n.as_dict(include_empty=False)) == {"title", "first", "last"} def test_initials_v1_semantics() -> None: assert HumanName("Sir Bob Andrew Dole").initials() == "B. A. D." assert HumanName("Sir Bob Andrew Dole").initials_list() == ["B", "A", "D"] n = HumanName("Doe, John A.", initials_delimiter="", initials_separator="") assert n.initials() == "J A D" # prefixes/conjunctions are filtered except in first names (v1 rule) assert HumanName("Juan de la Vega").initials() == "J. V." def test_initials_format_kwarg() -> None: n = HumanName("Sir Bob Andrew Dole", initials_format="{first} {middle}") assert n.initials() == "B. A." def test_render_default_setters_validate() -> None: n = HumanName("John Smith") with pytest.raises(TypeError, match="initials_delimiter"): n.initials_delimiter = 5 # type: ignore[assignment] with pytest.raises(TypeError, match="initials_format"): n.initials_format = 5 # type: ignore[assignment] with pytest.raises(TypeError, match="initials_separator"): n.initials_separator = None # type: ignore[assignment] with pytest.raises(TypeError, match="string_format"): n.string_format = 5 # type: ignore[assignment] with pytest.raises(TypeError, match="suffix_delimiter"): n.suffix_delimiter = 5 # type: ignore[assignment] n.string_format = None # None allowed for these two n.suffix_delimiter = None def test_capitalize_gate_and_force() -> None: n = HumanName("bob v. de la macdole-eisenhower phd") n.capitalize() assert str(n) == "Bob V. de la MacDole-Eisenhower Ph.D." m = HumanName("Shirley Maclaine") # mixed case: untouched m.capitalize() assert str(m) == "Shirley Maclaine" m.capitalize(force=True) assert str(m) == "Shirley MacLaine" def test_force_mixed_case_flag_feeds_default() -> None: c = Constants() c.force_mixed_case_capitalization = True n = HumanName("Shirley Maclaine", constants=c) n.capitalize() # force=None -> flag -> True assert str(n) == "Shirley MacLaine" def test_matches_and_comparison_key() -> None: # NB: the task spec's example compared "Dr. John A. Smith" against # "John Smith" -- but matches()/comparison_key() are component-wise # over all seven fields (v1 parity, verified against live 1.4), so a # title/middle-initial mismatch always fails the match; swapped in # case/order variations that actually exercise "case-insensitive, # component-wise" without dropping fields. n = HumanName("Dr. John A. Smith") assert n.matches("dr. john a. smith") assert n.matches(HumanName("smith, dr. john a.")) assert not n.matches("Jane Smith") assert n.comparison_key() == HumanName("DR. JOHN A. SMITH").comparison_key() def test_pickle_round_trip_preserves_components() -> None: n = HumanName("Dr. Juan de la Vega III") n.first = "Johan" # mutated state must survive loaded = pickle.loads(pickle.dumps(n)) assert loaded.first == "Johan" assert loaded.last == "de la Vega" assert loaded.original == "Dr. Juan de la Vega III" assert loaded.C is CONSTANTS # shared sentinel restored @pytest.mark.parametrize("bad", [ "John", # str: iterating yields characters {"John": 1}, # Mapping: iterating yields keys only b"John", # bytes: iterating yields ints ["John", None], # a non-str entry ]) def test_setstate_rejects_malformed_component_lists(bad: object) -> None: # _normset on the Lexicon side rejects str AND Mapping AND checks # entry types; this guard covered only str, so a dict silently # loaded its keys and bytes/None died later with an opaque # AttributeError instead of naming the field n = HumanName("John Smith") state = n.__getstate__() state["first_list"] = bad loaded = HumanName.__new__(HumanName) with pytest.raises(TypeError, match="first_list"): loaded.__setstate__(state) def test_setstate_restores_a_foreign_multi_word_suffix_entry() -> None: # the parametrized round-trip tests above go through v2's own # __getstate__, which is self-consistent by construction. The case # the fix actually protects is a 1.4-produced blob carrying a # multi-word suffix entry, so build that state directly. n = HumanName("John Smith") state = n.__getstate__() state["suffix_list"] = ["Ph. D."] loaded = HumanName.__new__(HumanName) loaded.__setstate__(state) assert loaded.suffix_list == ["Ph. D."] assert loaded.suffix == "Ph. D." #: Every name in the 486-name differential corpus whose HumanName pickle #: round-trip drifted before the __setstate__ element-boundary fix. Each #: carries a multi-word suffix entry -- a "joined" continuation token -- #: inside one comma segment. Verified against a live 1.4.0: all eight #: round-trip unchanged there, so drift here is a regression, not an #: inherited quirk. _MULTIWORD_SUFFIX_NAMES = [ "Andrew Perkins, Jr., Col. (Ret)", "Clarke, Kenneth, Q.C. M.P.", "Doe, John, MD PhD - FACS Fellow", "Franklin Washington, Jr. MD", "John Smith Ph. D.", "John Smith, Ph. D.", "John Smith, V Jr.", "John Smith, V MD", ] @pytest.mark.parametrize("raw", _MULTIWORD_SUFFIX_NAMES) def test_pickle_round_trip_preserves_multiword_suffix_entries(raw: str) -> None: # __setstate__ restores components from the pickled *_list values. # Joining them into one string and letting replace() re-split on # whitespace destroys the entry boundaries, and the suffix view's # ", " join then renders one credential as two ("Ph." + "D." -> # "Ph., D."). Entries must survive as entries. n = HumanName(raw) loaded = pickle.loads(pickle.dumps(n)) assert loaded.suffix_list == n.suffix_list assert loaded.suffix == n.suffix assert str(loaded) == str(n) assert loaded.as_dict() == n.as_dict() # -- Gap 2: parse_full_name() (v1's documented re-parse idiom) ------------- def test_parse_full_name_reparses_documented_idiom() -> None: c = Constants() n = HumanName("Zzqtitle Judy Dench", constants=c) assert n.title == "" # 'zzqtitle' not a default title c.titles.add("zzqtitle") # mutate C in place, no full_name reassignment n.parse_full_name() # documented v1 idiom assert n.title == "Zzqtitle" def test_subclass_overriding_parse_full_name_warns() -> None: # #280 class CustomReparse(HumanName): def parse_full_name(self) -> None: pass with pytest.deprecated_call(match="parse_full_name"): CustomReparse("John Smith") # -- Gap 3: HumanName.C setter (v1.4 #239) ---------------------------------- def test_c_setter_assigns_and_next_parse_honors_it() -> None: n = HumanName("John Smith") c2 = Constants() c2.titles.add("zzqtitle") n.C = c2 assert n.C is c2 n.full_name = "Zzqtitle Judy Dench" # next parse: v1's setter only stored, no reparse assert n.title == "Zzqtitle" def test_c_setter_none_raises_migration_hint() -> None: n = HumanName("John Smith") with pytest.raises(TypeError, match="constants"): n.C = None # type: ignore[assignment] def test_c_setter_non_constants_raises() -> None: n = HumanName("John Smith") with pytest.raises(TypeError): n.C = 42 # type: ignore[assignment] # -- Gap 4: matches() error message names the facade type ------------------- def test_matches_type_error_names_facade_type() -> None: n = HumanName("John Smith") with pytest.raises(TypeError, match="HumanName"): n.matches(None) # type: ignore[arg-type] def test_v14_humanname_blob_unpickles() -> None: # tests/v2/data/humanname_v14.pickle was produced by a real 1.4.0 # install (`uv run --no-project --with "nameparser==1.4.*" ...`) on # HumanName("Dr. Juan de la Vega III"); components come back exactly # as pickled, NOT re-parsed. Since the M11 swap replaced # nameparser.parser.HumanName with this facade, the blob's pickled # class reference now resolves here (mirrors # test_v14_constants_blob_unpickles_into_shim in test_config_shim.py). # pickle.load is safe here: humanname_v14.pickle is a repo-controlled # fixture generated by this task from a real 1.4.0 install (Step 2 # above), not data from an untrusted source. with open(_DATA_DIR / "humanname_v14.pickle", "rb") as f: loaded = pickle.load(f) assert isinstance(loaded, HumanName) assert loaded.first == "Juan" and loaded.last == "de la Vega" assert loaded.title == "Dr." and loaded.suffix == "III" assert loaded.original == "Dr. Juan de la Vega III" assert loaded.C is CONSTANTS # shared sentinel restored (v1.4's C: None) def test_suffix_delimiter_reassignment_after_construction() -> None: n = HumanName("Doe, John, RN - CRNA") assert n.suffix == "RN - CRNA" # no delimiter: one entry n.suffix_delimiter = " - " n.parse_full_name() assert n.suffix == "RN, CRNA" # setter invalidated the Policy n.suffix_delimiter = None n.parse_full_name() assert n.suffix == "RN - CRNA" # and back def test_c_swap_keeps_instance_suffix_delimiter() -> None: # both invalidation paths together: swapping C must not lose the # instance-level delimiter layered onto the new snapshot's Policy n = HumanName("Doe, John, RN - CRNA", suffix_delimiter=" - ") assert n.suffix == "RN, CRNA" n.C = Constants() n.parse_full_name() assert n.suffix == "RN, CRNA" def test_failed_reparse_leaves_state_consistent() -> None: # _resolve() raising must not leave full_name pointing at the new # value over the OLD parsed fields n = HumanName("John Smith") class _Boom(Exception): pass def explode() -> None: raise _Boom n._C = Constants() n._C._snapshot = explode # type: ignore[method-assign, assignment] n._snapshot_gen = -1 with pytest.raises(_Boom): n.full_name = "Jane Doe" assert n.full_name == "John Smith" assert n.first == "John" def test_middle_as_family_list_view_matches_string_view() -> None: # v1 PREPENDED middle_list to last_list; the list view must agree # with the string view, not with raw token order c = Constants() c.middle_name_as_last = True n = HumanName("Hassan, Mohamad Ahmad Ali", constants=c) assert n.last == "Ahmad Ali Hassan" assert n.last_list == ["Ahmad", "Ali", "Hassan"] def test_cold_import_order_config_first() -> None: # the _facade cycle-breaker (import nameparser.config first) is # order-dependent; this pins BOTH cold-start directions in fresh # interpreters so an import-sorting pass cannot silently break one import subprocess import sys for first in ("nameparser.config", "nameparser._facade"): proc = subprocess.run( [sys.executable, "-c", f"import {first}; import nameparser; " f"print(nameparser.HumanName('John Smith').last)"], capture_output=True, text=True) assert proc.returncode == 0, (first, proc.stderr) assert proc.stdout.strip() == "Smith" def test_facade_reads_wholly_han_names_family_first() -> None: # classified fix(#271): script_orders reaches the v1 surface. # Constants has no opt-out knob (v1 surface is frozen); the # migration path is v2 Policy(script_orders={}). n = HumanName("毛 泽东") assert (n.last, n.first) == ("毛", "泽东") # unsplit (Han segmentation is locales.ZH, v2-only), but the lone # wholly-Han token now takes the family role, not first assert HumanName("毛泽东").last == "毛泽东" assert HumanName("毛泽东").first == "" def test_facade_reads_kana_licensed_names_family_first() -> None: n = HumanName("高橋 みなみ") assert (n.last, n.first) == ("高橋", "みなみ") def test_facade_parses_unspaced_korean_by_default() -> None: # the default-behavior fix flows through the v1 facade (its # lexicon mirrors Lexicon.default() via the shim snapshot) n = HumanName("김민준") assert (n.last, n.first) == ("김", "민준") python-nameparser-2.1.0/tests/v2/test_facade_cases.py000066400000000000000000000150651523530431700226630ustar00rootroot00000000000000"""Facade runner (migration spec §5): the shared case table asserted through HumanName. Deleted wholesale in 3.0 with the facade.""" import dataclasses import pytest from nameparser import Policy from nameparser._config_shim import Constants from nameparser._facade import HumanName from .cases import CASES, Case _V1_KEY = {"given": "first", "family": "last"} # identity for the rest #: The parenthesis maiden shape. A row routing parens to maiden does #: NOT carry the default nickname set: Policy's maiden-wins #: canonicalization subtracts the pair from nickname_delimiters at #: CONSTRUCTION, so the shape to match is the default MINUS the pair. #: Comparing against the unsubtracted default is what kept this gate #: false for every row until #329. #: #: The v1 side has two spellings and only one of them is equivalent. #: ADDING parenthesis to the maiden bucket leaves the nickname bucket #: holding it too, and v1 gives a shared pair to nickname -- measured on #: "Baker (Johnson), Jenny", that spelling yields nickname Johnson and #: an empty maiden, the reverse of what maiden_delimiters_win_when_shared #: asserts. MOVING it (pop from nickname, assign to maiden) yields #: maiden Johnson and an empty nickname, matching. That is the idiom #: tests/test_nicknames.py already uses, and the one below. #: #: Which rows this leaves skipping is not described here but asserted, #: by id, in test_core_only_rows_are_the_declared_ones below. _MAIDEN_PARENS = frozenset({("(", ")")}) #: Policy fields _constants_for does not translate. A row moving one #: off its default SKIPS: the facade would otherwise run it under the #: field's inherited default and pass or fail on a knob the row never #: asked for. Some have no v1 spelling at all -- script_orders and #: segment_scripts, the v1 surface being frozen -- and the rest have #: none this runner has ever needed to write. _UNTRANSLATED = frozenset({ "name_order", "script_orders", "segment_scripts", "lenient_comma_suffixes", "strip_emoji", "strip_bidi", }) #: Expressible in exactly one shape, the parenthesis bucket move. #: BOTH delimiter fields are listed because the canonicalization that #: routes the pair to maiden is the same step that takes it out of #: nickname, so a row expressible this way necessarily differs from #: the default on both. _BUCKET_MOVE_ONLY = frozenset({ "nickname_delimiters", "maiden_delimiters", }) #: Fields _constants_for writes into the Constants it returns. _TRANSLATED = frozenset({ "patronymic_rules", "middle_as_family", "extra_suffix_delimiters", }) #: The non-locale rows the facade never sees. Held here rather than #: described in prose, because pytest reports a skipped row exactly #: like a row nobody wrote: reverting the _MAIDEN_PARENS shape above #: to the unsubtracted default pushes every parenthesis-maiden row #: into this set and turns nothing red. _CORE_ONLY_IDS = frozenset({ "maiden_marker_delimited_beside_a_nickname_clause", "maiden_marker_kyusei_delimited", "ko_honorific_period_under_strict_comma_suffixes", "ja_honorific_glued_family_comma_strict_knob", "ja_honorific_glued_family_comma_credential_pair_strict_knob", }) def _constants_for(case: Case) -> Constants | None: """Translate the row's Policy to a Constants, or None if the policy has no v1 spelling (those rows are core-only).""" policy = case.policy or Policy() default = Policy() c = Constants() maiden_via_bucket_move = ( policy.maiden_delimiters == _MAIDEN_PARENS and policy.nickname_delimiters == default.nickname_delimiters - _MAIDEN_PARENS ) # field by field, off the dataclass rather than by hand: a field # named in no table above would be admitted here and then run # under the facade's default for it (see # test_every_policy_field_is_translated_or_skipped). moved = {f.name for f in dataclasses.fields(Policy) if getattr(policy, f.name) != getattr(default, f.name)} if moved & _UNTRANSLATED or (moved & _BUCKET_MOVE_ONLY and not maiden_via_bucket_move): return None if policy.patronymic_rules: c.patronymic_name_order = True if policy.middle_as_family: c.middle_name_as_last = True if policy.extra_suffix_delimiters: c.suffix_delimiter = next(iter(policy.extra_suffix_delimiters)) if maiden_via_bucket_move: # pop, not assign: leaving parenthesis in the nickname bucket # would give v1's shared-pair reading, which goes to nickname. c.maiden_delimiters["parenthesis"] = c.nickname_delimiters.pop( "parenthesis") return c def test_every_policy_field_is_translated_or_skipped() -> None: # a Policy field in none of the three tables is neither rejected # nor written into the Constants, so a row setting it is ADMITTED # and then asserted under the facade's default value for that # field. segment_scripts sat in exactly that state until # 2026-08-03, invisibly, because no row set it. Same job # test_policy_patch_mirrors_policy_field_names does one file over. assert (_UNTRANSLATED | _BUCKET_MOVE_ONLY | _TRANSLATED == {f.name for f in dataclasses.fields(Policy)}) def test_core_only_rows_are_the_declared_ones() -> None: # the gate that decides this is one comparison away from claiming # no policy is expressible (see _MAIDEN_PARENS), and every row it # wrongly rejects leaves the suite green -- a skip reads as a pass. assert {case.id for case in CASES if case.locale is None and _constants_for(case) is None} == _CORE_ONLY_IDS @pytest.mark.parametrize("case", CASES, ids=lambda c: c.id) def test_facade_case(case: Case) -> None: if case.locale is not None: # v1 Constants' patronymic bool enables BOTH the East Slavic and # Turkic rules at once, so it cannot express a single-rule pack # faithfully -- these rows are core-only (proven via parser_for # in test_cases.py instead). pytest.skip("locale rows are core-only") constants = _constants_for(case) if constants is None: pytest.skip("policy not expressible through v1 Constants") name = HumanName(case.text, constants=constants) for field, expected in case.expect.items(): assert getattr(name, _V1_KEY.get(field, field)) == expected, ( f"{case.id}: {field}") for field in {"title", "given", "middle", "family", "suffix", "nickname", "maiden"} - set(case.expect): assert getattr(name, _V1_KEY.get(field, field)) == "", ( f"{case.id}: {field} expected empty") python-nameparser-2.1.0/tests/v2/test_layering.py000066400000000000000000000251421523530431700221110ustar00rootroot00000000000000"""Enforce the conventions doc's contracts mechanically: import layering, public exports, and the pickle layout guards.""" import ast import pathlib import nameparser PKG = pathlib.Path(nameparser.__file__).parent # ALLOWED keys whose module must exist on disk -- the exists() skip in # test_layering_contract is for entries that precede their module within # a plan, and without this list a renamed existing module would silently # drop out of enforcement. Later tasks add each stage file as it lands. _MUST_EXIST = {"_types.py", "_lexicon.py", "_policy.py", "_locale.py", "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py", "_pipeline/_extract.py", "_pipeline/_tokenize.py", "_pipeline/_vocab.py", "_pipeline/_script_segment.py", "_pipeline/_segment.py", "_pipeline/_classify.py", "_pipeline/_group.py", "_pipeline/_assign.py", "_pipeline/_post_rules.py", "_pipeline/_assemble.py", "_parser.py", "_facade.py", "_config_shim.py", "locales/__init__.py", "locales/ja.py", "locales/ru.py", "locales/tr_az.py", "locales/zh.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", # stages share _state plus in-package helpers (_vocab, _group's # piece predicates); the prefix still forbids _render/_locale/_parser "nameparser._pipeline.", ) # a locale pack: pure data over the base types, no pipeline or config # access (locales spec §2) -- one contract shared by every pack, like # _PIPELINE_STAGE_ALLOWED above, so tightening it is a one-line edit. # _types joined the three in #272: the ja pack's segmenter factory # constructs a Segmentation, and _types is the bottom of the graph # (Locale itself imports it), so this widens the base types, not the # reach. _LOCALE_PACK_ALLOWED = ( "nameparser._lexicon", "nameparser._locale", "nameparser._policy", "nameparser._types", ) # module -> prefixes it may import from within nameparser ALLOWED = { # call-time imports only (inside the rendering-delegate method # bodies); module level stays import-free. TYPE_CHECKING imports # are skipped by _nameparser_imports and need no entry. "_types.py": ("nameparser._render", "nameparser._parser"), # intent: DATA modules only, during 2.x -- mechanically this admits # any config submodule; "data only" holds by convention/review "_lexicon.py": ("nameparser.config.",), "_policy.py": ("nameparser._types",), # _types is a downward import (bottom of the graph): Locale shares # the _guarded_getstate/_guarded_setstate pickle functions "_locale.py": ("nameparser._types", "nameparser._lexicon", "nameparser._policy"), # _lexicon is needed at runtime: capitalized(lexicon=None) resolves # to Lexicon.default() "_render.py": ("nameparser._types", "nameparser._lexicon"), # every stage module: state + the three config/type modules "_pipeline/_state.py": ("nameparser._types", "nameparser._lexicon", "nameparser._policy"), "_pipeline/__init__.py": ("nameparser._pipeline.",), "_pipeline/_vocab.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_extract.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_tokenize.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_script_segment.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_segment.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_classify.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_group.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_assign.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_post_rules.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_assemble.py": _PIPELINE_STAGE_ALLOWED, # Parser sits on everything except _render and the facade "_parser.py": ("nameparser._types", "nameparser._lexicon", "nameparser._policy", "nameparser._locale", "nameparser._pipeline"), # facade layer (migration spec §2/§3): may import anything public # plus _render (unused yet) and each other. _facade delegates # parsing to the core Parser resolved from the bound Constants shim. # The bare "nameparser.config" import (not just its submodules) is # deliberate -- it resolves an import cycle, see _facade.py's comment. "_facade.py": ("nameparser._config_shim", "nameparser._lexicon", "nameparser._parser", "nameparser._types", "nameparser._render", "nameparser.util", "nameparser.config"), # the Constants/SetManager/TupleManager shim: config DATA modules # stay the single vocabulary source through 2.x (nameparser.config.*) "_config_shim.py": ("nameparser._lexicon", "nameparser._parser", "nameparser._policy", "nameparser.util", "nameparser.config"), # PEP 562 lazy loader: the only static import is _locale (for the # Locale return type); pack modules load via importlib.import_module # (a runtime string, invisible to the AST walker) -- the # "nameparser.locales." prefix documents that reach for humans even # though the walker never exercises it. "locales/__init__.py": ("nameparser._locale", "nameparser.locales."), "locales/ja.py": _LOCALE_PACK_ALLOWED, "locales/ru.py": _LOCALE_PACK_ALLOWED, "locales/tr_az.py": _LOCALE_PACK_ALLOWED, "locales/zh.py": _LOCALE_PACK_ALLOWED, # v1 import-path preservation: thin re-exports of the facade/shim "parser.py": ("nameparser._facade",), "config/__init__.py": ("nameparser._config_shim",), # CLI (migration spec §6): imports only the public package, same as # any other consumer -- no access to internal modules. "__main__.py": ("nameparser",), } def _is_type_checking(test: ast.expr) -> bool: return (isinstance(test, ast.Name) and test.id == "TYPE_CHECKING") or ( isinstance(test, ast.Attribute) and test.attr == "TYPE_CHECKING") def _nameparser_imports(path: pathlib.Path) -> list[str]: """All nameparser-internal imports in the module, at any nesting depth, EXCEPT those under `if TYPE_CHECKING:` -- annotation-only imports are not runtime dependencies.""" found: list[str] = [] def _record(node: ast.AST) -> None: # guard checked on EVERY node uniformly, so `elif TYPE_CHECKING:` # (a nested If in orelse) is skipped exactly like the plain form if isinstance(node, ast.If) and _is_type_checking(node.test): for stmt in node.orelse: _record(stmt) return if isinstance(node, ast.Import): found.extend(a.name for a in node.names) elif isinstance(node, ast.ImportFrom) and node.module: found.append(node.module) for child in ast.iter_child_nodes(node): _record(child) _record(ast.parse(path.read_text())) return [m for m in found if m.startswith("nameparser")] def _permitted(imported: str, allowed: tuple[str, ...]) -> bool: # An entry ending in "." is a pure prefix (subpackage contents only); # any other entry means that exact module or its submodules -- a bare # startswith would also admit siblings like nameparser._types_helpers. for entry in allowed: if entry.endswith("."): if imported.startswith(entry): return True elif imported == entry or imported.startswith(entry + "."): return True return False def test_layering_contract() -> None: for mod, allowed in ALLOWED.items(): path = PKG / mod if not path.exists(): assert mod not in _MUST_EXIST, ( f"{mod} keyed in ALLOWED but file is missing") continue # entries may precede their module within a plan for imported in _nameparser_imports(path): assert _permitted(imported, allowed), ( f"{mod} imports {imported}, which the layering contract " f"forbids (allowed prefixes: {allowed or 'none'})" ) def test_lexicon_never_imports_config_package_root_or_parser() -> None: for imported in _nameparser_imports(PKG / "_lexicon.py"): assert imported != "nameparser.config" assert not imported.startswith("nameparser.parser") def test_public_exports() -> None: expected = { "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "Segmentation", "Segmenter", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "Script", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "DEFAULT_NICKNAME_DELIMITERS", "DEFAULT_SCRIPT_ORDERS", "Locale", "Parser", "parse", "parser_for", } assert expected <= set(nameparser.__all__) for name in expected: assert getattr(nameparser, name) is not None def test_type_checking_imports_do_not_count(tmp_path: pathlib.Path) -> None: mod = tmp_path / "snippet.py" mod.write_text( "from typing import TYPE_CHECKING\n" "if TYPE_CHECKING:\n" " from nameparser._lexicon import Lexicon\n" "elif TYPE_CHECKING:\n" " import nameparser.never_runtime\n" "else:\n" " import nameparser.util\n" "def f():\n" " from nameparser import _render\n" ) assert _nameparser_imports(mod) == ["nameparser.util", "nameparser"] def test_every_frozen_dataclass_carries_the_pickle_guards() -> None: # Forgetting the two class-body assignments is SILENT -- # @dataclass(slots=True) installs its own working pickle methods # without skew detection. Lexicon keeps a private copy of the guard # (layering keeps _lexicon import-free of _types). import dataclasses import inspect import nameparser._lexicon import nameparser._locale import nameparser._parser import nameparser._policy import nameparser._types from nameparser._types import _guarded_getstate, _guarded_setstate # _pipeline internals (ParseState, WorkToken, ...) are exempt: they # are never pickled and are not public API. modules = (nameparser._types, nameparser._lexicon, nameparser._policy, nameparser._locale, nameparser._parser) for module in modules: for _, cls in inspect.getmembers(module, inspect.isclass): if cls.__module__ != module.__name__: continue if not dataclasses.is_dataclass(cls): continue if cls.__name__ == "Lexicon": assert "__getstate__" in cls.__dict__ assert "__setstate__" in cls.__dict__ continue assert cls.__dict__.get("__getstate__") is _guarded_getstate, cls assert cls.__dict__.get("__setstate__") is _guarded_setstate, cls python-nameparser-2.1.0/tests/v2/test_lexicon.py000066400000000000000000000673561523530431700217550ustar00rootroot00000000000000import dataclasses import pickle import warnings from collections.abc import Callable import pytest from nameparser import Parser from nameparser._lexicon import ( Lexicon, _VOCAB_FIELDS, _default_lexicon, _normalize, _title_key, ) from nameparser._policy import Script, _SCRIPT_RANGES def test_entries_are_normalized_at_construction() -> None: lex = Lexicon(titles=frozenset({"Dr.", "MR"})) assert lex.titles == frozenset({"dr", "mr"}) def test_default_sources_v1_vocabulary() -> None: lex = Lexicon.default() assert "dr" in lex.titles assert "van" in lex.particles assert "phd" in lex.suffix_acronyms # flipped model: 'dos' is never-given in v1, so NOT ambiguous here assert "dos" in lex.particles and "dos" not in lex.particles_ambiguous assert "van" in lex.particles_ambiguous # v1's CAPITALIZATION_EXCEPTIONS maps 'phd' -> 'Ph.D.' (verbatim, not # normalized -- only keys are lowercased/period-stripped at # construction, values pass through unchanged). assert lex.capitalization_exceptions_map["phd"] == "Ph.D." # maiden markers source from the same data-module pattern (#274); # non-colliding Cyrillic entries live in the default per the locales # design's sorting rule, and both ё/е spellings are listed because # case normalization does not fold them. assert "geborene" in lex.maiden_markers assert "урожденная" in lex.maiden_markers and "урождённая" in lex.maiden_markers # #269 fidelity: entries whose spelling casefold() would mutate must # arrive in the lexicon exactly as authored in the data modules # (final sigma intact, ß intact) -- the review that caught 'κοσ'. assert "κος" in lex.titles and "κοσ" not in lex.titles assert "großfürst" in lex.titles and "grossfürst" not in lex.titles def test_default_is_cached_single_instance() -> None: assert Lexicon.default() is Lexicon.default() def test_particles_ambiguous_must_be_subset_of_particles() -> None: with pytest.raises(ValueError, match="subset"): Lexicon(particles_ambiguous=frozenset({"van"})) def test_capitalization_exceptions_canonical_and_no_aliasing() -> None: exceptions = {"phd": "PhD", "ii": "II"} lex = Lexicon.empty() lex2 = dataclasses.replace(lex, capitalization_exceptions=exceptions) # type: ignore[arg-type] exceptions["iii"] = "III" # mutate caller's dict afterwards assert "iii" not in lex2.capitalization_exceptions_map # canonical: insertion order does not affect equality/hash lex3 = dataclasses.replace(lex, capitalization_exceptions=(("ii", "II"), ("phd", "PhD"))) assert lex2 == lex3 and hash(lex2) == hash(lex3) def test_lexicon_is_hashable() -> None: assert isinstance(hash(Lexicon.default()), int) def test_lexicon_rejects_bare_string_vocab() -> None: with pytest.raises(TypeError, match="bare string"): Lexicon(titles="dr") # type: ignore[arg-type] def test_lexicon_rejects_non_str_vocab_entries() -> None: with pytest.raises(TypeError, match="entries must be strings"): Lexicon(titles={"Dr.", 42}) # type: ignore[arg-type] def test_lexicon_rejects_non_iterable_vocab_field() -> None: # A non-iterable (an int) used to fall through to tuple(entries) # and surface as a bare "'int' object is not iterable" with no # hint which field was wrong. with pytest.raises(TypeError, match="titles.*iterable"): Lexicon(titles=5) # type: ignore[arg-type] with pytest.raises(TypeError, match="titles.*iterable"): Lexicon.empty().add(titles=5) # type: ignore[arg-type] def test_capitalization_exceptions_rejects_non_iterable() -> None: with pytest.raises(TypeError, match="capitalization_exceptions"): Lexicon(capitalization_exceptions=5) # type: ignore[arg-type] def test_entries_normalizing_to_empty_raise() -> None: # "." or "" is a data bug (stray split artifact, empty CSV cell); # dropping it silently would also let a future data-module typo # vanish instead of failing CI. One rule for vocab AND exception keys. with pytest.raises(ValueError, match="normalizes to empty"): Lexicon(titles=frozenset({"Dr.", "."})) with pytest.raises(ValueError, match="normalizes to empty"): Lexicon.empty().add(titles={" "}) with pytest.raises(ValueError, match="normalizes to empty"): Lexicon(capitalization_exceptions=(("...", "X"), ("phd", "PhD"))) def test_colliding_exception_keys_dedupe_last_wins() -> None: # 'Phd.' and 'phd' collide under edge-period normalization lex = Lexicon(capitalization_exceptions=(("Phd.", "A"), ("phd", "B"))) assert lex.capitalization_exceptions == (("phd", "B"),) rebuilt = Lexicon(capitalization_exceptions=lex.capitalization_exceptions_map) # type: ignore[arg-type] assert rebuilt == lex and hash(rebuilt) == hash(lex) def test_lexicon_rejects_non_str_exception_values() -> None: with pytest.raises(TypeError, match="str -> str"): Lexicon(capitalization_exceptions={"phd": 42}) # type: ignore[dict-item, arg-type] def test_add_and_remove_return_new_lexicons() -> None: # "zqtitle" is a synthetic word absent from v1's TITLES data (unlike # e.g. "dra", the feminine "dr." abbreviation, which is already there). base = Lexicon.default() lex = base.add(titles={"zqtitle"}).remove(suffix_words={"esquire"}) assert "zqtitle" in lex.titles and "zqtitle" not in base.titles # precondition, or the removal assertion passes vacuously (this is a # guard for the operation under test, not a vocabulary content pin) assert "esquire" in base.suffix_words assert "esquire" not in lex.suffix_words def test_add_unknown_field_raises_with_valid_names() -> None: with pytest.raises(TypeError, match="prefixes"): Lexicon.default().add(prefixes={"van"}) # v1 name: helpful error def test_add_capitalization_exceptions_raises_pointing_at_replace() -> None: with pytest.raises(TypeError, match="dataclasses.replace"): Lexicon.default().add(capitalization_exceptions={"x": "X"}) def test_union_is_fieldwise_and_right_biased_for_exceptions() -> None: a = dataclasses.replace(Lexicon.empty(), capitalization_exceptions=(("phd", "PhD"),)) a = a.add(titles={"dr"}) b = dataclasses.replace(Lexicon.empty(), capitalization_exceptions=(("phd", "Ph.D."),)) b = b.add(titles={"mr"}) u = a | b assert u.titles == frozenset({"dr", "mr"}) assert u.capitalization_exceptions_map["phd"] == "Ph.D." # right wins def test_remove_breaking_subset_invariant_raises() -> None: lex = Lexicon(particles=frozenset({"van"}), particles_ambiguous=frozenset({"van"})) with pytest.raises(ValueError, match="subset"): lex.remove(particles={"van"}) # would orphan particles_ambiguous def test_pickle_round_trip_preserves_equality_and_cap_map() -> None: # _cap_map holds a MappingProxyType, which pickle rejects; Lexicon # must round-trip anyway because Parser is picklable by construction # (spec: 2026-07-11-v2-core-api-design.md) and a Parser holds a Lexicon. import pickle for lex in (Lexicon.default(), Lexicon.empty().add(titles={"Dr."})): loaded = pickle.loads(pickle.dumps(lex)) assert loaded == lex assert hash(loaded) == hash(lex) assert (loaded.capitalization_exceptions_map == lex.capitalization_exceptions_map) def test_lexicon_rejects_mapping_for_plain_vocab_field() -> None: # A dict here almost always means the caller confused the field with # capitalization_exceptions; silently keeping just the keys would be # the lone quiet coercion on an otherwise fail-loud surface. with pytest.raises(TypeError, match="mapping"): Lexicon(titles={"Dr.": "Doctor"}) # type: ignore[arg-type] with pytest.raises(TypeError, match="mapping"): Lexicon.empty().add(titles={"Dr.": "Doctor"}) def test_capitalization_exceptions_rejects_malformed_shapes() -> None: with pytest.raises(TypeError, match="bare string"): Lexicon(capitalization_exceptions="ab") # type: ignore[arg-type] with pytest.raises(TypeError, match="pairs"): Lexicon(capitalization_exceptions=(("a", "b", "c"),)) # type: ignore[arg-type] with pytest.raises(TypeError, match="pairs"): # a 2-char string entry would unpack into two chars and silently # store {"a": "b"} -- reject str entries outright Lexicon(capitalization_exceptions=("ab",)) # type: ignore[arg-type] def test_setstate_rejects_mismatched_field_layout() -> None: # A pickle from a different Lexicon version (field added/renamed) # must fail at load time with a message naming the mismatch, not at # some later attribute read far from the unpickle site. lex = Lexicon.empty() good_state = lex.__getstate__() missing = dict(good_state) del missing["titles"] with pytest.raises(ValueError, match="titles"): Lexicon.__new__(Lexicon).__setstate__(missing) extra = dict(good_state) extra["zq_future_field"] = frozenset() with pytest.raises(ValueError, match="zq_future_field"): Lexicon.__new__(Lexicon).__setstate__(extra) def test_normalization_lowercases_and_keeps_interior_periods() -> None: # v1's lc() semantics: lowercase, EDGE periods trimmed, interior # periods KEPT ('J.R.' must not collapse to 'jr' and hit the # periodless vocabulary -- v1 parity, pinned live 2026-07-17). # Suffix-ACRONYM membership alone strips periods (see # _vocab.suffix_as_written). lex = Lexicon(titles=frozenset({"STRAßE", "Ph.D", "Dr."})) assert lex.titles == frozenset({"straße", "ph.d", "dr"}) def test_normalization_preserves_spellings_casefold_would_mutate() -> None: # lower(), not casefold(): casefold's caseless-matching folds would # rewrite stored vocabulary -- Greek final sigma flattens ('κος' -> # the misspelling 'κοσ') and German ß expands ('großfürst' -> # 'grossfürst'). lower() follows Unicode SpecialCasing contextually # ('ΚΟΣ' -> 'κος') and keeps entries as authored, matching v1's lc(). lex = Lexicon(titles=frozenset({"ΚΟΣ", "Großfürst"})) assert lex.titles == frozenset({"κος", "großfürst"}) def test_suffix_ambiguous_must_be_subset_of_acronyms() -> None: # same invariant as particles_ambiguous <= particles: the ambiguous # set marks a subset of suffix_acronyms, and classify's period gate # depends on the membership tests agreeing about it with pytest.raises(ValueError, match="subset"): Lexicon(suffix_acronyms_ambiguous=frozenset({"ma"})) def test_given_name_titles_may_hold_a_multi_word_phrase() -> None: # given_name_titles is looked up against the SPACE-JOINED title run # (_pipeline/_post_rules.py), not per token, so "grand duke" is a # legitimate entry. Its words must be titles -- that is what makes # the run a title in the first place -- but the phrase itself is # not, and must not be required to be. lex = Lexicon(titles=frozenset({"grand", "duke"}), given_name_titles=frozenset({"grand duke"})) assert "grand duke" in lex.given_name_titles @pytest.mark.parametrize("spelling", ["lt. col", "lt col", "Lt. Col."]) def test_given_name_titles_folds_per_word_so_abbreviations_match( spelling: str) -> None: # The key is built per word and rejoined, so an abbreviated entry # has to fold the same way the parse folds the title run. Storing # 'lt. col' verbatim kept the interior period and matched nothing -- # a silent no-op on the config surface, the failure this field is # most prone to (it has no validation by design). base = Lexicon.default() lex = dataclasses.replace( base, titles=base.titles | {"lt", "col"}, given_name_titles=base.given_name_titles | {spelling}) assert "lt col" in lex.given_name_titles assert Parser(lexicon=lex).parse("Lt. Col. Smith").given == "Smith" @pytest.mark.parametrize("entry", ["lt .", "lt . col", ". col", ". ."]) def test_given_name_titles_fold_is_a_fixed_point(entry: str) -> None: # Storage re-runs the fold on unpickle and on every # dataclasses.replace, so a value that changes under a second pass # is one Lexicon later rejects as "not written by this version". once = _title_key(entry.split()) assert _title_key(once.split()) == once assert once == once.strip() and " " not in once def test_a_title_word_that_folds_away_leaves_no_gap() -> None: # 'lt .' stored 'lt' before this field folded per word. A per-word # join that keeps the empty slot stores 'lt ', which match time can # never build -- silently inert config, the exact failure the # per-word fold was introduced to remove. base = Lexicon.default() lex = dataclasses.replace( base, titles=base.titles | {"lt", "col"}, given_name_titles=base.given_name_titles | {"lt ."}) assert "lt" in lex.given_name_titles assert Parser(lexicon=lex).parse("Lt. John").given == "John" def test_the_constructor_and_add_store_an_entry_identically() -> None: # add() goes through dataclasses.replace, which re-runs # __post_init__ -- a non-idempotent fold converges there and not in # the constructor, so the two paths silently disagree. base = dataclasses.replace(Lexicon.default(), titles=Lexicon.default().titles | {"lt", "col"}) built = dataclasses.replace( base, given_name_titles=base.given_name_titles | {"lt . col"}) assert built.given_name_titles == base.add( given_name_titles={"lt . col"}).given_name_titles def test_an_all_punctuation_given_name_title_is_rejected() -> None: # every other vocab field raises on this; this one must not be # special just because it folds per word with pytest.raises(ValueError, match="normalizes to empty"): Lexicon(given_name_titles=frozenset({". ."})) def test_given_name_titles_is_not_constrained_against_titles() -> None: # Deliberately unvalidated. The lookup key is the joined run of # Role.TITLE tokens, which the PARSE builds, and a conjunction # inside a run is title-tagged too -- so "sir and dame" is a # matchable key whose middle word is a conjunction, not a title. # A per-word check rejected it; a whole-entry check rejected # multi-word entries. An unreachable entry is inert rather than # harmful, so nothing is rejected here. assert Lexicon(titles=frozenset({"sir", "dame"}), given_name_titles=frozenset({"sir and dame"})) # and an entry nothing will ever consult is accepted, not rejected assert Lexicon(given_name_titles=frozenset({"zzqnotatitle"})) @pytest.mark.parametrize(("make", "marker", "base"), [ (lambda w: Lexicon(particles_ambiguous=w), "particles_ambiguous", "particles"), (lambda w: Lexicon(suffix_acronyms_ambiguous=w), "suffix_acronyms_ambiguous", "suffix_acronyms"), (lambda w: Lexicon(honorific_tails=w), "honorific_tails", "suffix_words"), ], ids=["particles_ambiguous", "suffix_acronyms_ambiguous", "honorific_tails"]) def test_subset_error_names_the_fix( make: Callable[[frozenset[str]], Lexicon], marker: str, base: str ) -> None: # the constraint alone leaves the reader to infer the remedy, and # the remedy is direction-dependent: a caller who was ADDING wants # to add to the base, one who was REMOVING wants to drop from the # marker. Naming both keeps the message right either way. with pytest.raises(ValueError, match=f"Add them to {base}, or drop " f"them from {marker}"): make(frozenset({"zzqnovel"})) def test_bound_given_name_that_is_a_particle_must_be_ambiguous() -> None: # nameparser/config/prefixes.py asserts this on its own data: # NON_FIRST_NAME_PREFIXES stays disjoint from BOUND_FIRST_NAMES. In # 2.0's complement model that reads as bound_given_names & particles # <= particles_ambiguous. A particle declared never-to-start-a-given- # name cannot simultaneously bind one; without the check, one of the # two contradictory declarations is silently discarded. with pytest.raises(ValueError, match="particles_ambiguous"): Lexicon(particles=frozenset({"dos"}), particles_ambiguous=frozenset(), bound_given_names=frozenset({"dos"})) def test_bound_given_name_may_overlap_an_ambiguous_particle() -> None: # the shipped case: "abu" is both, and legitimately so lex = Lexicon(particles=frozenset({"abu"}), particles_ambiguous=frozenset({"abu"}), bound_given_names=frozenset({"abu"})) assert "abu" in lex.bound_given_names # and a bound name that is not a particle at all is unconstrained assert "abdul" in Lexicon(bound_given_names=frozenset({"abdul"}) ).bound_given_names def test_default_lexicon_satisfies_the_bound_particle_invariant() -> None: d = Lexicon.default() assert not (d.bound_given_names & d.particles) - d.particles_ambiguous def test_removing_a_title_leaves_its_given_name_marker_alone() -> None: # the mirror of the above: removing from titles does not raise, and # the now-inert marker entry simply stops being reachable lex = Lexicon(titles=frozenset({"sheikh"}), given_name_titles=frozenset({"sheikh"})) lean = lex.remove(titles={"sheikh"}) assert lean.titles == frozenset() assert lean.given_name_titles == frozenset({"sheikh"}) @pytest.mark.parametrize("word", [ "dr.", " Dr. ", ". a .", ". .", "..x..", " .b. ", ]) def test_normalize_reaches_a_fixed_point(word: str) -> None: # strip() then strip(".") leaves periods-around-whitespace half # done: '. a .' -> ' a ' -> 'a'. Anything that stores a normalized # value and later re-normalizes it (unpickling, a second add) then # sees the value change under it, so the fold has to converge. once = _normalize(word) assert _normalize(once) == once def test_unpickling_revalidates_invariants() -> None: # the field-layout guard catches SHAPE skew, but the likeliest skew # is same-name/flipped-meaning: particles_ambiguous inverted sense # between v1's never-given set and v2's may-be-given set. A state # dict that violates an invariant must not load in silence. lex = Lexicon(particles=frozenset({"van"}), particles_ambiguous=frozenset({"van"})) state = lex.__getstate__() state["particles"] = frozenset() # orphans particles_ambiguous forged = Lexicon.__new__(Lexicon) with pytest.raises(ValueError, match="particles_ambiguous"): forged.__setstate__(state) def test_unpickling_rejects_unnormalized_entries() -> None: # a genuine pickle always holds normalized entries, so state that # does not is foreign. Correcting it silently would make the load # path a fourth place where caller data is rewritten without a # word; say so instead, at the site that can still name the entry. lex = Lexicon(titles=frozenset({"dr"})) state = lex.__getstate__() state["titles"] = {"Sir ", "DR."} restored = Lexicon.__new__(Lexicon) with pytest.raises(ValueError, match="not normalized"): restored.__setstate__(state) def test_unpickling_rejects_uncanonical_capitalization_exceptions() -> None: # the pair field gets the same treatment as the vocab sets: it is # re-canonicalized by __post_init__ (normalized keys, sorted, # deduped), so accepting a rewritten value would leave one field # silently corrected while ten raise lex = Lexicon(capitalization_exceptions=(("phd", "PhD"),)) state = lex.__getstate__() state["capitalization_exceptions"] = ((" PhD. ", "Ph.D."), ("aa", "AA")) restored = Lexicon.__new__(Lexicon) with pytest.raises(ValueError, match="not normalized"): restored.__setstate__(state) def test_unpickling_a_real_pickle_is_the_identity() -> None: # the corollary: anything this library produced round-trips # unchanged, including entries whose normalized form needed more # than one strip pass to converge for lex in (Lexicon.default(), Lexicon.empty(), Lexicon(titles=frozenset({". a ."}))): assert pickle.loads(pickle.dumps(lex)) == lex def test_remove_error_offers_the_removal_remedy_too() -> None: # a caller removing from the base does not want to be told to add # it back; the message names both directions lex = Lexicon(particles=frozenset({"van"}), particles_ambiguous=frozenset({"van"})) with pytest.raises(ValueError, match="drop them from particles_ambiguous"): lex.remove(particles={"van"}) @pytest.mark.parametrize("value", [b"dr", bytearray(b"dr"), memoryview(b"dr")]) @pytest.mark.parametrize("build", [ lambda v: Lexicon(titles=v), lambda v: Lexicon.empty().add(titles=v), lambda v: Lexicon(capitalization_exceptions=v), ]) def test_every_lexicon_entry_point_rejects_a_buffer_with_a_decode_hint( build: Callable[[object], object], value: object) -> None: # Binary sequences iterate to INTS, so the generic entry check # reported "entries must be strings, got 100" -- the byte value of # 'd', naming neither the cause nor the fix. v1 shipped a decode # hint for exactly this (#238). # # Every entry point, because the first version of this fix covered # the vocabulary fields and left capitalization_exceptions behind: # guarding one member of a family and not its siblings is the # single most repeated defect on this branch. with pytest.raises(TypeError, match="decode"): build(value) _PER_WORD_FIELDS = [f for f in _VOCAB_FIELDS if f != "given_name_titles"] @pytest.mark.parametrize("field", _PER_WORD_FIELDS) def test_multiword_entry_warns_per_field(field: str) -> None: # Guard the whole family: every per-word field warns, not one. with pytest.warns(UserWarning, match="matched one word at a time"): if field in ("particles_ambiguous", "suffix_acronyms_ambiguous", "honorific_tails"): # subset fields need their base populated to construct base = {"particles_ambiguous": "particles", "suffix_acronyms_ambiguous": "suffix_acronyms", "honorific_tails": "suffix_words"}[field] Lexicon.empty().add(**{base: ["zqx zqy"], field: ["zqx zqy"]}) else: Lexicon.empty().add(**{field: ["zqx zqy"]}) def test_multiword_given_name_title_does_not_warn() -> None: # given_name_titles matches space-joined runs -- multi-word entries # are the point (see the __post_init__ note in _lexicon.py). with warnings.catch_warnings(): warnings.simplefilter("error") Lexicon.empty().add(given_name_titles=["lt col"]) def test_multiword_capitalization_key_warns() -> None: with pytest.warns(UserWarning, match="matched one word at a time"): dataclasses.replace( Lexicon.empty(), capitalization_exceptions=(("zqx zqy", "ZqXZqY"),)) def test_default_lexicon_builds_warning_free() -> None: # Would have caught the eight dead entries the previous commit # repaired. _default_lexicon.cache_clear() try: with warnings.catch_warnings(): warnings.simplefilter("error") Lexicon.default() finally: _default_lexicon.cache_clear() def test_multiword_entry_warns_from_constructor_and_unpickle() -> None: with pytest.warns(UserWarning, match="matched one word at a time"): lex = Lexicon(titles=frozenset({"zqx zqy"})) with pytest.warns(UserWarning, match="matched one word at a time"): pickle.loads(pickle.dumps(lex)) def test_remove_does_not_warn() -> None: # remove() stores nothing; a dead-entry warning there would be a # false positive with wrong advice. with warnings.catch_warnings(): warnings.simplefilter("error") Lexicon.default().remove(titles=["zqx zqy"]) def test_multiword_warning_points_at_caller_line() -> None: # the frame walk is the feature: both entry depths must attribute # the warning to THIS file, not library internals with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") Lexicon(titles=frozenset({"zqx zqy"})) Lexicon.empty().add(titles=["zqx zqy"]) assert len(w) >= 2 assert all(x.filename == __file__ for x in w) def test_add_warns_exactly_once() -> None: # the warn=False plumbing exists to prevent the double warning # (_edit's pass + the new instance's __post_init__); count it with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") Lexicon.empty().add(titles=["zqx zqy"]) assert len([x for x in w if "matched one word" in str(x.message)]) == 1 def test_remove_of_a_stored_dead_entry_is_silent() -> None: # the scenario the design argues for: user got the warning, now # removes the dead entry -- silently. (An UNRELATED edit on a # lexicon still holding one re-warns; that is per-construction # warning, by design.) with pytest.warns(UserWarning): dirty = Lexicon.empty().add(titles=["zqx zqy"]) with warnings.catch_warnings(): warnings.simplefilter("error") dirty.remove(titles=["zqx zqy"]) def test_surnames_is_a_vocab_field() -> None: lex = Lexicon.empty().add(surnames={"毛", "欧阳"}) assert lex.surnames == frozenset({"毛", "欧阳"}) assert lex.remove(surnames={"毛"}).surnames == frozenset({"欧阳"}) merged = Lexicon(surnames=frozenset({"김"})) | Lexicon(surnames=frozenset({"박"})) assert merged.surnames == frozenset({"김", "박"}) def test_default_lexicon_ships_korean_surnames_only() -> None: lex = Lexicon.default() assert "김" in lex.surnames and "남궁" in lex.surnames # Han surnames are locales.ZH's cargo (segmentation opt-in), so # the DEFAULT set is wholly hangul -- structural check, not # content-pinning. The spans come from the shared table rather than # a second hand-copy of 0xAC00/0xD7A3 (test_locales.py's precedent). hangul = _SCRIPT_RANGES[Script.HANGUL] assert all(all(any(lo <= ord(c) <= hi for lo, hi in hangul) for c in s) and 1 <= len(s) <= 2 for s in lex.surnames) def test_honorific_tails_is_a_vocab_field() -> None: # every tail is also a suffix word -- the peeled piece is claimed # by suffix classification -- so each construction carries both lex = Lexicon.empty().add(honorific_tails={"씨", "さん"}, suffix_words={"씨", "さん"}) assert lex.honorific_tails == frozenset({"씨", "さん"}) assert lex.remove(honorific_tails={"씨"}).honorific_tails == \ frozenset({"さん"}) merged = (Lexicon(honorific_tails=frozenset({"씨"}), suffix_words=frozenset({"씨"})) | Lexicon(honorific_tails=frozenset({"様"}), suffix_words=frozenset({"様"}))) assert merged.honorific_tails == frozenset({"씨", "様"}) def test_removing_a_honorific_tail_is_the_peels_off_switch() -> None: # The off-switch docs/customize.rst documents, at parse level: # dropping a tail leaves the glued name unsplit while the SPACED # spelling still routes the honorific, since suffix_words is a # separate field and only the peel consults honorific_tails. # honorific_tails is the one marker field a caller can empty # without a matching base edit -- an orphan is what raises, and a # removal cannot make one. lex = Lexicon.default().remove(honorific_tails={"さん"}) p = Parser(lexicon=lex) glued = p.parse("田中さん") assert (glued.family, glued.suffix) == ("田中さん", "") spaced = p.parse("田中 さん") assert (spaced.family, spaced.suffix) == ("田中", "さん") # the baseline the removal is against, so the test cannot pass by # the peel being broken outright on = Parser().parse("田中さん") assert (on.family, on.suffix) == ("田中", "さん") python-nameparser-2.1.0/tests/v2/test_locale.py000066400000000000000000000055461523530431700215440ustar00rootroot00000000000000import pytest from nameparser._lexicon import Lexicon from nameparser._locale import Locale from nameparser._policy import PatronymicRule, PolicyPatch def test_locale_holds_code_lexicon_fragment_and_patch() -> None: ru = Locale( code="ru", lexicon=Lexicon.empty(), policy=PolicyPatch( patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})), ) assert ru.code == "ru" assert ru.policy.patronymic_rules == frozenset( {PatronymicRule.EAST_SLAVIC}) def test_locale_defaults_to_empty_patch() -> None: assert Locale(code="xx", lexicon=Lexicon.empty()).policy == PolicyPatch() def test_locale_code_must_be_nonempty_lowercase() -> None: with pytest.raises(ValueError, match="lowercase"): Locale(code="RU", lexicon=Lexicon.empty()) with pytest.raises(ValueError, match="non-empty"): Locale(code="", lexicon=Lexicon.empty()) def test_locale_code_rejects_whitespace() -> None: # caught by the [a-z0-9_]+ charset pin (a dedicated whitespace check # would be pure redundancy; the charset message is accurate) for bad in ("ru ", " ru", "ru\n", "r u"): with pytest.raises(ValueError, match="a-z0-9_"): Locale(code=bad, lexicon=Lexicon.empty()) def test_locale_is_hashable() -> None: loc = Locale(code="ru", lexicon=Lexicon.empty()) assert isinstance(hash(loc), int) def test_locale_validates_component_types() -> None: with pytest.raises(TypeError, match="Lexicon"): Locale(code="ru", lexicon={"titles": set()}) # type: ignore[arg-type] with pytest.raises(TypeError, match="PolicyPatch"): Locale(code="ru", lexicon=Lexicon.empty(), policy={"name_order": None}) # type: ignore[arg-type] with pytest.raises(TypeError, match="must be a str"): Locale(code=5, lexicon=Lexicon.empty()) # type: ignore[arg-type] def test_locale_with_lexicon_pickles_round_trip() -> None: import pickle loc = Locale(code="ru", lexicon=Lexicon.empty().add(titles={"Dr."})) assert pickle.loads(pickle.dumps(loc)) == loc def test_locale_code_is_pinned_to_registry_charset() -> None: # Codes become registry keys the moment parser_for and third-party # packs exist: every accepted character is supported forever, so pin # [a-z0-9_]+ now (matches ru/tr_az). One separator only -- allowing # both '-' and '_' would make tr-az and tr_az distinct keys. for bad in ("ru!", "tr-az", "тр", "zh/tw"): with pytest.raises(ValueError, match="a-z0-9_"): Locale(code=bad, lexicon=Lexicon.empty()) assert Locale(code="tr_az", lexicon=Lexicon.empty()).code == "tr_az" def test_setstate_rejects_layout_skew() -> None: loc = Locale(code="ru", lexicon=Lexicon.empty()) state = dict(loc.__getstate__()) del state["code"] with pytest.raises(ValueError, match="code"): Locale.__new__(Locale).__setstate__(state) python-nameparser-2.1.0/tests/v2/test_locales.py000066400000000000000000001642171523530431700217300ustar00rootroot00000000000000"""The locale pack layer (locales spec §2-3): lazy access, the shipped packs, composition, and the non-interference gate.""" import functools import importlib.util import re import sys import types from collections.abc import Callable, Iterable from types import ModuleType import pytest from nameparser import Locale, Parser, locales, parse, parser_for from nameparser._lexicon import _VOCAB_FIELDS, Lexicon from nameparser._policy import ( UNSET, PatronymicRule, Policy, Script, _SCRIPT_RANGES, ) from nameparser._types import AmbiguityKind from nameparser.locales import ja as _ja from nameparser.locales import ru as _ru from nameparser.locales import tr_az as _tr_az from nameparser.locales import zh as _zh from .conftest import differential_corpus _CORPUS = differential_corpus() # The registry is the pack contract (design note 2026-07-18, option C): # the DEVIATES requirement, the rotator lists, and the non-interference # gate below all iterate the registered packs, so adding a pack is one # registry entry plus one rotator list -- the contract meta-test fails # until both exist, instead of a human remembering to extend N tests. def _pack_modules() -> dict[str, ModuleType]: out: dict[str, ModuleType] = {} for modname, attr in locales._REGISTRY.values(): module = importlib.import_module(modname) out[getattr(module, attr).code] = module return out _PACKS = _pack_modules() # ja is the one pack whose behavior is not pack data: it activates # segmentation and supplies nothing to segment WITH, so every parse # through it needs the optional nameparser[ja] segmenter. Probed once # here; without the extra the ja code has no _PACKED entry, so every # test that PARSES through the pack skips -- its rotator rows and its # non-interference run included. What still runs either way is what # needs no parse: the registry contract (DEVIATES and a rotator list # exist), the range pin, the pack's contents, and the inertness test # below, which is the unsegmented pack's whole observable behavior. _JA_AVAILABLE = importlib.util.find_spec("namedivider") is not None _needs_ja = pytest.mark.skipif(not _JA_AVAILABLE, reason="needs nameparser[ja]") # shared across the gate and rotator tests: the default-parser baseline # is identical everywhere (only the packed side varies per test), so # parse each name once for the whole module instead of once per test _DEFAULT_PARSER = Parser() def _packed_parsers() -> dict[str, Parser]: out: dict[str, Parser] = {} for code in _PACKS: if code != "ja": out[code] = parser_for(locales.get(code)) elif _JA_AVAILABLE: out[code] = parser_for( locales.JA, segmenter=locales.ja_segmenter()) return out _PACKED = _packed_parsers() def _require_packed(code: str) -> Parser: """The pack's parser, skipping the test when it needs an optional extra that is not installed (ja only, today).""" parser = _PACKED.get(code) if parser is None: pytest.skip(f"the {code!r} pack needs nameparser[{code}], " f"which is not installed") return parser @functools.cache def _default_parse(name: str) -> dict: """Default-parser components for `name` (treat as read-only -- the dict is cached and shared across callers).""" return _DEFAULT_PARSER.parse(name).as_dict() def test_locales_module_attribute_access() -> None: assert isinstance(locales.RU, Locale) assert locales.RU.code == "ru" assert locales.RU is locales.RU # cached, not rebuilt def test_locales_get_and_available() -> None: assert locales.get("ru") is locales.RU assert set(locales.available()) == {"ja", "ru", "tr_az", "zh"} with pytest.raises(KeyError, match="ja, ru, tr_az, zh"): locales.get("xx") # the segmenter FACTORY is not a locale: it is an ordinary function # on the package, so it must not appear among the codes assert "ja_segmenter" not in locales.available() assert callable(locales.ja_segmenter) def test_tr_az_pack_contents() -> None: assert locales.TR_AZ.code == "tr_az" assert locales.TR_AZ.policy.patronymic_rules == frozenset( {PatronymicRule.TURKIC}) assert locales.TR_AZ.lexicon == Lexicon.empty() def test_zh_pack_contents() -> None: assert locales.ZH.code == "zh" # the pack activates Han segmentation and ships vocabulary -- # and does NOT touch order: script_orders already reads Han # family-first by default (amendment 2026-07-27) assert locales.ZH.policy.segment_scripts == frozenset({Script.HAN}) assert locales.ZH.policy.name_order is UNSET assert locales.ZH.policy.script_orders is UNSET assert "王" in locales.ZH.lexicon.surnames assert "欧阳" in locales.ZH.lexicon.surnames # compound assert "歐陽" in locales.ZH.lexicon.surnames # traditional assert _zh.DEVIATES("毛泽东") assert _zh.DEVIATES("𠮷田") # astral Han declared assert not _zh.DEVIATES("John Smith") assert not _zh.DEVIATES("김민준") # HAN alone, not a wider union def test_zh_surname_entries_are_wellformed() -> None: # structural integrity, not content-pinning: 1-2 chars each, # every char in the stage's own Han ranges (catches a stray # Latin char, a pasted full name, or a hangul entry) ranges = _SCRIPT_RANGES[Script.HAN] for entry in locales.ZH.lexicon.surnames: assert 1 <= len(entry) <= 2, entry assert all(any(lo <= ord(c) <= hi for lo, hi in ranges) for c in entry), entry def test_zh_parses_unspaced_names() -> None: p = _PACKED["zh"] n = p.parse("毛泽东") assert (n.family, n.given) == ("毛", "泽东") n = p.parse("欧阳修") # compound + 1-char given assert (n.family, n.given) == ("欧阳", "修") n = p.parse("司马相如") # compound + 2-char given assert (n.family, n.given) == ("司马", "相如") # the 肖/萧 census merger ships both simplified spellings n = p.parse("萧红") assert (n.family, n.given) == ("萧", "红") def test_zh_longest_match_prefers_compound_over_single_surname() -> None: # The longest-first tie-break, exercised by the PACK's own data # rather than by the synthetic lexicon the stage tests use. Most # compounds here begin with a character that is not itself a listed # surname (欧, 司, 诸 are all outside the top-100), so they would # split the same way under shortest-first and prove nothing. 夏侯 is # one of only three compounds that would not -- 夏 is rank 65 -- so # this is the pair that shows the shipped data actually reaches the # tie-break. p = _PACKED["zh"] n = p.parse("夏侯惇") assert (n.family, n.given) == ("夏侯", "惇") n = p.parse("夏雨") # the single-surname foil assert (n.family, n.given) == ("夏", "雨") # ...and the ambiguity contract on that same data: a decided fork # is REPORTED (first Han coverage of the SEGMENTATION emitter -- # the stage tests reach it through a synthetic lexicon, and the # default lexicon can only reach it through hangul), while the # foil, where only one split was ever possible, stays silent. assert [a.kind for a in p.parse("夏侯惇").ambiguities] == [ AmbiguityKind.SEGMENTATION] assert p.parse("夏雨").ambiguities == () # 毛泽东/欧阳修/夏侯惇/萧红 appear in _ROTATORS["zh"] below, and 毛泽东/ # 夏侯惇 in cases.py's zh rows, as well. Not deduplicated in either # direction, on purpose: the rotator tests assert only THAT the packed # parse differs from the default and that DEVIATES declares it -- they # never look at a field value -- and the case rows reach the same # packed parser only through the shared-table runners # (test_cases.py/test_facade_cases.py). These tests are where the zh # readings are pinned at unit level, against the stage directly. def test_zh_composes_with_korean_defaults() -> None: # the pack only ADDS (vocabulary + one union field): Korean # segmentation keeps working through a zh parser n = _PACKED["zh"].parse("김민준") assert (n.family, n.given) == ("김", "민준") def test_honorific_spellings_agree_where_the_vocabulary_divides() -> None: # usage.rst says a SPACED honorific is how you keep a family name # whole. That holds only where a SEGMENTER divides it -- the ja # tests below pin that half -- because the segmenter's precondition # counts any neighbour and a spaced honorific is one. The # VOCABULARY has no such precondition: the peel (#308) hands it the # same remainder whichever way the honorific was written, so the # two spellings agree exactly. Pinned in both scripts because the # reader's escape hatch differs -- the zh pack can be declined, # hangul segmentation is a DEFAULT and cannot, so a Korean-data # reader has no lever at all here. ko = _DEFAULT_PARSER.parse("김민준 씨") assert (ko.family, ko.given, ko.suffix) == ("김", "민준", "씨") assert _DEFAULT_PARSER.parse("김민준씨").as_dict() == ko.as_dict() zh = _PACKED["zh"].parse("王小明 先生") assert (zh.family, zh.given, zh.suffix) == ("王", "小明", "先生") assert _PACKED["zh"].parse("王小明先生").as_dict() == zh.as_dict() def test_ja_pack_contents() -> None: assert locales.JA.code == "ja" # segmentation activation ONLY: no vocabulary (no list settles a # kanji name) and no order (the kana license already reads # Japanese family-first by default, amendment 2026-07-29 §1) assert locales.JA.policy.segment_scripts == frozenset( {Script.HAN, Script.HIRAGANA}) assert locales.JA.policy.name_order is UNSET assert locales.JA.policy.script_orders is UNSET assert locales.JA.lexicon == Lexicon.empty() assert _ja.DEVIATES("山田太郎") assert _ja.DEVIATES("𠮷田") # astral Han declared # katakana declared (see the pack's repertoire note) assert _ja.DEVIATES("マイケル") assert _ja.DEVIATES("みなみ") # hiragana declared assert not _ja.DEVIATES("John Smith") assert not _ja.DEVIATES("김민준") # the three JA scripts, not a wider union def test_ja_segmenter_without_extra_raises_helpfully() -> None: if _JA_AVAILABLE: pytest.skip("namedivider installed; the error path is untestable") with pytest.raises(ImportError, match=r"nameparser\[ja\]"): locales.ja_segmenter() class _FakeDivided: """The shape of namedivider's DividedName, minus namedivider — lets the adapter's guard stack run in the plain (no-extra) suite, including the two defensive branches the real 0.4.x library can never reach (reconstruction failure, an empty side).""" def __init__(self, family: str, given: str, score: float) -> None: self.family, self.given, self.score = family, given, score def _fake_namedivider(monkeypatch: pytest.MonkeyPatch, divide: "Callable[[str], _FakeDivided]") -> list[str]: """Install a stub namedivider module and return the gbdt-call log (empty unless GBDTNameDivider was constructed).""" constructed: list[str] = [] class _Basic: def __init__(self) -> None: constructed.append("basic") def divide_name(self, text: str) -> _FakeDivided: return divide(text) class _GBDT(_Basic): def __init__(self) -> None: constructed.append("gbdt") stub = types.ModuleType("namedivider") stub.BasicNameDivider = _Basic # type: ignore[attr-defined] stub.GBDTNameDivider = _GBDT # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "namedivider", stub) return constructed def test_ja_adapter_guard_stack_against_a_stub( monkeypatch: pytest.MonkeyPatch) -> None: # The adapter's whole contract without the extra: this is what the # coverage-bearing CI job sees, so the guards are pinned here and # not only behind the nameparser[ja] integration tests. divide = _fake_namedivider( monkeypatch, lambda text: _FakeDivided(text[:2], text[2:], 0.5)) seg = locales.ja_segmenter() assert divide == ["basic"] answer = seg("山田太郎") assert answer is not None assert answer.splits == (2,) and answer.confidence == 0.5 assert seg("김민준") is None # outside the repertoire # contains JA chars but is not WHOLLY JA assert seg("Yamada太郎") is None assert seg("林") is None # namedivider raises below 2 def test_ja_adapter_declines_shime_tokens( monkeypatch: pytest.MonkeyPatch) -> None: # Since U+3006 classifies as Han, 〆木太郎 is wholly Japanese and # reaches the adapter, but the divider is never consulted for 〆 # tokens -- see the decline comment in ja.py for the measured # mis-cut it prevents. calls: list[str] = [] def divide(text: str) -> _FakeDivided: calls.append(text) return _FakeDivided(text[:1], text[1:], 1.0) _fake_namedivider(monkeypatch, divide) assert locales.ja_segmenter()("〆木太郎") is None assert calls == [] # the decline sits before divide_name def test_ja_adapter_defensive_branches_against_a_stub( monkeypatch: pytest.MonkeyPatch) -> None: # reconstruction failure -> decline; an empty side -> the stated # "confidently undivided" opinion; a score a hair over 1 -> clamped, # so Segmentation's [0, 1] validation cannot raise mid-parse _fake_namedivider( monkeypatch, lambda text: _FakeDivided("外", "れ", 0.5)) assert locales.ja_segmenter()("山田太郎") is None _fake_namedivider( monkeypatch, lambda text: _FakeDivided(text, "", 1.0000001)) whole = locales.ja_segmenter()("山田太郎") assert whole is not None assert whole.splits == () and whole.confidence == 1.0 def test_ja_adapter_declines_a_garbage_score( monkeypatch: pytest.MonkeyPatch) -> None: # The other half of the score handling, and the half the old # unconditional clamp got wrong: float noise at the edge of a # softmax is arithmetic around a real answer and clamps (above), # but 87.0 is not noise -- a divider scoring that is broken, and # its DIVISION is worth no more than its score. Clamping mapped it # to 1.0, i.e. "certain", which silenced the SEGMENTATION report at # exactly the point of maximum brokenness. Declining leaves the # token whole: safe, and observable by the report's absence. _fake_namedivider( monkeypatch, lambda text: _FakeDivided(text[:2], text[2:], 87.0)) assert locales.ja_segmenter()("山田太郎") is None _fake_namedivider( monkeypatch, lambda text: _FakeDivided(text[:2], text[2:], -0.5)) assert locales.ja_segmenter()("山田太郎") is None def test_ja_adapter_declines_a_nan_score( monkeypatch: pytest.MonkeyPatch) -> None: # NaN takes the garbage-score exit for free rather than needing one # of its own: every comparison against NaN is False, so it fails # BOTH bounds of the epsilon band. Worth a test precisely because # it is a property of the comparison rather than a written branch # -- under the old clamp, min/max propagated it straight into # Segmentation, whose own range check would then have raised # mid-parse. _fake_namedivider( monkeypatch, lambda text: _FakeDivided(text[:2], text[2:], float("nan"))) assert locales.ja_segmenter()("山田太郎") is None def test_ja_adapter_gbdt_flag_selects_the_gbdt_divider( monkeypatch: pytest.MonkeyPatch) -> None: divide = _fake_namedivider( monkeypatch, lambda text: _FakeDivided(text[:1], text[1:], 1.0)) locales.ja_segmenter(gbdt=True) assert divide == ["gbdt"] def test_ja_pack_alone_is_inert() -> None: # The pack's own claim, and the reason it is safe to register # without the extra: activating segmentation while shipping nothing # to segment WITH changes no parse at all. Pure pack data, so this # runs whether or not nameparser[ja] is installed -- and it is the # unsegmented pack's entire observable behavior. # construction now SAYS the pack alone cannot divide anything -- # the segmenterless-activation warning is the loud half of this # test's claim, and the inert parses below are the quiet half with pytest.warns(UserWarning, match="ja_segmenter"): p = parser_for(locales.JA) for name in _ROTATORS["ja"]: assert p.parse(name).as_dict() == _default_parse(name), name @_needs_ja def test_ja_end_to_end() -> None: p = _PACKED["ja"] n = p.parse("山田太郎") assert (n.family, n.given) == ("山田", "太郎") # the case a zh surname list corrupts (高 + 橋一郎), done right n = p.parse("高橋一郎") assert (n.family, n.given) == ("高橋", "一郎") # kana-licensed composite: gated in through the HIRAGANA entry n = p.parse("高橋みなみ") assert (n.family, n.given) == ("高橋", "みなみ") # the iteration mark 々 gates in as Han (Script=Han under UAX #24, # but outside the ideograph blocks the classifier spans, so it # rides in on a singleton range entry) and namedivider reads the # repeated character correctly -- verified against 0.4.1 rather # than assumed, since a divider could plausibly have choked on it # or cut beside it n = p.parse("佐々木健") assert (n.family, n.given) == ("佐々木", "健") @_needs_ja def test_ja_keeps_a_transcribed_name_in_its_source_order() -> None: # pure katakana is excluded from the pack's activation by design # (amendment §1: katakana is how Japanese writes FOREIGN names), so # the nakaguro form divides on the dot and keeps its given-first # source order instead of being read family-first n = _PACKED["ja"].parse("マイケル・ジャクソン") assert (n.given, n.family) == ("マイケル", "ジャクソン") @_needs_ja def test_ja_composes_with_zh() -> None: # vocabulary first, segmenter on decline (amendment §2) p = parser_for(locales.ZH, locales.JA, segmenter=locales.ja_segmenter()) assert p.parse("毛泽东").family == "毛" # zh surname wins assert p.parse("山田太郎").family == "山田" # zh declines, segmenter # ...and a Korean name still splits on the default vocabulary, # never reaching the segmenter (which would decline it anyway) assert p.parse("김민준").family == "김" @_needs_ja def test_ja_segmenter_declines_what_it_cannot_read() -> None: # The adapter's repertoire guard, which only a token that actually # REACHES the adapter can pin: a listed Korean surname is split by # the vocabulary first (the second case below), so this hangul # deliberately opens with a syllable no surname list carries. It # gates in on script, declines vocabulary, and reaches the adapter, # which must decline it rather than let namedivider divide Korean # -- without the guard it comes back as family 쿄 carrying a bogus # SEGMENTATION report. p = _PACKED["ja"] n = p.parse("쿄쿄쿄") assert (n.family, n.given) == ("쿄쿄쿄", "") assert n.ambiguities == () # ...and the vocabulary-wins case, which never reaches the adapter assert p.parse("김민준").family == "김" @_needs_ja def test_ja_segmenter_declines_a_lone_character() -> None: # namedivider RAISES below two characters, and a segmenter's # exceptions propagate by contract, so the adapter's length guard # is what keeps this parse total. It takes a LONE token to reach: # in "林 太郎" the stage's precondition stops the segmenter first. p = _PACKED["ja"] n = p.parse("林") assert (n.family, n.given) == ("林", "") assert n.ambiguities == () @_needs_ja def test_ja_leaves_spaced_names_to_the_default_order() -> None: # the segmenter's precondition (see test_parser.py): a spaced name # is already divided, and the kana/Han family-first defaults read # it correctly with no segmenter involved. Without this the pack # would wreck the commonest written form -- 山田 + 太郎 divided # again into 山 + 田 + 太郎. p = _PACKED["ja"] for name in ("山田 太郎", "高橋 みなみ", "高橋 エミ", "山田 太郎 Jr.", "山田 太郎 様"): assert p.parse(name).as_dict() == _default_parse(name), name assert p.parse("山田 太郎").family == "山田" @_needs_ja def test_ja_divides_a_glued_name_carrying_an_honorific() -> None: # #308 end to end, with the real divider: the honorific peels off # first, so namedivider is handed 山田太郎 rather than 山田太郎様 # -- which it would have divided somewhere wrong, answering for # any string it is given. GLUED only: the writer of 山田太郎様 drew # no boundary anywhere, so the tail the peel manufactured is none # either and the consult goes ahead. p = _PACKED["ja"] n = p.parse("山田太郎様") assert (n.family, n.given, n.suffix) == ("山田", "太郎", "様") # the kana-licensed composite too, whose division is rule-based n = p.parse("高橋みなみ様") assert (n.family, n.given, n.suffix) == ("高橋", "みなみ", "様") # and the spaced spelling is left alone, honorific and all: its # writer wrote 山田太郎 as a unit, which the pack respects the way # it respects "山田 太郎" n = p.parse("山田太郎 様") assert (n.family, n.given, n.suffix) == ("山田太郎", "", "様") @_needs_ja def test_ja_does_not_divide_a_spaced_surname_under_an_honorific() -> None: # A spaced honorific is a neighbour, and the segmenter is asked # only where an UNDIVIDED name divides, so it is never consulted # here. Only a tail this stage MANUFACTURED is exempt -- a glued # honorific means no boundary was typed anywhere, a spaced one # means one was, and by position a spaced honorific cannot be told # apart from a spaced name element besides. # The bare control is what makes any of that load-bearing: without # it the assertions pass vacuously, for any reason at all that the # segmenter went unconsulted. Each family name here DOES divide # standing alone, so the honorific is what stopped it -- and these # four are the measured price of counting spaced honorifics, paid # to decline the one division 山田太郎 様 (the test above). p = _PACKED["ja"] for name, family, divided in (("佐藤 氏", "佐藤", ("佐", "藤")), ("田中 様", "田中", ("田", "中")), ("鈴木 先生", "鈴木", ("鈴", "木")), ("中村 教授", "中村", ("中", "村"))): n = p.parse(name) assert (n.family, n.given) == (family, ""), name bare = p.parse(family) assert (bare.family, bare.given) == divided, family @_needs_ja def test_ja_divides_a_two_character_name_under_a_glued_honorific() -> None: # The consequence of the rule above on the commonest addressed # form, pinned so it is a recorded decision rather than a side # effect: with the peeled tail not blocking the consult, a # two-character remainder reaches namedivider, which divides it # one character each way BY RULE (score 1.0, so no SEGMENTATION # report). That is the same presumption locales/ja.py already # documents and test_ja_reports_statistical_divisions... pins for # 原恵 -- 田中さん is simply the shortest route to it. A caller # who wants 田中 kept whole is asking not to have unspaced names # divided, which is what declining the pack means -- or can write # the honorific spaced, which the test above pins. n = _PACKED["ja"].parse("田中さん") assert (n.family, n.given, n.suffix) == ("田", "中", "さん") assert n.ambiguities == () # rule-based, so nothing to report # the spaced twin, for the contrast in one place n = _PACKED["ja"].parse("田中 さん") assert (n.family, n.given, n.suffix) == ("田中", "", "さん") @_needs_ja def test_ja_reports_statistical_divisions_and_not_rule_based_ones() -> None: # the confidence floor's measured consequence (_script_segment.py's # constant): namedivider scores a rule-based division 1.0 and a # kanji-statistics division far below the floor, so the ambiguity # report separates exactly those two epistemic states p = _PACKED["ja"] assert [a.kind for a in p.parse("山田太郎").ambiguities] == [ AmbiguityKind.SEGMENTATION] assert p.parse("高橋みなみ").ambiguities == () # kana boundary rule assert p.parse("原恵").ambiguities == () # two-character rule @_needs_ja def test_ja_divides_a_lone_hiragana_token() -> None: # The whole-name presumption, applied to hiragana: at token level a # bare given name is indistinguishable from a full name, so the # stage gates みなみ in and namedivider divides it -- み + なみ, # which is wrong if the writer meant the given name alone and right # if they meant a (rare) all-kana full name. Exactly the acceptance # 田中 -> 田 + 中 already carries for kanji, pinned here so the kana # path's version is a recorded consequence rather than a surprise. # The statistical score puts a SEGMENTATION report on it, which is # the caller's handle for routing it to review. n = _PACKED["ja"].parse("みなみ") assert (n.family, n.given) == ("み", "なみ") assert [a.kind for a in n.ambiguities] == [AmbiguityKind.SEGMENTATION] @_needs_ja def test_ja_divides_an_astral_kanji_name() -> None: # 𠮷 (U+20BB7, the "tsuchiyoshi" 吉) is a real surname character in # the supplementary plane -- the reason _SCRIPT_RANGES carries the # astral Han span at all. It has to survive the whole chain as one # character and not two: the script gate, the adapter's repertoire # guard, namedivider itself, and the offset arithmetic that turns # the answer into spans (Python indexes by codepoint, so an offset # of 2 here spans four UTF-16 units -- a surrogate-pair bug would # land the split in the middle of the character). n = _PACKED["ja"].parse("𠮷田太郎") assert (n.family, n.given) == ("𠮷田", "太郎") assert [a.kind for a in n.ambiguities] == [AmbiguityKind.SEGMENTATION] @_needs_ja def test_namedivider_still_miscuts_shime_names() -> None: # The canary behind the adapter's 〆 decline: when this fails, # namedivider learned the mark -- revisit the decline in ja.py # rather than deleting this test. import namedivider result = namedivider.BasicNameDivider().divide_name("〆木太郎") assert (result.family, result.given) == ("〆", "木太郎") assert float(result.score) == 1.0 @_needs_ja def test_ja_stacked_on_zh_pays_the_zh_packs_documented_cost() -> None: # Stacking composes MECHANICALLY (the test above) but not for free, # and this pins the cost so it stays a documented trade rather than # a bug report. Vocabulary runs first, so a Japanese kanji name # opening on a listed CHINESE surname never reaches the segmenter: # 高 is a common Chinese surname, so 高橋一郎 splits 高 + 橋一郎 # under ZH+JA exactly as it does under ZH alone -- the same reading # cases.py's zh_japanese_kanji_tradeoff row records, and the reason # the docs call the two packs corpus alternatives and qualify the # stack (see _script_segment.py's module docstring, usage.rst and # the release log). JA alone reads it correctly (test_ja_end_to_end). p = parser_for(locales.ZH, locales.JA, segmenter=locales.ja_segmenter()) n = p.parse("高橋一郎") assert (n.family, n.given) == ("高", "橋一郎") # and silently: one vocabulary match is not a fork, so nothing here # flags the mis-split -- the pack choice is what has to be right assert n.ambiguities == () def test_pack_vocabulary_entries_are_single_words() -> None: # The shipped-vocabulary guard (test_lexicon.py's # test_default_lexicon_builds_warning_free), extended to the OTHER # vocabulary source: a locale pack's lexicon fields are matched one # word at a time same as Lexicon.default()'s, so a multi-word entry # in any field but given_name_titles would be dead. Checked # directly against field contents (not by capturing the # construction-time warning) because a pack module is imported -- # and its Locale built -- at most once per process; a later test in # this file may already have forced the import, which would make a # warning-capture version of this test pass even on a regression. # Iterates every registered pack, not just ru/tr_az by name, so a # future pack is covered automatically. zh is the pack that gives # the field loop something to chew on (ru/tr_az build on # Lexicon.empty()); the registry-non-empty assert below keeps the # test from passing vacuously if that ever stops being true. assert locales.available() for code in locales.available(): lexicon = locales.get(code).lexicon for field in _VOCAB_FIELDS: if field == "given_name_titles": continue for entry in getattr(lexicon, field): assert entry == "".join(entry.split()), ( f"{code}: {field} entry {entry!r} contains " f"whitespace and can never match") def test_pack_marker_regexes_stay_in_sync_with_post_rules() -> None: # ru.py/tr_az.py hand-copy their DEVIATES marker regexes from # _pipeline._post_rules (layering forbids a pack importing the # pipeline -- see each pack's module docstring). Assert pattern # equality mechanically so drift fails here, not at the L6 # non-interference gate. from nameparser._pipeline import _post_rules for pack_regex, rule_regex in ( (_ru._EAST_SLAVIC, _post_rules._EAST_SLAVIC), (_ru._EAST_SLAVIC_CYR, _post_rules._EAST_SLAVIC_CYR), (_tr_az._TURKIC, _post_rules._TURKIC), (_tr_az._TURKIC_CYR, _post_rules._TURKIC_CYR), ): assert pack_regex.pattern == rule_regex.pattern assert pack_regex.flags == rule_regex.flags def test_ru_deviates_scans_per_token() -> None: # Regression: the rule rotates on the family-POSITION token, so a # whole-string search missed a patronymic ending followed by a # suffix -- under-declaration, the unsafe direction for the # non-interference gate. The pack rotates this name; DEVIATES must # say so. assert _ru.DEVIATES("Ivan Petr Sidorovich Jr.") # Over-declaration is the accepted safe direction: the ending # matches a token although the rule's 1 given + 1 middle + # 1 family shape never fires on a two-token name. assert _ru.DEVIATES("Sidorovich Anna") assert not _ru.DEVIATES("John Smith") def test_tr_az_deviates_scans_per_token() -> None: # Mirror of the RU regression above: the Turkic marker regexes are # fullmatch-shaped (^...$), so anything but a per-token scan would # miss a marker followed by a suffix (under-declaration) -- and a # whole-string match would never fire at all on multi-token names. assert _tr_az.DEVIATES("Mammadova Aygun Ali kizi Jr.") # over-declaration stays the safe direction: a bare marker-shaped # token declares even where the rule won't rewrite anything assert _tr_az.DEVIATES("Kizi Anna") assert not _tr_az.DEVIATES("John Smith") # markers are whole-token: a name merely CONTAINING one must not # declare ("Ogluev" is not a marker) assert not _tr_az.DEVIATES("Ogluev Ivan") def test_locales_unknown_attribute() -> None: with pytest.raises(AttributeError, match="XX"): locales.XX def test_ru_plus_tr_az_unions_patronymic_rules() -> None: p = parser_for(locales.RU, locales.TR_AZ) assert p.policy.patronymic_rules == frozenset( {PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) # both rules live: one name from each pack's case segment ru = p.parse("Сидоров Иван Петрович") assert ru.given == "Иван" tr = p.parse("Mammadova Aygun Ali kizi") assert (tr.given, tr.middle, tr.family) == ( "Aygun", "Ali kizi", "Mammadova") def test_pack_over_custom_base() -> None: base = Parser(lexicon=Lexicon.default().add(titles={"zqxcustom"})) p = parser_for(locales.RU, base=base) n = p.parse("Zqxcustom Иван Петрович") assert n.title == "Zqxcustom" # base lexicon survives the fold def test_pack_preserves_custom_base_policy() -> None: # the lexicon-survival twin above has a policy counterpart: a # policy-only pack must ADD its patronymic rule without resetting # unrelated policy fields the base customized base = Parser(policy=Policy(strip_emoji=False)) p = parser_for(locales.RU, base=base) assert p.policy.strip_emoji is False assert p.policy.patronymic_rules == frozenset( {PatronymicRule.EAST_SLAVIC}) def test_parser_for_results_chain_as_bases() -> None: # parser_for's result is itself a valid base= -- packs applied in # two steps accumulate exactly like one call with both packs chained = parser_for(locales.TR_AZ, base=_PACKED["ru"]) assert chained.policy.patronymic_rules == frozenset( {PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) assert chained.parse("Сидоров Иван Петрович").given == "Иван" assert chained.parse("Mammadova Aygun Ali kizi").family == "Mammadova" def test_locales_import_is_lazy(monkeypatch: pytest.MonkeyPatch) -> None: # importing the package must not import any pack module; PEP 562 # loads them on first attribute access (spec §2: "importing # nameparser never pays for pack data"). monkeypatch snapshots # sys.modules AND the parent package attribute (the fresh import # below rebinds nameparser.locales), so everything rolls back even # if an assert fails mid-test (manual del + reload left other tests # running against a half-restored module cache on failure). import sys import nameparser as _np monkeypatch.setattr(_np, "locales", _np.locales) for mod in list(sys.modules): if mod.startswith("nameparser.locales"): monkeypatch.delitem(sys.modules, mod) import nameparser.locales assert "nameparser.locales.ru" not in sys.modules nameparser.locales.RU assert "nameparser.locales.ru" in sys.modules def _assert_non_interference( packed: Parser, deviates: Callable[[str], bool], corpus: Iterable[str], ) -> int: """Return the number of DECLARED deviations seen; fail on any undeclared one (spec §5.2 = the pack-acceptance rejection rule).""" declared = 0 for name in corpus: base = _default_parse(name) got = packed.parse(name).as_dict() if got != base: assert deviates(name), ( f"UNDECLARED deviation: {name!r}\n" f" default: {base}\n packed: {got}") declared += 1 return declared def _default_corpus() -> list[str]: from .cases import CASES return list(_CORPUS) + [c.text for c in CASES if c.locale is None and c.policy is None] # Synthetic rotator names, keyed by pack code: one per alternation # branch of each pack's marker regexes (both scripts), in the shape that # makes the rule fire (RU: family-first, patronymic last, plain middle; # TR_AZ: a standalone marker token). They serve the gate's POSITIVE side # -- without them the shared corpus exercises TR_AZ with a single name, # leaving the gate nearly vacuous (review finding, 2026-07-17). Branch # named per row, enforced by the branch-coverage test below. _ROTATORS: dict[str, list[str]] = {} _ROTATORS["ru"] = [ "Sidorov Ivan Petrovich", # ovich "Sidorova Anna Petrovna", # ovna "Karpov Oleg Sergeevich", # evich "Karpova Olga Sergeevna", # evna "Petrova Maria Nikitichna", # ichna "Ulyanov Vladimir Ilyich", # ilyich "Bulgakov Ivan Kuzmich", # kuzmich "Titov Pyotr Lukich", # lukich "Orlov Semyon Fomich", # fomich "Zaitsev Andrei Fokich", # fokich "Сидоров Иван Петрович", # ович "Сидорова Анна Петровна", # овна "Карпов Олег Сергеевич", # евич "Карпова Ольга Сергеевна", # евна "Петрова Мария Никитична", # ична "Ульянов Владимир Ильич", # ильич "Булгаков Иван Кузьмич", # кузьмич "Титов Пётр Лукич", # лукич "Орлов Семён Фомич", # фомич "Зайцев Андрей Фокич", # фокич ] _ROTATORS["tr_az"] = [ "Aliyev Ali Vali oglu", # oglu "Aliyev Rashad Vali oğlu", # oğlu "Mammadov Elchin Hasan ogly", # ogly "Karimov Aziz Vali ogli", # ogli "Karimov Alisher Vali o'g'li", # o['’ʻ]g['’ʻ]li, ASCII ' "Yusupov Botir Vali o’g’li", # o['’ʻ]g['’ʻ]li, typographic ’ "Mammadova Aygun Hasan qizi", # qizi "Aliyeva Leyla Vali qızı", # qızı "Mammadova Sevinj Ali kizi", # kizi "Asanova Aigul Bolot kyzy", # kyzy "Guliyeva Nigar Vali gyzy", # gyzy "Nazarbayev Nursultan Abish uly", # uly "Zheenbekov Sooronbay Sharip uulu", # uulu "Алиев Али Вали оглу", # оглу "Мамедов Эльчин Гасан оглы", # оглы "Алиев Рашад Вали оғлу", # оғлу "Каримов Алишер Вали ўғли", # ўғли "Юсупов Ботир Вали угли", # угли "Мамедова Айгюн Гасан кызы", # кызы "Алиева Лейла Вали гызы", # гызы "Назарбаева Дарига Нурсултан қызы", # қызы "Каримова Лола Вали қизи", # қизи "Назарбаев Нурсултан Абиш улы", # улы "Токаев Касым Кемел ұлы", # ұлы "Жээнбеков Сооронбай Шарип уулу", # уулу ] # zh has no marker regexes to rotate through: its rotators cover the # SEGMENTATION shapes instead (single/compound surname, simplified/ # traditional, 1- and 2-character given names). No spaced entry -- # spaced Han already parses family-first by default, so a spaced name # would not deviate from the default parser at all. _ROTATORS["zh"] = [ "毛泽东", # 1-char surname, unspaced "张伟", # 2-char name: 1-char surname + 1-char given "欧阳修", # compound surname (欧 is NOT itself listed) "夏侯惇", # compound beats single: 夏 IS listed too "司马相如", # compound + 2-char given "王力宏", # 1-char surname + 2-char given "諸葛亮", # traditional-script compound "萧红", # the 肖/萧 merger's second simplified spelling ] # ja has no marker regexes and no vocabulary either: its rotators cover # the shapes only the pack PLUS its segmenter changes. Every row is # UNSPACED on purpose -- a spaced kana-licensed name (高橋 みなみ) now # reads family-first by DEFAULT (amendment 2026-07-29 §1), so it would # not deviate and would fail the must-deviate assertion below. _ROTATORS["ja"] = [ "山田太郎", # 2-kanji family + 2-kanji given, the common shape "高橋一郎", # the name a zh surname list corrupts (高 + 橋一郎) "田中花子", # ditto (田 is a listed zh surname) "原辰徳", # 1-kanji family + 2-kanji given: the other fork "高橋みなみ", # kanji + hiragana: the kana license's carrier case "山田エミ", # kanji + katakana: licensed as a MIXED token, where # pure katakana (マイケル) is deliberately not "佐々木健", # the iteration mark 々 inside a top-20 surname ] def _marker_regexes(module: ModuleType) -> list[re.Pattern]: return [v for v in vars(module).values() if isinstance(v, re.Pattern)] # Packs whose DEVIATES surface is defined by marker REGEXES -- derived, # never listed, same rule as the registry itself: a pack qualifies by # having any module-level re.Pattern, so zh and ja (which declare by # codepoint range, derived from the shared table via _script_matcher, # so they hold no pattern of their own) drop out on their own and a # future marker-regex pack cannot silently miss branch coverage. _MARKER_REGEX_PACKS = tuple(code for code in sorted(_PACKS) if _marker_regexes(_PACKS[code])) def test_range_declaring_packs_stay_out_of_marker_classification() -> None: # Load-bearing since the packs derive from the shared table: zh/ja # hold no module-level re.Pattern because _script_matcher's closure # hides the compiled class. A pack author who "simplifies" one back # to a module-level pattern flips its classification into the # rotator branch-coverage sweep, which a codepoint-range predicate # cannot satisfy. assert "zh" not in _MARKER_REGEX_PACKS assert "ja" not in _MARKER_REGEX_PACKS def test_registry_is_the_pack_contract() -> None: # spec §2 authoring requirement 3, enforced structurally (design # note 2026-07-18, option C): every registered pack module must # declare its deviation surface, and every pack must feed the gate's # positive side -- a new pack fails HERE until it ships both, rather # than a reviewer remembering to extend the gate by hand. assert set(_PACKS) == set(locales.available()) for code, module in _PACKS.items(): assert callable(getattr(module, "DEVIATES", None)), ( f"pack {code!r} does not declare DEVIATES") # _MARKER_REGEX_PACKS selects on module-level patterns, so a # pack that declares by regex but keeps its patterns inside a # function would drop out of the branch-coverage sweep in # silence. Importing `re` is the tell. The tradeoff is # deliberate: a range-declaring pack that imports re for some # unrelated reason trips this loudly, which beats a silent # coverage gap -- and the fix is to expose the pattern. if "re" in vars(module): assert _marker_regexes(module), ( f"pack {code!r} imports re but exposes no module-level " f"pattern") assert set(_ROTATORS) == set(_PACKS), ( "every pack needs a rotator list (and only registered packs)") # the derivation is code, not a list, so an empty result would make # the branch-coverage test vanish into zero parametrizations rather # than fail. Assert it found something. assert _MARKER_REGEX_PACKS, "no pack declares marker regexes" def _alternation_branches(pattern: str) -> list[str]: """Split a marker regex of the shape '(a|b|...)$' / '^(a|b|...)$' into its alternatives. The packs' character classes contain no '|', so a flat split is safe (and the shape assert guards the day one stops being true).""" inner = pattern.removeprefix("^").removesuffix("$") assert inner.startswith("(") and inner.endswith(")") and "|" not in ( pattern.replace(inner, "")) return inner[1:-1].split("|") @pytest.mark.parametrize("code", _MARKER_REGEX_PACKS) def test_rotators_cover_every_marker_branch(code: str) -> None: # The rotator lists' per-row branch comments are a human convention; # this enforces them mechanically: every alternation branch of every # marker regex a pack defines (any module-level re.Pattern) must be # hit by some rotator token, so a regex edit that adds a branch # (kept in sync with _post_rules by the pattern-equality test above) # fails HERE until a rotator name covers it. Anchoring follows the # pattern's own shape: '^(...)$' = whole-token marker, '(...)$' = # token ending. tokens = [tok for name in _ROTATORS[code] for tok in name.split()] # non-empty by construction: having marker regexes is what put this # pack in _MARKER_REGEX_PACKS in the first place for regex in _marker_regexes(_PACKS[code]): for branch in _alternation_branches(regex.pattern): anchored = regex.pattern.startswith("^") branch_re = re.compile( f"^({branch})$" if anchored else f"({branch})$", re.I) assert any(branch_re.search(tok) for tok in tokens), ( f"no {code!r} rotator exercises marker branch {branch!r}") @pytest.mark.parametrize("code, name", [ (code, name) for code in sorted(_ROTATORS) for name in _ROTATORS[code]]) def test_rotator_deviates_and_declares(code: str, name: str) -> None: # every branch of the pack's marker regexes both FIRES (packed parse # differs from default) and is DECLARED (DEVIATES says so) -- the # per-name proof behind the gate's declared-count floor below packed = _require_packed(code) assert packed.parse(name).as_dict() != _default_parse(name) assert _PACKS[code].DEVIATES(name) @pytest.mark.parametrize("code", sorted(_PACKS)) def test_non_interference_each_pack(code: str) -> None: packed = _require_packed(code) corpus = _default_corpus() + _ROTATORS[code] declared = _assert_non_interference( packed, _PACKS[code].DEVIATES, corpus) # the positive side: every synthetic rotator (plus the corpus's own # in-scope names) must actually flow through the declared branch assert declared >= len(_ROTATORS[code]) def test_non_interference_all_packs_combined() -> None: # Every pack is applied whether or not its extra is installed (the # ja pack's policy patch is pure data); only the ROTATORS of a pack # that cannot act are dropped, since the count assertion below # requires each listed name to actually deviate. all_rotators = [n for code in sorted(_ROTATORS) if code in _PACKED for n in _ROTATORS[code]] corpus = _default_corpus() + all_rotators if _JA_AVAILABLE: packed = parser_for( *(locales.get(code) for code in sorted(_PACKS)), segmenter=locales.ja_segmenter()) else: # without the extra, the stack's hiragana activation is # unservable -- which construction now SAYS (the segmenterless # warning, pinned in test_parser.py); expected noise here, the # gate below is what this test is about with pytest.warns(UserWarning, match="ja_segmenter"): packed = parser_for( *(locales.get(code) for code in sorted(_PACKS))) declared = _assert_non_interference( packed, lambda n: any(m.DEVIATES(n) for m in _PACKS.values()), corpus) assert declared >= len(all_rotators) # -- #269: non-Latin default vocabulary (Cyrillic, Greek, Arabic, Hebrew) -- # # This is DEFAULT vocabulary (nameparser/config/titles.py, # conjunctions.py, prefixes.py), not a locale pack -- it lives here # because it's the non-Latin counterpart to the pack tests above. Every # row was verified live against a runtime-augmented Lexicon.default() # before the data landed (2026-07-17), so these pin actual observed # behavior, not guesses -- see the per-script comments below for the # rows that came out differently than a first guess would suggest. @pytest.mark.parametrize("name, field, expected", [ # Cyrillic (ru/uk) titles. ("г-н Иван Петров", "title", "г-н"), ("г-жа Мария Иванова", "title", "г-жа"), ("д-р Мария Иванова", "title", "д-р"), ("проф Петро Шевченко", "title", "проф"), ("акад Іван Франко", "title", "акад"), ("пан Тарас Шевченко", "title", "пан"), ("пані Марія Іванова", "title", "пані"), # Cyrillic conjunction "и": v1's single-letter carve-out (Google # Code issue 11, the "john e smith" bug) treats a bare # single-alphabetic-character conjunction in a short name as more # likely an initial (group._group_segment), so a 3-piece chain # ("проф и акад Іван Франко") does NOT join -- "и" reads as a given # name instead. The chain only joins once the segment has enough # rootname pieces (total >= 4); pinned against that actual behavior # with a 5-piece name instead of the shorter guess. ("проф и акад Тарас Григорович Шевченко", "title", "проф и акад"), ("проф та акад Іван Франко", "title", "проф та акад"), # Ukrainian "і" is single-character like "и", so the same # single-letter carve-out applies: pinned with a 5-piece name. ("проф і акад Тарас Григорович Шевченко", "title", "проф і акад"), # Ukrainian "й" is the euphonic alternate of "і" (the two swap by # surrounding vowel/consonant, so real data carries both) -- same # single-letter carve-out, pinned with the same 5-piece shape. ("проф й акад Тарас Григорович Шевченко", "title", "проф й акад"), # ...and joining two given names, the shape #267's reporter raised: # a bare Cyrillic letter between given names reads as the # conjunction, not an initial (Cyrillic typography writes initials # with periods -- see # test_267_dotted_cyrillic_initial_wins_over_conjunction). ("Олесь й Олена Коваленки", "given", "Олесь й Олена"), # Greek titles + conjunction (και has 3 letters, so the single-char # initial carve-out above never applies to it). Bare κ is NOT # shipped -- it collides with the initial+surname shape (see the # regression test below). # Match-time contract of the lower()-not-casefold() normalization # (review 2026-07-19): case variants that lower() folds still # match; the ASCII-SS spelling of ß does NOT (v1-parity -- casefold # would have matched it, at the cost of mutating stored spellings). ("ΚΟΣ Γιώργος Παπαδόπουλος", "title", "ΚΟΣ"), ("GROẞFÜRST Otto Schmidt", "title", "GROẞFÜRST"), ("GROSSFÜRST Otto Schmidt", "title", ""), ("δρ Νίκος Παπαδόπουλος", "title", "δρ"), ("κος Γιώργος Παπαδόπουλος", "title", "κος"), ("κα Μαρία Παπαδοπούλου", "title", "κα"), ("καθ και δρ Νίκος Παπαδόπουλος", "title", "καθ και δρ"), # Arabic patronymic/clan prefixes: non-leading "بن"/"بنت" chain onto # the family, mirroring the Latin "von"/"bin" prefix-chain rule. ("محمد بن سلمان", "family", "بن سلمان"), ("فاطمة بنت محمد", "family", "بنت محمد"), # "ابن" (alternate spelling) and the clan prefix "آل" are # never-given: non-leading they chain onto the family, and a # LEADING "آل" consumes the whole name as family (no given), like # "de Mesnil". ("محمد ابن رشد", "family", "ابن رشد"), ("محمد آل سعود", "family", "آل سعود"), ("آل سعود", "family", "آل سعود"), ("آل سعود", "given", ""), # The kunya "أبو"/"ابو" is AMBIGUOUS (it can begin a standalone # byname): leading it reads as the given name -- the "Van Johnson" # rule -- while non-leading it chains onto the family like any # particle. ("أبو مازن", "given", "أبو"), ("أحمد أبو خليل", "family", "أبو خليل"), ("علي ابو خالد", "family", "ابو خالد"), # "الشيخ" carries the FIRST_NAME_TITLES semantics of its # transliterated cousin 'sheikh': a single following name reads as # given, not family. ("الشيخ محمد", "given", "محمد"), # Arabic honorifics precede the GIVEN name, so all ship in # given_name_titles like الشيخ: article forms (never given names) # and the bare doctor/professor/engineer forms (not used as given # names). Deferred with the same collision rule as мл/ст: bare # سيد/شيخ/أمير/سلطان (common given names) and the 'د.' # abbreviation (bare 'د' would swallow initials, the bare-κ trap). ("الدكتور أحمد", "given", "أحمد"), ("الدكتور أحمد", "title", "الدكتور"), ("الدكتورة منى السيد", "title", "الدكتورة"), ("الدكتورة منى السيد", "family", "السيد"), ("دكتور أحمد", "given", "أحمد"), ("دكتورة منى", "given", "منى"), ("الأستاذ محمود", "given", "محمود"), ("أستاذ محمود", "given", "محمود"), ("أستاذة سلمى", "given", "سلمى"), ("الأستاذة سلمى", "given", "سلمى"), ("مهندس حسن", "given", "حسن"), ("الشيخة موزة", "given", "موزة"), ("الحاجة فاطمة", "given", "فاطمة"), # compound with the bound given name: title + bound join + family ("الحاج عبد الرحمن السيد", "title", "الحاج"), ("الحاج عبد الرحمن السيد", "given", "عبد الرحمن"), ("الحاج عبد الرحمن السيد", "family", "السيد"), # "و" ("and") joins title chains like Cyrillic "и"; single-char # conjunctions need the same single-letter carve-out headroom (enough # rootname pieces), so pinned with the 6-piece shape the и row uses ("الأستاذ و الدكتور طارق حسن السيد", "title", "الأستاذ و الدكتور"), # Hebrew patronymic prefixes: same non-leading chain-onto-family # behavior. ("דוד בן גוריון", "family", "בן גוריון"), ("שרה בת אברהם", "family", "בת אברהם"), # Hebrew "מר" title (plain title, not FIRST_NAME_TITLES -- like # 'mr', the following name reads as family). ("מר דוד לוי", "title", "מר"), # Hebrew title/suffix sweep (#269 follow-up): plain titles (Israeli # convention: family follows, the מר precedent); both geresh/ # gershayim spellings ship where an abbreviation carries one. ("גברת רות כהן", "title", "גברת"), ("פרופ' דוד לוי", "title", "פרופ'"), ("פרופ׳ דוד לוי", "title", "פרופ׳"), ("פרופסור רות כהן", "title", "פרופסור"), ('עו"ד דוד לוי', "title", 'עו"ד'), ("עו״ד דוד לוי", "title", "עו״ד"), ("הרב עובדיה יוסף", "title", "הרב"), # post-nominal Hebrew suffixes: ז"ל ("of blessed memory") and # שליט"א (honorific for living rabbis) -- suffix words, exact # token incl. the mid-word gershayim (inert in extraction, like # the ד"ר title) ('משה כהן ז"ל', "suffix", 'ז"ל'), ("משה כהן ז״ל", "suffix", "ז״ל"), ('הרב משה פיינשטיין שליט"א', "suffix", 'שליט"א'), ("הרב משה פיינשטיין שליט״א", "title", "הרב"), # Devanagari (hi/mr) titles -- a new #269 script: श्री/श्रीमती are # the Mr./Mrs. analogs, डॉ the Dr. abbreviation (edge-period # normalization makes "डॉ." match too). NO Latin twins on purpose: # transliterated sri/shri collide with real given names (Sri # Mulyani); the native script cannot. ("श्री राम शर्मा", "title", "श्री"), ("श्रीमती सीता शर्मा", "title", "श्रीमती"), ("डॉ राम शर्मा", "title", "डॉ"), ("डॉ. राम शर्मा", "title", "डॉ."), # Geresh/gershayim gate (#269 step 2): probed live against # extract_delimited's _open_ok/_close_ok boundary rules. Both the # ASCII-quote spelling and the typographic Unicode spelling of # "doctor"/"Mrs." survive extraction untouched -- the quote sits # mid-word (no preceding whitespace before '"', and for the # trailing "'" no following word boundary), so it never satisfies # the boundary-valid open/close test and is left as literal text. # Both spellings are shipped; see the titles.py comment. ('ד"ר דוד לוי', "title", 'ד"ר'), ("גב' דוד לוי", "title", "גב'"), ("ד״ר דוד לוי", "title", "ד״ר"), ("גב׳ דוד לוי", "title", "גב׳"), ]) def test_269_nonlatin_vocabulary_parses( name: str, field: str, expected: str) -> None: assert getattr(parse(name), field) == expected def test_269_vocabulary_reaches_the_v1_facade() -> None: # one row per script through HumanName: the facade resolves its # lexicon from the CONSTANTS shim snapshot, a different path from # parse()'s Lexicon.default() -- proves the #269 data modules feed # both surfaces from nameparser import HumanName assert HumanName("г-н Иван Петров").title == "г-н" assert HumanName("κος Γιώργος Παπαδόπουλος").title == "κος" assert HumanName("محمد بن سلمان").last == "بن سلمان" assert HumanName('ד"ר דוד לוי').title == 'ד"ר' # one row per NEW vocabulary category (2026-07-19 batches): bound # given name, Hebrew post-nominal suffix, Devanagari title assert HumanName("عبد الرحمن محمد").first == "عبد الرحمن" assert HumanName('משה כהן ז"ל').suffix == 'ז"ל' assert HumanName("डॉ. राम शर्मा").title == "डॉ." def test_269_bare_greek_kappa_not_a_title() -> None: # Regression for the deferred bare 'κ' entry: were it in TITLES, # _normalize's edge-period strip would make the abbreviated-initial # 'Κ.' match it, degrading the very common initial+surname shape to # title='Κ.' with an EMPTY given. Pin the correct reading. n = parse("Κ. Παπαδόπουλος") assert n.title == "" assert n.given == "Κ." assert n.family == "Παπαδόπουλος" def test_267_dotted_cyrillic_initial_wins_over_conjunction() -> None: # Blast-radius guard for the single-letter Cyrillic conjunctions # (и/і/й): each is also the first letter of real given names # (Игорь, Ірина, Йосип), so the shipped entries would be a trap if # membership could outrank a punctuated initial. It cannot -- the # initial regex's dotted branch is Unicode-aware -- and #267 settled # the BARE letter in the conjunction's favor precisely because # Cyrillic typography writes initials with the period. for dotted, family in (("И.", "Иванов"), ("І.", "Франко"), ("Й.", "Сліпий")): n = parse(f"{dotted} {family}") assert n.given == dotted assert n.family == family python-nameparser-2.1.0/tests/v2/test_parser.py000066400000000000000000001042521523530431700215730ustar00rootroot00000000000000import dataclasses import pickle import unicodedata import pytest from nameparser import ( Lexicon, Locale, Parser, Policy, PolicyPatch, locales, parse, parser_for, ) from nameparser._policy import ( FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, PatronymicRule, ) from nameparser._types import AmbiguityKind, Role, Segmentation def test_parser_defaults_and_properties() -> None: p = Parser() assert p.lexicon == Lexicon.default() assert p.policy == Policy() def test_parser_rejects_wrong_types_eagerly() -> None: with pytest.raises(TypeError, match="lexicon"): Parser(lexicon={"titles": set()}) # type: ignore[arg-type] with pytest.raises(TypeError, match="policy"): Parser(policy="strict") # type: ignore[arg-type] def test_parse_end_to_end_with_default_vocabulary() -> None: pn = parse("Dr. Juan de la Vega III") assert pn.title == "Dr." assert pn.given == "Juan" assert pn.family == "de la Vega" assert pn.suffix == "III" assert str(pn) == "Dr. Juan de la Vega III" def test_parse_rejects_non_str_with_decode_hint() -> None: with pytest.raises(TypeError, match="decode"): parse(b"John Smith") # type: ignore[arg-type] with pytest.raises(TypeError, match="str"): parse(None) # type: ignore[arg-type] def test_degenerate_inputs_are_total() -> None: # spec §5a table assert not parse("") assert not parse(" ") assert parse("").original == "" # an input with no alphanumeric character is not a name (v1 kept it) assert not parse(".,") assert not parse("- -") assert parse(".,").original == ".," # but the raw input is kept single = parse("John") assert single.given == "John" family_first = Parser(policy=Policy(name_order=FAMILY_FIRST)) assert family_first.parse("Yamada").family == "Yamada" title_only = parse("Dr.") assert title_only.title == "Dr." and not title_only.given unbalanced = parse('Jon "Nick Smith') kinds = {a.kind for a in unbalanced.ambiguities} assert AmbiguityKind.UNBALANCED_DELIMITER in kinds assert '"Nick' in [t.text for t in unbalanced.tokens] # literal def test_parser_is_picklable_and_frozen() -> None: p = Parser(policy=Policy(name_order=FAMILY_FIRST)) loaded = pickle.loads(pickle.dumps(p)) assert loaded == p assert loaded.parse("Yamada Taro").family == "Yamada" with pytest.raises(AttributeError): p.policy = Policy() # type: ignore[misc] def test_parser_repr_composes_component_reprs() -> None: assert repr(Parser()) == "Parser(Lexicon(default), Policy())" p = Parser(policy=Policy(name_order=FAMILY_FIRST)) assert repr(p) == "Parser(Lexicon(default), Policy(name_order=FAMILY_FIRST))" def test_parsedname_repr_includes_ambiguities_line() -> None: pn = parse("Van Johnson") r = repr(pn) assert "given: 'Van'" in r assert "ambiguities:" in r and "particle-or-given" in r def test_module_parse_reuses_the_default_parser() -> None: import nameparser._parser as parser_mod assert parser_mod._default_parser() is parser_mod._default_parser() def test_parser_for_stacks_locales() -> None: ru = Locale(code="ru", lexicon=Lexicon.empty().add(titles={"г-н"}), policy=PolicyPatch(patronymic_rules=frozenset( {PatronymicRule.EAST_SLAVIC}))) p = parser_for(ru) assert PatronymicRule.EAST_SLAVIC in p.policy.patronymic_rules pn = p.parse("г-н Сидоров Иван Петрович") assert pn.title == "г-н" assert pn.given == "Иван" assert pn.family == "Сидоров" def test_parser_for_rejects_non_locales() -> None: with pytest.raises(TypeError, match="Locale"): parser_for("ru") # type: ignore[arg-type] def test_parser_for_wraps_pack_errors_with_identity() -> None: # PolicyPatch validates lazily (by design), so an invalid value sits # latent in a perfectly constructible Locale until apply time bad = Locale(code="xx", lexicon=Lexicon.empty(), policy=PolicyPatch(name_order=(1, 2, 3))) # type: ignore[arg-type] # the rewrap preserves the taxonomy's exception type (here the # non-Role element TypeError) while adding the pack identity with pytest.raises(TypeError, match="while applying locale 'xx'"): parser_for(bad) def test_parser_for_warns_on_scalar_conflict() -> None: a = Locale(code="aa", lexicon=Lexicon.empty(), policy=PolicyPatch(strip_emoji=False)) b = Locale(code="bb", lexicon=Lexicon.empty(), policy=PolicyPatch(strip_emoji=True)) with pytest.warns(UserWarning, match="strip_emoji"): p = parser_for(a, b) assert p.policy.strip_emoji is True # later wins def test_matches_component_wise_case_insensitive() -> None: pn = parse("John Smith") assert pn.matches("JOHN SMITH") assert pn.matches(parse("john smith")) assert not pn.matches("John Smythe") with pytest.raises(TypeError, match="str or ParsedName"): pn.matches(42) # type: ignore[arg-type] def test_family_first_given_last_places_middle_between() -> None: # T1: the three-piece FAMILY_FIRST_GIVEN_LAST assignment -- family # from the front, given from the END, middle between (not a rotation # of FAMILY_FIRST) p = Parser(policy=Policy(name_order=FAMILY_FIRST_GIVEN_LAST)) pn = p.parse("Zeng Xiao Long") assert (pn.family, pn.middle, pn.given) == ("Zeng", "Xiao", "Long") def test_multiple_unbalanced_delimiters_each_reported() -> None: # T4: the extract scan continues past the first unmatched opener; # each one is reported and treated as literal text pn = parse('John "Jack (Smith') unbalanced = [a for a in pn.ambiguities if a.kind is AmbiguityKind.UNBALANCED_DELIMITER] assert len(unbalanced) == 2 assert pn.given == "John" and pn.family == "(Smith" assert not pn.nickname def test_matches_accepts_explicit_parser() -> None: family_first = Parser(policy=Policy(name_order=FAMILY_FIRST)) pn = family_first.parse("Yamada Taro") assert pn.matches("Yamada Taro", parser=family_first) assert not pn.matches("Yamada Taro") # default parser reads given-first def test_phd_split_heals_in_the_suffix_view() -> None: # v1 parity via fix_phd: the split credential renders as one suffix assert parse("John Ph. D.").suffix == "Ph. D." assert parse("John Smith PhD MD").suffix == "PhD, MD" # unchanged def test_phd_split_mid_name_is_a_suffix() -> None: # v1 parity: fix_phd extracted the credential BEFORE parsing, so # position never mattered; the merged piece is a suffix anywhere pn = parse("Dr. John Ph. D. Smith") assert pn.suffix == "Ph. D." assert pn.family == "Smith" assert pn.middle == "" def test_ambiguous_acronym_reports_the_reading_it_took() -> None: # 'ma' is both a post-nominal and a surname, so whichever way the # peel resolves it is a guess -- the same shape as the leading # ambiguous particle that already reports PARTICLE_OR_GIVEN took_suffix = parse("John Smith MA") assert took_suffix.suffix == "MA" assert [a.kind for a in took_suffix.ambiguities] == \ [AmbiguityKind.SUFFIX_OR_NAME] assert [t.text for t in took_suffix.ambiguities[0].tokens] == ["MA"] took_family = parse("Jack MA") assert took_family.family == "MA" assert [a.kind for a in took_family.ambiguities] == \ [AmbiguityKind.SUFFIX_OR_NAME] @pytest.mark.parametrize("text", [ "John Smith M.A.", # periods decide it; no guess "Ma, Jack", # a comma fixes the family "Joao da Silva do Amaral de Souza", # 'do' mid-name, never at the peel "John Smith PhD", # unambiguous vocabulary ]) def test_no_suffix_ambiguity_when_nothing_was_guessed(text: str) -> None: assert [a for a in parse(text).ambiguities if a.kind is AmbiguityKind.SUFFIX_OR_NAME] == [] def test_delimited_ambiguous_acronym_reports_suffix_or_nickname() -> None: # inside delimiters the competing readings are suffix and nickname: # "(MBA)" is unambiguously a credential and escapes to suffix, while # "(JD)" could be either, so it keeps the nickname reading -- a # guess, and until now a silent one n = parse("JEFFREY (JD) BRICKEN") assert n.nickname == "JD" assert [a.kind for a in n.ambiguities] == \ [AmbiguityKind.SUFFIX_OR_NICKNAME] assert [t.text for t in n.ambiguities[0].tokens] == ["JD"] # the unambiguous one decided on vocabulary, so it is not a guess assert parse("Andrew Perkins (MBA)").ambiguities == () def test_every_ambiguous_acronym_in_a_name_is_reported() -> None: # one coin-flip per acronym: a single-slot record dropped all but # the last, which defeats the point of reporting at all n = parse("John Smith MA JD") assert n.suffix == "MA, JD" assert [a.kind for a in n.ambiguities] == \ [AmbiguityKind.SUFFIX_OR_NAME] * 2 assert sorted(t.text for a in n.ambiguities for t in a.tokens) == \ ["JD", "MA"] def test_ambiguous_acronym_detail_names_the_role_it_got() -> None: # the unpeeled piece is the last NAME piece, which is the family # name only under GIVEN_FIRST -- FAMILY_FIRST puts it in given, so # the detail has to follow the role actually assigned fam_first = Parser(policy=Policy(name_order=FAMILY_FIRST)) n = fam_first.parse("Jack MA") assert (n.family, n.given) == ("Jack", "MA") assert "given name" in n.ambiguities[0].detail assert "family name" not in n.ambiguities[0].detail def test_trailing_roman_numeral_reports_the_fork() -> None: # a trailing single letter is a name part unless it happens to be a # roman numeral, in which case it is silently reclassified -- and # V/X/I are common middle initials, so this is a real coin-flip numeral = parse("John Smith V") assert numeral.suffix == "V" assert [a.kind for a in numeral.ambiguities] == \ [AmbiguityKind.SUFFIX_OR_NAME] assert [t.text for t in numeral.ambiguities[0].tokens] == ["V"] # the same shape with a non-numeral letter faces no fork assert parse("John Smith B").ambiguities == () # and a numeral after an initial is not treated as a suffix at all assert parse("John Q. V").ambiguities == () def test_ambiguous_particle_reports_both_branches_of_its_fork() -> None: # "Van Johnson" reads Van as a given name and says so. A leading # title shifts Van off index 0, the prefix-chain merge fires, and # Van becomes a particle instead -- the SAME fork, called the other # way. The two branches are taken in different stages (_assign vs # _group), so only the one with an emitter used to report. given_reading = parse("Van Johnson") assert given_reading.given == "Van" assert [a.kind for a in given_reading.ambiguities] == \ [AmbiguityKind.PARTICLE_OR_GIVEN] particle_reading = parse("Dr. Van Johnson") assert particle_reading.family == "Van Johnson" assert [a.kind for a in particle_reading.ambiguities] == \ [AmbiguityKind.PARTICLE_OR_GIVEN] assert [t.text for t in particle_reading.ambiguities[0].tokens] == ["Van"] def test_unambiguous_particle_chain_reports_nothing() -> None: # 'de' is never a given name, so merging it is not a fork assert parse("Dr. de la Vega").ambiguities == () @pytest.mark.parametrize("text", [ "Dr. Van Jr.", # the piece after the particle is a suffix, so "Dr. Van MD", # the chain scan never advances and merge() is "Dr. Do Jr.", # a no-op -- nothing was chained, no fork taken ]) def test_no_op_prefix_chain_is_not_a_fork(text: str) -> None: assert parse(text).ambiguities == () def test_a_fork_is_reported_by_exactly_one_stage() -> None: # the no-op merge left the particle a lone leading piece, which is # _assign's trigger, so both stages reported the same token n = parse("Dr. Van Jr Smith") assert n.given == "Van" assert len(n.ambiguities) == 1 def test_chained_particle_detail_does_not_claim_a_role() -> None: # _group runs before assignment, so it cannot know which field the # chained piece lands in -- "Dr. Van Johnson de la Cruz" puts it in # GIVEN. The detail must describe the decision, not guess a role. n = parse("Dr. Van Johnson de la Cruz") assert n.given == "Van Johnson" (amb,) = n.ambiguities assert "family name" not in amb.detail def test_each_suffix_or_name_branch_describes_itself() -> None: # one kind, two causes: the acronym branch turns on periods, the # roman-numeral branch turns on the letter being a numeral. Sharing # one template made "V written without periods" -- a distinction # that does not exist for V -- and hid which branch fired. acronym = parse("John Smith MA").ambiguities[0].detail assert "periods" in acronym and "post-nominal" in acronym numeral = parse("John Smith V").ambiguities[0].detail assert "periods" not in numeral assert "numeral" in numeral and "initial" in numeral def test_parser_matches_uses_its_own_config() -> None: p = Parser(lexicon=Lexicon.default().add(titles=["moff"])) a = p.parse("Moff Tarkin") # ParsedName.matches falls back to the DEFAULT parser for the str # argument, which reads "Moff" as a given name -- mismatch. assert not a.matches("Moff Tarkin") # Parser.matches parses the str with the same config -- match. assert p.matches(a, "Moff Tarkin") assert p.matches("Moff Tarkin", "MOFF TARKIN") def test_parser_matches_rejects_wrong_types() -> None: p = Parser() with pytest.raises(TypeError, match="takes str or ParsedName"): p.matches(42, "John Smith") # type: ignore[arg-type] with pytest.raises(TypeError, match="takes str or ParsedName"): p.matches("John Smith", 42) # type: ignore[arg-type] def test_parser_capitalized_uses_its_own_lexicon() -> None: # pair-tuples, not a dict: the field is tuple-annotated and typed # call sites pass canonical pairs (see _default_lexicon's note) exceptions = dict( Lexicon.default().capitalization_exceptions_map) | {"zqx": "ZqX"} lex = dataclasses.replace( Lexicon.default(), capitalization_exceptions=tuple(sorted(exceptions.items()))) p = Parser(lexicon=lex) n = p.parse("john zqx") assert p.capitalized(n).family == "ZqX" # ParsedName.capitalized() with no argument uses the DEFAULT # lexicon, which has no such exception. assert n.capitalized().family == "Zqx" def test_parser_capitalized_rejects_non_parsed_name() -> None: with pytest.raises(TypeError, match="takes a ParsedName"): Parser().capitalized("john smith") # type: ignore[arg-type] def test_revise_preserves_particle_tags() -> None: p = Parser() n = p.parse("Juan de la Vega") r = p.revise(n, family="de la Vega Smith") assert r.family == "de la Vega Smith" assert r.family_particles == "de la" assert r.initials() == "J. V. S." # particles contribute no initial def test_revise_keeps_multiword_suffix_one_credential() -> None: p = Parser() n = p.parse("John Smith Ph.D.") r = p.revise(n, suffix="Ph. D.") assert r.suffix == "Ph. D." # replace() would render "Ph., D." def test_revise_views_match_a_fresh_parse() -> None: p = Parser() r = p.revise(p.parse("John Smith"), family="de la Vega") f = p.parse("John de la Vega") for view in ("given", "family", "family_particles", "family_base"): assert getattr(r, view) == getattr(f, view) assert r.initials() == f.initials() def test_revise_replace_shared_semantics() -> None: p = Parser() n = p.parse("Dr. Juan de la Vega Jr.") r = p.revise(n, given="José", suffix="") assert r.given == "José" assert r.suffix == "" # empty value clears the field assert p.revise(n, suffix="()").suffix == "" # punctuation-only too assert r.original == n.original # provenance unchanged assert all(t.span is None for t in r.tokens_for("given")) assert r.title == "Dr." # untouched fields keep spans def test_revise_validation_matches_replace() -> None: p = Parser() n = p.parse("John Smith") with pytest.raises(TypeError, match="takes a ParsedName"): p.revise("John Smith", family="Doe") # type: ignore[arg-type] with pytest.raises(TypeError, match="unknown field 'last'"): p.revise(n, last="Doe") with pytest.raises(TypeError, match="must be a str"): p.revise(n, family=None) # type: ignore[arg-type] def test_revise_strips_the_fold_marker() -> None: # middle_as_family's fold tag must not survive the harvest: a # carried tag would make the family view reorder the value. p = Parser(policy=Policy(middle_as_family=True)) r = p.revise(p.parse("Juan Perez"), family="Gabriel García Márquez") assert r.family == "Gabriel García Márquez" def test_revise_sub_parse_structural_behavior() -> None: # the docstring's three structural promises, pinned: delimiters # never become tokens, marker words are consumed as in parsing, # and the sub-parse's ambiguities are discarded. p = Parser() n = p.parse("John Smith") assert p.revise(n, family="Smith (Jones)").family == "Smith Jones" revised = p.revise(n, family="Mary née Smith") assert revised.family == "Mary Smith" assert revised.maiden == "" assert p.revise(n, given="J.R. 'Bob'").given == "J.R. Bob" assert p.revise(n, family="Smith (Jones").ambiguities == () def test_revise_sub_parses_under_this_parsers_policy() -> None: # the sub-parse runs on SELF, not on a default Parser: revise's # docstring promises a marker LEADING a delimited value is # consumed, and only a policy routing that pair to maiden makes # the value delimited at all. Sub-parsing with Parser() instead # leaves the rest of the suite green -- every other revise # assertion uses a default parser, so nothing else can tell the # two apart. p = Parser(policy=Policy(maiden_delimiters=frozenset({("(", ")")}))) n = p.parse("John Smith") assert p.revise(n, family="(née Jones)").family == "Jones" # and the third leg: a leading marker in an UNDELIMITED value is # no marker at all, so the same words keep it (#329) assert p.revise(n, family="née Jones").family == "née Jones" def test_revise_forces_the_named_role_on_every_harvested_token() -> None: # the sub-parse reads "Dr." as a title and "Jr." as a suffix; the # named field's role must win for every token or the family view # silently drops them p = Parser() r = p.revise(p.parse("John Smith"), family="Dr. Vega Jr.") assert r.family == "Dr. Vega Jr." assert all(t.role is Role.FAMILY for t in r.tokens_for(Role.FAMILY)) def test_wholly_cjk_names_read_family_first_by_default() -> None: # the 2026-07-27 amendment: script determines the convention, so # no pack is needed -- release-log-classified fix (#271) n = parse("毛 泽东") assert (n.family, n.given) == ("毛", "泽东") n = parse("김 민준") assert (n.family, n.given) == ("김", "민준") # a lone wholly-CJK token takes the script order's first role: Han # segmentation is opt-in (locales.ZH), so the default parser leaves # this one token whole and reads it as the family name assert parse("毛泽东").family == "毛泽东" def test_nfd_korean_input_still_reads_family_first() -> None: # fix(#271) classification, landed via the #272 NFC-classification # amendment: NFD decomposes each Hangul syllable onto bare jamo, # entirely outside the HANGUL range, so raw NFD input used to miss # script_orders' family-first rule and fall back to the positional # default -- a live gap in #294's shipped behavior until # single_script started classifying an NFC-normalized copy. n = parse(unicodedata.normalize("NFD", "김 민준")) # classification-only: the rendered text is exactly what was # typed (still NFD, spans untouched), so compare NFC-normalized -- # the point under test is the ORDER (family first), not encoding assert (unicodedata.normalize("NFC", n.family), unicodedata.normalize("NFC", n.given)) == ("김", "민준") # pinned, not just stated: a future tokenize-level normalize (the # tempting over-fix, and an anti-#100 violation) must fail here assert n.family == unicodedata.normalize("NFD", "김") def test_kana_licensed_names_read_family_first_by_default() -> None: # the #272 amendment: hiragana identifies Japanese as certainly # as hangul identifies Korean -- release-log-classified fix n = parse("高橋 みなみ") assert (n.family, n.given) == ("高橋", "みなみ") n = parse("山田 エミ") # kanji piece + katakana piece: native assert (n.family, n.given) == ("山田", "エミ") # lone kana-licensed token takes the family role, unsplit assert parse("高橋みなみ").family == "高橋みなみ" def test_iteration_mark_counts_as_han() -> None: # 々 (U+3005) repeats the preceding kanji. It is Script=Han under # UAX #24 already, but it lives outside every CJK ideograph BLOCK, # and the classifier is a block table -- so without its singleton # entry a 佐々木 token was in no script at all: the name reversed, # and the token never reached the segmentation gate. The Script # property would have got this one right unaided; the block table # is what needed the special case. n = parse("佐々木 太郎") assert (n.family, n.given) == ("佐々木", "太郎") n = parse("野々村 真") assert (n.family, n.given) == ("野々村", "真") # and a lone one takes the family role like any other Han token assert parse("奈々").family == "奈々" def test_pure_katakana_stays_positional() -> None: # transcribed foreign names keep their original (usually Western) # order; katakana alone licenses nothing n = parse("マイケル ジャクソン") assert (n.given, n.family) == ("マイケル", "ジャクソン") def test_katakana_transcription_parses_by_its_divider() -> None: # the dot tells us where the parts meet; transcriptions keep the # source language's order, which the positional default provides n = parse("マイケル・ジャクソン") assert (n.given, n.family) == ("マイケル", "ジャクソン") def test_nakaguro_split_han_tokens_take_the_han_order() -> None: # different code path from the katakana case above: splitting on # the dot here produces two PURE-HAN tokens (no kana involved), so # script_orders' family-first HAN entry fires, same as any other # two-token Han name -- the dot only decides where the split falls n = parse("高橋・一郎") assert (n.family, n.given) == ("高橋", "一郎") def test_halfwidth_nakaguro_splits_at_parse_level_too() -> None: # decision, not accident: halfwidth kana classify as no script at # all (_SCRIPT_RANGES only covers the fullwidth blocks), so this # is order-agnostic positional fallback, not a script-order rule -- # the dot still divides the tokens regardless text = "ﾏｲｹﾙ･ｼﾞｬｸｿﾝ" n = parse(text) assert (n.given, n.family) == ( "ﾏｲｹﾙ", "ｼﾞｬｸｿﾝ") def test_nakaguro_inside_a_nickname_still_splits() -> None: # decision, not accident: delimited content tokenizes under the # same separator rules as the main stream, so a dot inside a # nickname still divides it -- and re-rendering a token stream # necessarily uses the render join, so the dot comes back as a # space, same as any other separator n = parse("山田 太郎 (マイケル・ジャクソン)") assert n.nickname == "マイケル ジャクソン" assert (n.family, n.given) == ("山田", "太郎") def test_latin_names_are_untouched_by_script_orders() -> None: n = parse("John Smith") assert (n.given, n.family) == ("John", "Smith") # mixed-script names fall back to name_order too n = parse("John 王") assert (n.given, n.family) == ("John", "王") def test_script_order_survives_latin_titles_and_suffixes() -> None: # The script test runs on the NAME pieces, after both peels, so a # Latin title or post-nominal cannot make the name look mixed. n = parse("Dr. 毛 泽东") assert (n.title, n.family, n.given) == ("Dr.", "毛", "泽东") n = parse("毛 泽东, PhD") assert (n.family, n.given, n.suffix) == ("毛", "泽东", "PhD") def test_a_comma_still_decides_the_family_name_for_cjk() -> None: # The family-comma structure fixes the family before any positional # read, so the table is never consulted -- same rule name_order has. n = parse("泽东, 毛") assert (n.family, n.given) == ("泽东", "毛") def test_three_cjk_pieces_take_the_script_order_middles() -> None: n = parse("毛 泽东 泽民") assert (n.family, n.given, n.middle) == ("毛", "泽东", "泽民") def test_two_cjk_scripts_fall_back_even_though_both_read_family_first() -> None: # The rule is one script, or the Han/kana repertoire the #272 # license covers -- not "the entries agree": Han+Hangul is written # in neither tradition, so it takes the positional default. n = parse("毛 김") assert (n.given, n.family) == ("毛", "김") def test_a_hyphen_in_a_name_piece_declines_the_script_order() -> None: # Documenting the conservative direction, not proposing it: ANY # non-CJK character in a name piece (here the hyphen) puts that # piece in no script, so the piece set has two members and # script_orders declines in favour of the positional default. n = parse("毛 泽东-泽民") assert (n.given, n.family) == ("毛", "泽东-泽民") def test_script_orders_opt_out_restores_positional_reading() -> None: p = Parser(policy=Policy(script_orders={})) # type: ignore[arg-type] n = p.parse("毛 泽东") assert (n.given, n.family) == ("毛", "泽东") def test_script_order_beats_explicit_global_name_order() -> None: # the script entry is the more specific rule; opting out means # script_orders={}, not a different name_order p = Parser(policy=Policy(name_order=FAMILY_FIRST_GIVEN_LAST)) n = p.parse("김 민준") assert (n.family, n.given) == ("김", "민준") def test_unspaced_korean_names_parse_by_default() -> None: # the whole point of shipping the census list as default # vocabulary (#271): no pack, no config n = parse("김민준") assert (n.family, n.given) == ("김", "민준") n = parse("남궁민수") # two-syllable surname beats single 남 assert (n.family, n.given) == ("남궁", "민수") # -- the segmenter hook (#272) ------------------------------------------ def _module_level_decline(text: str) -> Segmentation | None: return None # module-level so pickle can find it # Inert sentinels for the plumbing tests below, which only ever compare # identity. Two of them, because the override test needs the loser and # the winner to be DISTINCT objects or its assertion is vacuous. def _decline_a(text: str) -> Segmentation | None: return None def _decline_b(text: str) -> Segmentation | None: return None def test_parser_segmenter_is_keyword_only_and_validated() -> None: assert Parser().segmenter is None with pytest.raises(TypeError, match="callable"): Parser(segmenter=5) # type: ignore[arg-type] with pytest.raises(TypeError, match="positional"): Parser(Lexicon.default(), Policy(), None) # type: ignore[misc] # positional: rejected def test_parser_for_carries_the_base_segmenter() -> None: # not given: the base's carries through unchanged p = parser_for(locales.get("zh"), base=Parser(segmenter=_decline_a)) assert p.segmenter is _decline_a def test_parser_for_rejects_a_non_parser_base() -> None: with pytest.raises(TypeError, match="base must be a Parser"): parser_for(locales.get("zh"), base=5) # type: ignore[arg-type] def test_parser_for_takes_a_segmenter_keyword() -> None: p = parser_for(locales.get("zh"), segmenter=_decline_a) assert p.segmenter is _decline_a assert parser_for(locales.get("zh")).segmenter is None def test_parser_for_segmenter_keyword_overrides_the_base() -> None: # later wins, the same rule scalar policy fields follow p = parser_for(locales.get("zh"), base=Parser(segmenter=_decline_a), segmenter=_decline_b) assert p.segmenter is _decline_b def test_parser_for_segmenter_none_clears_the_base() -> None: # the third state UNSET buys (#272 review): None is a VALUE here, # not an absence, so passing it explicitly drops the base's # segmenter instead of inheriting the very thing it was asked to # remove. Omitting the keyword is what carries the base's through # (the test above), and this is how you derive an unsegmented # parser from a segmented one without rebuilding its lexicon and # policy by hand. base = Parser(segmenter=_decline_a) pack = locales.get("zh") assert parser_for(pack, base=base, segmenter=None).segmenter is None # ...and the base is untouched: parser_for builds a fresh Parser assert base.segmenter is _decline_a def test_parser_picklability_is_conditional_on_the_segmenter() -> None: # declared contract (locales spec section 4): Parser pickles iff # its segmenter does -- like any callable-holding object p = pickle.loads(pickle.dumps(Parser(segmenter=_module_level_decline))) assert p.segmenter is _module_level_decline unpicklable = Parser(segmenter=lambda t: None) # constructs fine with pytest.raises(Exception): # pickle's exception type varies pickle.dumps(unpicklable) # only pickling fails def test_segmenter_exceptions_propagate() -> None: # the ONE exception to parse-totality (locales spec section 4, # declared 2026-07-11): a user-supplied callable's own error is a # user-code error, not a content error, and must not be swallowed def boom(text: str) -> Segmentation | None: raise RuntimeError("segmenter bug") p = parser_for(locales.get("zh"), base=Parser(segmenter=boom)) with pytest.raises(RuntimeError, match="segmenter bug"): p.parse("阿明") # zh pack active, 阿 unmatched by vocabulary -> # the stage consults the segmenter def test_the_segmenter_sees_only_an_undivided_name() -> None: # its precondition (#272 Task 5): a segmenter answers where an # UNDIVIDED name divides, so a name part carrying a second # script-written token is already divided and the segmenter is not # consulted -- otherwise "山田 太郎" would have its family divided # again. A Latin title/suffix is not such a boundary. def always(text: str) -> Segmentation | None: return Segmentation((1,), confidence=1.0) p = parser_for(locales.get("zh"), base=Parser(segmenter=always)) assert p.parse("阿明").family == "阿" # one token: consulted n = p.parse("阿明 日月") # already divided assert (n.family, n.given) == ("阿明", "日月") assert p.parse("Dr 阿明").family == "阿" # a title is not a split def test_a_segmenter_split_reaches_the_fields() -> None: # end to end: the sub-slices the stage makes are ordinary tokens # from there on, so the pack's family-first order reads them like # any vocabulary split, and a low-confidence answer surfaces def two(text: str) -> Segmentation | None: return Segmentation((1,), confidence=0.5) p = parser_for(locales.get("zh"), base=Parser(segmenter=two)) n = p.parse("阿明") assert (n.family, n.given) == ("阿", "明") assert [a.kind for a in n.ambiguities] == [AmbiguityKind.SEGMENTATION] # two cuts, three pieces -- and the token indices the comma # structure recorded still name the right words afterwards def three(text: str) -> Segmentation | None: return Segmentation((1, 2)) q = parser_for(locales.get("zh"), base=Parser(segmenter=three)) n3 = q.parse("Dr 阿明日, Jr.") assert (n3.title, n3.family, n3.given, n3.middle, n3.suffix) == ( "Dr", "阿", "明", "日", "Jr.") def _noop_segmenter(text: str) -> None: return None def test_segmenterless_activation_without_vocabulary_warns() -> None: # The JA pack activates HAN and HIRAGANA segmentation but ships no # vocabulary, and no bundled list could serve those scripts -- so # without a segmenter, Japanese names can never divide. That is a # CONFIGURATION gap, not a fact about any name, and it was silent: # the misconfigured parser behaved identically to a working one # minus the feature. Statically detectable at construction, so # warn there. with pytest.warns(UserWarning, match=r"ja_segmenter"): parser_for(locales.JA) def test_the_activation_warning_offers_a_spelling_that_type_checks() -> None: # The message's actionable half is the deactivation it offers, and # a user pastes it verbatim. nameparser ships py.typed, so that # paste has to survive a type checker: Policy(segment_scripts=()) # is an arg-type error, since the field is annotated with what it # STORES (a frozenset) rather than everything the constructor # accepts. Nothing pinned this half of the message before. with pytest.warns(UserWarning) as caught: parser_for(locales.JA) message = str(caught[0].message) assert "Policy(segment_scripts=frozenset())" in message assert "Policy(segment_scripts=())" not in message def test_a_segmenter_silences_the_activation_warning() -> None: # any segmenter counts: the gap is "nothing can divide these", # not "you did not use namedivider" parser_for(locales.JA, segmenter=_noop_segmenter) def test_covered_activation_does_not_warn() -> None: # HANGUL is served by the census surnames; the zh pack ships the # vocabulary its own activation needs Parser() parser_for(locales.ZH) def test_stacked_activation_warns_only_for_uncovered_scripts() -> None: # zh covers HAN and the default vocabulary covers HANGUL; only # HIRAGANA is left unservable, and the message must say WHICH # scripts are dead rather than naming the whole activation set with pytest.warns(UserWarning) as caught: parser_for(locales.ZH, locales.JA) # select by content: pack application can emit its own warnings # ahead of construction, so positional indexing is order-fragile message = next(str(w.message) for w in caught if "segment_scripts activates" in str(w.message)) assert "hiragana" in message assert "hangul" not in message python-nameparser-2.1.0/tests/v2/test_policy.py000066400000000000000000001026411523530431700215760ustar00rootroot00000000000000import dataclasses from collections.abc import Iterator import pytest from nameparser._policy import ( DEFAULT_SCRIPT_ORDERS, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, PatronymicRule, Policy, PolicyPatch, Script, UNSET, _SCRIPT_RANGES, _script_matcher, apply_patch, ) from nameparser._types import Role def test_order_constants_read_as_their_contents() -> None: assert GIVEN_FIRST == (Role.GIVEN, Role.MIDDLE, Role.FAMILY) assert FAMILY_FIRST == (Role.FAMILY, Role.GIVEN, Role.MIDDLE) assert FAMILY_FIRST_GIVEN_LAST == (Role.FAMILY, Role.MIDDLE, Role.GIVEN) def test_policy_defaults() -> None: p = Policy() assert p.name_order == GIVEN_FIRST assert p.patronymic_rules == frozenset() assert ("(", ")") in p.nickname_delimiters assert p.maiden_delimiters == frozenset() assert p.strip_emoji and p.strip_bidi and p.lenient_comma_suffixes def test_policy_is_hashable_and_replaceable() -> None: p = dataclasses.replace(Policy(), name_order=FAMILY_FIRST) assert p.name_order == FAMILY_FIRST assert isinstance(hash(p), int) def test_name_order_must_be_permutation_and_error_names_constants() -> None: with pytest.raises(ValueError, match="FAMILY_FIRST_GIVEN_LAST"): Policy(name_order=(Role.TITLE, Role.GIVEN, Role.FAMILY)) with pytest.raises(ValueError, match="GIVEN_FIRST"): Policy(name_order=(Role.GIVEN, Role.GIVEN, Role.FAMILY)) def test_name_order_restricted_to_the_three_exported_orders() -> None: # _name_positions only implements the three exported semantics; the # unnamed permutations would silently misassign (PR review I7). # Pre-2.0 strictness is free: relaxing later is compatible. with pytest.raises(ValueError, match="GIVEN_FIRST"): Policy(name_order=(Role.MIDDLE, Role.GIVEN, Role.FAMILY)) def test_name_order_rejects_non_role_elements_with_type_error() -> None: # taxonomy: wrong element type -> TypeError, not the permutation # ValueError (PR review polish) with pytest.raises(TypeError, match="Role"): Policy(name_order=(1, 2, 3)) # type: ignore[arg-type] def test_name_order_rejects_plain_string_tuples() -> None: # Role is a StrEnum, so ("given", ...) == GIVEN_FIRST -- the # isinstance loop is the only guard rejecting string elements. with pytest.raises(TypeError, match="must be Role members"): Policy(name_order=("given", "middle", "family")) # type: ignore[arg-type] def test_script_values_are_the_public_names() -> None: assert Script.HAN == "han" and Script.HANGUL == "hangul" assert Script.HIRAGANA == "hiragana" and Script.KATAKANA == "katakana" def test_script_matcher_contains_any() -> None: has_han = _script_matcher(Script.HAN) assert has_han("毛泽东") assert has_han("Bruce 李") # mixed input: CONTAINS is enough assert not has_han("Bruce Lee") assert not has_han("") def test_script_matcher_whole() -> None: wholly_ja = _script_matcher( Script.HAN, Script.HIRAGANA, Script.KATAKANA, whole=True) assert wholly_ja("高橋みなみ") assert wholly_ja("マイケル") # katakana is in this union assert not wholly_ja("高橋 みなみ") # the space is outside it assert not wholly_ja("") # empty is not WHOLLY anything # the same union in contains mode accepts what wholly rejects assert _script_matcher( Script.HAN, Script.HIRAGANA, Script.KATAKANA)("高橋 みなみ") def test_script_matcher_rejects_zero_scripts() -> None: # an empty union would compile "[]" and raise a cryptic pattern # error far from the mistake with pytest.raises(ValueError, match="at least one Script"): _script_matcher() def test_script_matcher_reaches_astral_han() -> None: # the supplementary-plane span must survive the class compilation # (\U escapes, not \u): 𠮷 is U+20BB7 assert _script_matcher(Script.HAN)("𠮷田") assert _script_matcher(Script.HAN, whole=True)("𠮷田") def test_every_script_member_has_a_range_entry() -> None: # a member added to the enum alone would be accepted by # Policy(segment_scripts=...) yet never classified by # single_script -- a silent no-op until this fails assert set(_SCRIPT_RANGES) == set(Script) def test_patronymic_rules_coerce_and_reject() -> None: p = Policy(patronymic_rules=frozenset({"east-slavic"})) # type: ignore[arg-type] assert p.patronymic_rules == frozenset({PatronymicRule.EAST_SLAVIC}) with pytest.raises(ValueError, match="east-slavic, turkic"): Policy(patronymic_rules=frozenset({"klingon"})) # type: ignore[arg-type] def test_delimiter_pairs_must_be_nonempty_string_pairs() -> None: with pytest.raises(ValueError, match="non-empty"): Policy(nickname_delimiters=frozenset({("", ")")})) def test_delimiter_pair_rejects_two_char_string() -> None: with pytest.raises(TypeError, match="tuples"): Policy(nickname_delimiters=frozenset({"()"})) # type: ignore[arg-type] def test_patronymic_rules_rejects_bare_string_and_non_iterable() -> None: with pytest.raises(TypeError, match="bare string"): Policy(patronymic_rules="east-slavic") # type: ignore[arg-type] with pytest.raises(TypeError, match="iterable"): Policy(patronymic_rules=5) # type: ignore[arg-type] def test_patronymic_rules_true_points_at_the_v1_migration() -> None: # v1's patronymic_name_order was a BOOL flag, so True is the single # likeliest wrong value a migrator passes here -- the message must # name the v1 flag and the working replacements, not just say # "not iterable". Same guard on the PolicyPatch path (packs). with pytest.raises(TypeError, match="patronymic_name_order"): Policy(patronymic_rules=True) # type: ignore[arg-type] with pytest.raises(TypeError, match="patronymic_name_order"): PolicyPatch(patronymic_rules=True) # type: ignore[arg-type] # non-patronymic union fields get the generic curated message with pytest.raises(TypeError, match="extra_suffix_delimiters"): PolicyPatch(extra_suffix_delimiters=True) # type: ignore[arg-type] def test_maiden_delimiters_win_over_nickname_defaults() -> None: # A pair routes to exactly one field, and listing it in # maiden_delimiters is the specific intent -- so the one-liner works # without knowing (or rebuilding) the nickname default set. The # review that prompted this: routing parens to maiden used to # require both buckets edited in tandem. p = Policy(maiden_delimiters=frozenset({("(", ")")})) assert ("(", ")") in p.maiden_delimiters assert ("(", ")") not in p.nickname_delimiters # canonicalization: the explicit-removal spelling (the documented # set-math idiom) converges to the same value (equal AND same hash # -- cache keys agree) from nameparser import DEFAULT_NICKNAME_DELIMITERS explicit = Policy( nickname_delimiters=DEFAULT_NICKNAME_DELIMITERS - {("(", ")")}, maiden_delimiters=frozenset({("(", ")")}), ) assert p == explicit and hash(p) == hash(explicit) def test_maiden_wins_applies_to_typographic_pairs_too() -> None: # the subtraction is pair-agnostic; pin one #273 pair end-to-end from nameparser import Parser p = Policy(maiden_delimiters=frozenset({("«", "»")})) assert ("«", "»") not in p.nickname_delimiters n = Parser(policy=p).parse("Jean «Dupont» Martin") assert n.maiden == "Dupont" and n.nickname == "" def test_maiden_precedence_applies_through_policy_patch() -> None: # apply_patch re-runs Policy's constructor, so a patch (e.g. from a # locale pack) adding a maiden pair gets the same subtraction patched = apply_patch( Policy(), PolicyPatch(maiden_delimiters=frozenset({("(", ")")}))) assert ("(", ")") in patched.maiden_delimiters assert ("(", ")") not in patched.nickname_delimiters def test_default_nickname_delimiters_constant_is_the_default() -> None: from nameparser import DEFAULT_NICKNAME_DELIMITERS assert Policy().nickname_delimiters == DEFAULT_NICKNAME_DELIMITERS # the documented set-math idiom stays valid input p = Policy(nickname_delimiters=DEFAULT_NICKNAME_DELIMITERS | {("«", "»")}) assert ("«", "»") in p.nickname_delimiters def test_policy_delimiters_coerce_to_frozensets() -> None: p = Policy(nickname_delimiters=[("(", ")")]) # type: ignore[arg-type] assert isinstance(p.nickname_delimiters, frozenset) assert isinstance(hash(p), int) assert p == Policy(nickname_delimiters=frozenset({("(", ")")})) def test_policy_delimiters_do_not_alias_caller_containers() -> None: source = {("(", ")")} p = Policy(nickname_delimiters=source) # type: ignore[arg-type] source.add(("'", "'")) assert ("'", "'") not in p.nickname_delimiters def test_extra_suffix_delimiters_validated_and_coerced() -> None: with pytest.raises(TypeError, match="bare string"): Policy(extra_suffix_delimiters="ab") # type: ignore[arg-type] with pytest.raises(TypeError, match="must be strings"): Policy(extra_suffix_delimiters=frozenset({5})) # type: ignore[arg-type] with pytest.raises(ValueError, match="non-empty strings"): Policy(extra_suffix_delimiters=frozenset({""})) p = Policy(extra_suffix_delimiters=["-"]) # type: ignore[arg-type] assert p.extra_suffix_delimiters == frozenset({"-"}) assert isinstance(hash(p), int) def test_non_iterable_fields_name_the_offending_field() -> None: # Non-iterables (an int, a bool) used to fall through to tuple()/ # frozenset() and surface as a bare "'int' object is not iterable", # with no hint which field was wrong -- unlike patronymic_rules, # which has always named itself. Every iterable-typed Policy field # gets the same treatment. with pytest.raises(TypeError, match="name_order"): Policy(name_order=5) # type: ignore[arg-type] with pytest.raises(TypeError, match="nickname_delimiters"): Policy(nickname_delimiters=5) # type: ignore[arg-type] with pytest.raises(TypeError, match="maiden_delimiters"): Policy(maiden_delimiters=5) # type: ignore[arg-type] with pytest.raises(TypeError, match="extra_suffix_delimiters"): Policy(extra_suffix_delimiters=5) # type: ignore[arg-type] def test_policy_patch_mirrors_policy_field_names() -> None: policy_fields = {f.name for f in dataclasses.fields(Policy)} patch_fields = {f.name for f in dataclasses.fields(PolicyPatch)} assert policy_fields == patch_fields def test_policy_patch_mirrors_policy_field_types() -> None: for f in dataclasses.fields(Policy): patch_annotation = PolicyPatch.__dataclass_fields__[f.name].type assert patch_annotation == f"{f.type} | _Unset" def test_policy_patch_canonicalizes_union_fields() -> None: p = PolicyPatch(extra_suffix_delimiters=frozenset({"-"})) assert isinstance(p.extra_suffix_delimiters, frozenset) assert isinstance(hash(p), int) out = apply_patch(Policy(), PolicyPatch(extra_suffix_delimiters=["-"])) # type: ignore[arg-type] assert out.extra_suffix_delimiters == frozenset({"-"}) def test_policy_patch_rejects_bare_string_union_fields() -> None: with pytest.raises(TypeError, match="bare string"): PolicyPatch(extra_suffix_delimiters="ab") # type: ignore[arg-type] def test_apply_patch_overrides_scalars_and_unions_sets() -> None: base = Policy(patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})) patch = PolicyPatch( name_order=FAMILY_FIRST, patronymic_rules=frozenset({PatronymicRule.TURKIC}), ) out = apply_patch(base, patch) assert out.name_order == FAMILY_FIRST # override assert out.patronymic_rules == frozenset( # union {PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) assert out.strip_emoji is True # untouched def test_apply_patch_with_empty_patch_returns_same_policy() -> None: base = Policy() assert apply_patch(base, PolicyPatch()) is base def test_unset_fields_are_distinguishable_from_defaults() -> None: patch = PolicyPatch(strip_emoji=True) # explicitly set to the default value assert patch.strip_emoji is True assert PolicyPatch().strip_emoji is UNSET def test_policy_rejects_non_bool_flags() -> None: # "no" and "false" are truthy: storing them would silently invert # the caller's intent downstream. for flag in ("middle_as_family", "lenient_comma_suffixes", "strip_emoji", "strip_bidi"): with pytest.raises(TypeError, match="must be a bool"): Policy(**{flag: "no"}) # type: ignore[arg-type] def test_patronymic_rules_generator_errors_propagate_untouched() -> None: # A ValueError raised inside the caller's own generator must not be # rewritten as "unknown patronymic rule" with the traceback erased. def bad_loader(): # noqa: ANN202 yield "east-slavic" raise ValueError("config line 7: bad int") with pytest.raises(ValueError, match="config line 7"): Policy(patronymic_rules=bad_loader()) # type: ignore[arg-type] def test_unknown_patronymic_rule_error_names_the_offender() -> None: with pytest.raises(ValueError, match="klingon"): Policy(patronymic_rules=iter(["east-slavic", "klingon"])) # type: ignore[arg-type] def test_policy_patch_canonicalizes_scalar_name_order() -> None: # A list name_order stored as-is made the patch -- and any Locale # holding it -- unhashable, failing far from the construction site. p = PolicyPatch(name_order=[Role.FAMILY, Role.GIVEN, Role.MIDDLE]) # type: ignore[arg-type] assert p.name_order == (Role.FAMILY, Role.GIVEN, Role.MIDDLE) assert isinstance(hash(p), int) def test_policy_patch_canonicalizes_script_orders_all_the_way_down() -> None: # Same failure as name_order above, one level deeper: tuple-izing # only the (Script, order) pair left the ORDER a list, so the patch # -- and any Locale holding it -- was still unhashable. p = PolicyPatch(script_orders={Script.HAN: [Role.FAMILY, Role.GIVEN, # type: ignore[arg-type] Role.MIDDLE]}) assert p.script_orders == ((Script.HAN, FAMILY_FIRST),) assert isinstance(hash(p), int) def test_apply_patch_overrides_script_orders_as_a_scalar() -> None: # script_orders composes as a SCALAR: a patch REPLACES the table # rather than merging into it, so an empty one is how a pack turns # the default off. An UNSET patch must leave the default alone -- # the distinction a union field would blur. cleared = apply_patch(Policy(), PolicyPatch(script_orders={})) # type: ignore[arg-type] assert cleared.script_orders == () untouched = apply_patch(Policy(), PolicyPatch()) assert untouched.script_orders == DEFAULT_SCRIPT_ORDERS replaced = apply_patch( Policy(), PolicyPatch(script_orders={Script.HANGUL: GIVEN_FIRST})) # type: ignore[arg-type] assert replaced.script_orders == ((Script.HANGUL, GIVEN_FIRST),) def test_apply_patch_revalidates_deferred_script_orders() -> None: # The value half defers exactly like name_order's, and the error # must name script_orders -- not name_order, whose message the # check is shared with. bad = PolicyPatch(script_orders={Script.HAN: (Role.MIDDLE, Role.GIVEN, # type: ignore[arg-type] Role.FAMILY)}) with pytest.raises(ValueError, match="script_orders must be one of"): apply_patch(Policy(), bad) def test_policy_patch_defers_malformed_script_orders_verbatim() -> None: # The canonicalization must not eat the evidence: a bare string # would shred to character tuples (its elements are themselves # tuple-izable, so nothing raises to stop it) and bytes to ints, # leaving Policy's curated messages nothing of the caller's to # quote. Both shapes must survive the patch and raise at APPLY. with pytest.raises(TypeError, match="bare string"): apply_patch(Policy(), PolicyPatch(script_orders="han")) # type: ignore[arg-type] with pytest.raises(TypeError, match="decode first"): apply_patch(Policy(), PolicyPatch(script_orders=b"han")) # type: ignore[arg-type] def test_apply_patch_revalidates_deferred_values() -> None: # PolicyPatch documents lazy validation: invalid values sit latent in # the patch and must fail when applied, not silently flow into Policy. bad_order = PolicyPatch(name_order=(Role.TITLE, Role.GIVEN, Role.FAMILY)) with pytest.raises(ValueError, match="exported orders"): apply_patch(Policy(), bad_order) bad_rules = PolicyPatch(patronymic_rules=frozenset({"klingon"})) # type: ignore[arg-type] with pytest.raises(ValueError, match="valid rules"): apply_patch(Policy(), bad_rules) def test_all_set_valued_patch_fields_declare_union_composition() -> None: # apply_patch is driven by this metadata; dropping it from one field # would silently flip locale layering from union to override. union_fields = { f.name for f in dataclasses.fields(PolicyPatch) if f.metadata.get("compose") == "union" } assert union_fields == { "patronymic_rules", "nickname_delimiters", "maiden_delimiters", "extra_suffix_delimiters", "segment_scripts", } def test_policy_and_patch_pickle_round_trip_preserves_unset_identity() -> None: import pickle p = Policy(patronymic_rules=frozenset({PatronymicRule.TURKIC})) assert pickle.loads(pickle.dumps(p)) == p patch = PolicyPatch(strip_emoji=False) loaded = pickle.loads(pickle.dumps(patch)) assert loaded == patch # apply_patch gates on 'value is UNSET'; an unpickled patch is only # correct because Enum members round-trip BY IDENTITY. A plain # object() sentinel would break this silently. assert loaded.name_order is UNSET assert loaded.strip_emoji is False def test_setstate_rejects_layout_skew() -> None: state = dict(Policy().__getstate__()) del state["name_order"] with pytest.raises(ValueError, match="name_order"): Policy.__new__(Policy).__setstate__(state) def test_name_order_rejects_bare_string() -> None: # tuple("gmf") is ("g","m","f"): without the guard a bare string # fell through to the permutation ValueError instead of the # taxonomy's bare-string TypeError every other iterable field raises with pytest.raises(TypeError, match="bare string"): Policy(name_order="gmf") # type: ignore[arg-type] with pytest.raises(TypeError, match="bare string"): PolicyPatch(name_order="gmf") # type: ignore[arg-type] # Policy's collection fields had no guard against the two shapes # Lexicon._normset rejects by name: a bare string (iterates into # characters) and a Mapping (contributes only its keys). Both were # accepted silently, storing a value the caller never wrote -- the # expensive failure this library guards against elsewhere. # # Parametrized over BOTH classes on purpose. PolicyPatch had its own # inline copy of the bare-string half and never grew the mapping half, # so the two silently diverged; a patch is also what a Locale pack # ships, and its frozenset() coercion destroys the evidence before # Policy could catch it at apply time. _GUARDED = ["nickname_delimiters", "maiden_delimiters", "extra_suffix_delimiters", "patronymic_rules", "segment_scripts"] @pytest.mark.parametrize("cls", [Policy, PolicyPatch]) @pytest.mark.parametrize("field", _GUARDED) @pytest.mark.parametrize("value", [{"dr": "Dr"}, {}]) def test_collection_fields_reject_a_mapping( cls: type, field: str, value: dict) -> None: # {} is the sharp one: it is Python's empty-set trap, so a caller # disabling delimiters writes it and silently gets what they meant # -- until the day they write a non-empty one. with pytest.raises(TypeError, match="mapping"): cls(**{field: value}) @pytest.mark.parametrize("cls", [Policy, PolicyPatch]) @pytest.mark.parametrize("field", _GUARDED) @pytest.mark.parametrize("value", ["", "()", "dr"]) def test_collection_fields_reject_a_bare_string( cls: type, field: str, value: str) -> None: # '' iterates to nothing, so it silently stored an empty frozenset # rather than failing. with pytest.raises(TypeError, match="bare string"): cls(**{field: value}) @pytest.mark.parametrize("cls", [Policy, PolicyPatch]) @pytest.mark.parametrize("field,items,expected", [ ("nickname_delimiters", [("<", ">")], frozenset({("<", ">")})), ("maiden_delimiters", [("[", "]")], frozenset({("[", "]")})), ("extra_suffix_delimiters", ["/"], frozenset({"/"})), ("patronymic_rules", ["turkic"], frozenset({PatronymicRule.TURKIC})), ("segment_scripts", [Script.HAN], frozenset({Script.HAN})), ]) def test_legitimate_iterables_are_accepted_and_stored_intact( cls: type, field: str, items: list, expected: frozenset) -> None: """The guards must not over-reject -- and must not consume. Asserts the STORED value, not merely that construction returned. `assert cls(...) is not None` is a tautology for a dataclass: it passes for a guard that accepts a one-shot iterable, exhausts it, and stores frozenset(). That is the same "stored a value the caller never wrote" failure the guards exist to prevent, so the generator and iter() entries below are the ones that matter. All four fields, both classes: an over-strict guard on maiden_delimiters was invisible to the entire suite, because every other call site happens to pass it a frozenset. """ for value in (frozenset(items), set(items), list(items), tuple(items), (x for x in items), iter(items)): stored = getattr(cls(**{field: value}), field) assert stored == expected, ( f"{cls.__name__}.{field} stored {stored!r} for " f"{type(value).__name__}") @pytest.mark.parametrize("cls", [Policy, PolicyPatch]) def test_a_pair_mapping_is_accepted_through_items(cls: type) -> None: # The escape the mapping guard's message recommends. It works only # because ItemsView is a Set rather than a Mapping; if that ever # stops being true, the advice becomes wrong and this fails. stored = cls(nickname_delimiters={"<": ">"}.items()).nickname_delimiters assert stored == frozenset({("<", ">")}) @pytest.mark.parametrize("cls", [Policy, PolicyPatch]) @pytest.mark.parametrize("value", [b" - ", bytearray(b" - "), memoryview(b" - ")]) @pytest.mark.parametrize( "field", ["extra_suffix_delimiters", "nickname_delimiters", "maiden_delimiters", "patronymic_rules", "name_order", "segment_scripts"]) def test_every_field_rejects_a_buffer_with_a_decode_hint( cls: type, field: str, value: object) -> None: # Same cryptic int message Lexicon had, and name_order was the one # field the first pass missed. with pytest.raises(TypeError, match="decode"): cls(**{field: value}) def test_patched_unions_set_valued_fields() -> None: base = Policy(extra_suffix_delimiters={"|"}) # type: ignore[arg-type] out = base.patched( PolicyPatch(extra_suffix_delimiters={"/"})) # type: ignore[arg-type] assert out.extra_suffix_delimiters == frozenset({"|", "/"}) def test_patched_overrides_scalars_and_leaves_unset_alone() -> None: base = Policy(middle_as_family=True, lenient_comma_suffixes=False) out = base.patched(PolicyPatch(strip_emoji=False)) assert out.strip_emoji is False assert out.middle_as_family is True assert out.lenient_comma_suffixes is False def test_patched_with_empty_patch_returns_same_policy() -> None: base = Policy(middle_as_family=True) assert base.patched(PolicyPatch()) is base def test_patched_validates_patch_values_at_apply_time() -> None: # PolicyPatch scalars validate lazily by design; the error surfaces # when the patch is applied. patch = PolicyPatch(strip_emoji="off") # type: ignore[arg-type] with pytest.raises(TypeError, match="strip_emoji must be a bool"): Policy().patched(patch) def test_patched_rejects_non_patch() -> None: with pytest.raises(TypeError, match="takes a PolicyPatch"): Policy().patched({"strip_emoji": False}) # type: ignore[arg-type] def test_script_orders_default_and_canonical_storage() -> None: p = Policy() assert p.script_orders == DEFAULT_SCRIPT_ORDERS # three entries now (#272 adds HIRAGANA), sorted by Script value: # "han" < "hangul" < "hiragana" assert p.script_orders == ( (Script.HAN, FAMILY_FIRST), (Script.HANGUL, FAMILY_FIRST), (Script.HIRAGANA, FAMILY_FIRST), ) # constructor tolerates a Mapping; storage is the sorted pair # tuple (hashability, the capitalization_exceptions precedent -- # which is also why the mapping spelling needs the ignore: the # field is annotated with what it STORES) q = Policy(script_orders={Script.HANGUL: FAMILY_FIRST, # type: ignore[arg-type] Script.HAN: FAMILY_FIRST, Script.HIRAGANA: FAMILY_FIRST}) assert q == p and hash(q) == hash(p) assert Policy(script_orders={}).script_orders == () # type: ignore[arg-type] def test_a_later_script_orders_entry_replaces_an_earlier_one() -> None: """Appending is how you override a single script, and the docs now say so (modules.rst, the script_orders entry). The behavior falls out of _validated_script_orders building a dict before sorting, so it was emergent rather than contracted -- and prose that tells readers to append is prose that depends on it. The type-clean spelling for "everything as shipped, but Han reads given-first" is exactly this, which is why it is worth pinning. """ overridden = Policy( script_orders=(*DEFAULT_SCRIPT_ORDERS, (Script.HAN, GIVEN_FIRST))) assert dict(overridden.script_orders)[Script.HAN] == GIVEN_FIRST # the other shipped entries survive untouched assert dict(overridden.script_orders)[Script.HANGUL] == FAMILY_FIRST assert dict(overridden.script_orders)[Script.HIRAGANA] == FAMILY_FIRST # one entry per script, whatever the input carried keys = [s for s, _ in overridden.script_orders] assert len(keys) == len(set(keys)) == len(DEFAULT_SCRIPT_ORDERS) # and the mapping spelling it replaces still means the same thing assert overridden == Policy( script_orders=dict(DEFAULT_SCRIPT_ORDERS) # type: ignore[arg-type] | {Script.HAN: GIVEN_FIRST}) def test_script_orders_validates_keys_and_values() -> None: with pytest.raises(ValueError, match="han, hangul"): Policy(script_orders={"klingon": FAMILY_FIRST}) # type: ignore[arg-type] with pytest.raises(ValueError, match="exported orders"): Policy(script_orders={Script.HAN: (Role.MIDDLE, Role.GIVEN, # type: ignore[arg-type] Role.FAMILY)}) with pytest.raises(TypeError, match="bare string"): Policy(script_orders="han") # type: ignore[arg-type] def test_segment_scripts_defaults_to_hangul_and_coerces() -> None: # hangul segmentation is default-on (amendment 2026-07-27): # hangul is unambiguously Korean and the surname set is closed assert Policy().segment_scripts == frozenset({Script.HANGUL}) p = Policy(segment_scripts=["han", Script.HANGUL]) # type: ignore[arg-type] assert p.segment_scripts == frozenset({Script.HAN, Script.HANGUL}) assert Policy(segment_scripts=()).segment_scripts == frozenset() # type: ignore[arg-type] def test_segment_scripts_rejects_unknown_script() -> None: # bare-string/mapping rejection is covered by the _GUARDED family # sweep (test_collection_fields_reject_a_bare_string / # test_collection_fields_reject_a_mapping); this pins the one # check unique to segment_scripts -- the unknown-script message. with pytest.raises(ValueError, match="han, hangul"): Policy(segment_scripts={"klingon"}) # type: ignore[arg-type] def test_segment_scripts_rejects_non_iterable_with_curated_message() -> None: # parallel to patronymic_rules: name the expected shape rather than # letting _require_iterable's default "an iterable" phrasing stand with pytest.raises(TypeError, match="segment_scripts must be an iterable of " "Script members, got True"): Policy(segment_scripts=True) # type: ignore[arg-type] def test_segment_scripts_patch_unions() -> None: patched = apply_patch( Policy(), PolicyPatch(segment_scripts={Script.HAN})) # type: ignore[arg-type] assert patched.segment_scripts == frozenset( {Script.HAN, Script.HANGUL}) stringly = apply_patch( Policy(), PolicyPatch(segment_scripts={"han"})) # type: ignore[arg-type] assert all(isinstance(s, Script) for s in stringly.segment_scripts) # One item per field, used to build a bad_iter() below that yields a # legitimate first entry, then raises -- forcing consumption past the # iter()-probe stage so a rewrite-vs-propagate bug can only be caught # by actually running the generator. _PROPAGATION_FIELD_VALUES = { "segment_scripts": Script.HAN, "patronymic_rules": PatronymicRule.TURKIC, "script_orders": (Script.HAN, FAMILY_FIRST), "nickname_delimiters": ("<", ">"), "maiden_delimiters": ("[", "]"), "extra_suffix_delimiters": "/", } @pytest.mark.parametrize("cls,field", [ (Policy, "segment_scripts"), (Policy, "patronymic_rules"), (Policy, "script_orders"), (Policy, "nickname_delimiters"), (Policy, "maiden_delimiters"), (Policy, "extra_suffix_delimiters"), (PolicyPatch, "segment_scripts"), (PolicyPatch, "patronymic_rules"), (PolicyPatch, "script_orders"), (PolicyPatch, "nickname_delimiters"), (PolicyPatch, "maiden_delimiters"), (PolicyPatch, "extra_suffix_delimiters"), ]) def test_caller_generator_errors_propagate_untouched( cls: type, field: str) -> None: """A TypeError raised INSIDE a caller's generator, after it has already yielded a legitimate value, must reach the caller with its own message -- not be rewritten as "not an iterable" by whichever guard happens to consume the generator to check its shape. Probe-only-in-try is what makes this true: iter() succeeds (so the guard doesn't fire), and only the SEPARATE, unguarded consumption step sees the generator's own exception. PolicyPatch.script_orders is the scalar (non-union) field with its own canonicalization block, distinct from the shared union loop the other fields go through below -- it needs the identical probe/consume split, since deferring a caller-generator error to apply time is impossible once the generator is already exhausted (verified: the error would otherwise be lost for good, not merely delayed).""" def bad_iter() -> Iterator[object]: yield _PROPAGATION_FIELD_VALUES[field] raise TypeError("boom") with pytest.raises(TypeError, match="boom"): cls(**{field: bad_iter()}) def test_policy_patch_script_orders_non_iterable_still_defers_to_apply() -> None: # The deferral contract this fix must NOT break: a genuinely # non-iterable script_orders value is not a caller-generator error # -- it fails the iter() probe itself, so PolicyPatch construction # stays lazy (and hashable) and Policy's own guard raises the # curated message once the patch is actually applied. patch = PolicyPatch(script_orders=5) # type: ignore[arg-type] assert isinstance(hash(patch), int) with pytest.raises(TypeError, match="script_orders must be a mapping"): apply_patch(Policy(), patch) def test_canonical_script_pair_defers_each_malformed_shape() -> None: # _canonical_script_pair's three per-pair deferral branches, each # constructible (hashable) and each quoted by Policy's own guard at # apply time: a pair of the wrong arity; a str order value (which # tuple() would shred to characters); a non-iterable order value. arity = PolicyPatch( script_orders=[(Script.HAN, FAMILY_FIRST, "extra")]) # type: ignore[arg-type] assert isinstance(hash(arity), int) with pytest.raises(TypeError, match=r"\(Script, order\) pairs"): apply_patch(Policy(), arity) stringly = PolicyPatch(script_orders={Script.HAN: "gmf"}) # type: ignore[arg-type] assert isinstance(hash(stringly), int) with pytest.raises(TypeError, match="bare string"): apply_patch(Policy(), stringly) lone = PolicyPatch(script_orders={Script.HAN: 5}) # type: ignore[arg-type] assert isinstance(hash(lone), int) with pytest.raises(TypeError, match="must be an iterable, got 5"): apply_patch(Policy(), lone) def test_policy_patch_one_shot_bad_tail_defers_without_silent_drop() -> None: # The silent-drop repro: a ONE-SHOT generator whose first item is a # good pair and whose second is a bad shape (not caller-generator # code raising -- just a malformed entry). PolicyPatch construction # must not raise (this is a shape problem, deferred to apply time, # like the bytes case), but the exhausted-generator hole meant # apply_patch used to silently store () -- "opt out of per-script # ordering" -- with NO error anywhere, dropping the Han/Hangul # family-first defaults. The list form of the identical input # already raised correctly; only the one-shot form leaked. gen = (x for x in [(Script.HAN, FAMILY_FIRST), 5]) patch = PolicyPatch(script_orders=gen) # type: ignore[arg-type] # Materialized into a re-iterable tuple, not left as the exhausted # generator -- Policy can still quote the caller's bad entry (5). assert patch.script_orders == ((Script.HAN, FAMILY_FIRST), 5) with pytest.raises(TypeError, match=r"script_orders entries must be .* got 5"): apply_patch(Policy(), patch) python-nameparser-2.1.0/tests/v2/test_properties.py000066400000000000000000000650651523530431700225030ustar00rootroot00000000000000"""Property layer (core spec §7.3). Hypothesis is a dev dependency only. The alphabet is punctuation-heavy on purpose: plain st.text() spreads over all of Unicode, so commas, quotes, and delimiters almost never appear and the interesting planes go unexercised. derandomize=True keeps runs reproducible on shared CI runners -- this layer guards against regressions; exploratory fuzzing happened during review. """ import dataclasses import warnings import pytest from hypothesis import given, settings from hypothesis import strategies as st from nameparser import ( DEFAULT_SCRIPT_ORDERS, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, Lexicon, Parser, PatronymicRule, Policy, Script, parse, ) from nameparser._lexicon import _VOCAB_FIELDS from nameparser._pipeline import run from nameparser._pipeline._state import ParseState from nameparser._pipeline._vocab import effective_script from nameparser._types import AmbiguityKind, Role from .conftest import differential_corpus _ALPHABET = st.sampled_from( 'abcdefgh ABC 12 .,،，\'"()«»‏‏\U0001f600éñßЖ-') _FORK_CORPUS = differential_corpus() @given(st.text(alphabet=_ALPHABET, max_size=200)) @settings(max_examples=300, deadline=None, derandomize=True) def test_parse_never_raises_on_str(text: str) -> None: parse(text) @given(st.text(alphabet=_ALPHABET, max_size=200)) @settings(max_examples=300, deadline=None, derandomize=True) def test_provenance_for_parser_produced_names(text: str) -> None: pn = parse(text) for t in pn.tokens: assert t.span is not None assert t.text == pn.original[t.span.start:t.span.end] @given(st.text(alphabet=_ALPHABET, max_size=100)) @settings(max_examples=200, deadline=None, derandomize=True) def test_capitalized_idempotent(text: str) -> None: once = parse(text).capitalized() assert once.capitalized() == once @given(st.text(alphabet=_ALPHABET, max_size=100)) @settings(max_examples=200, deadline=None, derandomize=True) def test_render_reparse_reaches_fixpoint(text: str) -> None: # render/reparse legitimately takes several rounds to stabilize on # comma-heavy input (each round can re-segment); the invariant is # BOUNDED CONVERGENCE, not one-step idempotence s = str(parse(text)) for _ in range(10): nxt = str(parse(s)) if nxt == s: break s = nxt assert str(parse(s)) == s, f"no fixpoint within 10 rounds: {s!r}" @given(st.text(alphabet=_ALPHABET, max_size=100)) @settings(max_examples=300, deadline=None, derandomize=True) def test_every_original_char_is_accounted_for(text: str) -> None: # Reverse coverage (the dual of provenance): no character of the # input silently vanishes. Every char lies in a token span, a # masked delimited span, or is individually ignorable -- whitespace, # a structural comma, or a char the strip options remove. Checked on # the pre-assembly state because dropped/extracted tokens keep their # spans there. state = run(ParseState(original=text, lexicon=Lexicon.default(), policy=Policy())) covered: set[int] = set() for tok in state.tokens: covered.update(range(tok.span.start, tok.span.end)) for span in state.masked: covered.update(range(span.start, span.end)) ignorable = {",", "،", "，", "\U0001f600", "‏"} for i, ch in enumerate(text): if i in covered or ch.isspace() or ch in ignorable: continue raise AssertionError( f"char {ch!r} at {i} in {text!r} is unaccounted for") _NAME_ROLES = (Role.GIVEN, Role.MIDDLE, Role.FAMILY) def _fork_count(state: ParseState) -> int: return sum(a.kind is AmbiguityKind.PARTICLE_OR_GIVEN for a in state.ambiguities) def test_a_leading_ambiguous_particle_is_reported_once_and_only_once( ) -> None: """PARTICLE_OR_GIVEN is the one kind two stages emit: _group takes the particle branch when a title shifts the particle off index 0 and the chain claims something ("Dr. Van Johnson"), _assign takes the given branch when it stays a lone leading piece ("Van Johnson"). Each reports the side it decides -- see the ParseState docstring -- but they coordinate only through _group's `j > k + 1` guard, which mirrors _assign's reachability by hand. Nothing checked the mirror. The shape that separates them is the one real-name corpora have least reason to contain: a suffix straight after the particle, where the chain is a no-op and only _assign should speak. Generate it, over every ambiguous particle, so a vocabulary addition is covered too. Deliberately asserts the COUNT and not which stage spoke. Whether a given fork was decided by a vocabulary merge or by position is not recoverable from the finished parse -- 'Dr. aan Johnson Jr.' and 'أبو بكر أحمد' end with the same roles and the same tags, and only one is a fork -- so any reconstruction here would have to re-implement _group rather than check it. """ lex = Lexicon.default() # bound-given prefixes are excluded, not overlooked: 'abu' is both # an ambiguous particle and a bound given prefix, so whether it # forks depends on whether the bound join fired -- a second rule, # covered by the case corpus rather than by this sweep particles = sorted(lex.particles_ambiguous - lex.bound_given_names) assert particles, "no ambiguous particles to exercise" failures = [] for particle in particles: for lead in ("", "Dr. ", "Dr. Ann "): for body in ("", "Johnson ", "Johnson Smith "): for tail in ("", "Jr.", "MD", "III"): text = f"{lead}{particle} {body}{tail}".strip() state = run(ParseState(original=text, lexicon=lex, policy=Policy())) names = [t for t in state.tokens if t.role in _NAME_ROLES] # Leading = no name part precedes it. With a given # name in front ("Dr. Ann van Johnson") the particle # sits mid-name, where nothing has to choose -- the # decision-not-a-word rule, so no report. And a lone # name part is the whole name, not a coin flip. leads = bool(names) and "vocab:particle-ambiguous" \ in names[0].tags want = 1 if leads and len(names) >= 2 else 0 got = _fork_count(state) if got != want: failures.append( f"{text!r}: {got} report(s), expected {want} " f"({len(names)} name tokens, " f"leading={leads})") assert not failures, ( f"{len(failures)} shape(s) disagree:\n" + "\n".join(failures[:15])) @pytest.mark.parametrize("text", _FORK_CORPUS) def test_a_fork_is_never_reported_twice_on_a_real_name(text: str) -> None: state = run(ParseState(original=text, lexicon=Lexicon.default(), policy=Policy())) assert _fork_count(state) <= 1 @given(st.text(alphabet=_ALPHABET, max_size=120)) @settings(max_examples=400, deadline=None, derandomize=True) def test_particle_fork_is_never_double_reported(text: str) -> None: # The half of the invariant that needs no reconstruction: whatever # the two emitters decide, they must not both fire on one parse. state = run(ParseState(original=text, lexicon=Lexicon.default(), policy=Policy())) assert _fork_count(state) <= 1, ( f"{text!r} reported the same fork more than once") # ---------------------------------------------------------------- config # Everything above fuzzes the INPUT STRING against the default # configuration. That leaves 2.0's largest new surface -- Lexicon and # Policy -- covered only by hand-written cases, which is backwards: the # vocabulary and the switches are the parts a user is invited to # change, so they are the parts most likely to be given a combination # nobody tried. # Deliberately mixed: real vocabulary, one-letter words that collide # with initials, an interior period, and non-Latin entries. Nothing # here normalizes to empty, which Lexicon rejects outright. No # multi-word phrase: every field below except given_name_titles is # matched one word at a time, so a multi-word draw in a per-word field # would be a dead entry that trips Lexicon's multi-word warning. # The CJK tail (#271) is what lets a drawn `surnames` set activate # script_segment at all: hangul is the script segmented by default, so # "김"/"남궁"/"남" are what make the stage fire (see _names_using, which # supplies the unspaced token to fire it ON -- and, since the drawn # policy picks its own segment_scripts, only when that policy # activates hangul too). "남" is there for the stage's multi-match # FORK, which nothing else in this pool can reach: it is a proper # PREFIX of "남궁", so a lexicon drawn with both makes "남궁민준" match # twice, and longest-first then has to choose and report the reading # it passed over. Reachable is all it is -- both entries have to land # in the same drawn `surnames`, 0.8% of draws, so the fork fires # roughly once in 900 examples (42 over 36000 measured) and the # committed 250-example seed does not reach it at all; a randomized # run is what sees it. The Han rows ride along for script_orders, and # are not inert here either -- `_policies` draws segment_scripts # freely, so HAN is activated in 37.7% of drawn policies (measured # over 20000 draws), and an activated Han token STANDING EARLIER takes # the surname site: "欧阳 김민준" does not split, where "김민준 欧阳" # does. What these policies never draw is a locale PACK, which is the # only way shipped configuration turns Han segmentation on. _VOCAB = st.sampled_from([ "van", "de", "la", "bin", "abdul", "abu", "dr", "sir", "prof", "md", "jr", "iii", "esq", "ma", "do", "and", "y", "née", "geb", "a", "b", "ph.d", "عبد", "фон", "μεγα", "김", "남궁", "남", "毛", "欧阳", ]) # given_name_titles is the one field matched as a space-joined run, so # a multi-word phrase there is meaningful (not dead) and must not warn. # Derived from _VOCAB rather than duplicated, so the two pools cannot # drift apart. _TITLE_VOCAB = st.one_of(_VOCAB, st.just("grand duke")) def _fix_invariants(**fields: frozenset[str]) -> dict[str, frozenset[str]]: """Repair a random draw into a legal Lexicon instead of generating one legally. Drawing dependent subsets directly (particles_ambiguous from whatever particles happened to be drawn) makes the strategy tree deep and mostly rejects; intersecting after the fact keeps every draw usable and still reaches every shape. The five rules are Lexicon's own, restated here on purpose -- if one changes, this fails loudly rather than silently fuzzing a narrower space. """ fields["particles_ambiguous"] &= fields["particles"] fields["suffix_acronyms_ambiguous"] &= fields["suffix_acronyms"] fields["suffix_words"] -= fields["suffix_acronyms_ambiguous"] # order matters: must run AFTER the suffix_words subtraction above, # or that later subtraction could re-orphan a tail this repair just # fixed fields["honorific_tails"] &= fields["suffix_words"] fields["bound_given_names"] -= ( fields["particles"] - fields["particles_ambiguous"]) return fields # Derived, never listed: a new vocabulary field must be fuzzed the day # it is added, and a hand-copied list would leave it silently unfuzzed # -- the invisible gap this whole layer exists to prevent. _SET_FIELDS = _VOCAB_FIELDS @st.composite def _lexicons(draw: st.DrawFn) -> Lexicon: fields = {name: draw(st.frozensets( _TITLE_VOCAB if name == "given_name_titles" else _VOCAB, max_size=5)) for name in _SET_FIELDS} caps = draw(st.lists(st.tuples(_VOCAB, _VOCAB), max_size=3)) return Lexicon(capitalization_exceptions=tuple(caps), **_fix_invariants(**fields)) # script_orders' legal values are as restricted as name_order's (only # the three exported orders, keyed by Script), so they are sampled # rather than generated. The five cover the axes that matter: the # shipped default, the full opt-out, one script alone, an order that # disagrees with the default -- FAMILY_FIRST_GIVEN_LAST on hangul # reads "김민준 수" differently from every other entry here, which is # what makes a script table that is merely PRESENT distinguishable # from one that is actually consulted -- and a table keyed on the kana # license's carrier (#272). That last row is the only one whose key is # reached INDIRECTLY: a mixed kanji+kana name resolves to HIRAGANA # rather than to any script it is literally written in, and pointing # the carrier back at GIVEN_FIRST makes a consulted table visibly # different from the default it would otherwise agree with. _SCRIPT_ORDER_TABLES = [ DEFAULT_SCRIPT_ORDERS, (), ((Script.HAN, FAMILY_FIRST),), ((Script.HANGUL, FAMILY_FIRST_GIVEN_LAST),), ((Script.HIRAGANA, GIVEN_FIRST),), ] @st.composite def _policies(draw: st.DrawFn) -> Policy: pairs = st.sampled_from([("(", ")"), ('"', '"'), ("'", "'"), ("[", "]"), ("«", "»")]) nickname = draw(st.frozensets(pairs, max_size=2)) # a pair may not sit in both buckets; Policy canonicalizes overlap # away, but constructing the contradiction is not what this fuzzes maiden = draw(st.frozensets(pairs, max_size=2)) - nickname return Policy( name_order=draw(st.sampled_from( [GIVEN_FIRST, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST])), script_orders=draw(st.sampled_from(_SCRIPT_ORDER_TABLES)), # no max_size: Script has four members, so the unbounded draw # still reaches every subset -- including the empty one, which # is the documented segmentation opt-out, and the full one, # which turns on the Han and kana activation that only # locales.ZH and locales.JA turn on in shipped configuration segment_scripts=draw(st.frozensets(st.sampled_from(list(Script)))), patronymic_rules=draw(st.frozensets( st.sampled_from(list(PatronymicRule)), max_size=2)), middle_as_family=draw(st.booleans()), nickname_delimiters=nickname, maiden_delimiters=maiden, extra_suffix_delimiters=draw( st.frozensets(st.sampled_from(["/", ";", "|"]), max_size=2)), lenient_comma_suffixes=draw(st.booleans()), strip_emoji=draw(st.booleans()), strip_bidi=draw(st.booleans()), ) @st.composite def _names_using(draw: st.DrawFn, lexicon: Lexicon, policy: Policy) -> str: """Build the input out of the lexicon's OWN words. Fuzzing configuration while feeding unrelated text tests almost nothing: a randomly generated string essentially never contains a randomly generated vocabulary entry, so every configured set would sit unused and the parse would take the same path every time. Takes the POLICY as well as the lexicon because one of the shapes below is only reachable when the two agree -- see the segmentation note. """ vocab = sorted({w for name in _SET_FIELDS for w in getattr(lexicon, name)}) # script_segment (#271, #308) holds the two halves a space-joined # name can never reach, and BOTH need their token derived from the # draw: waiting for a drawn entry and a matching literal to # coincide leaves the stage unexercised. The surname half splits an # unspaced token whose PREFIX is a drawn surname; the peel splits a # listed tail off the END of one. The peel's line was added after a # mutation pass asked the same question of it and instrumentation # answered: 14 fires across this file, every one under the DEFAULT # lexicon and none under a drawn one, because no generated token # ended in a drawn tail. Deriving it moves that off zero. # A Latin entry is useful input in its own right: it makes a # mixed-script token the surname half correctly declines, and for # the peel it is the one shape that reaches an ASCII tail at all -- # the stage bails on a wholly-ASCII original, so the non-Latin stem # is what admits '민준jr'. Every peel fire observed under a drawn # lexicon is of exactly that shape -- the committed run's is # '민준de'. # What the forced insertion below is worth, in the one number that # cannot rot: instrument _split under this test's committed # derandomize=True seed -- peel and surname split are told apart by # its tail_tag argument -- and origin/master's version of this # strategy counts ZERO surname splits against two peels, while this # one counts 23 against one. That figure is reproducible by anyone # in one command and moves only when these strategies do. # Being in the POOL is not the same as being in the NAME, and for # the surname half that gap was the whole story. Two things must # coincide before the shape is legal at all -- a HANGUL surname # drawn AND a policy that activates hangul, together 5.5% of draws # over 20000 -- and the token then has to win a place in a 1-8 # piece name against a pool of median size 23. Merely offered, it # reached the name a couple of times per 250. The shortfall was # sampling, never the stage. # What that buys is COVERAGE, not detection, and the two are worth # separating because the first is the easier to oversell. This # layer asserts totality and span-exactness only, so every # BEHAVIORAL mutant in the stage -- the site's last-token scan, # shortest-first, the whole-token guard, the activation gate, the # post-nominal decline, the prefix cap, the segment remap -- passes # here with the insertion and without it, and is killed by # tests/v2/pipeline/test_script_segment.py instead. The one defect # class this layer CAN catch is span arithmetic inside _split, and # the peel already reached that path. Writing `base + end` as # `base + end + 1` and giving each tree ten randomized runs of 250: # origin/master finds it in 8 runs of 10, at a median 1561 shrink # calls and 14.6 seconds; this tree finds it in 10 of 10, at 346 # calls and 3.6 seconds, and shrinks to '김민준 김' where master # shrinks to the peel's '민준van'. st.integers shrinks the # insertion index toward 0, which walks the token into the position # likeliest to fire instead of away from it. # Alignment is still not a guarantee: with the token forced in the # split fires in about four aligned examples in five (350 of 450, # over 24 randomized runs of 250). Every decline is the stage # deciding, not waste. A FAMILY_COMMA opts the stage out whole -- a # ',' piece is in the pool. Otherwise an EARLIER script-written # token takes the surname site: a bare drawn surname ('김 김민준' # leaves 김민준 unsplit -- the whole-token guard), a drawn hangul # post-nominal (the leading post-nominal decline), or a Han token # under a policy that activated HAN. A LATIN word never takes it -- # 'John 김민준' still splits, zero occurrences in 1082 aligned # examples -- which is why the insertion index is drawn rather than # pinned to 0. # Two earlier versions of this comment overstated this half from # small samples -- one calling it "structurally inert, not luck" on # a single derandomize=True run reporting zero, one putting the # conversion above at 100%. So: a derandomize=True run is one # sample and cannot disagree with itself, and every randomized # figure quoted here was taken under derandomize=False instead, # which is how to re-measure them. The structural claim is real but # narrower than it was made: `w + "민준"` on a NON-hangul surname is # a mixed-script token whose effective_script is None, so those # candidates can never be a site whatever the policy says. Only a # drawn hangul surname makes a usable one, which is what # `activatable` selects. # Two shapes the insertion costs, both small, neither zero. It is # unconditional once lexicon and policy align, so the stage's # i-is-None early return under a LIVE configuration fell from 1.3% # of examples to 0.1%: it survives only where a drawn quote pair # carries the token off into a nickname, leaving segments[0] with # nothing in an activated script ("prof ' Smith 남민준 '"). And the # inserted token can SUPPRESS a peel, the peel site being the last # non-post-nominal token and this token being no post-nominal: # '김민준씨 김민준' leaves 씨 glued where '김민준 김민준씨' peels # it. # The peel gets no forced piece of its own -- it is saturated by # case rows and stage tests, so a second one was not worth the # distribution shift -- and its count here is a lottery either way # (0-7 per randomized run of 250). On net the insertion nudges it # UP, by a route worth knowing: a hangul token makes the original # non-ASCII, which lifts the stage's ASCII bail off Latin tails # that are otherwise unreachable -- under honorific_tails={'a'}, # 'John la' does not peel and '김민준 la' does. # sorted for the same reason `vocab` above is: frozenset iteration # order is not stable across runs, and an unsorted pool shifts # every index sampled_from draws -- which would defeat # derandomize=True on the whole strategy, not just this slice. unspaced = sorted(w + "민준" for w in lexicon.surnames) glued = sorted("민준" + w for w in lexicon.honorific_tails) # plain names and structure characters are always available, so the # pool is never empty even for an empty lexicon pieces = st.sampled_from( vocab + unspaced + glued + ["John", "Smith", "Q.", ",", "(", "'"]) drawn = draw(st.lists(pieces, min_size=1, max_size=8)) # The one shape the pool does not deliver RELIABLY -- it is in # there, it just loses the draw. Inserted at a drawn position # rather than the front: the stage takes the first ACTIVATED-script # token, not the first token, so a leading Latin word cannot hide # it -- and both placements are worth covering. activatable = sorted( w + "민준" for w in lexicon.surnames if effective_script(w + "민준") in policy.segment_scripts) if activatable: # rebuilt rather than list.insert()d: st.lists hands back a # fresh list today, so mutating it is safe today, and a # strategy that ever memoized one would make this a bug in the # fuzzer rather than in the code under test at = draw(st.integers(0, len(drawn))) token = draw(st.sampled_from(activatable)) drawn = [*drawn[:at], token, *drawn[at:]] return " ".join(drawn) def _quiet_parser(**kwargs: object) -> Parser: """Parser construction with the segmenterless-activation warning ignored: a drawn segment_scripts with a drawn vocabulary that cannot serve it is exactly the misconfiguration the warning names, so drawn configs hit it legitimately and constantly. The fuzz here targets parse behavior, not construction diagnostics -- tests/v2/test_parser.py pins the warning itself.""" with warnings.catch_warnings(): # message-scoped, not a blanket UserWarning ignore: a future # unrelated construction diagnostic should still fail the fuzz warnings.filterwarnings( "ignore", message=r"Policy\.segment_scripts activates") return Parser(**kwargs) # type: ignore[arg-type] @given(_lexicons(), _policies(), st.data()) @settings(max_examples=250, deadline=None, derandomize=True) def test_any_valid_config_still_parses_totally( lexicon: Lexicon, policy: Policy, data: st.DataObject) -> None: # Building the parser is part of the contract: a Lexicon and Policy # that each constructed must also combine. parser = _quiet_parser(lexicon=lexicon, policy=policy) text = data.draw(_names_using(lexicon, policy)) parsed = parser.parse(text) # must not raise, ever # the anti-#100 invariant, under configuration rather than under # the default vocabulary: spans index the original exactly for token in parsed.tokens: assert token.span is not None assert token.text == parsed.original[ token.span.start:token.span.end] # rendering is downstream of every config choice above assert isinstance(str(parsed), str) assert isinstance(parsed.capitalized().given, str) assert isinstance(parsed.initials(), str) @given(_lexicons(), _policies()) @settings(max_examples=100, deadline=None, derandomize=True) def test_config_values_are_hashable_and_reusable( lexicon: Lexicon, policy: Policy) -> None: # The docs promise these are frozen values: safe as dict keys, and # safe to build a parser from more than once. assert hash(lexicon) == hash(lexicon) assert {lexicon: 1, policy: 2} assert _quiet_parser(lexicon=lexicon, policy=policy) == \ _quiet_parser(lexicon=lexicon, policy=policy) # Values a real caller plausibly passes by mistake: the bare string that # iterates into characters, a mapping confused for a set, bytes, None, # and entries that normalize to nothing. _HOSTILE = st.sampled_from([ None, 0, 1, True, "", "dr", " ", ".", b"dr", ["dr", None], {"dr": "Dr"}, {1, 2}, [("a", "b")], [[]], object(), ]) # Same rule as _SET_FIELDS; test_policy.py already reflects this way. _POLICY_FIELDS = tuple(f.name for f in dataclasses.fields(Policy)) @given(st.sampled_from(_SET_FIELDS + ("capitalization_exceptions",)), _HOSTILE) @settings(max_examples=200, deadline=None, derandomize=True) def test_bad_lexicon_field_fails_cleanly(field: str, value: object) -> None: """A rejected configuration must be a DOCUMENTED rejection. ValueError and TypeError are the contract; anything else means the bad value got past validation and blew up somewhere downstream, where the message no longer names the field the caller typed. """ try: lexicon = Lexicon(**{field: value}) # type: ignore[arg-type] except (ValueError, TypeError): return # Accepted, so it has to survive an actual parse -- construction # succeeding while parsing dies is the same bug one stage later. _quiet_parser(lexicon=lexicon).parse("Dr. John de la Vega III") @given(st.sampled_from(_POLICY_FIELDS), _HOSTILE) @settings(max_examples=200, deadline=None, derandomize=True) def test_bad_policy_field_fails_cleanly(field: str, value: object) -> None: try: policy = Policy(**{field: value}) # type: ignore[arg-type] except (ValueError, TypeError): return _quiet_parser(policy=policy).parse("Dr. John de la Vega III") python-nameparser-2.1.0/tests/v2/test_regex_sync.py000066400000000000000000000420451523530431700224460ustar00rootroot00000000000000"""Pins the pipeline's hand-duplicated regex/table copies to their nameparser.config.regexes source of truth. The 2.0 layering rule forbids nameparser._pipeline/_render importing nameparser.config directly, so several patterns and tables are copied by hand into the modules that need them, each with a "keep in sync by hand" comment. Nothing previously enforced that promise: if config/regexes.py changed, the copies would silently diverge with no CI signal. Tests may legally import both sides (test_layering.py's own convention), so this module is where the promise gets checked. Layering is the usual reason for a copy but not the only one, so this module's scope is the PROMISE rather than that one pair of packages: the comma-set pin below reads _pipeline._state instead of config, and the last three tests reach outside the package altogether -- two to a TOML file that could not import a Python constant if it wanted to, one to a generated corpus whose generator can, and must stay run. """ import importlib.util import json import re import tomllib from pathlib import Path import pytest from nameparser.config import regexes as _config from nameparser._pipeline import _assign, _post_rules, _tokenize, _vocab from nameparser import _policy from nameparser._policy import Script from nameparser import _render def test_emoji_ranges_match_config() -> None: assert _tokenize._EMOJI_RANGES == _config._EMOJI_RANGES def test_bidi_pattern_matches_config() -> None: assert _tokenize._BIDI.pattern == _config.re_bidi.pattern assert _tokenize._BIDI.flags == _config.re_bidi.flags def test_period_not_at_end_matches_config() -> None: source = _config.REGEXES["period_not_at_end"] assert _vocab._PERIOD_NOT_AT_END.pattern == source.pattern assert _vocab._PERIOD_NOT_AT_END.flags == source.flags def test_period_abbreviation_matches_config() -> None: source = _config.REGEXES["period_abbreviation"] assert _assign._PERIOD_ABBREV.pattern == source.pattern assert _assign._PERIOD_ABBREV.flags == source.flags def test_roman_numeral_matches_config() -> None: source = _config.REGEXES["roman_numeral"] assert _assign._ROMAN.pattern == source.pattern assert _assign._ROMAN.flags == source.flags def test_patronymic_patterns_match_config() -> None: pairs = ( (_post_rules._EAST_SLAVIC, "east_slavic_patronymic"), (_post_rules._EAST_SLAVIC_CYR, "east_slavic_patronymic_cyrillic"), (_post_rules._TURKIC, "turkic_patronymic_marker"), (_post_rules._TURKIC_CYR, "turkic_patronymic_marker_cyrillic"), ) for copy, key in pairs: source = _config.REGEXES[key] assert copy.pattern == source.pattern, key assert copy.flags == source.flags, key def test_initial_copies_agree_with_each_other_and_config() -> None: # _vocab._INITIAL and _render._INITIAL are both v1's "initial" # pattern minus its trailing "?" (documented at _render.py's # _INITIAL definition: the two call sites always fullmatch a # non-empty token, so the empty-string alternative is dropped on # purpose). Assert both the internal-copy agreement and the exact, # documented relationship to the config source, so a future edit to # either side that breaks the relationship fails loudly here # instead of silently drifting. assert _vocab._INITIAL.pattern == _render._INITIAL.pattern source = _config.REGEXES["initial"] # config's pattern is the pipeline copy with an extra "?" spliced in # just before the trailing "$", making the whole group optional. trimmed = _vocab._INITIAL.pattern reconstructed = trimmed[:-1] + "?" + trimmed[-1:] assert source.pattern == reconstructed assert source.flags == _vocab._INITIAL.flags # The roster above grew one test at a time, and four hand-copies were # never added to it -- _render's _SPACES, _SPACE_BEFORE_COMMA, _MAC and # _WORD. They had not diverged, but nothing would have said so, which # is the exact promise this module exists to keep. All four mirror a # config key whose only other reader was v1's parser.py, deleted at the # M11 swap, so nothing else touches them either. # # Declared as a roster rather than one test per copy, with a # completeness check below: adding a pattern without declaring its # source now fails here instead of being silently unpinned. _SOURCES: dict[tuple[str, str], str | None] = { ("_assign", "_PERIOD_ABBREV"): "period_abbreviation", ("_assign", "_ROMAN"): "roman_numeral", ("_post_rules", "_EAST_SLAVIC"): "east_slavic_patronymic", ("_post_rules", "_EAST_SLAVIC_CYR"): "east_slavic_patronymic_cyrillic", ("_post_rules", "_TURKIC"): "turkic_patronymic_marker", ("_post_rules", "_TURKIC_CYR"): "turkic_patronymic_marker_cyrillic", ("_render", "_SPACES"): "spaces", ("_render", "_SPACE_BEFORE_COMMA"): "space_before_comma", ("_render", "_MAC"): "mac", ("_render", "_WORD"): "word", ("_vocab", "_PERIOD_NOT_AT_END"): "period_not_at_end", # Deliberately NOT a straight copy -- pinned by the dedicated tests # above, which assert the documented RELATIONSHIP instead: ("_render", "_INITIAL"): None, # config's pattern minus one "?" ("_vocab", "_INITIAL"): None, # same ("_tokenize", "_BIDI"): None, # re_bidi, not a REGEXES key # Mirrors _pipeline._state.COMMA_CHARS, not nameparser.config ("_render", "_COMMA_CHAR"): None, } _MODULES = {"_assign": _assign, "_post_rules": _post_rules, "_render": _render, "_tokenize": _tokenize, "_vocab": _vocab} @pytest.mark.parametrize( "where,key", [(w, k) for w, k in _SOURCES.items() if k is not None], ids=lambda v: v if isinstance(v, str) else f"{v[0]}.{v[1]}") def test_declared_copy_matches_its_config_source( where: tuple[str, str], key: str) -> None: copy = getattr(_MODULES[where[0]], where[1]) source = _config.REGEXES[key] assert copy.pattern == source.pattern, f"{where[1]} vs REGEXES[{key!r}]" assert copy.flags == source.flags, f"{where[1]} vs REGEXES[{key!r}]" def test_every_hand_copied_pattern_is_declared() -> None: """The roster must cover every compiled pattern in these modules. Without this, the roster is just another list that a new constant can be left out of -- which is how the four above went unpinned. """ undeclared = [ (name, attr) for name, mod in _MODULES.items() for attr, value in vars(mod).items() if attr.startswith("_") and not attr.startswith("__") and isinstance(value, re.Pattern) and (name, attr) not in _SOURCES ] assert not undeclared, ( f"compiled patterns missing from _SOURCES: {undeclared}. Add each " f"with its nameparser.config.regexes key, or None if it has no " f"config counterpart.") def test_comma_char_matches_the_pipeline_comma_set() -> None: # _render splits on the same comma characters segment does; the set # lives in _state, so this one is pinned against that, not config. from nameparser._pipeline._state import COMMA_CHARS assert set(_render._COMMA_CHAR.pattern.strip("[]")) == set(COMMA_CHARS) # The one sanctioned divergence between the differential rules' # character classes and _SCRIPT_RANGES: the halfwidth middle dot # separates tokens without being classified (halfwidth kana stays out # of the table on purpose). U+00B7 is deliberately NOT here -- its # flank guard means every name it can change matches through a # classified flanking character already. Single-sourced: both span # pins below read this set. _SANCTIONED_EXTRAS = frozenset({(0xFF65, 0xFF65)}) def _declared_spans(name_regex: str) -> set[tuple[int, int]]: """The \\uXXXX-\\uXXXX span pairs a rule's character class declares.""" return { (int(lo, 16), int(hi, 16)) for lo, hi in re.findall(r"\\u([0-9A-Fa-f]{4})-\\u([0-9A-Fa-f]{4})", name_regex)} def _expected_bmp_spans() -> set[tuple[int, int]]: """What a full CJK character class in the toml must declare: the table's BMP spans plus the sanctioned extras.""" return {span for spans in _policy._SCRIPT_RANGES.values() for span in spans if span[1] <= 0xFFFF} | set(_SANCTIONED_EXTRAS) def test_differential_cjk_rule_matches_the_script_ranges() -> None: """The CJK rule in tools/differential/expected_since_1.4.0.toml hand- copies the script spans from _policy._SCRIPT_RANGES into a character class. A TOML file cannot import the constant, so this is the one copy with no possible alternative -- and the one whose divergence is quietest, because the harness is run by hand rather than in CI. Both failure directions matter, which is why this compares sets rather than checking coverage. A span MISSING from the class turns an intended #271/#272 change into an UNEXPLAINED diff (a release blocker for the wrong reason); a span that should not be there silently classifies a real regression as intended, which is the failure the whole harness exists to prevent. Every table entry is in scope. The rule covered HAN and HANGUL alone while the kana members existed only for classification, but #272 gave HIRAGANA a default order entry and made the kana blocks part of the same first/middle/last diff shape the rule explains, so scoping it by issue no longer draws a real line. Comparing against the whole table is also the stronger promise: a script added to _SCRIPT_RANGES for ANY reason fails here until someone decides, in writing, whether the rule should cover it. Han's astral block is the single exception, out of scope on both sides. The rule omits it deliberately -- no corpus name reaches it, see the comment there -- so the comparison runs over the BMP spans only. The rule is also WIDER than the table by exactly one span, which the equality has to know about or it would just fail forever. The halfwidth middle dot U+FF65 changes parses without being classified as anything: tokenize separates on it, so a halfwidth transcription splits where 1.4 kept one token, while halfwidth kana stays out of _SCRIPT_RANGES on purpose. U+00B7 -- the context-sensitive 间隔号 (#298) -- also changes parses yet is deliberately NOT an extra: its flank guard means every name it can change matches the class through a flanking character already, and a B7 span's only actual effect would be letting the rule claim diffs on punt-volat Latin names (Gal·la), pre-excusing a regression on exactly the guarded class. Naming the sanctioned span here rather than relaxing the comparison to a subset check is what keeps the pin honest in both directions: an unsanctioned source of divergence still fails, and each sanctioned difference has to be written down to exist. """ toml_path = (Path(__file__).parents[2] / "tools" / "differential" / "expected_since_1.4.0.toml") rules = tomllib.loads(toml_path.read_text())["change"] matched = [r for r in rules if "#271" in r["issue"] or "#272" in r["issue"]] assert len(matched) == 1, ( f"expected exactly one CJK rule in {toml_path.name}, " f"found {len(matched)}") # A new table entry must force an explicit decision rather than # quietly widening (or failing to widen) the rule above. assert set(_policy._SCRIPT_RANGES) == { Script.HAN, Script.HANGUL, Script.HIRAGANA, Script.KATAKANA}, ( "a Script joined _SCRIPT_RANGES: decide whether the " f"differential rule in {toml_path.name} should cover it, then " "update this assertion") declared = _declared_spans(matched[0]["name_regex"]) # the extras must stay UNclassified, or they belong in the table for xlo, xhi in _SANCTIONED_EXTRAS: assert not any(lo <= xhi and xlo <= hi for spans in _policy._SCRIPT_RANGES.values() for lo, hi in spans), ( f"U+{xlo:04X}-U+{xhi:04X} is classified now; drop it " "from _SANCTIONED_EXTRAS") expected = _expected_bmp_spans() assert declared == expected, ( f"{toml_path.name}'s CJK name_regex declares {sorted(declared)}; " f"_SCRIPT_RANGES' BMP spans are {sorted(expected)}") def test_every_span_bearing_rule_matches_the_script_ranges() -> None: """Auto-discovered pin for every FURTHER hand copy of the script spans in the toml: any rule whose character class declares spans intersecting _SCRIPT_RANGES must declare the whole expected class (table BMP spans + sanctioned extras). The compound rules' require-a-classified-codepoint lookaheads exist so their trigger sets alone (delimiters; a comma) cannot claim a Latin name's regression -- and each such lookahead is a copy nothing else checks. Discovery replaces a hand-maintained slug roster: a new compound rule's copy is pinned by existing here, not by an author remembering to enroll it. Rules whose spans touch OTHER scripts (Cyrillic, say) are out of scope and skipped by the intersection test. Selection note for future rule authors: the canonical-rule pin above selects by the literal '#271'/'#272' substrings and asserts uniqueness -- compound slugs must avoid them. """ toml_path = (Path(__file__).parents[2] / "tools" / "differential" / "expected_since_1.4.0.toml") rules = tomllib.loads(toml_path.read_text())["change"] table_spans = _expected_bmp_spans() checked = [] for rule in rules: regex = rule.get("name_regex") if not isinstance(regex, str): continue declared = _declared_spans(regex) if not declared & table_spans: continue checked.append(rule["issue"]) assert declared == table_spans, ( f"{rule['issue']!r} declares {sorted(declared)}; expected " f"{sorted(table_spans)}") # the canonical rule plus both compound rules, today -- if this # count drops, a hand copy fell out of discovery's reach assert len(checked) >= 3, checked # the delimiter compound's trigger set is its own decision surface nickname = [r for r in rules if "cjk-delimited-nickname" in r["issue"]] assert len(nickname) == 1 assert "[\u300C\u300D\u300E\u300F\u30FB\uFF65]" in nickname[0][ "name_regex"] or "[「」『』・･]" in nickname[0]["name_regex"], ( "the compound rule's delimiter set changed; decide deliberately") def test_cjk_corpus_matches_the_case_table() -> None: """corpus_cjk.jsonl is GENERATED, not curated (#295): every distinct case-table text bearing a codepoint the script table classifies, sorted -- see build_cjk_corpus.py for why the other two corpora cannot carry these names. The checked-in file must equal what the generator would write, so a CJK case row added without regenerating fails HERE instead of silently narrowing the differential gate back toward the blind spot #295 closed. Same promise as the toml pin above, aimed at a generated artifact instead of a hand copy. """ tools = Path(__file__).parents[2] / "tools" / "differential" spec = importlib.util.spec_from_file_location( "build_cjk_corpus", tools / "build_cjk_corpus.py") assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) checked_in = [json.loads(line) for line in (tools / "corpus_cjk.jsonl") .read_text(encoding="utf-8").splitlines()] assert checked_in == module.selected_names(), ( "corpus_cjk.jsonl is stale: regenerate with " "`uv run python tools/differential/build_cjk_corpus.py`") def test_differential_honorific_rule_matches_the_suffix_vocabulary() -> None: """The fix(cjk-honorific-suffix) rule's alternation is a hand copy of SUFFIX_NOT_ACRONYMS' CJK entries (#307) -- the toml cannot import them. The expected set is DERIVED from the config by script membership (a classified codepoint anywhere in the entry), so adding a CJK honorific without widening the rule -- or widening the rule with something the vocabulary does not ship -- fails here. The span-bearing pins above skip this rule on purpose: its trigger is the alternation, not a character class. """ from nameparser.config.suffixes import SUFFIX_NOT_ACRONYMS toml_path = (Path(__file__).parents[2] / "tools" / "differential" / "expected_since_1.4.0.toml") rules = tomllib.loads(toml_path.read_text())["change"] matched = [r for r in rules if "cjk-honorific-suffix" in r["issue"]] assert len(matched) == 1 regex = matched[0]["name_regex"] prefix = "(?:^| )(?:" assert regex.startswith(prefix) and regex.endswith(")$"), ( "the honorific rule's shape changed; update this parser") declared = set(regex[len(prefix):-2].split("|")) has_classified = _policy._script_matcher(*_policy._SCRIPT_RANGES) expected = {entry for entry in SUFFIX_NOT_ACRONYMS if has_classified(entry)} assert declared == expected, ( f"rule declares {sorted(declared)}; the config's CJK suffix " f"entries are {sorted(expected)}") python-nameparser-2.1.0/tests/v2/test_render.py000066400000000000000000000264511523530431700215620ustar00rootroot00000000000000import pytest from nameparser._lexicon import Lexicon from nameparser._render import _collapse, render from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token def test_collapse_is_the_254_algorithm() -> None: # normative: leading/trailing whitespace, doubled spaces, # space-before-comma, one trailing comma char (incl. Arabic/CJK), # leading/trailing ', ' debris, and empty-wrapper artifacts from # empty fields are removed assert _collapse(" John Smith ") == "John Smith" assert _collapse("Smith , John") == "Smith, John" assert _collapse("John Smith ,") == "John Smith" assert _collapse("John Smith،") == "John Smith" # Arabic comma assert _collapse("John Smith，") == "John Smith" # fullwidth comma assert _collapse(", John Smith, ") == "John Smith" assert _collapse("John Smith ()") == "John Smith" assert _collapse("John Smith ''") == "John Smith" assert _collapse('John Smith ""') == "John Smith" assert _collapse("") == "" def _pn(original: str, tokens: list[Token]) -> ParsedName: return ParsedName(original=original, tokens=tuple(tokens)) def _delavega() -> ParsedName: # "Dr. Juan de la Vega III" -- spans verified by hand # 0123456789012345678901234 return _pn("Dr. Juan de la Vega III", [ Token("Dr.", Span(0, 3), Role.TITLE), Token("Juan", Span(4, 8), Role.GIVEN), Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})), Token("la", Span(12, 14), Role.FAMILY, frozenset({"particle"})), Token("Vega", Span(15, 19), Role.FAMILY), Token("III", Span(20, 23), Role.SUFFIX), ]) def test_render_fills_fields_and_collapses() -> None: pn = _delavega() assert render(pn, "{title} {given} {middle} {family} {suffix}") \ == "Dr. Juan de la Vega III" # empty middle collapses; comma survives correctly assert render(pn, "{family}, {given} {middle}") == "de la Vega, Juan" def test_render_accepts_derived_view_keys() -> None: assert render(_delavega(), "{family_base}, {given} {family_particles}") \ == "Vega, Juan de la" assert render(_delavega(), "{surnames}") == "de la Vega" assert render(_delavega(), "{given_names}") == "Juan" def test_render_every_role_key_is_valid() -> None: pn = _delavega() for role in Role: render(pn, f"{{{role.value}}}") # must not raise def test_render_unknown_key_raises_enriched_keyerror() -> None: with pytest.raises(KeyError, match="valid fields"): render(_delavega(), "{first}") # v1 spelling: redirected loudly def test_render_empty_parse_is_empty_string() -> None: assert render(_pn("", []), "{title} {given} {middle} {family} {suffix}") == "" def test_default_str_includes_nickname_and_maiden() -> None: # Derek's terminal find (2026-07-19): str(parse('Von Johnson # (smith)')) dropped the nickname; the default spec now shows all # seven fields -- nickname quoted after the given name, maiden # parenthesized after the family name (Derek's chosen format). # The nickname decoration round-trips exactly; the maiden parens # re-extract as a NICKNAME on reparse (documented trade-off: # presentation over lossless round-trip). from nameparser import parse assert str(parse("Von Johnson (smith)")) == 'Von "smith" Johnson' assert str(parse("Jane Smith née Jones")) == "Jane Smith (Jones)" both = parse("Jane (Janie) Smith née Jones") assert str(both) == 'Jane "Janie" Smith (Jones)' # no orphaned decoration when the fields are empty assert str(parse("Dr. Juan Q. Xavier de la Vega III")) == ( "Dr. Juan Q. Xavier de la Vega III") def test_parsedname_render_and_str_delegate() -> None: pn = _delavega() assert pn.render() == "Dr. Juan de la Vega III" assert pn.render("{family}, {given}") == "de la Vega, Juan" assert str(pn) == pn.render() assert str(_pn("", [])) == "" def _bobdole() -> ParsedName: # "Sir Bob Andrew Dole" # 01234567890123456789 return _pn("Sir Bob Andrew Dole", [ Token("Sir", Span(0, 3), Role.TITLE), Token("Bob", Span(4, 7), Role.GIVEN), Token("Andrew", Span(8, 14), Role.MIDDLE), Token("Dole", Span(15, 19), Role.FAMILY), ]) def test_initials_default_spec() -> None: assert _bobdole().initials() == "B. A. D." def test_initials_skips_tagged_particles_outside_given() -> None: # family "de la Vega" with particle tags -> only V contributes; # a given-name token always contributes even if tagged assert _delavega().initials() == "J. V." # conjunction tag skips too pn = _pn("Mr. and Mrs. Smith", [ Token("Mr.", Span(0, 3), Role.TITLE), Token("and", Span(4, 7), Role.FAMILY, frozenset({"conjunction"})), Token("Smith", Span(13, 18), Role.FAMILY), ]) assert pn.initials("{family}") == "S." def test_initials_custom_delimiter_and_separator() -> None: assert _bobdole().initials(delimiter="", separator="") == "B A D" def test_initials_multiword_group_joins_within_group() -> None: pn = _pn("Mary Jane Watson", [ Token("Mary", Span(0, 4), Role.GIVEN), Token("Jane", Span(5, 9), Role.GIVEN), Token("Watson", Span(10, 16), Role.FAMILY), ]) assert pn.initials() == "M. J. W." def test_initials_custom_spec_and_unknown_key() -> None: assert _bobdole().initials("{given} {middle}") == "B. A." with pytest.raises(KeyError, match="valid fields"): _bobdole().initials("{title}") def test_initials_already_initial_words() -> None: pn = _pn("J. Doe", [ Token("J.", Span(0, 2), Role.GIVEN), Token("Doe", Span(3, 6), Role.FAMILY), ]) assert pn.initials() == "J. D." def test_initials_empty_group_renders_empty() -> None: # v2 returns "" for an empty result -- no v1-style # empty_attribute_default fallback assert _bobdole().initials("{middle}") == "A." assert _pn("Cher", [Token("Cher", Span(0, 4), Role.GIVEN)]).initials("{middle}") == "" def _lowercase_mac() -> ParsedName: # v1 capitalize() doctest input: 'bob v. de la macdole-eisenhower phd' # 0123456789012345678901234567890123456 return _pn("bob v. de la macdole-eisenhower phd", [ Token("bob", Span(0, 3), Role.GIVEN), Token("v.", Span(4, 6), Role.MIDDLE), Token("de", Span(7, 9), Role.FAMILY), Token("la", Span(10, 12), Role.FAMILY), Token("macdole-eisenhower", Span(13, 31), Role.FAMILY), Token("phd", Span(32, 35), Role.SUFFIX), ]) def test_capitalized_all_lower_input_v1_parity() -> None: out = _lowercase_mac().capitalized() assert out.given == "Bob" assert out.middle == "V." assert out.family == "de la MacDole-Eisenhower" # particles stay lower assert out.suffix == "Ph.D." # exceptions map, verbatim # same spans, new texts (provenance is a documented non-invariant) assert [t.span for t in out.tokens] == [t.span for t in _lowercase_mac().tokens] def test_capitalized_all_upper_input() -> None: pn = _pn("JOHN SMITH", [ Token("JOHN", Span(0, 4), Role.GIVEN), Token("SMITH", Span(5, 10), Role.FAMILY), ]) assert str(pn.capitalized()) == "John Smith" def test_capitalized_preserves_mixed_case_unless_forced() -> None: pn = _pn("Shirley Maclaine", [ Token("Shirley", Span(0, 7), Role.GIVEN), Token("Maclaine", Span(8, 16), Role.FAMILY), ]) assert pn.capitalized() == pn # untouched assert pn.capitalized(force=True).family == "MacLaine" def test_capitalized_is_idempotent() -> None: once = _lowercase_mac().capitalized() assert once.capitalized() == once assert once.capitalized(force=True) == once def test_capitalized_with_explicit_lexicon() -> None: # empty lexicon: no particle rule, no exceptions -> plain capitalize out = _lowercase_mac().capitalized(Lexicon.empty()) assert out.family == "De La MacDole-Eisenhower" assert out.suffix == "Phd" def test_capitalized_lowers_conjunctions_but_not_initial_shapes() -> None: # v1's is_conjunction excludes initial-shaped words: a lowercase # 'y' lowers, but an uppercase 'Y' looks like an initial and # capitalizes ('JOSE ORTEGA Y GASSET' -> 'Jose Ortega Y Gasset', # pinned live against v1.4 2026-07-17) # 01234567890123456789 pn = _pn("juan ortega y gasset", [ Token("juan", Span(0, 4), Role.GIVEN), Token("ortega", Span(5, 11), Role.FAMILY), Token("y", Span(12, 13), Role.FAMILY, frozenset({"conjunction"})), Token("gasset", Span(14, 20), Role.FAMILY), ]) out = pn.capitalized(force=True) assert out.family == "Ortega y Gasset" upper = _pn("JUAN ORTEGA Y GASSET", [ Token("JUAN", Span(0, 4), Role.GIVEN), Token("ORTEGA", Span(5, 11), Role.FAMILY), Token("Y", Span(12, 13), Role.FAMILY, frozenset({"conjunction"})), Token("GASSET", Span(14, 20), Role.FAMILY), ]) assert upper.capitalized(force=True).family == "Ortega Y Gasset" def test_capitalized_rebuilds_ambiguity_tokens() -> None: tok = Token("van", Span(0, 3), Role.GIVEN, frozenset({"particle"})) pn = ParsedName( original="van johnson", tokens=(tok, Token("johnson", Span(4, 11), Role.FAMILY)), ambiguities=(Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "leading 'van' may be a particle", (tok,)),), ) out = pn.capitalized() # the ambiguity references the NEW capitalized token (subset invariant) assert out.ambiguities[0].tokens[0] is out.tokens[0] assert out.ambiguities[0].tokens[0].text == "Van" def test_render_and_initials_reject_non_str_arguments() -> None: # eager, like every constructor: not an AttributeError frames deep pn = _delavega() with pytest.raises(TypeError, match="spec must be a str"): pn.render(7) # type: ignore[arg-type] with pytest.raises(TypeError, match="spec must be a str"): pn.initials(7) # type: ignore[arg-type] with pytest.raises(TypeError, match="delimiter must be a str"): pn.initials(delimiter=None) # type: ignore[arg-type] with pytest.raises(TypeError, match="separator must be a str"): pn.initials(separator=0) # type: ignore[arg-type] def test_capitalized_rejects_non_lexicon_argument() -> None: # previously a silent no-op on mixed-case input and a deep # AttributeError on single-case input with pytest.raises(TypeError, match="must be a Lexicon"): _delavega().capitalized("garbage") # type: ignore[arg-type] with pytest.raises(TypeError, match="must be a Lexicon"): _lowercase_mac().capitalized({"titles": set()}) # type: ignore[arg-type] def test_initials_given_tokens_ignore_skip_tags() -> None: # documented: a given-name token contributes even when tagged (the # PARTICLE_OR_GIVEN case -- 'van' read as a given name) pn = _pn("van Johnson", [ Token("van", Span(0, 3), Role.GIVEN, frozenset({"particle"})), Token("Johnson", Span(4, 11), Role.FAMILY), ]) assert pn.initials() == "v. J." def test_render_malformed_specs_surface_raw_format_errors() -> None: # documented contract: only unknown KEYS get the enriched KeyError; # positional fields and bad conversions raise str.format's own error pn = _delavega() with pytest.raises(IndexError): pn.render("{}") with pytest.raises(ValueError): pn.render("{given!q}") python-nameparser-2.1.0/tests/v2/test_reprs.py000066400000000000000000000151061523530431700214310ustar00rootroot00000000000000import dataclasses import functools from typing import cast import pytest from nameparser._lexicon import Lexicon from nameparser._locale import Locale from nameparser._parser import Parser from nameparser._policy import FAMILY_FIRST, PatronymicRule, Policy, PolicyPatch from nameparser._types import ( Ambiguity, AmbiguityKind, ParsedName, Role, Segmentation, Span, Token, ) def _decline(text: str, blob: str = "") -> None: return None # a stand-in segmenter for the Parser repr cases def test_token_repr_is_compact() -> None: t = Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})) assert repr(t) == "Token('de' @9:11 FAMILY {particle})" assert repr(Token("Jane", None, Role.GIVEN)) == "Token('Jane' @synthetic GIVEN)" def test_ambiguity_repr_shows_kind_and_token_texts() -> None: van = Token("Van", Span(0, 3), Role.GIVEN) a = Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "detail", (van,)) assert repr(a) == "Ambiguity('particle-or-given': 'Van')" def test_parsedname_repr_lists_nonempty_fields_in_canonical_order() -> None: pn = ParsedName("John Smith", ( Token("John", Span(0, 4), Role.GIVEN), Token("Smith", Span(5, 10), Role.FAMILY), )) assert repr(pn) == ( "" ) def test_parsedname_repr_includes_ambiguities_line_when_present() -> None: van = Token("Van", Span(0, 3), Role.GIVEN) pn = ParsedName("Van Johnson", (van, Token("Johnson", Span(4, 11), Role.FAMILY)), (Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (van,)),)) assert "ambiguities: ['particle-or-given']" in repr(pn) def test_empty_parsedname_repr() -> None: assert repr(ParsedName("", ())) == "" def test_policy_repr_shows_only_nondefault_fields() -> None: assert repr(Policy()) == "Policy()" p = Policy(name_order=FAMILY_FIRST, strip_bidi=False) assert repr(p) == "Policy(name_order=FAMILY_FIRST, strip_bidi=False)" def test_lexicon_repr_is_bounded() -> None: assert repr(Lexicon.default()) == "Lexicon(default)" lex = Lexicon.default().add(titles={"zqx", "zqy"}) assert repr(lex) == "Lexicon(default + titles: +2)" assert "zqx" not in repr(lex) # never dump contents def test_locale_repr_shows_code_and_patched_fields() -> None: ru = Locale("ru", Lexicon.empty(), PolicyPatch(patronymic_rules=frozenset( {PatronymicRule.EAST_SLAVIC}))) assert repr(ru) == "Locale('ru': patronymic_rules)" assert repr(Locale("xx", Lexicon.empty())) == "Locale('xx')" def test_lexicon_repr_diffs_against_nearer_baseline() -> None: # An empty()-built Lexicon must not render as default() minus ~700 # entries -- diff against whichever named constructor is nearer. assert repr(Lexicon.empty()) == "Lexicon(empty)" lex = Lexicon.empty().add(titles={"zqx"}) assert repr(lex) == "Lexicon(empty + titles: +1)" def test_policy_patch_repr_shows_only_set_fields() -> None: assert repr(PolicyPatch()) == "PolicyPatch()" assert repr(PolicyPatch(middle_as_family=True)) == \ "PolicyPatch(middle_as_family=True)" assert repr(PolicyPatch(name_order=FAMILY_FIRST)) == \ "PolicyPatch(name_order=FAMILY_FIRST)" @pytest.mark.parametrize("obj", [ Policy(), Policy(middle_as_family=True), PolicyPatch(), PolicyPatch(strip_emoji=False), Locale("xx", Lexicon.empty()), Locale("xx", Lexicon.empty(), PolicyPatch(middle_as_family=True)), Lexicon.default(), Lexicon.empty(), Parser(), Parser(segmenter=functools.partial(_decline, blob="x" * 500)), ]) def test_config_reprs_never_leak_the_unset_sentinel(obj: object) -> None: # Guard the whole family: every config type's repr is bounded and # deviation-only; the UNSET sentinel must never appear. assert "UNSET" not in repr(obj) assert "_Unset" not in repr(obj) def test_parser_repr_names_the_segmenter_without_its_contents() -> None: # A callable is the one component whose own repr is unbounded: a # partial spells out its bound arguments, a callable instance its # memory address. Neither may reach Parser's repr, so the name comes # from __qualname__ when there is one and the TYPE otherwise. plain = repr(Parser(segmenter=_decline)) assert "segmenter=_decline" in plain partial = repr(Parser(segmenter=functools.partial(_decline, blob="x" * 500))) assert "segmenter=partial" in partial assert "xxx" not in partial and len(partial) < len(plain) + 20 class Div: # a callable INSTANCE: no __qualname__, so the type def __call__(self, text: str) -> Segmentation | None: return None instance = repr(Parser(segmenter=Div())) assert "segmenter=Div" in instance and "0x" not in instance def test_policy_patch_repr_survives_unvalidated_name_order() -> None: # PolicyPatch defers name_order validation to apply time, so a # patch can hold a non-canonical or even garbage tuple -- its repr # must describe it, never raise. patch = PolicyPatch(name_order=(Role.GIVEN, Role.FAMILY, Role.MIDDLE)) assert repr(patch) == "PolicyPatch(name_order=(GIVEN, FAMILY, MIDDLE))" garbage = PolicyPatch(name_order=("given", "family", "middle")) # type: ignore[arg-type] assert "given" in repr(garbage) # described, not crashed unhashable = PolicyPatch(name_order=([1], "y", "z")) # type: ignore[arg-type] repr(unhashable) # must not raise def test_order_repr_survives_smuggled_setstate_values() -> None: # __setstate__ validates layout, not values (the pickling rule), so # the reprs must survive shapes no constructor can produce: an # all-Role list (passes an element check, unhashable), a # non-iterable, and a tuple of Role-EQUAL plain strings (StrEnum # equality would hit the named lookup and lie). for garbage in ([Role.GIVEN, Role.MIDDLE, Role.FAMILY], 5, ("given", "middle", "family")): for cls in (Policy, PolicyPatch): obj = cls() state = {f.name: getattr(obj, f.name) for f in dataclasses.fields(obj)} state["name_order"] = garbage smuggled = cast("Policy | PolicyPatch", cls.__new__(cls)) smuggled.__setstate__(state) rendered = repr(smuggled) # must not raise if isinstance(garbage, tuple): # a smuggled string tuple must not render as the # named constant it merely compares equal to assert "GIVEN_FIRST" not in rendered python-nameparser-2.1.0/tests/v2/test_types.py000066400000000000000000000510711523530431700214430ustar00rootroot00000000000000import dataclasses from collections.abc import Iterable, Iterator import pytest from nameparser import parse from nameparser._types import ( STABLE_TAGS, Ambiguity, AmbiguityKind, ParsedName, Role, Segmentation, Span, Token, ) def test_role_declaration_order_is_canonical_field_order() -> None: assert [r.value for r in Role] == [ "title", "given", "middle", "family", "suffix", "nickname", "maiden", ] def test_role_members_are_their_string_values() -> None: # Role is a StrEnum, like AmbiguityKind: members compare, hash, # and stringify as their field names. assert Role.GIVEN == "given" assert str(Role.FAMILY) == "family" d: dict[str, int] = {Role.GIVEN: 1} assert d["given"] == 1 assert isinstance(Role.GIVEN, str) assert repr(Role.GIVEN) == "" # repr keeps .name form assert Role("maiden") is Role.MAIDEN def test_token_construction_and_span_coercion() -> None: t = Token("Juan", (0, 4), Role.GIVEN) # type: ignore[arg-type] assert t.span == Span(0, 4) assert isinstance(t.span, Span) assert t.span.start == 0 and t.span.end == 4 assert t.tags == frozenset() def test_synthetic_token_has_no_span() -> None: t = Token("Jane", None, Role.GIVEN) assert t.span is None def test_token_rejects_empty_text() -> None: with pytest.raises(ValueError, match="non-empty"): Token("", Span(0, 0), Role.GIVEN) def test_token_rejects_inverted_span() -> None: with pytest.raises(ValueError, match="start <= end"): Token("x", Span(5, 2), Role.GIVEN) def test_token_rejects_negative_span() -> None: with pytest.raises(ValueError, match="start <= end"): Token("x", Span(-1, 1), Role.GIVEN) def test_token_rejects_malformed_span_shapes() -> None: with pytest.raises(TypeError, match="expected a \\(start, end\\) pair"): Token("x", (0, 4, 9), Role.GIVEN) # type: ignore[arg-type] with pytest.raises(TypeError, match="expected a \\(start, end\\) pair"): Token("x", 5, Role.GIVEN) # type: ignore[arg-type] def test_token_rejects_non_string_text() -> None: with pytest.raises(TypeError, match="got None"): Token(None, None, Role.GIVEN) # type: ignore[arg-type] def test_token_is_frozen_and_hashable() -> None: t = Token("Juan", Span(0, 4), Role.GIVEN) with pytest.raises(AttributeError): t.text = "X" # type: ignore[misc] assert hash(t) == hash(Token("Juan", Span(0, 4), Role.GIVEN)) def test_segmentation_validates_splits() -> None: s = Segmentation((2,), confidence=0.97) assert s.splits == (2,) and s.confidence == 0.97 assert Segmentation(()).confidence is None with pytest.raises(ValueError, match="ascending"): Segmentation((3, 2)) with pytest.raises(ValueError, match="interior"): Segmentation((0,)) with pytest.raises(TypeError, match="integers"): Segmentation(("2",)) # type: ignore[arg-type] with pytest.raises(ValueError, match="confidence"): Segmentation((2,), confidence=1.5) # wrong TYPE is the TypeError branch, distinct from the range check with pytest.raises(TypeError, match="confidence"): Segmentation((2,), confidence="high") # type: ignore[arg-type] def test_segmentation_rejects_mappings_and_non_iterables() -> None: # Token.tags' guards, on the other collection field: a mapping # would silently contribute only its keys, and a bare int would # surface as an uncurated "not iterable" naming nothing with pytest.raises(TypeError, match="Segmentation.splits"): Segmentation({2: "a", 3: "b"}) # type: ignore[arg-type] with pytest.raises(TypeError, match="Segmentation.splits"): Segmentation(5) # type: ignore[arg-type] # ...and the bare string, the third member of that family and the # one with teeth: "" is iterable and empty, so without the guard # Segmentation("") became "confidently one token" -- an opinion # nobody stated -- while "23" failed one character deep naming '2' # instead of the argument. Both must name the field. for bad in ("", "23"): with pytest.raises(TypeError, match="Segmentation.splits"): Segmentation(bad) # type: ignore[arg-type] def test_segmentation_rejects_bools_in_both_fields() -> None: # bool is an int subclass in both slots: True as an offset is a # comparison result leaking into an index, True as a confidence one # leaking into a score. Behavioral twin of Token's span guard. with pytest.raises(TypeError, match="Segmentation.splits"): Segmentation((True,)) # type: ignore[list-item] with pytest.raises(TypeError, match="Segmentation.confidence"): Segmentation((2,), confidence=True) # type: ignore[arg-type] def test_segmentation_lets_a_generator_raise_its_own_error() -> None: # the non-iterable guard probes iter(), which cannot run a # generator's body; wrapping the tuple() call instead would catch a # TypeError raised INSIDE the user's generator and relabel it as # "splits must be an iterable", pointing the reader at the wrong bug def boom() -> Iterator[int]: yield 1 raise TypeError("boom") with pytest.raises(TypeError) as caught: Segmentation(boom()) # type: ignore[arg-type] # the message is the generator's own, whole -- a `match=` would # pass on the relabeled text too, since the generator's repr names # the function assert str(caught.value) == "boom" def test_segmentation_is_frozen_and_hashable() -> None: s = Segmentation((2,)) assert isinstance(hash(s), int) with pytest.raises(dataclasses.FrozenInstanceError): s.splits = (3,) # type: ignore[misc] def test_ambiguity_kind_members_are_their_string_values() -> None: assert AmbiguityKind.PARTICLE_OR_GIVEN == "particle-or-given" assert AmbiguityKind("order") is AmbiguityKind.ORDER def test_ambiguity_construction_coerces_kind_string() -> None: t = Token("Van", Span(0, 3), Role.GIVEN, frozenset({"particle"})) a = Ambiguity("particle-or-given", "leading 'van' may be a particle", (t,)) # type: ignore[arg-type] assert a.kind is AmbiguityKind.PARTICLE_OR_GIVEN assert a.tokens == (t,) def test_ambiguity_rejects_unknown_kind() -> None: with pytest.raises(ValueError, match="particle-or-given"): Ambiguity("no-such-kind", "detail", ()) # type: ignore[arg-type] def test_ambiguity_rejects_non_token_elements() -> None: with pytest.raises(TypeError, match="only Token instances"): Ambiguity("order", "detail", ("not-a-token",)) # type: ignore[arg-type] def test_ambiguity_rejects_empty_detail() -> None: with pytest.raises(ValueError, match="non-empty string"): Ambiguity(AmbiguityKind.ORDER, "", ()) def test_ambiguity_rejects_non_str_detail() -> None: with pytest.raises(TypeError, match="must be a str"): Ambiguity(AmbiguityKind.ORDER, None, ()) # type: ignore[arg-type] def _pn(original: str, tokens: Iterable[Token], ambiguities: Iterable[Ambiguity] = ()) -> ParsedName: return ParsedName(original=original, tokens=tuple(tokens), ambiguities=tuple(ambiguities)) def test_parsedname_accepts_valid_spans_and_is_truthy() -> None: pn = _pn("John Smith", [ Token("John", Span(0, 4), Role.GIVEN), Token("Smith", Span(5, 10), Role.FAMILY), ]) assert bool(pn) is True def test_empty_parse_is_falsy() -> None: assert bool(_pn("", [])) is False assert bool(_pn(" ", [])) is False def test_parsedname_rejects_out_of_bounds_span() -> None: with pytest.raises(ValueError, match="out of bounds"): _pn("John", [Token("Johnny", Span(0, 6), Role.GIVEN)]) def test_parsedname_rejects_overlapping_spans() -> None: with pytest.raises(ValueError, match="ascending"): _pn("John Smith", [ Token("John", Span(0, 4), Role.GIVEN), Token("ohn S", Span(1, 6), Role.FAMILY), ]) def test_parsedname_rejects_descending_spans() -> None: with pytest.raises(ValueError, match="ascending"): _pn("John Smith", [ Token("Smith", Span(5, 10), Role.FAMILY), Token("John", Span(0, 4), Role.GIVEN), ]) def test_synthetic_tokens_skip_span_checks() -> None: pn = _pn("John Smith", [ Token("John", Span(0, 4), Role.GIVEN), Token("Qux", None, Role.MIDDLE), Token("Smith", Span(5, 10), Role.FAMILY), ]) assert len(pn.tokens) == 3 def test_ambiguity_tokens_must_be_subset_of_parse_tokens() -> None: inside = Token("Van", Span(0, 3), Role.GIVEN) outside = Token("Zzz", None, Role.GIVEN) with pytest.raises(ValueError, match="subset"): _pn("Van Johnson", [inside, Token("Johnson", Span(4, 11), Role.FAMILY)], [Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (outside,))]) def test_parsedname_equality_is_strict_structural() -> None: a = _pn("John", [Token("John", Span(0, 4), Role.GIVEN)]) b = _pn("John", [Token("John", Span(0, 4), Role.GIVEN)]) c = _pn("John ", [Token("John", Span(0, 4), Role.GIVEN)]) assert a == b and hash(a) == hash(b) assert a != c # different original: not interchangeable def test_parsedname_rejects_non_str_original() -> None: with pytest.raises(TypeError, match="must be a str"): _pn(None, []) # type: ignore[arg-type] def test_parsedname_rejects_non_token_and_non_ambiguity_elements() -> None: with pytest.raises(TypeError, match="only Token instances"): _pn("x", ["not-a-token"]) # type: ignore[list-item] with pytest.raises(TypeError, match="only Ambiguity instances"): _pn("John", [Token("John", Span(0, 4), Role.GIVEN)], ["nope"]) # type: ignore[list-item] def _delavega() -> ParsedName: # "Dr. Juan de la Vega III" -- hand-built, spans verified by hand # 0123456789012345678901234 return _pn("Dr. Juan de la Vega III", [ Token("Dr.", Span(0, 3), Role.TITLE), Token("Juan", Span(4, 8), Role.GIVEN), Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})), Token("la", Span(12, 14), Role.FAMILY, frozenset({"particle"})), Token("Vega", Span(15, 19), Role.FAMILY), Token("III", Span(20, 23), Role.SUFFIX), ]) def test_string_properties_join_by_role() -> None: pn = _delavega() assert pn.title == "Dr." assert pn.given == "Juan" assert pn.middle == "" assert pn.family == "de la Vega" assert pn.suffix == "III" assert pn.nickname == "" assert pn.maiden == "" def test_suffix_joins_with_comma_space() -> None: pn = _pn("John Smith PhD MD", [ Token("John", Span(0, 4), Role.GIVEN), Token("Smith", Span(5, 10), Role.FAMILY), Token("PhD", Span(11, 14), Role.SUFFIX), Token("MD", Span(15, 17), Role.SUFFIX), ]) assert pn.suffix == "PhD, MD" def test_derived_views_filter_on_stable_particle_tag() -> None: # Pin the hard-coded "particle" string in _text_for to the published # contract until parser tag-emission contract tests land. assert "particle" in STABLE_TAGS pn = _delavega() assert pn.family_particles == "de la" assert pn.family_base == "Vega" assert pn.surnames == "de la Vega" # middle + family assert pn.given_names == "Juan" # given + middle def test_tokens_for_preserves_order() -> None: pn = _delavega() assert [t.text for t in pn.tokens_for(Role.FAMILY)] == ["de", "la", "Vega"] assert pn.tokens_for(Role.NICKNAME) == () def test_as_dict_canonical_order_and_empty_filtering() -> None: pn = _delavega() d = pn.as_dict() assert list(d) == ["title", "given", "middle", "family", "suffix", "nickname", "maiden"] assert d["family"] == "de la Vega" and d["middle"] == "" d2 = pn.as_dict(include_empty=False) assert list(d2) == ["title", "given", "family", "suffix"] def test_replace_swaps_field_with_synthetic_tokens_in_place() -> None: pn = _delavega() pn2 = pn.replace(given="Jean Paul") assert pn2.given == "Jean Paul" assert pn2.family == "de la Vega" # untouched assert pn2.original == pn.original # provenance unchanged assert all(t.span is None for t in pn2.tokens_for(Role.GIVEN)) assert pn.given == "Juan" # source object unchanged # positional: synthetic given tokens sit where the old ones were assert [t.role for t in pn2.tokens][:3] == [Role.TITLE, Role.GIVEN, Role.GIVEN] def test_replace_adds_missing_field_at_end() -> None: pn = _pn("John Smith", [ Token("John", Span(0, 4), Role.GIVEN), Token("Smith", Span(5, 10), Role.FAMILY), ]) pn2 = pn.replace(suffix="Jr") assert pn2.suffix == "Jr" assert pn2.tokens[-1].role is Role.SUFFIX def test_replace_with_empty_string_clears_field() -> None: pn = _delavega() assert pn.replace(title="").title == "" def test_replace_rejects_unknown_field() -> None: with pytest.raises(TypeError, match="given"): _delavega().replace(firstname="X") def test_replace_drops_ambiguities_referencing_removed_tokens() -> None: van = Token("Van", Span(0, 3), Role.GIVEN) pn = _pn("Van Johnson", [van, Token("Johnson", Span(4, 11), Role.FAMILY)], [Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (van,))]) assert pn.replace(given="Bob").ambiguities == () assert pn.replace(family="Smith").ambiguities != () def test_replace_rejects_non_str_value() -> None: with pytest.raises(TypeError, match="must be a str"): _delavega().replace(given=None) # type: ignore[arg-type] def test_replace_appends_missing_roles_in_canonical_order() -> None: pn = _pn("John", [Token("John", Span(0, 4), Role.GIVEN)]) pn2 = pn.replace(maiden="X", suffix="Y") assert [t.role for t in pn2.tokens] == [Role.GIVEN, Role.SUFFIX, Role.MAIDEN] def test_comparison_key_is_casefolded_canonical_seven_tuple() -> None: pn = _delavega() assert pn.comparison_key() == ( "dr.", "juan", "", "de la vega", "iii", "", "", ) upper = _pn("JUAN DE LA VEGA", [ Token("JUAN", Span(0, 4), Role.GIVEN), Token("DE", Span(5, 7), Role.FAMILY, frozenset({"particle"})), Token("LA", Span(8, 10), Role.FAMILY, frozenset({"particle"})), Token("VEGA", Span(11, 15), Role.FAMILY), ]) lower = _pn("juan de la vega", [ Token("juan", Span(0, 4), Role.GIVEN), Token("de", Span(5, 7), Role.FAMILY, frozenset({"particle"})), Token("la", Span(8, 10), Role.FAMILY, frozenset({"particle"})), Token("vega", Span(11, 15), Role.FAMILY), ]) assert upper.comparison_key() == lower.comparison_key() def test_matches_casefolds_unicode_case_pairs() -> None: # Deliberate 1.4 deviation (release log): comparison folds with # casefold() where 1.4 used lower(). Comparison is the one surface # where casefold's aggressive folds are WANTED -- ß/SS and final- # sigma forms are the same name under different case conventions, # and the key is opaque, so the spelling-mutation bug casefold # caused in vocabulary storage (which stays lower(), v1 lc parity # -- see _lexicon._normalize) cannot happen here. from nameparser import HumanName, parse assert parse("Hans Straße").matches("HANS STRASSE") # ß <-> SS assert parse("Νίκος Παπαδόπουλος").matches("Νίκοσ Παπαδόπουλοσ") # ς <-> σ # same fold on the facade path -- both APIs share the deviation assert HumanName("Hans Straße").matches("HANS STRASSE") key = parse("Hans Straße").comparison_key() assert key == parse("HANS STRASSE").comparison_key() def test_token_rejects_bare_string_and_mapping_tags() -> None: # frozenset("particle") is the set(str) footgun: eight single chars. with pytest.raises(TypeError, match="bare string"): Token("Van", None, Role.GIVEN, tags="particle") # type: ignore[arg-type] with pytest.raises(TypeError, match="mapping"): Token("Van", None, Role.GIVEN, tags={"particle": 1}) # type: ignore[arg-type] def test_token_rejects_non_str_tags() -> None: with pytest.raises(TypeError, match="tags must contain only strings"): Token("Van", None, Role.GIVEN, tags=frozenset({1})) # type: ignore[arg-type] def test_token_coerces_role_string_and_rejects_unknown() -> None: # mirror Ambiguity.kind: coerce the string form, ValueError for any # failed enum lookup (stdlib EnumType precedent). assert Token("Juan", None, "given").role is Role.GIVEN # type: ignore[arg-type] with pytest.raises(ValueError, match="title, given"): Token("Juan", None, "chief") # type: ignore[arg-type] with pytest.raises(ValueError, match="title, given"): Token("Juan", None, 5) # type: ignore[arg-type] def test_types_pickle_round_trip() -> None: import pickle pn = _delavega() assert pickle.loads(pickle.dumps(pn)) == pn amb = Ambiguity(AmbiguityKind.ORDER, "two-comma structure", ()) assert pickle.loads(pickle.dumps(amb)) == amb tok = Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})) assert pickle.loads(pickle.dumps(tok)) == tok # the family sweep is the point of this test, so a frozen type # added later belongs in it -- Segmentation (#272) carries the same # guarded __getstate__/__setstate__ pair as its three siblings seg = Segmentation((2,), confidence=0.42) assert pickle.loads(pickle.dumps(seg)) == seg def test_span_add_is_blocked() -> None: # NamedTuple + would concatenate into a 4-tuple, the natural but # wrong spelling of "covering span" (a real cover() arrives with the # pipeline's join stage, its consumer). with pytest.raises(TypeError, match="covering span"): Span(0, 2) + Span(3, 4) # type: ignore[operator] def test_token_rejects_bool_span_coordinates() -> None: # bool is an int subclass; (False, True) is a comparison result # leaking into a coordinate slot, not a span. with pytest.raises(TypeError, match="pair of ints"): Token("x", (False, True), Role.GIVEN) # type: ignore[arg-type] def test_setstate_rejects_layout_skew_on_frozen_types() -> None: # version-skewed pickles must fail at the LOAD site, naming the # mismatch -- not at a distant attribute read (same policy as # Lexicon; values are deliberately NOT re-validated: pickle is not # a security boundary) tok = Token("Juan", Span(0, 4), Role.GIVEN) state = tok.__getstate__() bad = dict(state) del bad["tags"] with pytest.raises(ValueError, match="tags"): Token.__new__(Token).__setstate__(bad) pn = _pn("Juan", [tok]) bad_pn = dict(pn.__getstate__()) bad_pn["zq_future"] = () with pytest.raises(ValueError, match="zq_future"): ParsedName.__new__(ParsedName).__setstate__(bad_pn) def test_tokens_for_accepts_role_string() -> None: name = parse("Juan de la Vega") assert name.tokens_for("family") == name.tokens_for(Role.FAMILY) assert name.tokens_for("given") == name.tokens_for(Role.GIVEN) def test_tokens_for_unknown_role_raises_listing_roles() -> None: with pytest.raises(ValueError, match=r"unknown Role 'last'.*valid roles: title, given, middle"): parse("John Smith").tokens_for("last") def test_tokens_for_non_coercible_raises() -> None: with pytest.raises(ValueError, match="unknown Role 3"): parse("John Smith").tokens_for(3) # type: ignore[arg-type] def test_stable_tags_is_public_api() -> None: import nameparser assert "STABLE_TAGS" in nameparser.__all__ # the documented stable tag vocabulary -- these four values are API assert nameparser.STABLE_TAGS == frozenset( {"particle", "conjunction", "initial", "joined"}) def test_as_dict_include_empty_is_keyword_only() -> None: with pytest.raises(TypeError): parse("John Smith").as_dict(False) # type: ignore[misc] def test_particle_tag_marks_vocabulary_membership_not_role() -> None: # the tag follows the WORD, not the field: a leading ambiguous # particle read as a given name still carries it (STABLE_TAGS docs) tok = parse("Van Johnson").tokens_for(Role.GIVEN)[0] assert "particle" in tok.tags def test_str_enum_value_sets_are_pairwise_disjoint() -> None: # three StrEnums cross-compare by value; a future collision would # make unrelated members equal from nameparser import AmbiguityKind, PatronymicRule role = {m.value for m in Role} kind = {m.value for m in AmbiguityKind} rule = {m.value for m in PatronymicRule} assert not (role & kind) and not (role & rule) and not (kind & rule) def test_with_field_tokens_rejects_mismatched_roles() -> None: name = parse("John Smith") with pytest.raises(ValueError, match="has role family, not given"): name._with_field_tokens( {Role.GIVEN: (Token("x", None, Role.FAMILY),)}) python-nameparser-2.1.0/tools/000077500000000000000000000000001523530431700163315ustar00rootroot00000000000000python-nameparser-2.1.0/tools/differential/000077500000000000000000000000001523530431700207655ustar00rootroot00000000000000python-nameparser-2.1.0/tools/differential/README.md000066400000000000000000000363431523530431700222550ustar00rootroot00000000000000# Differential harness (a released baseline vs the working tree) Dev-only tooling for the 2.x line (migration plan S5). Not shipped (excluded from the wheel by the packaging config -- only `nameparser/` is packaged) and not CI-gated. Run it by hand when touching parsing behavior, and before cutting a release. Two processes, two environments: - A **baseline worker** runs under a pinned nameparser installed fresh from PyPI via a PEP 723 inline script. It is not a checked-in file: `compare.py` renders it from a template with the version pin substituted and writes it to a temp directory outside the worktree. That placement is a safety mechanism rather than plumbing -- it is what makes the invocation traps below unreachable rather than merely documented. - `compare.py` runs in the project's own dev environment and imports `nameparser` normally: the working tree, on whichever surfaces the baseline supports. ## Running it ``` uv run python tools/differential/build_corpus.py --ref > tools/differential/corpus.jsonl # only when regenerating uv run python tools/differential/compare.py --baseline 1.4.0 uv run python tools/differential/compare.py --baseline 2.0.0 ``` `compare.py` spawns the worker as a subprocess, feeds it every corpus name as a line of JSON, and diffs the two sides field by field. Every diff is checked against that baseline's ledger: - Matches a rule -> counted as an intentional, classified change. - Matches no rule -> printed under `UNEXPLAINED` and the run exits 1. An unexplained diff means either a real parity bug (fix it, don't allowlist it) or a known change whose ledger rule needs widening. The run must exit 0 at every baseline you claim before a release; the classified summary it prints is the source for the "Behavior Changes" section of `docs/release_log.rst`. ## Baselines `--baseline VERSION` chooses what the tree is compared against, and two things follow from it: which ledger is read (`expected_since_.toml`, a hard error if absent) and which surfaces are compared (the facade alone below 2.0, which has no v2 API; both from 2.0 on, ambiguity kinds included). Run both before cutting a release: - `--baseline 1.4.0` — the v1 compat contract. - `--baseline ` — what changes for a user upgrading. The worker is generated per run, with the pin substituted, into a temp directory outside the worktree. Its first output line is a version tell, and `compare.py` aborts before comparing anything if the wrong version answered or if the module resolved inside the checkout. **Both** sides are proved, not just the baseline. `compare.py` also checks that its own `nameparser` is this checkout's source package and prints it on a `tree:` line — see the third trap below for why a bare import was not enough. A rule's `fields` names roles the way `Role` does, whichever surface the diff came from, plus the pseudo-field `_ambiguities` for a change in reported `AmbiguityKind`s. The roster is not restated here: it is `Role`'s members, `validate_rules` rejects anything outside them, and a copy in prose is a copy that goes stale when a role is added. The facade reports `first`/`last`; those are canonicalized on the way in, and the `UNEXPLAINED` block prints the canonical name so what you read is what you write. ## The three invocation traps None of the three below is hypothetical -- the second was reproduced twice while working on #320, and the third was demonstrated during review of this harness's own generalization. The first two are recorded here because the generated worker's cwd and script path are what disarm them: a later change that runs the worker from a cwd inside the project reopens both. The analysis is the reason for the design, so it outlives the bug. **Without `--no-project`,** `uv` installs the working tree as an editable dependency and the version pin never takes effect, silently comparing the tree against itself. `compare.py` passes `--no-project`, and runs the worker from a temp cwd where there is no project to discover in the first place. **With `python` in front of the script path** -- `uv run --no-project python