pax_global_header 0000666 0000000 0000000 00000000064 15160554312 0014514 g ustar 00root root 0000000 0000000 52 comment=a98f9d6f8efd875fbd158965793334f5b25f9eeb
explosion-confection-a98f9d6/ 0000775 0000000 0000000 00000000000 15160554312 0016315 5 ustar 00root root 0000000 0000000 explosion-confection-a98f9d6/.github/ 0000775 0000000 0000000 00000000000 15160554312 0017655 5 ustar 00root root 0000000 0000000 explosion-confection-a98f9d6/.github/workflows/ 0000775 0000000 0000000 00000000000 15160554312 0021712 5 ustar 00root root 0000000 0000000 explosion-confection-a98f9d6/.github/workflows/cibuildwheel.yml 0000664 0000000 0000000 00000001405 15160554312 0025075 0 ustar 00root root 0000000 0000000 name: Build
on:
push:
tags:
# ytf did they invent their own syntax that's almost regex?
# ** matches 'zero or more of any character'
- 'release-v[0-9]+.[0-9]+.[0-9]+**'
- 'prerelease-v[0-9]+.[0-9]+.[0-9]+**'
workflow_dispatch: # allows you to trigger manually
permissions: {}
jobs:
build_wheels:
uses: explosion/gha-cibuildwheel/.github/workflows/cibuildwheel.yml@2c98f757f13d112cf73fcf4b627249f1fffb5aae # main
permissions:
contents: write
actions: read
with:
wheel-name-pattern: "confection-*.whl"
pure-python: true
create-release: ${{ startsWith(github.ref, 'refs/tags/release-') || startsWith(github.ref, 'refs/tags/prerelease-') }}
secrets:
gh-token: ${{ secrets.GITHUB_TOKEN }}
explosion-confection-a98f9d6/.github/workflows/tests.yml 0000664 0000000 0000000 00000003767 15160554312 0023614 0 ustar 00root root 0000000 0000000 name: tests
on:
push:
branches: [main]
pull_request:
branches: ["*"]
workflow_dispatch:
permissions: {}
concurrency:
group: tests-${{ github.ref }}
cancel-in-progress: true
jobs:
lint:
name: Lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
with:
python-version: "3.13"
- run: pip install ruff pyright
- run: ruff check confection
- run: ruff format confection --check
- run: pyright confection
tests:
name: Test
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
python_version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
runs-on: ${{ matrix.os }}
steps:
- name: Check out repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
- name: Configure Python version
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
with:
python-version: ${{ matrix.python_version }}
- name: Build sdist
run: |
python -m pip install -U build pip setuptools
python -m pip install -U -r requirements.txt
python -m build --sdist
- name: Delete source directory
run: rm -rf confection
shell: bash
- name: Uninstall all packages
run: |
python -m pip freeze > installed.txt
python -m pip uninstall -y -r installed.txt
- name: Install from sdist
run: |
SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1)
python -m pip install dist/$SDIST
shell: bash
- name: Test import
run: python -c "import confection" -Werror
- name: Install test requirements
run: |
python -m pip install -U -r requirements.txt
- name: Run tests
run: |
python -m pytest tests/
explosion-confection-a98f9d6/.gitignore 0000664 0000000 0000000 00000001546 15160554312 0020313 0 ustar 00root root 0000000 0000000 tmp/
.pytest_cache
.vscode
.mypy_cache
.prettierrc
.python-version
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
.env/
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/
# Translations
*.mo
*.pot
# Django stuff:
*.log
# Sphinx documentation
docs/_build/
# PyBuilder
target/
#Ipython Notebook
.ipynb_checkpoints
# Pycharm project files
*.idea
explosion-confection-a98f9d6/LICENSE 0000664 0000000 0000000 00000002061 15160554312 0017321 0 ustar 00root root 0000000 0000000 MIT License
Copyright (c) 2019 ExplosionAI GmbH
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
explosion-confection-a98f9d6/MANIFEST.in 0000664 0000000 0000000 00000000054 15160554312 0020052 0 ustar 00root root 0000000 0000000 include LICENSE
include confection/py.typed
explosion-confection-a98f9d6/README.md 0000664 0000000 0000000 00000043573 15160554312 0017610 0 ustar 00root root 0000000 0000000 
# Confection: The sweetest config system for Python
`confection` :candy: is a lightweight library that offers a **configuration
system** letting you conveniently describe arbitrary trees of objects.
Configuration is a huge challenge for machine-learning code because you may want
to expose almost any detail of any function as a hyperparameter. The setting you
want to expose might be arbitrarily far down in your call stack, so it might
need to pass all the way through the CLI or REST API, through any number of
intermediate functions, affecting the interface of everything along the way. And
then once those settings are added, they become hard to remove later. Default
values also become hard to change without breaking backwards compatibility.
To solve this problem, `confection` offers a config system that lets you easily
describe arbitrary trees of objects. The objects can be created via function
calls you register using a simple decorator syntax. You can even version the
functions you create, allowing you to make improvements without breaking
backwards compatibility. The most similar config system we’re aware of is
[Gin](https://github.com/google/gin-config), which uses a similar syntax, and
also allows you to link the configuration system to functions in your code using
a decorator. `confection`'s config system is simpler and emphasizes a different
workflow via a subset of Gin’s functionality.
[](https://github.com/explosion/confection/actions/workflows/tests.yml)
[](https://github.com/explosion/confection/releases)
[](https://pypi.org/project/confection/)
[](https://anaconda.org/conda-forge/confection)
[](https://github.com/ambv/black)
## ⏳ Installation
```bash
pip install confection
```
```bash
conda install -c conda-forge confection
```
## 👩💻 Usage
The configuration system parses a `.cfg` file like
```ini
[training]
patience = 10
dropout = 0.2
use_vectors = false
[training.logging]
level = "INFO"
[nlp]
# This uses the value of training.use_vectors
use_vectors = ${training.use_vectors}
lang = "en"
```
and resolves it to a `Dict`:
```json
{
"training": {
"patience": 10,
"dropout": 0.2,
"use_vectors": false,
"logging": {
"level": "INFO"
}
},
"nlp": {
"use_vectors": false,
"lang": "en"
}
}
```
The config is divided into sections, with the section name in square brackets –
for example, `[training]`. Within the sections, config values can be assigned to
keys using `=`. Values can also be referenced from other sections using the dot
notation and placeholders indicated by the dollar sign and curly braces. For
example, `${training.use_vectors}` will receive the value of use_vectors in the
training block. This is useful for settings that are shared across components.
The config format has three main differences from Python’s built-in
`configparser`:
1. JSON-formatted values. `confection` passes all values through `json.loads` to
interpret them. You can use atomic values like strings, floats, integers or
booleans, or you can use complex objects such as lists or maps.
2. Structured sections. `confection` uses a dot notation to build nested
sections. If you have a section named `[section.subsection]`, `confection`
will parse that into a nested structure, placing subsection within section.
3. References to registry functions. If a key starts with `@`, `confection` will
interpret its value as the name of a function registry, load the function
registered for that name and pass in the rest of the block as arguments. If
type hints are available on the function, the argument values (and return
value of the function) will be validated against them. This lets you express
complex configurations, like a training pipeline where `batch_size` is
populated by a function that yields floats.
There’s no pre-defined scheme you have to follow; how you set up the top-level
sections is up to you. At the end of it, you’ll receive a dictionary with the
values that you can use in your script – whether it’s complete initialized
functions, or just basic settings.
For instance, let’s say you want to define a new optimizer. You'd define its
arguments in `config.cfg` like so:
```ini
[optimizer]
@optimizers = "my_cool_optimizer.v1"
learn_rate = 0.001
gamma = 1e-8
```
To load and parse this configuration using a `catalogue` registry (install
[`catalogue`](https://github.com/explosion/catalogue) separately):
```python
import dataclasses
from typing import Union, Iterable
import catalogue
from confection import registry, Config
# Create a new registry.
registry.optimizers = catalogue.create("confection", "optimizers", entry_points=False)
# Define a dummy optimizer class.
@dataclasses.dataclass
class MyCoolOptimizer:
learn_rate: float
gamma: float
@registry.optimizers.register("my_cool_optimizer.v1")
def make_my_optimizer(learn_rate: Union[float, Iterable[float]], gamma: float):
return MyCoolOptimizer(learn_rate, gamma)
# Load the config file from disk, resolve it and fetch the instantiated optimizer object.
config = Config().from_disk("./config.cfg")
resolved = registry.resolve(config)
optimizer = resolved["optimizer"] # MyCoolOptimizer(learn_rate=0.001, gamma=1e-08)
```
> ⚠️ Caution: Type-checkers such as `mypy` will mark adding new attributes to `registry` this way - i. e.
> `registry.new_attr = ...` - as errors. This is because a new attribute is added to the class after initialization. If
> you are using typecheckers, you can either ignore this (e. g. with `# type: ignore` for `mypy`) or use a typesafe
> alternative: instead of `registry.new_attr = ...`, use `setattr(registry, "new_attr", ...)`.
Under the hood, `confection` will look up the `"my_cool_optimizer.v1"` function
in the "optimizers" registry and then call it with the arguments `learn_rate`
and `gamma`. If the function has type annotations, it will also validate the
input. For instance, if `learn_rate` is annotated as a float and the config
defines a string, `confection` will raise an error.
The Thinc documentation offers further information on the configuration system:
- [recursive blocks](https://thinc.ai/docs/usage-config#registry-recursive)
- [defining variable positional arguments](https://thinc.ai/docs/usage-config#registries-args)
- [using interpolation](https://thinc.ai/docs/usage-config#config-interpolation)
- [using custom registries](https://thinc.ai/docs/usage-config#registries-custom)
- [advanced type annotations with Pydantic](https://thinc.ai/docs/usage-config#advanced-types)
- [using base schemas](https://thinc.ai/docs/usage-config#advanced-types-base-schema)
- [filling a configuration with defaults](https://thinc.ai/docs/usage-config#advanced-types-fill-defaults)
## 🎛 API
### class `Config`
This class holds the model and training
[configuration](https://thinc.ai/docs/usage-config) and can load and save the
INI-style configuration format from/to a string, file or bytes. The `Config`
class is a subclass of `dict` and uses Python’s `ConfigParser` under the hood.
#### method `Config.__init__`
Initialize a new `Config` object with optional data.
```python
from confection import Config
config = Config({"training": {"patience": 10, "dropout": 0.2}})
```
| Argument | Type | Description |
| ----------------- | ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `data` | `Optional[Union[Dict[str, Any], Config]]` | Optional data to initialize the config with. |
| `section_order` | `Optional[List[str]]` | Top-level section names, in order, used to sort the saved and loaded config. All other sections will be sorted alphabetically. |
| `is_interpolated` | `Optional[bool]` | Whether the config is interpolated or whether it contains variables. Read from the `data` if it’s an instance of `Config` and otherwise defaults to `True`. |
#### method `Config.from_str`
Load the config from a string.
```python
from confection import Config
config_str = """
[training]
patience = 10
dropout = 0.2
"""
config = Config().from_str(config_str)
print(config["training"]) # {'patience': 10, 'dropout': 0.2}}
```
| Argument | Type | Description |
| ------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------- |
| `text` | `str` | The string config to load. |
| `interpolate` | `bool` | Whether to interpolate variables like `${section.key}`. Defaults to `True`. |
| `overrides` | `Dict[str, Any]` | Overrides for values and sections. Keys are provided in dot notation, e.g. `"training.dropout"` mapped to the value. |
| **RETURNS** | `Config` | The loaded config. |
#### method `Config.to_str`
Load the config from a string.
```python
from confection import Config
config = Config({"training": {"patience": 10, "dropout": 0.2}})
print(config.to_str()) # '[training]\npatience = 10\n\ndropout = 0.2'
```
| Argument | Type | Description |
| ------------- | ------ | --------------------------------------------------------------------------- |
| `interpolate` | `bool` | Whether to interpolate variables like `${section.key}`. Defaults to `True`. |
| **RETURNS** | `str` | The string config. |
#### method `Config.to_bytes`
Serialize the config to a byte string.
```python
from confection import Config
config = Config({"training": {"patience": 10, "dropout": 0.2}})
config_bytes = config.to_bytes()
print(config_bytes) # b'[training]\npatience = 10\n\ndropout = 0.2'
```
| Argument | Type | Description |
| ------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------- |
| `interpolate` | `bool` | Whether to interpolate variables like `${section.key}`. Defaults to `True`. |
| `overrides` | `Dict[str, Any]` | Overrides for values and sections. Keys are provided in dot notation, e.g. `"training.dropout"` mapped to the value. |
| **RETURNS** | `str` | The serialized config. |
#### method `Config.from_bytes`
Load the config from a byte string.
```python
from confection import Config
config = Config({"training": {"patience": 10, "dropout": 0.2}})
config_bytes = config.to_bytes()
new_config = Config().from_bytes(config_bytes)
```
| Argument | Type | Description |
| ------------- | -------- | --------------------------------------------------------------------------- |
| `bytes_data` | `bool` | The data to load. |
| `interpolate` | `bool` | Whether to interpolate variables like `${section.key}`. Defaults to `True`. |
| **RETURNS** | `Config` | The loaded config. |
#### method `Config.to_disk`
Serialize the config to a file.
```python
from confection import Config
config = Config({"training": {"patience": 10, "dropout": 0.2}})
config.to_disk("./config.cfg")
```
| Argument | Type | Description |
| ------------- | ------------------ | --------------------------------------------------------------------------- |
| `path` | `Union[Path, str]` | The file path. |
| `interpolate` | `bool` | Whether to interpolate variables like `${section.key}`. Defaults to `True`. |
#### method `Config.from_disk`
Load the config from a file.
```python
from confection import Config
config = Config({"training": {"patience": 10, "dropout": 0.2}})
config.to_disk("./config.cfg")
new_config = Config().from_disk("./config.cfg")
```
| Argument | Type | Description |
| ------------- | ------------------ | -------------------------------------------------------------------------------------------------------------------- |
| `path` | `Union[Path, str]` | The file path. |
| `interpolate` | `bool` | Whether to interpolate variables like `${section.key}`. Defaults to `True`. |
| `overrides` | `Dict[str, Any]` | Overrides for values and sections. Keys are provided in dot notation, e.g. `"training.dropout"` mapped to the value. |
| **RETURNS** | `Config` | The loaded config. |
#### method `Config.copy`
Deep-copy the config.
| Argument | Type | Description |
| ----------- | -------- | ------------------ |
| **RETURNS** | `Config` | The copied config. |
#### method `Config.interpolate`
Interpolate variables like `${section.value}` or `${section.subsection}` and
return a copy of the config with interpolated values. Can be used if a config is
loaded with `interpolate=False`, e.g. via `Config.from_str`.
```python
from confection import Config
config_str = """
[hyper_params]
dropout = 0.2
[training]
dropout = ${hyper_params.dropout}
"""
config = Config().from_str(config_str, interpolate=False)
print(config["training"]) # {'dropout': '${hyper_params.dropout}'}}
config = config.interpolate()
print(config["training"]) # {'dropout': 0.2}}
```
| Argument | Type | Description |
| ----------- | -------- | ---------------------------------------------- |
| **RETURNS** | `Config` | A copy of the config with interpolated values. |
##### method `Config.merge`
Deep-merge two config objects, using the current config as the default. Only
merges sections and dictionaries and not other values like lists. Values that
are provided in the updates are overwritten in the base config, and any new
values or sections are added. If a config value is a variable like
`${section.key}` (e.g. if the config was loaded with `interpolate=False)`, **the
variable is preferred**, even if the updates provide a different value. This
ensures that variable references aren’t destroyed by a merge.
> :warning: Note that blocks that refer to registered functions using the `@`
> syntax are only merged if they are referring to the same functions. Otherwise,
> merging could easily produce invalid configs, since different functions can
> take different arguments. If a block refers to a different function, it’s
> overwritten.
```python
from confection import Config
base_config_str = """
[training]
patience = 10
dropout = 0.2
"""
update_config_str = """
[training]
dropout = 0.1
max_epochs = 2000
"""
base_config = Config().from_str(base_config_str)
update_config = Config().from_str(update_config_str)
merged = Config(base_config).merge(update_config)
print(merged["training"]) # {'patience': 10, 'dropout': 0.1, 'max_epochs': 2000}
```
| Argument | Type | Description |
| ----------- | ------------------------------- | --------------------------------------------------- |
| `overrides` | `Union[Dict[str, Any], Config]` | The updates to merge into the config. |
| **RETURNS** | `Config` | A new config instance containing the merged config. |
### Config Attributes
| Argument | Type | Description |
| ----------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `is_interpolated` | `bool` | Whether the config values have been interpolated. Defaults to `True` and is set to `False` if a config is loaded with `interpolate=False`, e.g. using `Config.from_str`. |
explosion-confection-a98f9d6/bin/ 0000775 0000000 0000000 00000000000 15160554312 0017065 5 ustar 00root root 0000000 0000000 explosion-confection-a98f9d6/bin/push-tags.sh 0000664 0000000 0000000 00000000537 15160554312 0021341 0 ustar 00root root 0000000 0000000 #!/usr/bin/env bash
set -e
# Insist repository is clean
git diff-index --quiet HEAD
git checkout $1
git pull origin $1
git push origin $1
version=$(grep "version = " setup.cfg)
version=${version/version = }
version=${version/\'/}
version=${version/\'/}
version=${version/\"/}
version=${version/\"/}
git tag "v$version"
git push origin "v$version" explosion-confection-a98f9d6/bin/release.sh 0000775 0000000 0000000 00000000663 15160554312 0021051 0 ustar 00root root 0000000 0000000 #!/usr/bin/env bash
set -e
# Insist repository is clean
git diff-index --quiet HEAD
version=$(grep "version = " setup.cfg)
version=${version/version = }
version=${version/\'/}
version=${version/\'/}
version=${version/\"/}
version=${version/\"/}
echo "Pushing release-v"$version
git tag -d release-v$version || true
sudo gitw push origin :release-v$version || true
git tag release-v$version
sudo gitw push origin release-v$version
explosion-confection-a98f9d6/confection/ 0000775 0000000 0000000 00000000000 15160554312 0020444 5 ustar 00root root 0000000 0000000 explosion-confection-a98f9d6/confection/__init__.py 0000664 0000000 0000000 00000001177 15160554312 0022563 0 ustar 00root root 0000000 0000000 # FIXME some symbols are not in __all__; can we remove them?
from ._config import Config # noqa: F401
from ._constants import ( # noqa: F401
ARGS_FIELD,
ARGS_FIELD_ALIAS,
RESERVED_FIELDS,
SECTION_PREFIX,
VARIABLE_RE,
)
from ._errors import ConfigValidationError
from ._registry import Promise, registry
from .util import ( # noqa: F401
SimpleFrozenDict,
SimpleFrozenList,
try_dump_json,
try_load_json,
)
from .validation import Schema # noqa: F401
__all__ = [
"Config",
"registry",
"ConfigValidationError",
"Promise",
"Schema",
"SimpleFrozenDict",
"SimpleFrozenList",
]
explosion-confection-a98f9d6/confection/_config.py 0000664 0000000 0000000 00000023440 15160554312 0022425 0 ustar 00root root 0000000 0000000 from __future__ import annotations
import copy
import sys
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
if sys.version_info >= (3, 11):
from typing import Self
else:
from typing_extensions import Self
from ._errors import ConfectionError, ConfigValidationError
from ._parser import parse_config, serialize_config
from .validation import ValidationError, ensure_schema
class Config(dict):
# TODO: Improve doc string
"""Dict subclass to save TOML-style configuration format from/to string, file
or bytes.
"""
is_interpolated: bool
def __init__(
self,
data: Optional[Union[Dict[str, Any], "Config"]] = None,
*,
is_interpolated: Optional[bool] = None,
section_order: Optional[List[str]] = None,
) -> None:
"""Initialize a new Config object with optional data."""
dict.__init__(self)
if data is None:
data = {}
if not isinstance(data, (dict, Config)):
raise ConfectionError(
f"Can't initialize Config with data. Expected dict or "
f"Config but got: {type(data)}"
)
# Whether the config has been interpolated. We can use this to check
# whether we need to interpolate again when it's resolved. We assume
# that a config is interpolated by default.
if is_interpolated is not None:
self.is_interpolated = is_interpolated
elif isinstance(data, Config):
self.is_interpolated = data.is_interpolated
else:
self.is_interpolated = True
if section_order is not None:
self.section_order = section_order
elif isinstance(data, Config):
self.section_order = data.section_order
else:
self.section_order = []
# Update with data
self.update(data)
def interpolate(self) -> Self:
"""Interpolate (resolve var references) a config.
Returns a copy of the object.
"""
# This is currently the most effective way because we need our custom
# to_str logic to run in order to re-serialize the values so we can
# interpolate them again. ConfigParser.read_dict will just call str()
# on all values, which isn't enough.
return type(self)().from_str(self.to_str())
def copy(self) -> Self:
"""Deepcopy the config."""
config = copy.deepcopy(self)
return type(self)(
config,
is_interpolated=self.is_interpolated,
section_order=self.section_order,
)
def merge(
self, updates: Union[Dict[str, Any], "Config"], remove_extra: bool = False
) -> Self:
"""Deep merge the config with updates, using current as defaults."""
defaults = self.copy()
updates = Config(updates).copy()
merged = deep_merge_configs(updates, defaults, remove_extra=remove_extra)
return type(self)(
merged,
is_interpolated=defaults.is_interpolated and updates.is_interpolated,
section_order=defaults.section_order,
)
def validate(self, schema) -> Self:
"""Validate the config against a schema. Raises ConfigValidationError
if validation fails.
"""
schema = ensure_schema(schema)
_validate_recursive(dict(self), schema, self)
return self
def fill_defaults(self, schema) -> Self:
"""Fill in missing values from schema defaults and remove extra
fields if the schema forbids them. Modifies in place and returns self.
"""
schema = ensure_schema(schema)
extra = schema.model_config.get("extra", "allow")
# Fill defaults
for name, field in schema.model_fields.items():
if name not in self and not field.is_required():
self[name] = field.default
elif name in self and isinstance(self[name], dict):
field_schema = field.annotation
if isinstance(field_schema, type) and hasattr(
field_schema, "model_fields"
):
sub_schema = ensure_schema(field_schema)
_fill_defaults_recursive(self[name], sub_schema)
# Strip extras
if extra == "forbid":
known = set(schema.model_fields.keys())
for key in list(self.keys()):
if key not in known:
del self[key]
return self
def from_str(
self,
text: str,
*,
interpolate: bool = True,
overrides: Dict[str, Any] = {},
schema=None,
) -> Self:
"""Load the config from a string."""
self.clear()
self.update(parse_config(text, interpolate=interpolate, overrides=overrides))
if overrides and interpolate:
# Re-interpolate now that overrides are applied. The recursive
# from_str call will have no overrides, so this doesn't loop.
self = self.interpolate()
self.is_interpolated = interpolate
if schema is not None:
self.fill_defaults(schema)
self.validate(schema)
return self
def to_str(self, *, interpolate: bool = True) -> str:
"""Write the config to a string."""
return serialize_config(self, interpolate=interpolate)
def to_bytes(self, *, interpolate: bool = True) -> bytes:
"""Serialize the config to a byte string."""
return self.to_str(interpolate=interpolate).encode("utf8")
def from_bytes(
self,
bytes_data: bytes,
*,
interpolate: bool = True,
overrides: Dict[str, Any] = {},
) -> Self:
"""Load the config from a byte string."""
return self.from_str(
bytes_data.decode("utf8"), interpolate=interpolate, overrides=overrides
)
def to_disk(self, path: Union[str, Path], *, interpolate: bool = True) -> None:
"""Serialize the config to a file."""
path = Path(path) if isinstance(path, str) else path
with path.open("w", encoding="utf8") as file_:
file_.write(self.to_str(interpolate=interpolate))
def from_disk(
self,
path: Union[str, Path],
*,
interpolate: bool = True,
overrides: Dict[str, Any] = {},
) -> Self:
"""Load config from a file."""
path = Path(path) if isinstance(path, str) else path
with path.open("r", encoding="utf8") as file_:
text = file_.read()
return self.from_str(text, interpolate=interpolate, overrides=overrides)
def _fill_defaults_recursive(data, schema):
"""Fill defaults and strip extras recursively for nested schemas."""
extra = schema.model_config.get("extra", "allow")
for name, field in schema.model_fields.items():
if name not in data and not field.is_required():
data[name] = field.default
elif name in data and isinstance(data[name], dict):
field_schema = field.annotation
if isinstance(field_schema, type) and hasattr(field_schema, "model_fields"):
_fill_defaults_recursive(data[name], ensure_schema(field_schema))
if extra == "forbid":
known = set(schema.model_fields.keys())
for key in list(data.keys()):
if key not in known:
del data[key]
def _validate_recursive(data, schema, config, parent=""):
"""Validate data against a schema, recursing into nested schemas."""
try:
schema.model_validate(data)
except ValidationError as e:
section = f" in [{parent}]" if parent else ""
raise ConfigValidationError(
config=config,
errors=e.errors(),
title=f"Config validation error{section}",
) from None
# Recurse into fields that are themselves schemas
for name, field in schema.model_fields.items():
annotation = field.annotation
if (
isinstance(annotation, type)
and hasattr(annotation, "model_validate")
and name in data
and isinstance(data[name], dict)
):
child_parent = f"{parent}.{name}" if parent else name
_validate_recursive(data[name], annotation, config, parent=child_parent)
def deep_merge_configs(
config: Union[Dict[str, Any], Config],
defaults: Union[Dict[str, Any], Config],
*,
remove_extra: bool = False,
) -> Union[Dict[str, Any], Config]:
"""Deep merge two configs."""
if remove_extra:
# Filter out values in the original config that are not in defaults
keys = list(config.keys())
for key in keys:
if key not in defaults:
del config[key]
for key, value in defaults.items():
if isinstance(value, dict):
node = config.setdefault(key, {})
if not isinstance(node, dict):
continue
value_promises = [k for k in value if k.startswith("@")]
value_promise = value_promises[0] if value_promises else None
node_promises = [k for k in node if k.startswith("@")] if node else []
node_promise = node_promises[0] if node_promises else None
# We only update the block from defaults if it refers to the same
# registered function
if (
value_promise
and node_promise
and (
value_promise in node
and node[value_promise] != value[value_promise]
)
):
continue
if node_promise and (
node_promise not in value or node[node_promise] != value[node_promise]
):
continue
defaults = deep_merge_configs(node, value, remove_extra=remove_extra)
elif key not in config:
config[key] = value
return config
explosion-confection-a98f9d6/confection/_constants.py 0000664 0000000 0000000 00000001516 15160554312 0023174 0 ustar 00root root 0000000 0000000 import re
# Regex to detect whether a value contains a variable
VARIABLE_RE = re.compile(r"\$\{[\w\.:]+\}")
# Internal prefix used to mark section references for custom interpolation
SECTION_PREFIX = "__SECTION__:"
# Field used for positional arguments, e.g. [section.*.xyz]. The alias is
# required for the schema (shouldn't clash with user-defined arg names)
ARGS_FIELD = "*"
ARGS_FIELD_ALIAS = "VARIABLE_POSITIONAL_ARGS"
# Aliases for fields that would otherwise shadow pydantic attributes. Can be any
# string, so we're using name + space so it looks the same in error messages etc.
RESERVED_FIELDS = {
"validate": "validate\u0020",
"model_config": "model_config\u0020",
"model_validate": "model_validate\u2020",
"model_fields": "model_fields\u2020",
}
RESERVED_FIELDS_REVERSE = {v: k for k, v in RESERVED_FIELDS.items()}
explosion-confection-a98f9d6/confection/_errors.py 0000664 0000000 0000000 00000010641 15160554312 0022473 0 ustar 00root root 0000000 0000000 from typing import Optional
class ConfectionError(ValueError): ...
class ConfigValidationError(ConfectionError):
def __init__(
self,
*,
config=None,
errors=None,
title: Optional[str] = "Config validation error",
desc: Optional[str] = None,
parent: Optional[str] = None,
show_config: bool = True,
) -> None:
"""Custom error for validating configs.
config (Union[Config, Dict[str, Dict[str, Any]], str]): The
config the validation error refers to.
errors (Union[Sequence[Mapping[str, Any]], Iterable[Dict[str, Any]]]):
A list of errors as dicts with keys "loc" (list of strings
describing the path of the value), "msg" (validation message
to show) and optional "type" (mostly internals).
Same format as produced by pydantic's validation error (e.errors()).
title (str): The error title.
desc (str): Optional error description, displayed below the title.
parent (str): Optional parent to use as prefix for all error locations.
For example, parent "element" will result in "element -> a -> b".
show_config (bool): Whether to print the whole config with the error.
ATTRIBUTES:
config (Union[Config, Dict[str, Dict[str, Any]], str]): The config.
errors (Iterable[Dict[str, Any]]): The errors.
error_types (Set[str]): All "type" values defined in the errors, if
available. This is most relevant for the pydantic errors that define
types like "type_error.integer". This attribute makes it easy to
check if a config validation error includes errors of a certain
type, e.g. to log additional information or custom help messages.
title (str): The title.
desc (str): The description.
parent (str): The parent.
show_config (bool): Whether to show the config.
text (str): The formatted error text.
"""
self.config = config
self.errors = errors
self.title = title
self.desc = desc
self.parent = parent
self.show_config = show_config
self.error_types = set()
if self.errors:
for error in self.errors:
err_type = error.get("type")
if err_type:
self.error_types.add(err_type)
self.text = self._format()
ValueError.__init__(self, self.text)
@classmethod
def from_error(
cls,
err: "ConfigValidationError",
title: Optional[str] = None,
desc: Optional[str] = None,
parent: Optional[str] = None,
show_config: Optional[bool] = None,
) -> "ConfigValidationError":
"""Create a new ConfigValidationError based on an existing error, e.g.
to re-raise it with different settings. If no overrides are provided,
the values from the original error are used.
err (ConfigValidationError): The original error.
title (str): Overwrite error title.
desc (str): Overwrite error description.
parent (str): Overwrite error parent.
show_config (bool): Overwrite whether to show config.
RETURNS (ConfigValidationError): The new error.
"""
return cls(
config=err.config,
errors=err.errors,
title=title if title is not None else err.title,
desc=desc if desc is not None else err.desc,
parent=parent if parent is not None else err.parent,
show_config=show_config if show_config is not None else err.show_config,
)
def _format(self) -> str:
"""Format the error message."""
loc_divider = "->"
data = []
if self.errors:
for error in self.errors:
err_loc = f" {loc_divider} ".join(
[str(p) for p in error.get("loc", [])]
)
if self.parent:
err_loc = f"{self.parent} {loc_divider} {err_loc}"
data.append((err_loc, error.get("msg")))
result = []
if self.title:
result.append(self.title)
if self.desc:
result.append(self.desc)
if data:
result.append("\n".join([f"{entry[0]}\t{entry[1]}" for entry in data]))
if self.config and self.show_config:
result.append(f"{self.config}")
return "\n\n" + "\n".join(result)
explosion-confection-a98f9d6/confection/_parser.py 0000664 0000000 0000000 00000042263 15160554312 0022460 0 ustar 00root root 0000000 0000000 import io
import json
import warnings
from configparser import (
MAX_INTERPOLATION_DEPTH,
ConfigParser,
ExtendedInterpolation,
InterpolationDepthError,
InterpolationMissingOptionError,
InterpolationSyntaxError,
NoOptionError,
NoSectionError,
ParsingError,
)
from typing import Any, Dict, List
from ._constants import SECTION_PREFIX
from ._errors import ConfigValidationError
from .util import VARIABLE_RE, try_dump_json, try_load_json
def parse_config(
text: str,
*,
interpolate: bool = True,
overrides: Dict[str, Any] = {},
) -> dict[str, Any]:
"""Parse a config string into a nested dict.
Handles the full pipeline: parse with ConfigParser, validate structure,
apply overrides, interpret values, and resolve section references.
Returns the nested dict and whether a second interpolation pass is needed
(when overrides were applied with interpolation enabled).
"""
config_parser = _get_configparser(interpolate=interpolate and not overrides)
try:
config_parser.read_string(text)
except ParsingError as e:
desc = f"Make sure the sections and values are formatted correctly.\n\n{e}"
raise ConfigValidationError(desc=desc) from None
errors = _validate_configparser(config_parser)
if errors:
raise errors[0]
errors = _validate_overrides(config_parser, overrides)
if errors:
raise errors[0]
result: dict[str, Any] = {}
section_parts = [section.split(".") for section in config_parser.sections()]
# Phase 1: Build the skeleton of nested dicts from section names.
for parts in section_parts:
node = result
for part in parts[:-1]:
node = node.setdefault(part, {}) if part == "*" else node[part]
node.setdefault(parts[-1], {})
# Phase 2: Fill in values, processing breadth-first by section depth.
for section, values in sorted(
config_parser.items(), key=lambda x: len(x[0].split("."))
):
if section == "DEFAULT":
continue
parts = section.split(".")
node = result
for part in parts:
node = node[part]
for key in values:
node[key] = _interpret_value(config_parser.get(section, key))
# Phase 3: Apply overrides on the nested dict.
_apply_overrides(result, overrides)
# Phase 4: Replace section reference placeholders with actual dicts.
_replace_section_refs(result, result)
return result
def _apply_overrides(result: dict[str, Any], overrides: Dict[str, Any]) -> None:
"""Apply dot-notation overrides to a nested dict.
Override paths have already been validated by _validate_overrides.
"""
for key, value in overrides.items():
path = key.split(".")
node = result
for part in path[:-1]:
node = node[part]
node[path[-1]] = value
def serialize_config(
data: dict[str, Any],
*,
interpolate: bool = True,
inline_paths: frozenset[str] = frozenset(),
) -> str:
"""Serialize a nested config dict to a config string.
inline_paths: dotted paths whose values should be serialized as inline
JSON rather than expanded into subsections. For example, if "a.b" is
in inline_paths, data["a"]["b"] (even if it's a dict) will be
serialized as ``b = {"key": "value"}`` under [a].
"""
flattened = _get_configparser(interpolate=interpolate)
queue: list[tuple[tuple, dict[str, Any]]] = [(tuple(), data)]
for path, node in queue:
section_name = ".".join(path)
is_kwarg = path and path[-1] != "*"
has_leaves = any(not hasattr(v, "items") for v in node.values())
if path and has_leaves and not flattened.has_section(section_name):
# Create sections that have leaf values (including * sections).
flattened.add_section(section_name)
elif is_kwarg and not flattened.has_section(section_name):
flattened.add_section(section_name)
for key, value in node.items():
child_path = f"{section_name}.{key}" if section_name else key
if hasattr(value, "items") and child_path not in inline_paths:
queue.append((path + (key,), value))
else:
flattened.set(section_name, key, try_dump_json(value, node))
string_io = io.StringIO()
flattened.write(string_io)
return string_io.getvalue().strip()
def _get_configparser(interpolate: bool = True) -> ConfigParser:
config = ConfigParser(interpolation=_CustomInterpolation() if interpolate else None)
# Preserve case of keys: https://stackoverflow.com/a/1611877/6400719
config.optionxform = str # type: ignore
return config
def _validate_configparser(config_parser: ConfigParser) -> list[ConfigValidationError]:
"""Validate a configparser's structure before interpreting it into a Config.
Checks that:
- No values leak into the DEFAULT section (top-level values without a section)
- All parent sections exist for dotted section names (e.g. "a.b" requires "a")
- No key in a section conflicts with a child section name
- No interpolation errors in values
"""
errors = []
default_section = config_parser.defaults()
if default_section: # pragma: no cover -- configparser raises ParsingError first
err_title = "Found config values without a top-level section"
err_msg = "not part of a section"
err = [{"loc": [k], "msg": err_msg} for k in default_section]
errors.append(ConfigValidationError(errors=err, title=err_title))
section_names = set(config_parser.sections())
for section in config_parser.sections():
path = section.split(".")
for i in range(1, len(path)):
# "*" is an implicit list section — it doesn't need a parent
# section header, and paths through it are always valid.
if path[i - 1] == "*":
continue
parent = ".".join(path[:i])
# A parent is valid if it's a declared section OR if the path
# goes through a "*" component (which is implicitly created).
if parent not in section_names and "*" not in parent.split("."):
err_title = (
"Error parsing config section. Perhaps a section name is wrong?"
)
err = [{"loc": path, "msg": f"Section '{path[i - 1]}' is not defined"}]
errors.append(ConfigValidationError(errors=err, title=err_title))
break
keys = set(config_parser.options(section))
for other in section_names:
if other.startswith(section + "."):
child = other[len(section) + 1 :].split(".")[0]
if child in keys:
err = [{"loc": other.split("."), "msg": "found conflicting values"}]
errors.append(ConfigValidationError(errors=err))
return errors
def _validate_overrides(
config_parser: ConfigParser, overrides: dict[str, Any]
) -> list[ConfigValidationError]:
errors = []
err_title = "Error parsing config overrides"
for key in overrides:
err_msg = "not a section value that can be overridden"
err = [{"loc": key.split("."), "msg": err_msg}]
if "." not in key:
errors.append(ConfigValidationError(errors=err, title=err_title))
continue
section, _ = key.rsplit(".", 1)
# Check for section and accept if option not in config[section]
if section not in config_parser:
errors.append(ConfigValidationError(errors=err, title=err_title))
# TODO: Are we supposed to chek for the *option*?
return errors
def _interpret_value(value: Any) -> Any:
"""Interpret a single config value."""
result = try_load_json(value)
# If value is a string and it contains a variable, use original value
# (not interpreted string, which could lead to double quotes:
# ${x.y} -> "${x.y}" -> "'${x.y}'"). Make sure to check it's a string,
# so we're not keeping lists as strings.
# NOTE: This currently can't handle uninterpolated values like [${x.y}]!
if isinstance(result, str) and VARIABLE_RE.search(value):
result = value
return result
def _replace_section_refs(
root: dict[str, Any], node: dict[str, Any], parent: str = ""
) -> None:
"""Replace section reference placeholders with actual dicts."""
for key, value in node.items():
key_parent = f"{parent}.{key}".strip(".")
if isinstance(value, dict):
_replace_section_refs(root, value, parent=key_parent)
elif isinstance(value, list):
node[key] = [_get_section_ref(root, v, parent=[parent, key]) for v in value]
else:
node[key] = _get_section_ref(root, value, parent=[parent, key])
def _get_section_ref(
root: Dict[str, Any], value: Any, *, parent: List[str] = []
) -> Any:
"""Resolve a single section reference placeholder, or return value as-is."""
if isinstance(value, str) and value.startswith(
f'"{SECTION_PREFIX}'
): # pragma: no cover
value = try_load_json(value) # pragma: no cover
if (
isinstance(value, str)
and value.startswith(SECTION_PREFIX)
and value != SECTION_PREFIX
):
parts = value.replace(SECTION_PREFIX, "", 1).split(".")
result = root
for item in parts:
result = result[item]
return result
elif isinstance(value, str) and SECTION_PREFIX in value and value != SECTION_PREFIX:
err_desc = (
"Can't reference whole sections or return values of function "
"blocks inside a string or list\n\nYou can change your variable to "
"reference a value instead. Keep in mind that it's not "
"possible to interpolate the return value of a registered "
"function, since variables are interpolated when the config "
"is loaded, and registered functions are resolved afterwards."
)
err = [{"loc": parent, "msg": "uses section variable in string or list"}]
raise ConfigValidationError(errors=err, desc=err_desc)
return value
class _CustomInterpolation(ExtendedInterpolation):
def before_read( # pyright: ignore[reportIncompatibleMethodOverride]
self, parser: ConfigParser, section: str, option: str, value: str
) -> str:
# Warn about single-quoted strings (common mistake)
if value and value[0] == value[-1] == "'":
warnings.warn(
f"The value [{value}] seems to be single-quoted, but values "
"use JSON formatting, which requires double quotes."
)
return super().before_read(parser, section, option, value)
def _coerce_for_string_context(self, v: str) -> str:
"""Coerce a raw config value for use in a compound string expression."""
# Don't coerce section references - they need to stay quoted for JSON
if SECTION_PREFIX in v:
return v
try:
parsed = json.loads(v)
except json.JSONDecodeError:
return v # Not valid JSON, already a plain string
if isinstance(parsed, str):
return parsed # Unwrap JSON string
# Use json.dumps() for non-strings, escaping inner quotes so they don't
# conflict with the outer JSON string quotes
return json.dumps(parsed).replace('"', '\\"')
def before_get( # pyright: ignore[reportIncompatibleMethodOverride]
self,
parser: ConfigParser,
section: str,
option: str,
value: str,
defaults: Dict[str, str],
) -> str:
# Mostly copy-pasted from the built-in configparser implementation.
# The interpolate() method resolves ${...} references and appends pieces
# to L. For a bare reference like ${x}, L has one element. For compound
# expressions like "hello ${x}", L has multiple pieces that we join.
# Compound results stay as strings (coerced via _coerce_for_string_context),
# while bare references keep their JSON type for _interpret_value to parse.
L: List[str] = []
self.interpolate(parser, option, L, value, section, defaults, 1)
if len(L) == 1:
return L[0]
return "".join(self._coerce_for_string_context(piece) for piece in L)
def interpolate(
self,
parser: ConfigParser,
option: str,
accum: List[str],
rest: str,
section: str,
map: Dict[str, str],
depth: int,
) -> None:
"""Resolve variable references like ${foo.bar}"""
# Mostly copy-pasted from the built-in configparser implementation.
# We need to overwrite this method so we can add special handling for
# block references :( All values produced here should be strings –
# we need to wait until the whole config is interpreted anyways so
# filling in incomplete values here is pointless. All we need is the
# section reference so we can fetch it later.
rawval = parser.get(section, option, raw=True, fallback=rest)
if depth > MAX_INTERPOLATION_DEPTH:
raise InterpolationDepthError(option, section, rawval)
while rest:
p = rest.find("$")
if p < 0:
accum.append(rest)
return
if p > 0:
accum.append(rest[:p])
rest = rest[p:]
# p is no longer used
c = rest[1:2]
if c == "$":
accum.append("$")
rest = rest[2:]
elif c == "{":
# We want to treat both ${a:b} and ${a.b} the same
m = self._KEYCRE.match(rest) # type: ignore[attr-defined]
if m is None:
err = f"bad interpolation variable reference {rest}"
raise InterpolationSyntaxError(option, section, err)
orig_var = m.group(1)
path = orig_var.replace(":", ".").rsplit(".", 1)
rest = rest[m.end() :]
sect = section
opt = option
try:
if len(path) == 1:
opt = parser.optionxform(path[0])
# Check if the variable references a section rather
# than a key in the current section. If the key
# exists in the current map but its raw value is the
# same interpolation variable (self-reference), or if
# the key doesn't exist in the map, treat it as a
# section reference.
is_section_ref = opt not in map
if not is_section_ref:
raw = map[opt]
is_section_ref = raw.strip() == rawval.strip()
if not is_section_ref:
v = map[opt]
else:
# We have block reference, store it as a special key
section_name = parser[parser.optionxform(path[0])].name
v = self._get_section_name(section_name)
elif len(path) == 2:
sect = path[0]
opt = parser.optionxform(path[1])
fallback = "__FALLBACK__"
v = parser.get(sect, opt, raw=True, fallback=fallback)
# If a variable doesn't exist, try again and treat the
# reference as a section
if v == fallback:
v = self._get_section_name(parser[f"{sect}.{opt}"].name)
else: # pragma: no cover
# Dead code: rsplit(".", 1) produces at most 2 elements
err = f"More than one ':' found: {rest}"
raise InterpolationSyntaxError(option, section, err)
except (KeyError, NoSectionError, NoOptionError):
raise InterpolationMissingOptionError(
option, section, rawval, orig_var
) from None
if "$" in v:
new_map = dict(parser.items(sect, raw=True))
self.interpolate(parser, opt, accum, v, sect, new_map, depth + 1)
else:
accum.append(v)
else:
err = "'$' must be followed by '$' or '{', found: %r" % (rest,)
raise InterpolationSyntaxError(option, section, err)
def _get_section_name(self, name: str) -> str:
"""Generate the name of a section. Note that we use a quoted string here
so we can use section references within lists and load the list as
JSON. Since section references can't be used within strings, we don't
need the quoted vs. unquoted distinction like we do for variables.
Examples (assuming section = {"foo": 1}):
- value: ${section.foo} -> value: 1
- value: "hello ${section.foo}" -> value: "hello 1"
- value: ${section} -> value: {"foo": 1}
- value: "${section}" -> value: {"foo": 1}
- value: "hello ${section}" -> invalid
"""
return f'"{SECTION_PREFIX}{name}"'
__all__ = ["parse_config", "serialize_config"]
explosion-confection-a98f9d6/confection/_registry.py 0000664 0000000 0000000 00000037374 15160554312 0023043 0 ustar 00root root 0000000 0000000 import inspect
from dataclasses import dataclass
from typing import (
Any,
Callable,
Dict,
Generic,
List,
Optional,
Tuple,
TypeVar,
Union,
)
from ._config import Config
from ._constants import (
ARGS_FIELD,
ARGS_FIELD_ALIAS,
RESERVED_FIELDS_REVERSE,
)
from ._errors import ConfigValidationError
from .util import is_promise
from .validation import Schema
_PromisedType = TypeVar("_PromisedType")
@dataclass
class Promise(Generic[_PromisedType]):
registry: str
name: str
var_args: List[Any]
kwargs: Dict[str, Any]
getter: Union[Callable[..., _PromisedType], Exception]
@property
def return_type(self) -> _PromisedType:
if isinstance(self.getter, Exception): # pragma: no cover
raise self.getter # pragma: no cover
signature = inspect.signature(self.getter)
return signature.return_annotation
def resolve(self) -> Any:
if isinstance(self.getter, Exception): # pragma: no cover
raise self.getter # pragma: no cover
kwargs = _recursive_resolve(self.kwargs)
args = _recursive_resolve(self.var_args)
args = list(args.values()) if isinstance(args, dict) else args
kwargs = _coerce_basemodel_args(self.getter, kwargs)
return self.getter(*args, **kwargs) # type: ignore
@classmethod
def from_dict(cls, registry, values) -> "Promise":
reg_name, func_name = registry.get_constructor(values)
var_args, kwargs = registry.parse_args(values)
try:
getter = registry.get(reg_name, func_name)
except Exception as e: # pragma: no cover
getter = e # pragma: no cover
output = cls(
registry=reg_name,
name=func_name,
var_args=var_args,
kwargs=kwargs,
getter=getter,
)
return output
def _coerce_basemodel_args(func, kwargs):
"""Coerce dict kwargs to BaseModel instances where the function signature
expects a BaseModel subclass. This lets registered functions receive
constructed model instances instead of raw dicts (issue #58).
"""
try:
from typing import get_type_hints
hints = get_type_hints(func)
except Exception:
return kwargs
result = dict(kwargs)
for name, value in result.items():
if not isinstance(value, dict):
continue
hint = hints.get(name)
if hint is None or not isinstance(hint, type):
continue
# Check for pydantic BaseModel (v1 or v2)
if hasattr(hint, "model_validate"):
result[name] = hint.model_validate(value)
elif hasattr(hint, "parse_obj"):
result[name] = hint.parse_obj(value)
return result
def _recursive_resolve(obj):
if isinstance(obj, list):
return [_recursive_resolve(v) for v in obj]
elif isinstance(obj, dict):
return {k: _recursive_resolve(v) for k, v in obj.items()}
elif isinstance(obj, Promise):
return obj.resolve()
else:
return obj
class registry:
@classmethod
def has(cls, registry_name: str, func_name: str) -> bool:
"""Check whether a function is available in a registry."""
if not hasattr(cls, registry_name):
return False
reg = getattr(cls, registry_name)
return func_name in reg
@classmethod
def get(cls, registry_name: str, func_name: str) -> Callable:
"""Get a registered function from a given registry."""
if not hasattr(cls, registry_name):
raise ValueError(f"Unknown registry: '{registry_name}'")
reg = getattr(cls, registry_name)
func = reg.get(func_name)
if func is None: # pragma: no cover -- catalogue raises RegistryError first
raise ValueError(f"Could not find '{func_name}' in '{registry_name}'")
return func
@classmethod
def resolve(
cls,
config: Union[Config, Dict[str, Dict[str, Any]]],
*,
schema=None,
overrides: Dict[str, Any] = {},
validate: bool = True,
) -> Dict[str, Any]:
config = cls.fill(
config,
schema=schema,
overrides=overrides,
interpolate=True,
)
promised = insert_promises(cls, config, resolve=True)
resolved = resolve_promises(promised)
fixed = fix_positionals(resolved)
assert isinstance(fixed, dict)
if schema is not None and validate:
Config(fixed).validate(schema)
return fixed
@classmethod
def fill(
cls,
config: Union[Config, Dict[str, Dict[str, Any]]],
*,
schema=None,
overrides: Dict[str, Any] = {},
interpolate: bool = False,
validate: bool = True,
) -> Config:
if cls.is_promise(config):
err_msg = "The top-level config object can't be a reference to a registered function."
raise ConfigValidationError(config=config, errors=[{"msg": err_msg}])
# If a Config was loaded with interpolate=False, we assume it needs to
# be interpolated first, otherwise we take it at face value
is_interpolated = not isinstance(config, Config) or config.is_interpolated
section_order = config.section_order if isinstance(config, Config) else None
orig_config = config
if not is_interpolated:
config = Config(orig_config).interpolate()
filled = fill_config(cls, config, overrides=overrides, validate=validate)
filled = Config(filled, section_order=section_order)
# Merge the original config back to preserve variables if we started
# with a config that wasn't interpolated. Here, we prefer variables to
# allow auto-filling a non-interpolated config without destroying
# variable references.
if not interpolate and not is_interpolated:
filled = filled.merge(
Config(orig_config, is_interpolated=False), remove_extra=True
)
if schema is not None:
filled.fill_defaults(schema)
return filled
@classmethod
def is_promise(cls, obj: Any) -> bool:
"""Check whether an object is a "promise", i.e. contains a reference
to a registered function (via a key starting with `"@"`.
"""
return is_promise(obj)
@classmethod
def get_constructor(cls, obj: Dict[str, Any]) -> Tuple[str, str]:
id_keys = [k for k in obj.keys() if k.startswith("@")]
if len(id_keys) != 1:
err_msg = f"A block can only contain one function registry reference. Got: {id_keys}"
raise ConfigValidationError(config=obj, errors=[{"msg": err_msg}])
else:
key = id_keys[0]
value = obj[key]
return (key[1:], value)
@classmethod
def parse_args(cls, obj: Dict[str, Any]) -> Tuple[List[Any], Dict[str, Any]]:
args = []
kwargs = {}
for key, value in obj.items():
if not key.startswith("@"):
if key == ARGS_FIELD:
args = value
else:
kwargs[key] = value
return args, kwargs
def _is_config_section(obj) -> bool:
"""Check if a dict is a config section (all string keys) vs a data value."""
if not isinstance(obj, dict):
return False
return all(isinstance(k, str) for k in obj.keys())
def alias_generator(name: str) -> str:
"""Generate field aliases in promise schema."""
# Underscore fields are not allowed in model, so use alias
if name == ARGS_FIELD_ALIAS:
return ARGS_FIELD
# Auto-alias fields that shadow base model attributes
return RESERVED_FIELDS_REVERSE.get(name, name)
def fill_config(
registry,
config: Dict[str, Any],
*,
overrides: Dict[str, Dict[str, Any]] = {},
validate: bool = True,
) -> Dict[str, Any]:
overrided = apply_overrides(dict(config), overrides)
return _fill_defaults(registry, overrided, validate=validate)
def _fill_defaults(
registry, config: Dict[str, Any], *, validate: bool = True
) -> Dict[str, Any]:
"""Recursively fill default values from registered function signatures."""
output = dict(config)
for key, value in output.items():
if is_promise(value):
output[key] = _fill_promise_defaults(
registry, value, validate=validate, parent=key
)
elif isinstance(value, dict):
output[key] = _fill_defaults(registry, value, validate=validate)
return output
def _fill_promise_defaults(
registry,
promise_dict: Dict[str, Any],
*,
validate: bool = True,
parent: str = "",
) -> Dict[str, Any]:
"""Fill default argument values for a promise block from the function signature."""
reg_name, func_name = registry.get_constructor(promise_dict)
if not registry.has(reg_name, func_name):
# Can't fill defaults for unknown functions — they'll error at resolve time
return dict(promise_dict)
func = registry.get(reg_name, func_name)
schema = Schema.from_function(func)
filled = dict(promise_dict)
# Fill in defaults from the schema
for param_name, field in schema.model_fields.items():
if param_name not in filled and not field.is_required():
filled[param_name] = field.default
# Validate: check for missing required args and type errors
if validate:
_validate_promise_args(filled, schema, func_name, parent)
# Recurse into nested values (which may themselves be promises)
for key, value in filled.items():
if key.startswith("@"):
continue
if is_promise(value):
filled[key] = _fill_promise_defaults(
registry, value, validate=validate, parent=f"{parent}.{key}"
)
elif isinstance(value, dict):
filled[key] = _fill_defaults(registry, value, validate=validate)
return filled
def _validate_promise_args(
filled: Dict[str, Any],
schema,
func_name: str,
parent: str,
) -> None:
"""Validate promise arguments against the function schema."""
from .validation import validate_type
errors = []
for param_name, field in schema.model_fields.items():
# Positional args (*args) are stored under the "*" key in the config
# as a dict of named entries — skip type validation since the dict
# form doesn't match Sequence[T] yet (fix_positionals converts later)
effective_name = param_name
if param_name not in filled and ARGS_FIELD in filled:
effective_name = ARGS_FIELD
if effective_name not in filled:
if field.is_required():
errors.append(
{
"loc": [parent, param_name] if parent else [param_name],
"msg": f"missing required argument: '{param_name}'",
}
)
elif effective_name == ARGS_FIELD:
pass # Skip validation — dict of positional args, not final form
elif not is_promise(filled[effective_name]):
# Only validate non-promise values — promises will be validated
# when they're resolved
err = validate_type(filled[effective_name], field.annotation)
if err:
errors.append(
{
"loc": [parent, param_name] if parent else [param_name],
"msg": err,
}
)
# Check for unexpected arguments (@ keys are registry refs, * is positional args)
known = (
set(schema.model_fields.keys())
| {k for k in filled if k.startswith("@")}
| {ARGS_FIELD}
)
for key in filled:
if key not in known:
errors.append(
{
"loc": [parent, key] if parent else [key],
"msg": f"unexpected argument: '{key}'",
}
)
if errors:
raise ConfigValidationError(
config=filled,
errors=errors,
title=f"Config error for '{func_name}'",
)
def insert_promises(
registry, config: Dict[str, Dict[str, Any]], resolve: bool
) -> Dict[str, Dict[str, Any]]:
"""Create a version of a config dict where promises are recognised and replaced by
Promise dataclasses
"""
output = {}
for key, value in config.items():
if registry.is_promise(value):
value = insert_promises(registry, value, resolve=resolve)
output[key] = Promise.from_dict(
registry,
value,
)
elif isinstance(value, dict):
output[key] = insert_promises(registry, value, resolve=resolve)
else:
output[key] = value
return output
def resolve_promises(config: Dict[str, Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
output = {}
for key, value in config.items():
if isinstance(value, dict):
output[key] = resolve_promises(value)
elif isinstance(value, Promise):
output[key] = value.resolve()
else:
output[key] = value
return output
def fix_positionals(config):
"""Ensure positionals are provided as a tuple, rather than a dict."""
if isinstance(config, dict):
output = {}
for key, value in config.items():
if key == ARGS_FIELD and isinstance(value, dict):
value = tuple(value.values())
if isinstance(value, dict):
value = fix_positionals(value)
elif isinstance(value, list) or isinstance(value, tuple):
value = fix_positionals(value)
output[key] = value
return output
elif isinstance(config, list):
return [fix_positionals(v) for v in config]
elif isinstance(config, tuple):
return tuple([fix_positionals(v) for v in config])
else:
return config
def _deep_copy_with_uncopyable(obj: Any, memo: Optional[Dict[int, Any]] = None) -> Any:
"""Deep copy that passes through objects that can't be copied (like generators)."""
if memo is None:
memo = {}
obj_id = id(obj)
if obj_id in memo:
return memo[obj_id]
if isinstance(obj, dict):
result = {}
memo[obj_id] = result
for k, v in obj.items():
result[_deep_copy_with_uncopyable(k, memo)] = _deep_copy_with_uncopyable(
v, memo
)
return result
elif isinstance(obj, list):
result = []
memo[obj_id] = result
for item in obj:
result.append(_deep_copy_with_uncopyable(item, memo))
return result
elif isinstance(obj, tuple):
# Tuples are immutable, but we still need to copy their contents
return tuple(_deep_copy_with_uncopyable(item, memo) for item in obj)
else:
return obj
def apply_overrides(
config: Dict[str, Dict[str, Any]],
overrides: Dict[str, Dict[str, Any]],
) -> Dict[str, Dict[str, Any]]:
"""Build first representation of the config:"""
output = dict(config)
for key, value in overrides.items():
path = key.split(".")
err_title = "Error parsing config overrides"
err_msg = "not a section value that can be overridden"
err = [{"loc": path, "msg": err_msg}]
node = output
for subkey in path[:-1]:
if (
not isinstance(node, dict) or subkey not in node
): # pragma: no cover -- overrides validated in _parser
raise ConfigValidationError(
errors=err, title=err_title
) # pragma: no cover
node = node[subkey]
if path[-1] not in node: # pragma: no cover
raise ConfigValidationError(errors=err, title=err_title) # pragma: no cover
node[path[-1]] = value # pragma: no cover
return output
explosion-confection-a98f9d6/confection/py.typed 0000664 0000000 0000000 00000000000 15160554312 0022131 0 ustar 00root root 0000000 0000000 explosion-confection-a98f9d6/confection/typechecker.py 0000664 0000000 0000000 00000047043 15160554312 0023334 0 ustar 00root root 0000000 0000000 """
A structural type checker with clean separation of concerns.
The checker validates values against type annotations without requiring
the types to be instantiated. It supports standard library types, generics,
Union/Optional, Literal, Annotated, TypeVar, NewType, dataclasses, and
pydantic-compatible custom types.
Architecture:
check_type - entry point, dispatches to custom handlers or standard path
get_annot_branches - peels Union/Optional into flat alternatives
check_branch - outer_match then decompose+recurse
outer_match - does the value match this annotation at the top level?
decompose - yield (child_value, child_annotation, child_ctx) triples
"""
from __future__ import annotations
import collections.abc
import inspect
import types
from dataclasses import dataclass, field, is_dataclass
from dataclasses import fields as dataclass_fields
from enum import Enum
from pathlib import PurePath
from types import GeneratorType
from typing import (
Annotated,
Any,
Callable,
Dict,
ForwardRef,
Iterator,
Literal,
Optional,
Tuple,
TypeVar,
Union,
get_args,
get_origin,
)
# ---------------------------------------------------------------------------
# Error accumulation
# ---------------------------------------------------------------------------
@dataclass
class TypeCheckError:
"""A single type-check failure, recording the path, value, and expected type."""
path: Tuple[Any, ...]
value: Any
annotation: Any
def __str__(self) -> str:
path_str = " → ".join(str(p) for p in self.path) if self.path else "root"
return f"at {path_str}: {self.value!r} is not {self.annotation}"
@dataclass
class Ctx:
"""Accumulates errors during a type-check traversal.
All recursive calls share the same ``errors`` list via ``child()``,
so errors from any depth are collected in one place.
"""
path: Tuple[Any, ...] = ()
errors: list[TypeCheckError] = field(default_factory=list)
def child(self, segment: Any) -> Ctx:
"""Create a child context with *segment* appended to the path."""
return Ctx(self.path + (segment,), self.errors) # shared errors list
def fail(self, value: Any, annotation: Any) -> None:
"""Record a type-check failure at the current path."""
self.errors.append(TypeCheckError(self.path, value, annotation))
# ---------------------------------------------------------------------------
# Main entry points
# ---------------------------------------------------------------------------
def check_type(
value: Any,
annotation: Any,
custom_handlers: Optional[Dict[type, Callable[..., bool]]] = None,
ctx: Optional[Ctx] = None,
) -> bool:
"""Check whether *value* is compatible with *annotation*.
Returns ``True`` if the value matches, ``False`` otherwise. Errors are
accumulated in *ctx* (created automatically if not provided).
*custom_handlers* maps ``type(value)`` to a callable
``(value, annotation, handlers, ctx) -> bool`` that overrides the
default checking logic for that runtime type.
"""
if custom_handlers is None:
custom_handlers = {}
if ctx is None:
ctx = Ctx()
if type(value) in custom_handlers:
return custom_handlers[type(value)](value, annotation, custom_handlers, ctx)
return any(
check_branch(value, branch, custom_handlers, ctx)
for branch in get_annot_branches(annotation)
)
def check_branch(
value: Any,
annotation: Any,
custom_handlers: Dict[type, Callable[..., bool]],
ctx: Ctx,
) -> bool:
"""Check *value* against a single (non-Union) annotation branch.
First checks the top-level match via ``outer_match``, then recursively
checks children yielded by ``decompose``.
"""
if not outer_match(value, annotation):
ctx.fail(value, annotation)
return False
return all(
check_type(v, a, custom_handlers, child_ctx)
for v, a, child_ctx in decompose(value, annotation, ctx)
)
# ---------------------------------------------------------------------------
# get_annot_branches: peel Union/Optional into flat alternatives
# ---------------------------------------------------------------------------
def get_annot_branches(annotation: Any) -> Tuple[Any, ...]:
"""Split a (possibly Union) annotation into individual branches.
``Union[int, str]`` becomes ``(int, str)``. ``Optional[X]`` becomes
``(X, NoneType)``. ``X | Y`` (Python 3.10+) is handled via
``types.UnionType``. Non-union annotations are returned as a
single-element tuple.
"""
origin = get_origin(annotation)
# Union[X, Y] and Optional[X] (which is Union[X, None])
if origin is Union:
return get_args(annotation)
# types.UnionType handles X | Y syntax (Python 3.10+)
if origin is types.UnionType or isinstance(annotation, types.UnionType):
return get_args(annotation)
return (annotation,)
# ---------------------------------------------------------------------------
# outer_match: does the value match at this level, ignoring children?
# ---------------------------------------------------------------------------
#: Map from typing generic origins to the runtime types used for isinstance.
ORIGIN_TO_BUILTIN: Dict[Any, Any] = {
list: list,
dict: dict,
tuple: tuple,
set: set,
frozenset: frozenset,
collections.abc.Sequence: collections.abc.Sequence,
collections.abc.MutableSequence: collections.abc.MutableSequence,
collections.abc.Set: collections.abc.Set,
collections.abc.MutableSet: collections.abc.MutableSet,
collections.abc.Mapping: collections.abc.Mapping,
collections.abc.MutableMapping: collections.abc.MutableMapping,
collections.abc.Callable: collections.abc.Callable,
collections.abc.Iterable: collections.abc.Iterable,
collections.abc.Iterator: collections.abc.Iterator,
}
def outer_match(value: Any, annotation: Any) -> bool:
"""Check whether *value* matches *annotation* at the top level.
This does **not** recurse into container elements — that is the job of
``decompose`` + ``check_branch``. Coercion rules:
* ``bool`` requires an exact bool (``0``/``1`` are rejected).
* ``int`` accepts ints and parseable strings, but rejects bools.
* ``float`` accepts ints, floats, and parseable strings, but rejects bools.
* ``Path``/``PurePath`` accept strings.
* ``str`` enums accept valid member value strings.
* Generators / iterators always pass (to avoid consuming them).
* Unresolved ``ForwardRef`` and string annotations always pass.
"""
# Any / Parameter.empty matches everything
if annotation is Any or annotation is inspect.Parameter.empty:
return True
# None/NoneType
if annotation is None or annotation is type(None):
return value is None
# Generators pass through without consumption
if isinstance(value, (GeneratorType, collections.abc.Iterator)) and not isinstance(
value, (str, bytes)
):
return True
# Literal[v1, v2, ...]
if get_origin(annotation) is Literal:
return value in get_args(annotation)
# Annotated[T, ...] — unwrap to inner type
if get_origin(annotation) is Annotated:
inner = get_args(annotation)[0]
metadata = get_args(annotation)[1:]
if _has_strict_metadata(metadata):
return _strict_match(value, inner)
return outer_match(value, inner)
# Type[X] — value should be a class that is a subclass of X
if get_origin(annotation) is type:
args = get_args(annotation)
if not isinstance(value, type):
return False
if args and args[0] is not Any:
try:
return issubclass(value, args[0])
except (
TypeError
): # pragma: no cover -- modern Python handles Union in issubclass
return True # pragma: no cover
return True
# Callable — just check callability here, signature checking is hard
# and arguably belongs in a custom handler
if get_origin(annotation) is collections.abc.Callable:
return callable(value)
# Generic types: List[int], Dict[str, int], etc.
origin = get_origin(annotation)
if origin is not None:
check_against = ORIGIN_TO_BUILTIN.get(origin, origin)
try:
if not isinstance(value, check_against):
return False
except TypeError: # pragma: no cover -- custom generics with non-type origins
return True # pragma: no cover
# Fixed-length tuple: check length here
if origin is tuple:
args = get_args(annotation)
if args and not (len(args) == 2 and args[1] is Ellipsis):
if len(value) != len(args):
return False
return True
# NewType — unwrap to supertype
if callable(annotation) and hasattr(annotation, "__supertype__"):
return outer_match(value, annotation.__supertype__) # pyright: ignore[reportFunctionMemberAccess]
# TypeVar
if isinstance(annotation, TypeVar):
bound = annotation.__bound__
constraints = annotation.__constraints__
if bound:
try:
return isinstance(value, bound)
except TypeError:
# bound contains unresolved ForwardRefs or complex generics;
# use check_type which handles Union via get_annot_branches
return check_type(value, bound)
if constraints:
return any(check_type(value, c) for c in constraints)
return True
# Forward references — can't resolve, accept
if isinstance(annotation, (str, ForwardRef)):
return True
# --- Plain types with coercion ---
if isinstance(annotation, type):
# bool: exact type (don't accept int 0/1)
if annotation is bool:
return isinstance(value, bool)
# int: accept ints (not bools), reject strings that don't parse
if annotation is int:
if isinstance(value, bool):
return False
if isinstance(value, int):
return True
if isinstance(value, str):
try:
int(value)
return True
except (ValueError, TypeError):
return False
return False
# float: accept int/float (not bools), strings that parse
if annotation is float:
if isinstance(value, bool):
return False
if isinstance(value, (int, float)):
return True
if isinstance(value, str):
try:
float(value)
return True
except (ValueError, TypeError):
return False
return False
# str: straightforward
if annotation is str:
return isinstance(value, str)
# Path: accept strings
if issubclass(annotation, PurePath):
return isinstance(value, (str, PurePath))
# str enums: accept plain strings that are valid member values
if issubclass(annotation, str) and issubclass(annotation, Enum):
try:
annotation(value)
return True
except (ValueError, KeyError):
return False
# Dataclass / Schema with dict value: accept for decompose
if is_dataclass(annotation) and isinstance(value, dict):
return True
if hasattr(annotation, "model_fields") and isinstance(value, dict):
return True
# Pydantic v2 validator hook
if hasattr(annotation, "__get_pydantic_core_schema__"):
return _pydantic_v2_match(value, annotation)
# Pydantic v1 validator hook
if hasattr(annotation, "__get_validators__"):
return _pydantic_v1_match(value, annotation)
# Default isinstance
return isinstance(value, annotation)
return False
# ---------------------------------------------------------------------------
# Helpers for outer_match
# ---------------------------------------------------------------------------
def _resolve_dataclass_hints(cls: type) -> Dict[str, Any]:
"""Resolve forward references in a dataclass's type annotations.
Uses ``get_type_hints`` with the class's module globals so that
forward references like ``ForwardRef('Floats3d')`` are resolved to
actual types. Returns an empty dict on failure.
"""
import sys
from typing import get_type_hints
mod = sys.modules.get(cls.__module__)
globalns = vars(mod) if mod else None
try:
return get_type_hints(cls, globalns=globalns)
except (NameError, AttributeError, TypeError, RecursionError): # pragma: no cover
return {} # pragma: no cover
def _has_strict_metadata(metadata: Tuple[Any, ...]) -> bool:
"""Check if ``Annotated`` metadata contains a ``Strict()`` marker."""
return any(getattr(m, "strict", False) for m in metadata if hasattr(m, "strict"))
def _strict_match(value: Any, inner_type: type) -> bool:
"""Exact type match for ``Annotated[X, Strict()]``.
Unlike the normal coercion rules, strict matching requires
``type(value)`` to be exactly the annotated type.
"""
if inner_type is int:
return type(value) is int and not isinstance(value, bool)
if inner_type is float:
return type(value) is float
if inner_type is str:
return type(value) is str
if inner_type is bool:
return type(value) is bool
return isinstance(value, inner_type)
class _AnySchemaHandler:
"""Minimal stand-in for pydantic's ``GetCoreSchemaHandler``."""
def __call__(
self, _source_type: Any
) -> Dict[str, Any]: # pragma: no cover -- called internally by pydantic hooks
return {"type": "any"} # pragma: no cover
def _pydantic_v2_match(value: Any, annotation: type) -> bool:
"""Check *value* against a type with ``__get_pydantic_core_schema__``.
Extracts the validator function from the pydantic core schema and calls
it. Falls back to ``isinstance`` if the value is already an instance.
"""
if isinstance(value, annotation):
return True
try:
schema = annotation.__get_pydantic_core_schema__(
annotation, _AnySchemaHandler()
)
fn_entry = schema.get("function", {})
validator = fn_entry.get("function") if isinstance(fn_entry, dict) else None
if callable(validator):
validator(value)
return True
except (ValueError, TypeError, AssertionError):
return False
return False
class _PydanticV1FieldShim:
"""Minimal shim providing ``field.type_`` for pydantic v1 validators."""
def __init__(self, typ: type) -> None:
self.type_ = typ
def _pydantic_v1_match(value: Any, annotation: type) -> bool:
"""Check *value* against a type with ``__get_validators__``.
Iterates through the validators yielded by the type's
``__get_validators__`` classmethod. Validators with more than
2 parameters are skipped (they require a ``config`` argument we
don't have).
"""
if isinstance(value, annotation):
return True
shim = _PydanticV1FieldShim(annotation)
for validator in annotation.__get_validators__():
try:
nparams = len(inspect.signature(validator).parameters)
except (ValueError, TypeError):
nparams = 1
if nparams > 2:
continue
try:
value = validator(value) if nparams == 1 else validator(value, shim)
except (ValueError, TypeError, AssertionError):
return False
return True
# ---------------------------------------------------------------------------
# decompose: yield (child_value, child_annotation, child_ctx) triples
# ---------------------------------------------------------------------------
#: Origins that are sequence-like: one type arg, fan across elements.
SEQUENCE_ORIGINS: set[Any] = {
list,
set,
frozenset,
collections.abc.Sequence,
collections.abc.MutableSequence,
collections.abc.Set,
collections.abc.MutableSet,
collections.abc.Iterable,
collections.abc.Iterator,
}
#: Origins that are mapping-like: two type args (key, value).
MAPPING_ORIGINS: set[Any] = {
dict,
collections.abc.Mapping,
collections.abc.MutableMapping,
}
def decompose(value: Any, annotation: Any, ctx: Ctx) -> Iterator[Tuple[Any, Any, Ctx]]:
"""Yield ``(child_value, child_annotation, child_ctx)`` triples.
This is the recursive engine of the type checker. For container types
it fans out over elements; for dataclasses and schemas it fans out
over fields. ``outer_match`` has already confirmed the top-level
match, so we only need to check the children here.
"""
# Annotated[T, ...] — unwrap
if get_origin(annotation) is Annotated:
annotation = get_args(annotation)[0]
origin = get_origin(annotation)
args = get_args(annotation)
# Schema / model_fields annotation with dict value — fan out over fields
if (
isinstance(annotation, type)
and hasattr(annotation, "model_fields")
and isinstance(value, dict)
):
for name, field_info in annotation.model_fields.items():
data_key = name
if hasattr(field_info, "alias") and field_info.alias is not None:
data_key = field_info.alias
if data_key in value:
yield (value[data_key], field_info.annotation, ctx.child(data_key))
return
# Dataclass annotation with dict value — fan out over fields
if (
isinstance(annotation, type)
and is_dataclass(annotation)
and isinstance(value, dict)
):
resolved_hints = _resolve_dataclass_hints(annotation)
for f in dataclass_fields(annotation):
if f.name in value:
hint = resolved_hints.get(f.name, f.type)
yield (value[f.name], hint, ctx.child(f.name))
return
# Dataclass annotation with dataclass value — match fields
if (
isinstance(annotation, type)
and is_dataclass(annotation)
and is_dataclass(value)
):
resolved_hints = _resolve_dataclass_hints(annotation)
for f in dataclass_fields(annotation):
if hasattr(value, f.name):
hint = resolved_hints.get(f.name, f.type)
yield (getattr(value, f.name), hint, ctx.child(f.name))
return
# No type args means nothing to recurse into for generics
if not args:
return
# Tuple: fixed-length or variable-length
if origin is tuple:
if len(args) == 2 and args[1] is Ellipsis:
# Tuple[int, ...] — variable length, one type
for i, elem in enumerate(value):
yield (elem, args[0], ctx.child(i))
else:
# Tuple[int, str, float] — fixed length
# Length already verified in outer_match
for i, (elem, arg) in enumerate(zip(value, args)):
yield (elem, arg, ctx.child(i))
return
# Sequence-like
if origin in SEQUENCE_ORIGINS:
arg = args[0]
for i, elem in enumerate(value):
yield (elem, arg, ctx.child(i))
return
# Mapping-like
if origin in MAPPING_ORIGINS:
k_ann, v_ann = args[0], args[1]
for k in value:
yield (k, k_ann, ctx.child(f"key({k!r})"))
yield (value[k], v_ann, ctx.child(k))
return
# Type[X], Callable, TypeVar, Literal — no children to decompose
# (fully resolved in outer_match)
return
explosion-confection-a98f9d6/confection/util.py 0000664 0000000 0000000 00000012377 15160554312 0022005 0 ustar 00root root 0000000 0000000 import functools
import json
import re
from copy import deepcopy
from typing import Any, Callable, Protocol, TypeVar
from ._constants import VARIABLE_RE
from ._errors import ConfigValidationError
_DIn = TypeVar("_DIn")
class Decorator(Protocol):
"""Protocol to mark a function as returning its child with identical signature."""
def __call__(self, name: str) -> Callable[[_DIn], _DIn]: ...
# This is how functools.partials seems to do it, too, to retain the return type
PartialT = TypeVar("PartialT")
def partial(
func: Callable[..., PartialT], *args: Any, **kwargs: Any
) -> Callable[..., PartialT]:
"""Wrapper around functools.partial that retains docstrings and can include
other workarounds if needed.
"""
partial_func = functools.partial(func, *args, **kwargs)
partial_func.__doc__ = func.__doc__
return partial_func
DEFAULT_FROZEN_DICT_ERROR = (
"Can't write to frozen dictionary. This is likely an internal "
"error. Are you writing to a default function argument?"
)
DEFAULT_FROZEN_LIST_ERROR = (
"Can't write to frozen list. Maybe you're trying to modify a computed "
"property or default function argument?"
)
class SimpleFrozenDict(dict):
"""Simplified implementation of a frozen dict, mainly used as default
function or method argument (for arguments that should default to empty
dictionary). Will raise an error if the user attempts to add to dict.
"""
def __init__(
self,
*args,
error: str = DEFAULT_FROZEN_DICT_ERROR,
**kwargs,
) -> None:
"""Initialize the frozen dict. Can be initialized with pre-defined
values.
error (str): The error message when user tries to assign to dict.
"""
super().__init__(*args, **kwargs)
self.error = error
def __setitem__(self, key, value):
raise NotImplementedError(self.error)
def pop(self, key, default=None):
raise NotImplementedError(self.error)
def update(self, other=(), /, **kwargs): # pyright: ignore[reportIncompatibleMethodOverride]
raise NotImplementedError(self.error)
def __deepcopy__(self, memo):
return self.__class__(deepcopy({k: v for k, v in self.items()}))
class SimpleFrozenList(list):
"""Wrapper class around a list that lets us raise custom errors if certain
attributes/methods are accessed. Mostly used for properties that return an
immutable list (and that we don't want to convert to a tuple to not break
too much backwards compatibility). If a user accidentally calls
frozen_list.append(), we can raise a more helpful error.
"""
def __init__(
self,
*args,
error: str = DEFAULT_FROZEN_LIST_ERROR,
) -> None:
"""Initialize the frozen list.
error (str): The error message when user tries to mutate the list.
"""
self.error = error
super().__init__(*args)
def append(self, *args, **kwargs):
raise NotImplementedError(self.error)
def clear(self, *args, **kwargs):
raise NotImplementedError(self.error)
def extend(self, *args, **kwargs):
raise NotImplementedError(self.error)
def insert(self, *args, **kwargs):
raise NotImplementedError(self.error)
def pop(self, *args, **kwargs):
raise NotImplementedError(self.error)
def remove(self, *args, **kwargs):
raise NotImplementedError(self.error)
def reverse(self, *args, **kwargs):
raise NotImplementedError(self.error)
def sort(self, *args, **kwargs):
raise NotImplementedError(self.error)
def __deepcopy__(self, memo):
return self.__class__(deepcopy(v) for v in self)
def is_promise(obj) -> bool:
if not hasattr(obj, "keys"):
return False
id_keys = [k for k in obj.keys() if isinstance(k, str) and k.startswith("@")]
if len(id_keys):
return True
return False
_PYTHON_LITERALS = {"True": True, "False": False, "None": None}
def try_load_json(value: str) -> Any:
"""Load a JSON string if possible, otherwise default to original value.
Also handles Python-style literals ``True``, ``False``, and ``None``
which are not valid JSON but are commonly used in config files.
"""
try:
return json.loads(value)
except Exception:
if value in _PYTHON_LITERALS:
return _PYTHON_LITERALS[value]
return value
def try_dump_json(value: Any, data: dict[str, dict] | str = "") -> str:
"""Dump a config value as JSON and output user-friendly error if it fails."""
# Special case if we have a variable: it's already a string so don't dump
# to preserve ${x:y} vs. "${x:y}"
if isinstance(value, str) and VARIABLE_RE.search(value):
return value
try:
value = json.dumps(value)
except Exception as e:
err_msg = (
f"Couldn't serialize config value of type {type(value)}: {e}. Make "
f"sure all values in your config are JSON-serializable. If you want "
f"to include Python objects, use a registered function that returns "
f"the object instead."
)
raise ConfigValidationError(config=data, desc=err_msg) from e
# Escape $ to $$ for configparser, but preserve ${...} variable references
return re.sub(r"\$(?!\{)", "$$", value)
explosion-confection-a98f9d6/confection/validation.py 0000664 0000000 0000000 00000046704 15160554312 0023163 0 ustar 00root root 0000000 0000000 """Lightweight type validation system replacing Pydantic.
Provides Schema base class, dynamic schema creation, and type validation
for config values against function signatures.
"""
import inspect
import sys
from typing import Any, Optional, get_type_hints
from .typechecker import Ctx
from .typechecker import check_type as _tc2_check_type
# Optional pydantic imports — confection doesn't depend on pydantic,
# but if it's installed we can detect and convert BaseModel schemas.
# Skip pydantic.v1 on Python 3.14+ where it is unsupported.
if sys.version_info >= (3, 14):
_PydanticV1BaseModel = None # type: ignore[assignment,misc]
_PydanticV1ValidationError = None # type: ignore[assignment,misc]
else:
try:
from pydantic.v1 import (
BaseModel as _PydanticV1BaseModel, # pyright: ignore[reportMissingImports]
)
from pydantic.v1 import (
ValidationError as _PydanticV1ValidationError, # pyright: ignore[reportMissingImports]
)
except (ImportError, ModuleNotFoundError): # pragma: no cover
_PydanticV1BaseModel = None # type: ignore[assignment,misc]
_PydanticV1ValidationError = None # type: ignore[assignment,misc]
try:
from pydantic import (
BaseModel as _PydanticV2BaseModel, # pyright: ignore[reportMissingImports]
)
from pydantic import (
ValidationError as _PydanticV2ValidationError, # pyright: ignore[reportMissingImports]
)
except (ImportError, ModuleNotFoundError): # pragma: no cover
_PydanticV2BaseModel = None # type: ignore[assignment,misc]
_PydanticV2ValidationError = None # type: ignore[assignment,misc]
# === Constrained Types ===
class StrictBool:
"""Only accepts actual bool values (not int 0/1)."""
pass
class PositiveInt:
"""Only accepts positive integers (> 0, not bool)."""
pass
class StrictFloat:
"""Only accepts actual float values (not int)."""
pass
# === Field Info ===
class FieldInfo:
"""Information about a schema field."""
__slots__ = ("default", "alias", "annotation")
def __init__(self, default=..., *, alias=None):
self.default = default
self.alias = alias
self.annotation: Any = None
def is_required(self):
return self.default is ...
def Field(default=..., *, alias=None):
"""Create a field definition."""
return FieldInfo(default=default, alias=alias)
# === Validation Error ===
class ValidationError(Exception):
"""Raised when schema validation fails."""
def __init__(self, error_list):
self._errors = error_list
msgs = "; ".join(e.get("msg", "") for e in error_list)
super().__init__(msgs)
def errors(self):
return self._errors
# === Schema ===
class _ValidatedResult:
"""Attribute-accessible result from model_validate."""
def __init__(self, data):
self.__dict__.update(data)
class Schema:
"""Base class for config validation schemas. Replaces pydantic.BaseModel."""
model_config: dict = {"extra": "allow", "arbitrary_types_allowed": True}
model_fields: dict = {}
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
fields = {}
all_hints = {}
for base in reversed(cls.__mro__):
base_annotations = getattr(base, "__annotations__", {})
all_hints.update(base_annotations)
for name, annotation in all_hints.items():
if name in ("model_config", "model_fields") or name.startswith("_"):
continue
default = ...
alias = None
for klass in cls.__mro__:
if name in klass.__dict__:
val = klass.__dict__[name]
if isinstance(val, FieldInfo):
default = val.default
alias = val.alias
elif not isinstance(
val, (type, classmethod, staticmethod, property)
):
if not callable(val):
default = val
break
field = FieldInfo(default=default, alias=alias)
field.annotation = annotation
fields[name] = field
cls.model_fields = fields
def __init__(self, **kwargs):
for name, field in self.__class__.model_fields.items():
if name in kwargs:
setattr(self, name, kwargs[name])
elif not field.is_required():
setattr(self, name, field.default)
@classmethod
def model_validate(cls, data):
"""Validate a dict against this schema."""
alias_gen = cls.model_config.get("alias_generator")
errors = _validate_schema(data, cls.model_fields, cls.model_config, alias_gen)
if errors:
raise ValidationError(errors)
# Build result with defaults filled in
result_data = dict(data)
for name, field in cls.model_fields.items():
data_key = name
if field.alias is not None:
data_key = field.alias
elif alias_gen:
data_key = alias_gen(name)
if data_key not in result_data and not field.is_required():
result_data[data_key] = field.default
return _ValidatedResult(result_data)
@classmethod
def from_function(
cls,
func,
*,
config=None,
):
"""Build a Schema subclass from a function's signature.
Each parameter becomes a field. The annotation is used as the type
(defaulting to ``Any`` when missing) and the default value is
preserved (parameters without defaults become required fields).
``*args`` parameters are wrapped in ``Sequence[annotation]`` and
stored under the ``VARIABLE_POSITIONAL_ARGS`` field name.
Forward-reference annotations are resolved via
``typing.get_type_hints`` against the function's module namespace.
"""
from typing import Sequence as _Seq
if config is None:
config = {"extra": "forbid", "arbitrary_types_allowed": True}
resolved = resolve_type_hints(func)
fields = {}
for param in inspect.signature(func).parameters.values():
annotation = resolved.get(param.name, param.annotation)
if annotation is inspect.Parameter.empty:
annotation = Any
if param.default is inspect.Parameter.empty:
default = ...
else:
default = param.default
if param.kind == inspect.Parameter.VAR_POSITIONAL:
annotation = _Seq[annotation] # type: ignore[valid-type]
if param.kind == inspect.Parameter.VAR_KEYWORD:
continue
field = FieldInfo(default=default)
field.annotation = annotation
fields[param.name] = field
return create_schema(
func.__name__,
__config__=config,
**{name: (f.annotation, f) for name, f in fields.items()},
)
def model_dump(self):
"""Convert instance to dict."""
result = {}
for name in self.__class__.model_fields:
if hasattr(self, name):
val = getattr(self, name)
if isinstance(val, Schema):
result[name] = val.model_dump()
else:
result[name] = val
return result
def create_schema(__name, __config__=None, **fields):
"""Dynamically create a Schema subclass.
Each field value should be a (annotation, FieldInfo) tuple.
"""
if __config__ is None:
__config__ = {"extra": "allow"}
processed = {}
annotations = {}
defaults = {}
for name, field_def in fields.items():
if isinstance(field_def, tuple) and len(field_def) == 2:
annotation, field_info = field_def
if not isinstance(field_info, FieldInfo):
field_info = FieldInfo(default=field_info)
else:
raise ValueError(f"Field {name} must be (annotation, FieldInfo) tuple")
field_info.annotation = annotation
processed[name] = field_info
annotations[name] = annotation
if not field_info.is_required():
defaults[name] = field_info.default
namespace = {
"__annotations__": annotations,
"model_config": __config__,
}
namespace.update(defaults)
cls = type(__name, (Schema,), namespace)
# Apply alias_generator to fields that don't have explicit aliases
alias_gen = __config__.get("alias_generator") if __config__ else None
if alias_gen and callable(alias_gen):
for name, field in processed.items():
if field.alias is None:
field.alias = alias_gen(name)
# Override with our processed fields (preserving aliases)
cls.model_fields = processed
return cls
# === Resolve forward references ===
def resolve_type_hints(func):
"""Resolve type hints for a function, handling forward references.
Falls back to raw annotations if resolution fails.
"""
try:
mod_name = getattr(func, "__module__", None)
module = sys.modules.get(mod_name) if mod_name else None
globalns = vars(module) if module else None
return get_type_hints(func, globalns=globalns)
except (NameError, AttributeError, TypeError, RecursionError):
# NameError: unresolvable forward reference
# AttributeError: module without expected attributes
# TypeError: invalid annotation object
# RecursionError: self-referential types (Python 3.13+)
return {}
# === Type Validation ===
def _error_type_for(annotation):
"""Get an error type string for an annotation."""
if annotation is int or annotation is PositiveInt:
return "int_parsing"
elif annotation is str:
return "string_type"
elif annotation is float or annotation is StrictFloat:
return "float_parsing"
elif annotation is bool or annotation is StrictBool:
return "bool_type"
return "value_error"
def validate_type(value, annotation):
"""Validate value against a type annotation.
Returns None if valid, or an error message string if invalid.
"""
ctx = Ctx()
if _tc2_check_type(value, annotation, ctx=ctx):
return None
if ctx.errors:
return str(ctx.errors[0])
return f"{value!r} does not match {annotation}" # pragma: no cover -- defensive fallback
# === Schema Validation ===
def _validate_schema(data, fields, config, alias_generator=None):
"""Validate a data dict against schema fields.
Returns list of error dicts (empty if valid).
"""
errors = []
extra_mode = config.get("extra", "allow")
# Build mapping: data_key -> (field_name, FieldInfo)
key_to_field = {}
known_keys = set()
for name, field in fields.items():
if field.alias is not None:
data_key = field.alias
elif alias_generator:
data_key = alias_generator(name)
else:
data_key = name
known_keys.add(data_key)
key_to_field[data_key] = (name, field)
# Check extra fields
if extra_mode == "forbid":
for key in data:
if key not in known_keys:
errors.append(
{
"loc": (key,),
"msg": "Extra inputs are not permitted",
"type": "extra_forbidden",
}
)
# Validate each field
for data_key, (name, field) in key_to_field.items():
if data_key in data:
value = data[data_key]
err = validate_type(value, field.annotation)
if err:
errors.append(
{
"loc": (data_key,),
"msg": err,
"type": _error_type_for(field.annotation),
}
)
elif field.is_required():
errors.append(
{
"loc": (data_key,),
"msg": "Field required",
"type": "missing",
}
)
return errors
# === Pydantic Compatibility Shim ===
_pydantic_cache: dict = {}
def _get_pydantic_validation_error():
"""Return the pydantic ValidationError class(es) to catch.
Tries both pydantic.v1 and pydantic so we catch the right exception
regardless of which API the caller's model was built with.
"""
errors = []
if _PydanticV1ValidationError is not None:
errors.append(_PydanticV1ValidationError)
if _PydanticV2ValidationError is not None:
errors.append(_PydanticV2ValidationError)
if errors:
return tuple(errors)
# Should never happen — we only get here if someone passed a pydantic
# model, which means pydantic is installed. Fall back to Exception so
# the except clause still works rather than crashing.
return (Exception,) # pragma: no cover
def _is_pydantic_model(cls):
"""Check if cls is a pydantic BaseModel class (v1 or v2) without hard-depending
on pydantic. Returns False if pydantic is not installed."""
if not isinstance(cls, type):
return False
if issubclass(cls, Schema):
return False
if _PydanticV1BaseModel is not None and issubclass(cls, _PydanticV1BaseModel):
return True
if _PydanticV2BaseModel is not None and issubclass(cls, _PydanticV2BaseModel):
return True
return False
def _pydantic_instance_to_dict(obj):
"""Convert a pydantic model instance to a dict."""
if hasattr(obj, "model_dump"):
return obj.model_dump()
if hasattr(obj, "dict"):
return obj.dict()
return obj
def _extract_pydantic_fields(pydantic_cls):
"""Extract field definitions from a pydantic BaseModel class (v1 or v2)."""
fields = {}
if hasattr(pydantic_cls, "model_fields"):
# pydantic v2 interface (check first — v2 also exposes __fields__
# as a deprecated shim, so we must not fall into the v1 branch)
for name, pyd_field in pydantic_cls.model_fields.items():
annotation = pyd_field.annotation
if pyd_field.is_required():
default = ...
else:
default = pyd_field.default
alias = pyd_field.alias
if isinstance(annotation, type) and _is_pydantic_model(annotation):
annotation = ensure_schema(annotation)
if default is not ... and hasattr(default, "model_dump"):
default = _pydantic_instance_to_dict(default)
field = FieldInfo(default=default, alias=alias)
field.annotation = annotation
fields[name] = field
elif hasattr(pydantic_cls, "__fields__"):
# pydantic v1 interface
for name, pyd_field in pydantic_cls.__fields__.items():
annotation = pyd_field.outer_type_
# pydantic v1 unwraps Optional[X] into outer_type_=X +
# allow_none=True. Re-wrap so our validator sees the Union.
if getattr(pyd_field, "allow_none", False):
annotation = Optional[annotation]
if pyd_field.required:
default = ...
else:
default = pyd_field.default
alias = pyd_field.alias if pyd_field.alias != name else None
# Recursively convert nested pydantic model annotations
if isinstance(annotation, type) and _is_pydantic_model(annotation):
annotation = ensure_schema(annotation)
# Convert pydantic instance defaults to dicts
if default is not ... and hasattr(default, "__fields__"):
default = _pydantic_instance_to_dict(default)
field = FieldInfo(default=default, alias=alias)
field.annotation = annotation
fields[name] = field
return fields
def _extract_pydantic_config(pydantic_cls):
"""Extract model config from a pydantic BaseModel class (v1 or v2)."""
config = {"extra": "allow"}
if hasattr(pydantic_cls, "__config__"):
# pydantic v1: inner class Config
cfg = pydantic_cls.__config__
extra = getattr(cfg, "extra", "allow")
# v1 may use an enum (e.g. Extra.forbid); extract the .value
if hasattr(extra, "value"):
extra = extra.value # pyright: ignore[reportAttributeAccessIssue]
config["extra"] = extra if isinstance(extra, str) else str(extra)
if hasattr(cfg, "arbitrary_types_allowed"):
config["arbitrary_types_allowed"] = cfg.arbitrary_types_allowed
elif hasattr(pydantic_cls, "model_config") and isinstance(
pydantic_cls.model_config, dict
):
# pydantic v2: dict
config = dict(pydantic_cls.model_config)
return config
def ensure_schema(schema_cls):
"""Ensure *schema_cls* satisfies the Schema interface.
If it already is a Schema subclass, return it unchanged.
If it is a pydantic BaseModel (v1 or v2), build a thin Schema wrapper
that exposes the same ``model_fields`` / ``model_config`` and delegates
``model_validate`` to the original pydantic class so that pydantic
validators, strict types, constrained types etc. keep working.
This allows downstream libraries (spaCy, thinc, …) to keep passing
pydantic schemas to ``registry.resolve()`` / ``registry.fill()`` even
though confection itself no longer depends on pydantic.
"""
if isinstance(schema_cls, type) and issubclass(schema_cls, Schema):
return schema_cls
if not _is_pydantic_model(schema_cls):
return schema_cls
# Return cached conversion if available
if schema_cls in _pydantic_cache:
return _pydantic_cache[schema_cls]
fields = _extract_pydantic_fields(schema_cls)
config = _extract_pydantic_config(schema_cls)
# Build wrapper class that inherits from Schema
pyd_cls = schema_cls # capture for closure
wrapper = type(pydantic_cls_name(schema_cls), (Schema,), {})
wrapper.model_fields = fields
wrapper.model_config = config
# Delegate model_validate to the original pydantic model so that
# pydantic-level validators / strict types / constraints keep working.
@classmethod # type: ignore[misc]
def _pydantic_model_validate(cls, data):
# Resolve the concrete pydantic ValidationError class once so the
# except clause is as narrow as possible.
pyd_validation_err = _get_pydantic_validation_error()
try:
if hasattr(pyd_cls, "model_validate"):
pyd_cls.model_validate(data)
elif hasattr(pyd_cls, "parse_obj"):
pyd_cls.parse_obj(data)
else: # pragma: no cover -- all pydantic versions have model_validate or parse_obj
pyd_cls(**data) # pragma: no cover
except pyd_validation_err as e:
raise ValidationError(
e.errors() # pyright: ignore[reportAttributeAccessIssue]
) from None
# Return attribute-accessible result with defaults filled in
result_data = dict(data)
for name, field in cls.model_fields.items():
data_key = field.alias if field.alias is not None else name
if data_key not in result_data and not field.is_required():
result_data[data_key] = field.default
return _ValidatedResult(result_data)
wrapper.model_validate = _pydantic_model_validate
_pydantic_cache[schema_cls] = wrapper
return wrapper
def pydantic_cls_name(cls):
return getattr(cls, "__name__", "PydanticSchema")
explosion-confection-a98f9d6/pyproject.toml 0000664 0000000 0000000 00000000550 15160554312 0021231 0 ustar 00root root 0000000 0000000 [build-system]
requires = [
"setuptools",
]
build-backend = "setuptools.build_meta"
[tool.ruff]
line-length = 88
[tool.ruff.lint]
select = ["E", "F", "W", "I"]
ignore = ["E203", "E501", "E402", "E731"]
[tool.ruff.lint.isort]
known-first-party = ["confection"]
[tool.pyright]
include = ["confection"]
exclude = ["confection/tests"]
pythonVersion = "3.10"
explosion-confection-a98f9d6/pyrightconfig.json 0000664 0000000 0000000 00000000113 15160554312 0022057 0 ustar 00root root 0000000 0000000 {
"exclude": ["confection/tests"],
"reportMissingImports": "warning"
}
explosion-confection-a98f9d6/requirements.txt 0000664 0000000 0000000 00000000260 15160554312 0021577 0 ustar 00root root 0000000 0000000 # Development/test requirements
catalogue>=2.0.3,<2.1.0
pathy>=0.3.5
pytest>=5.2.0,!=7.1.0
numpy>=1.15.0
hypothesis
pydantic>=1.7.4,<3.0.0 # for ensure_schema shim tests
ruff
explosion-confection-a98f9d6/setup.cfg 0000664 0000000 0000000 00000002073 15160554312 0020140 0 ustar 00root root 0000000 0000000 [metadata]
version = 1.3.3
description = The sweetest config system for Python
url = https://github.com/explosion/confection
author = Explosion
author_email = contact@explosion.ai
license = MIT
long_description = file: README.md
long_description_content_type = text/markdown
classifiers =
Development Status :: 5 - Production/Stable
Environment :: Console
Intended Audience :: Developers
Intended Audience :: Science/Research
License :: OSI Approved :: MIT License
Operating System :: POSIX :: Linux
Operating System :: MacOS :: MacOS X
Operating System :: Microsoft :: Windows
Programming Language :: Python :: 3
Programming Language :: Python :: 3.10
Programming Language :: Python :: 3.11
Programming Language :: Python :: 3.12
Programming Language :: Python :: 3.13
Programming Language :: Python :: 3.14
Topic :: Scientific/Engineering
[options]
zip_safe = true
include_package_data = true
python_requires = >=3.10
install_requires =
typing_extensions>=4.0,<5.0; python_version < "3.11"
[sdist]
formats = gztar
explosion-confection-a98f9d6/setup.py 0000664 0000000 0000000 00000000232 15160554312 0020024 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
if __name__ == "__main__":
from setuptools import find_packages, setup
setup(name="confection", packages=find_packages())
explosion-confection-a98f9d6/tests/ 0000775 0000000 0000000 00000000000 15160554312 0017457 5 ustar 00root root 0000000 0000000 explosion-confection-a98f9d6/tests/conftest.py 0000664 0000000 0000000 00000000000 15160554312 0021644 0 ustar 00root root 0000000 0000000 explosion-confection-a98f9d6/tests/strategies.py 0000664 0000000 0000000 00000021425 15160554312 0022207 0 ustar 00root root 0000000 0000000 import json
from hypothesis import strategies as st
# Valid config keys: simple identifiers, no dots or special configparser chars
config_keys = st.from_regex(r"[a-z][a-z0-9_]{0,15}", fullmatch=True)
# Strings safe for config values (no $ to avoid interpolation, no " or \ to
# avoid JSON escaping issues)
config_strings = st.text(
st.characters(whitelist_categories=("L", "N", "Z"), blacklist_characters='$"\\'),
min_size=0,
max_size=20,
)
# Scalar leaf values: str, int, float, bool, None
scalar_leaves = st.one_of(
config_strings,
st.integers(min_value=-(2**31), max_value=2**31),
st.floats(allow_nan=False, allow_infinity=False),
st.booleans(),
st.none(),
)
# JSON leaf values: scalars, or lists/dicts of scalars (arbitrarily nested).
# These are values that get serialized as a single JSON-encoded string in the
# config, rather than expanded into subsections.
_json_values = st.recursive(
scalar_leaves,
lambda children: st.one_of(
st.lists(children, max_size=5),
st.dictionaries(config_keys, children, max_size=5),
),
max_leaves=10,
)
# Only the non-scalar cases are interesting as "json leaves" — scalars are
# already covered by scalar_leaves.
json_leaves = st.one_of(
st.lists(scalar_leaves, min_size=0, max_size=5),
st.dictionaries(config_keys, _json_values, min_size=0, max_size=5),
)
# Basic config leaves (no JSON-encoded collections)
config_leaves = scalar_leaves
# A config node is either a leaf or a dict of config nodes.
config_nodes = st.recursive(
config_leaves,
lambda children: st.dictionaries(config_keys, children, min_size=0, max_size=4),
max_leaves=15,
)
# A valid config must have sections at the top level (all values must be dicts).
config_dicts = st.dictionaries(
config_keys,
st.dictionaries(config_keys, config_nodes, min_size=0, max_size=5),
min_size=1,
max_size=5,
)
def _leaf_to_str(value):
"""Serialize a leaf value the way confection's config format expects."""
if value is None:
return "null"
elif isinstance(value, bool):
return "true" if value else "false"
elif isinstance(value, str):
return json.dumps(value)
elif isinstance(value, float):
return json.dumps(value)
elif isinstance(value, int):
return str(value)
elif isinstance(value, (list, dict)):
return json.dumps(value)
raise TypeError(f"Unexpected leaf type: {type(value)}")
def _flatten_sections(data, prefix="", inline_paths=frozenset()):
"""Convert a nested dict into a list of (section_path, {key: leaf_str}) pairs.
Dicts at paths in inline_paths are serialized as inline JSON values rather
than expanded into subsections.
"""
sections = []
leaves = {}
for key, value in data.items():
key_path = f"{prefix}.{key}" if prefix else key
if isinstance(value, dict) and key_path not in inline_paths:
sections.extend(_flatten_sections(value, key_path, inline_paths))
else:
leaves[key] = _leaf_to_str(value)
if prefix:
sections.insert(0, (prefix, leaves))
return sections
def serialize_with_inline(data, inline_paths=frozenset()):
"""Serialize a nested dict to a config string, inlining dicts at the given paths."""
sections = _flatten_sections(data, inline_paths=inline_paths)
parts = []
for section_name, leaves in sections:
parts.append(f"[{section_name}]")
for key, value_str in leaves.items():
parts.append(f"{key} = {value_str}")
parts.append("")
return "\n".join(parts).strip()
@st.composite
def json_config_dicts(draw):
"""Strategy that produces (data, inline_paths) pairs.
The data is a nested dict suitable for Config. inline_paths is a set of
dotted paths where the value is a JSON-encoded leaf (list or dict) rather
than a config subsection. This distinction matters for serialization: the
library always expands dicts into subsections, but a valid config string
could also have them as inline JSON values.
"""
# Start with a basic config dict (sections with scalar leaves)
base = draw(config_dicts)
inline_paths = set()
# Sprinkle some JSON leaf values into the config
for section_key, section in list(base.items()):
for key in list(section.keys()):
if draw(st.booleans()):
# Replace some leaves with JSON-encoded values
value = draw(json_leaves)
section[key] = value
inline_paths.add(f"{section_key}.{key}")
return base, inline_paths
def _collect_scalar_paths(data, prefix=""):
"""Collect all (dotted_path, value) pairs for scalar leaves in a config dict."""
paths = []
for key, value in data.items():
path = f"{prefix}.{key}" if prefix else key
if isinstance(value, dict):
paths.extend(_collect_scalar_paths(value, path))
elif not isinstance(value, (list, dict)):
paths.append((path, value))
return paths
def _set_at_path(data, path, value):
"""Set a value at a dotted path in a nested dict."""
parts = path.split(".")
node = data
for part in parts[:-1]:
node = node[part]
node[parts[-1]] = value
def _deep_copy(obj):
"""Simple deep copy for nested dicts/lists of scalars."""
if isinstance(obj, dict):
return {k: _deep_copy(v) for k, v in obj.items()}
elif isinstance(obj, list):
return [_deep_copy(v) for v in obj]
return obj
def _make_interpolated_config_str(sections, replacements):
"""Build a config string from sections, applying variable replacements."""
modified = {name: dict(leaves) for name, leaves in sections}
for replace_path, target_path in replacements.items():
section, key = replace_path.rsplit(".", 1)
if section in modified:
modified[section][key] = "${" + target_path + "}"
parts = []
for section_name, _ in sections:
parts.append(f"[{section_name}]")
for key, val in modified.get(section_name, {}).items():
parts.append(f"{key} = {val}")
parts.append("")
return "\n".join(parts).strip()
@st.composite
def interpolated_config(draw):
"""Strategy producing (config_str, expected_dict) with variable interpolation.
Replaces some scalar leaves with ${section.key} references. Avoids cycles
by only referencing values that are never themselves replaced.
"""
base = draw(config_dicts)
scalar_paths = _collect_scalar_paths(base)
# Filter out bools — they serialize as true/false which can't be
# referenced via ${} (configparser treats them specially)
scalar_paths = [(p, v) for p, v in scalar_paths if not isinstance(v, bool)]
if len(scalar_paths) < 2:
return serialize_with_inline(base), base
sections = _flatten_sections(base)
expected = _deep_copy(base)
# Split paths into targets (stable values to reference) and candidates
# (values that may be replaced with refs). A path can't be both.
# Use random subset selection instead of permutations (which is O(n!)).
target_flags = draw(
st.lists(
st.booleans(),
min_size=len(scalar_paths),
max_size=len(scalar_paths),
)
)
targets = [sp for sp, flag in zip(scalar_paths, target_flags) if flag]
candidates = [sp for sp, flag in zip(scalar_paths, target_flags) if not flag]
if not targets or not candidates:
return serialize_with_inline(base), base
replacements = {}
for cand_path, _ in candidates:
if draw(st.booleans()):
target_path, target_value = draw(st.sampled_from(targets))
replacements[cand_path] = target_path
_set_at_path(expected, cand_path, target_value)
config_str = _make_interpolated_config_str(sections, replacements)
return config_str, expected
@st.composite
def circular_interpolated_config(draw):
"""Strategy producing config strings with circular variable references."""
base = draw(config_dicts)
scalar_paths = _collect_scalar_paths(base)
scalar_paths = [(p, v) for p, v in scalar_paths if not isinstance(v, bool)]
if len(scalar_paths) < 2:
from hypothesis import assume
assume(False)
sections = _flatten_sections(base)
# Pick 2+ paths and create a cycle: a -> b -> ... -> a
cycle_len = draw(st.integers(min_value=2, max_value=min(4, len(scalar_paths))))
cycle_indices = draw(
st.lists(
st.sampled_from(range(len(scalar_paths))),
min_size=cycle_len,
max_size=cycle_len,
unique=True,
)
)
cycle_paths = [scalar_paths[i][0] for i in cycle_indices]
replacements = {}
for i in range(cycle_len):
replacements[cycle_paths[i]] = cycle_paths[(i + 1) % cycle_len]
return _make_interpolated_config_str(sections, replacements)
explosion-confection-a98f9d6/tests/test_config_basic.py 0000664 0000000 0000000 00000011503 15160554312 0023476 0 ustar 00root root 0000000 0000000 """Test basic config sections.
* No variable interpolation
* No promises
Just basic structure and JSON-encoded values.
"""
from configparser import InterpolationDepthError
import pytest
from hypothesis import HealthCheck, given, settings
from confection import Config
from tests.strategies import (
circular_interpolated_config,
config_dicts,
interpolated_config,
json_config_dicts,
serialize_with_inline,
)
@given(config_dicts)
def test_roundtrip(data):
"""Config.from_str(config.to_str()) should reproduce the original dict."""
config = Config(data)
serialized = config.to_str(interpolate=False)
restored = Config().from_str(serialized, interpolate=False)
assert dict_equal(restored, data)
@given(json_config_dicts())
def test_json_leaves_parse(pair):
"""Config strings with inline JSON values (lists, dicts) should parse
to the same nested dict structure."""
data, inline_paths = pair
config_str = serialize_with_inline(data, inline_paths)
restored = Config().from_str(config_str, interpolate=False)
assert dict_equal(restored, data)
@given(interpolated_config())
def test_variable_interpolation(pair):
"""Config strings with ${section.key} variable references should resolve
to the referenced values after interpolation."""
config_str, expected = pair
restored = Config().from_str(config_str, interpolate=True)
assert dict_equal(restored, expected)
@settings(suppress_health_check=[HealthCheck.too_slow])
@given(circular_interpolated_config())
def test_circular_interpolation_raises(config_str):
"""Circular variable references should raise an error."""
with pytest.raises((InterpolationDepthError, Exception)):
Config().from_str(config_str, interpolate=True)
def test_star_sections_parse():
"""[section.*.name] creates a dict under the "*" key."""
result = Config().from_str(
"""
[section]
[section.*.first]
x = 1
[section.*.second]
x = 2
""",
interpolate=False,
)
assert result["section"]["*"] == {"first": {"x": 1}, "second": {"x": 2}}
def test_star_sections_roundtrip():
"""Configs with * sections roundtrip through to_str/from_str."""
original = Config().from_str(
"""
[section]
[section.*.a]
x = 1
y = "hello"
[section.*.b]
x = 2
y = "world"
""",
interpolate=False,
)
serialized = original.to_str(interpolate=False)
restored = Config().from_str(serialized, interpolate=False)
assert dict_equal(restored, original)
def test_star_sections_nested():
"""* sections can appear at different levels of nesting."""
result = Config().from_str(
"""
[top]
[top.*.item]
val = 1
[top.*.item.sub]
val = 2
""",
interpolate=False,
)
assert result["top"]["*"]["item"]["val"] == 1
assert result["top"]["*"]["item"]["sub"] == {"val": 2}
def test_star_with_interpolation():
"""Variable interpolation works across * sections."""
result = Config().from_str(
"""
[settings]
lr = 0.001
[models]
[models.*.first]
learning_rate = ${settings.lr}
""",
interpolate=True,
)
assert result["models"]["*"]["first"]["learning_rate"] == 0.001
@pytest.mark.parametrize(
"text,expected_value,expected_type",
[
("True", True, bool),
("False", False, bool),
("true", True, bool),
("false", False, bool),
("None", None, type(None)),
("null", None, type(None)),
],
)
def test_python_style_literals(text, expected_value, expected_type):
"""Python-style True/False/None must be parsed the same as JSON-style
true/false/null. Many existing spaCy configs use the capitalised form."""
cfg = f"[section]\nval = {text}\n"
config = Config().from_str(cfg)
result = config["section"]["val"]
assert result is expected_value, (
f"Expected {expected_value!r} ({expected_type.__name__}), "
f"got {result!r} ({type(result).__name__})"
)
assert type(result) is expected_type
@pytest.mark.parametrize("text", ['"True"', '"False"', '"None"'])
def test_quoted_python_literals_stay_strings(text):
"""JSON-quoted strings like '"True"' must remain strings, not be coerced
to booleans/None."""
cfg = f"[section]\nval = {text}\n"
config = Config().from_str(cfg)
result = config["section"]["val"]
assert isinstance(result, str), (
f"Expected str, got {type(result).__name__}: {result!r}"
)
def dict_equal(a, b) -> bool:
"""Recursively compare two nested dicts, treating empty dicts as equal."""
if type(a) is not type(b) and not (isinstance(a, dict) and isinstance(b, dict)):
return a == b
if isinstance(a, dict):
all_keys = set(a) | set(b)
for key in all_keys:
av = a.get(key, {})
bv = b.get(key, {})
if not dict_equal(av, bv):
return False
return True
return a == b
explosion-confection-a98f9d6/tests/test_config_errors.py 0000664 0000000 0000000 00000013006 15160554312 0023731 0 ustar 00root root 0000000 0000000 """Test error cases in config parsing and validation."""
import pytest
from confection import Config
from confection._errors import ConfigValidationError
def test_malformed_config_string():
"""Malformed config syntax should raise ConfigValidationError."""
with pytest.raises(ConfigValidationError, match="formatted correctly"):
Config().from_str("[[invalid", interpolate=False)
def test_malformed_missing_section_header():
"""Values without any section header are malformed."""
with pytest.raises(ConfigValidationError, match="formatted correctly"):
Config().from_str("key = value", interpolate=False)
def test_top_level_values_without_section():
"""Top-level values that leak into DEFAULT should be caught."""
# configparser treats values before any section header as defaults.
# This is caught by read_string as a MissingSectionHeaderError -> ParsingError.
with pytest.raises(ConfigValidationError):
Config().from_str("x = 1\n[section]\ny = 2", interpolate=False)
def test_missing_parent_section():
"""Dotted section names require all parent sections to exist."""
with pytest.raises(ConfigValidationError, match="not defined"):
Config().from_str("[a.b.c]\nx = 1", interpolate=False)
def test_missing_intermediate_section():
"""[a] and [a.b.c] without [a.b] should fail."""
with pytest.raises(ConfigValidationError, match="not defined"):
Config().from_str("[a]\nx = 1\n\n[a.b.c]\ny = 2", interpolate=False)
def test_key_subsection_conflict():
"""A key that conflicts with a subsection name should be caught."""
with pytest.raises(ConfigValidationError, match="conflicting"):
Config().from_str("[a]\nb = 1\n\n[a.b]\nc = 2", interpolate=False)
def test_override_without_dot():
"""Overrides must have a dotted path (section.key)."""
with pytest.raises(ConfigValidationError, match="overrid"):
Config().from_str("[a]\nx = 1", interpolate=False, overrides={"x": 2})
def test_override_nonexistent_section():
"""Overrides for nonexistent sections should fail."""
with pytest.raises(ConfigValidationError, match="overrid"):
Config().from_str("[a]\nx = 1", interpolate=False, overrides={"b.x": 2})
def test_override_applies():
"""Valid overrides should replace values."""
result = Config().from_str("[a]\nx = 1", interpolate=False, overrides={"a.x": 99})
assert result["a"]["x"] == 99
def test_interpolation_missing_variable():
"""Referencing a nonexistent variable should raise."""
with pytest.raises(Exception):
Config().from_str("[a]\nx = ${b.y}", interpolate=True)
def test_section_ref_in_string():
"""Referencing a whole section inside a string should raise."""
with pytest.raises(ConfigValidationError, match="Can't reference whole sections"):
Config().from_str(
'[defaults]\nlr = 0.001\n\n[a]\nx = "hello ${defaults}"',
interpolate=True,
)
def test_section_reference_resolves():
"""${section} references should resolve to the section dict."""
result = Config().from_str(
"[defaults]\nlr = 0.001\n\n[a]\nsettings = ${defaults}",
interpolate=True,
)
assert result["a"]["settings"] == {"lr": 0.001}
def test_override_replaces_promise_section():
"""Overriding a promise section with a different promise should replace it."""
result = Config().from_str(
"""
[a]
[a.scorer]
@scorers = "old_scorer.v1"
""",
interpolate=False,
overrides={"a.scorer": {"@scorers": "new_scorer.v1"}},
)
assert result["a"]["scorer"]["@scorers"] == "new_scorer.v1"
def test_uninterpolated_variable_preserved():
"""With interpolate=False, variable references should stay as strings."""
result = Config().from_str("[a]\nx = 1\n\n[b]\ny = ${a.x}", interpolate=False)
assert result["b"]["y"] == "${a.x}"
def test_single_quoted_string_warns():
"""Single-quoted values should emit a warning about JSON formatting."""
import warnings
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
Config().from_str("[a]\nx = 'hello'", interpolate=True)
assert any("single-quoted" in str(warning.message) for warning in w)
def test_dollar_dollar_escape():
"""$$ in config values should produce a literal $."""
result = Config().from_str('[a]\nx = "$$100"', interpolate=True)
assert result["a"]["x"] == "$100"
def test_bad_interpolation_syntax():
"""Malformed ${...} reference should raise."""
with pytest.raises(Exception):
Config().from_str("[a]\nx = ${", interpolate=True)
def test_bare_dollar_raises():
"""A bare $ not followed by $ or { should raise."""
with pytest.raises(Exception):
Config().from_str('[a]\nx = "$x"', interpolate=True)
def test_same_section_variable():
"""${key} without a section prefix references the same section."""
result = Config().from_str("[a]\nx = 1\ny = ${x}", interpolate=True)
assert result["a"]["y"] == 1
def test_string_interpolation_coerces_values():
"""Non-string values interpolated into strings should be coerced."""
result = Config().from_str(
'[a]\nx = 42\n\n[b]\ny = "value is ${a.x}"',
interpolate=True,
)
assert result["b"]["y"] == "value is 42"
def test_string_interpolation_unwraps_json_strings():
"""JSON strings interpolated into compound expressions should be unwrapped."""
result = Config().from_str(
'[a]\nx = "hello"\n\n[b]\ny = "${a.x} world"',
interpolate=True,
)
assert result["b"]["y"] == "hello world"
explosion-confection-a98f9d6/tests/test_config_serialization.py 0000664 0000000 0000000 00000015053 15160554312 0025276 0 ustar 00root root 0000000 0000000 """Test Config serialization methods: to_str, from_str, to_bytes, from_bytes,
to_disk, from_disk, copy, merge, interpolate."""
import pytest
from confection import Config
from confection._errors import ConfectionError
# -- to_str / from_str basics (beyond the hypothesis tests) --
def test_to_str_from_str_roundtrip():
data = {"training": {"lr": 0.001, "epochs": 10}, "model": {"name": "cnn"}}
config = Config(data)
restored = Config().from_str(config.to_str(interpolate=False), interpolate=False)
assert dict(restored["training"]) == {"lr": 0.001, "epochs": 10}
assert dict(restored["model"]) == {"name": "cnn"}
def test_empty_config():
config = Config()
assert config.to_str(interpolate=False) == ""
restored = Config().from_str("", interpolate=False)
assert dict(restored) == {}
# -- to_bytes / from_bytes --
def test_to_bytes():
config = Config({"a": {"x": 1}})
b = config.to_bytes(interpolate=False)
assert isinstance(b, bytes)
assert b"[a]" in b
assert b"x = 1" in b
def test_from_bytes_roundtrip():
config = Config({"a": {"x": 1, "y": "hello"}})
b = config.to_bytes(interpolate=False)
restored = Config().from_bytes(b, interpolate=False)
assert restored["a"]["x"] == 1
assert restored["a"]["y"] == "hello"
def test_from_bytes_with_overrides():
config = Config({"a": {"x": 1}})
b = config.to_bytes(interpolate=False)
restored = Config().from_bytes(b, interpolate=False, overrides={"a.x": 99})
assert restored["a"]["x"] == 99
# -- to_disk / from_disk --
def test_to_disk_from_disk_roundtrip(tmp_path):
config = Config({"section": {"key": "value", "num": 42}})
path = tmp_path / "config.cfg"
config.to_disk(path, interpolate=False)
assert path.exists()
restored = Config().from_disk(path, interpolate=False)
assert restored["section"]["key"] == "value"
assert restored["section"]["num"] == 42
def test_to_disk_str_path(tmp_path):
config = Config({"a": {"x": 1}})
path = str(tmp_path / "config.cfg")
config.to_disk(path, interpolate=False)
restored = Config().from_disk(path, interpolate=False)
assert restored["a"]["x"] == 1
def test_from_disk_with_overrides(tmp_path):
config = Config({"a": {"x": 1}})
path = tmp_path / "config.cfg"
config.to_disk(path, interpolate=False)
restored = Config().from_disk(path, interpolate=False, overrides={"a.x": 99})
assert restored["a"]["x"] == 99
# -- copy --
def test_copy_is_deep():
config = Config({"a": {"x": [1, 2, 3]}})
copied = config.copy()
copied["a"]["x"].append(4)
assert config["a"]["x"] == [1, 2, 3]
assert copied["a"]["x"] == [1, 2, 3, 4]
def test_copy_preserves_metadata():
config = Config({"a": {"x": 1}}, is_interpolated=False, section_order=["a"])
copied = config.copy()
assert copied.is_interpolated is False
assert copied.section_order == ["a"]
# -- interpolate --
def test_interpolate():
config = Config().from_str(
"[a]\nx = 1\n\n[b]\ny = ${a.x}",
interpolate=False,
)
assert config["b"]["y"] == "${a.x}"
interpolated = config.interpolate()
assert interpolated["b"]["y"] == 1
# Original should be unchanged
assert config["b"]["y"] == "${a.x}"
def test_interpolate_returns_new_config():
config = Config().from_str("[a]\nx = 1", interpolate=False)
interpolated = config.interpolate()
assert interpolated is not config
# -- merge --
def test_merge_basic():
base = Config({"a": {"x": 1, "y": 2}})
updates = {"a": {"x": 99}}
merged = base.merge(updates)
assert merged["a"]["x"] == 99
assert merged["a"]["y"] == 2
def test_merge_adds_new_keys():
base = Config({"a": {"x": 1}})
updates = {"a": {"x": 1, "y": 2}}
merged = base.merge(updates)
assert merged["a"]["y"] == 2
def test_merge_remove_extra():
"""remove_extra filters keys from updates that aren't in defaults."""
base = Config({"a": {"x": 1}})
updates = {"a": {"x": 99, "extra": "gone"}}
merged = base.merge(updates, remove_extra=True)
assert merged["a"]["x"] == 99
assert "extra" not in merged["a"]
def test_merge_does_not_mutate_original():
base = Config({"a": {"x": 1}})
updates = {"a": {"x": 99}}
base.merge(updates)
assert base["a"]["x"] == 1
def test_merge_deep():
base = Config({"a": {"sub": {"x": 1, "y": 2}}})
updates = {"a": {"sub": {"x": 99}}}
merged = base.merge(updates)
assert merged["a"]["sub"]["x"] == 99
assert merged["a"]["sub"]["y"] == 2
# -- __init__ --
def test_init_from_dict():
config = Config({"a": {"x": 1}})
assert config["a"]["x"] == 1
assert config.is_interpolated is True
def test_init_from_config():
original = Config({"a": {"x": 1}}, is_interpolated=False, section_order=["a"])
copy = Config(original)
assert copy["a"]["x"] == 1
assert copy.is_interpolated is False
assert copy.section_order == ["a"]
def test_init_bad_data():
with pytest.raises(ConfectionError, match="Expected dict"):
Config([1, 2, 3])
def test_init_override_metadata():
original = Config({"a": {"x": 1}}, is_interpolated=False)
copy = Config(original, is_interpolated=True)
assert copy.is_interpolated is True
# -- overrides with interpolation --
# -- deep_merge_configs edge cases --
def test_merge_leaf_vs_dict():
"""When config has a leaf where defaults has a dict, skip merging."""
base = Config({"a": {"sub": {"x": 1}}})
updates = Config({"a": {"sub": "leaf_value"}})
merged = base.merge(updates)
assert merged["a"]["sub"] == "leaf_value"
def test_merge_different_promise():
"""Blocks with different @registry functions should not merge."""
base = Config({"a": {"@cats": "meow.v1", "x": 1}})
updates = Config({"a": {"@cats": "woof.v1", "y": 2}})
merged = base.merge(updates)
# Updates win — base defaults not merged in because different function
assert merged["a"]["@cats"] == "woof.v1"
assert merged["a"]["y"] == 2
assert "x" not in merged["a"]
def test_merge_promise_not_in_defaults():
"""Promise in updates but not in defaults should not merge defaults."""
base = Config({"a": {"x": 1}})
updates = Config({"a": {"@cats": "meow.v1", "y": 2}})
merged = base.merge(updates)
assert merged["a"]["@cats"] == "meow.v1"
assert "x" not in merged["a"]
# -- overrides with interpolation --
def test_overrides_with_interpolation():
result = Config().from_str(
"[a]\nx = 1\n\n[b]\ny = ${a.x}",
interpolate=True,
overrides={"a.x": 42},
)
assert result["a"]["x"] == 42
assert result["b"]["y"] == 42
explosion-confection-a98f9d6/tests/test_errors.py 0000664 0000000 0000000 00000001247 15160554312 0022410 0 ustar 00root root 0000000 0000000 """Test _errors.py edge cases."""
from confection._errors import ConfigValidationError
def test_from_error():
original = ConfigValidationError(
errors=[{"loc": ["a"], "msg": "bad"}],
title="Original",
desc="original desc",
)
new = ConfigValidationError.from_error(original, title="New title")
assert new.title == "New title"
assert new.desc == "original desc"
assert new.errors == original.errors
def test_format_with_parent():
err = ConfigValidationError(
errors=[{"loc": ["x"], "msg": "bad"}],
title="Error",
parent="section",
)
assert "section" in err.text
assert "-> x" in err.text
explosion-confection-a98f9d6/tests/test_issue52.py 0000664 0000000 0000000 00000001402 15160554312 0022364 0 ustar 00root root 0000000 0000000 """Test for issue #52: Config with `*` (positional) args should roundtrip
to str and back.
"""
from confection import Config
def test_config_star_args_to_str():
"""Config.to_str() should not crash when sections contain * args."""
test_cfg = """[test]
foo = "bar"
[test.*]
bar = "foo"
"""
cfg = Config().from_str(test_cfg)
# This should not raise NoSectionError
result = cfg.to_str()
assert "[test]" in result
def test_config_star_args_roundtrip():
"""Config with * args should survive a from_str -> to_str -> from_str cycle."""
test_cfg = """[test]
@cats = "catsie.v1"
evil = false
[test.*.foo]
x = 1
"""
cfg = Config().from_str(test_cfg)
cfg_str = cfg.to_str()
cfg2 = Config().from_str(cfg_str)
assert cfg == cfg2
explosion-confection-a98f9d6/tests/test_issue54.py 0000664 0000000 0000000 00000002733 15160554312 0022376 0 ustar 00root root 0000000 0000000 """Test for issue #54: Error when pydantic validator coerces str to dict.
A custom type with __get_validators__ that converts a string to a dict
should not crash _update_from_parsed.
"""
import sys
import pytest
pydantic = pytest.importorskip("pydantic")
if sys.version_info >= (3, 14):
pytest.skip(
"pydantic v1 is not compatible with Python 3.14+", allow_module_level=True
)
try:
from pydantic.v1 import BaseModel
except ImportError:
from pydantic import BaseModel # type: ignore
from confection import Config, ConfigValidationError, registry
class CastStrAsDict:
@classmethod
def validate(cls, value):
if isinstance(value, str):
return {value: True}
return value
@classmethod
def __get_validators__(cls):
yield cls.validate
class SectionSchema(BaseModel):
value: CastStrAsDict
class MainSchema(BaseModel):
section: SectionSchema
def test_fill_with_validator_str_to_dict():
"""fill() should not crash when a validator coerces str -> dict."""
cfg = Config().from_str(
"""
[section]
value = "ok"
"""
)
# This should not raise TypeError: string indices must be integers
try:
result = registry.fill(cfg, schema=MainSchema)
# If it succeeds, the section should still be present
assert "section" in result
except ConfigValidationError:
# A validation error is acceptable (the type is unusual),
# but TypeError is not.
pass
explosion-confection-a98f9d6/tests/test_issue58.py 0000664 0000000 0000000 00000002434 15160554312 0022400 0 ustar 00root root 0000000 0000000 """Test for issue #58: Registered functions with pydantic BaseModel arguments
should receive constructed model instances, not raw dicts.
"""
import sys
import catalogue
import pytest
pydantic = pytest.importorskip("pydantic")
if sys.version_info >= (3, 14):
pytest.skip(
"pydantic v1 is not compatible with Python 3.14+", allow_module_level=True
)
try:
from pydantic.v1 import BaseModel
except ImportError:
from pydantic import BaseModel # type: ignore
from confection import Config, registry
class Resource(BaseModel):
url: str
@pytest.fixture(autouse=True)
def _register():
if not hasattr(registry, "resources"):
registry.resources = catalogue.create(
"confection", "resources", entry_points=False
)
registry.resources.register("Resource.v1", func=make_resource)
yield
def make_resource(resource: Resource) -> str:
return resource.url
def test_basemodel_arg_receives_model_instance():
"""A registered function with a BaseModel param should receive
the constructed model, not a plain dict."""
config = Config().from_str(
"""
[test]
@resources = "Resource.v1"
[test.resource]
url = "http://example.com"
"""
)
result = registry.resolve(config)
assert result["test"] == "http://example.com"
explosion-confection-a98f9d6/tests/test_issue59.py 0000664 0000000 0000000 00000004126 15160554312 0022401 0 ustar 00root root 0000000 0000000 """Test for issue #59: Registered functions with pydantic class arguments
should receive constructed model instances, not have them flattened to dicts.
This is closely related to #58 but tests the case where the BaseModel arg
is itself resolved from a registered function (nested promise).
"""
import sys
import catalogue
import pytest
pydantic = pytest.importorskip("pydantic")
if sys.version_info >= (3, 14):
pytest.skip(
"pydantic v1 is not compatible with Python 3.14+", allow_module_level=True
)
try:
from pydantic.v1 import BaseModel
except ImportError:
from pydantic import BaseModel # type: ignore
from confection import Config, registry
class Dim(BaseModel):
length: int
@pytest.fixture(autouse=True)
def _register():
if not hasattr(registry, "dims"):
registry.dims = catalogue.create("confection", "dims", entry_points=False)
if not hasattr(registry, "architectures"):
registry.architectures = catalogue.create(
"confection", "architectures", entry_points=False
)
registry.dims.register("Dim.v1", func=make_dim)
registry.architectures.register("arch.v1", func=load_architecture)
yield
def make_dim(length: int) -> Dim:
return Dim(length=length)
def load_architecture(foo: str, dim: Dim) -> str:
# dim should be a Dim instance, not a dict
return f"{foo}:{dim.length}"
def test_nested_basemodel_arg():
"""A registered function receiving a BaseModel resolved from another
registered function should get the model instance."""
config = Config().from_str(
"""
[model]
@architectures = "arch.v1"
foo = "test"
[model.dim]
@dims = "Dim.v1"
length = 6
"""
)
result = registry.resolve(config)
assert result["model"] == "test:6"
def test_direct_basemodel_arg():
"""A registered function receiving a BaseModel from config keys
(not a promise) should get the model instance."""
config = Config().from_str(
"""
[model]
@architectures = "arch.v1"
foo = "test"
[model.dim]
length = 6
"""
)
result = registry.resolve(config)
assert result["model"] == "test:6"
explosion-confection-a98f9d6/tests/test_issue61.py 0000664 0000000 0000000 00000002066 15160554312 0022373 0 ustar 00root root 0000000 0000000 """Test for issue #61: Top-level section references in interpolation.
Users want to reference a top-level section (e.g. ${logger}) from within
another section, instead of having to nest it (${logger.logger}).
Currently this causes an InterpolationDepthError because configparser's
ExtendedInterpolation only resolves ${section.key} not ${section}.
"""
from confection import Config
def test_top_level_section_reference():
"""Referencing a top-level section via ${section} should work."""
config_str = """
[logger]
name = "test_logger"
[training]
batch_size = 32
logger = ${logger}
"""
# This currently raises InterpolationDepthError
cfg = Config().from_str(config_str)
assert cfg["training"]["logger"]["name"] == "test_logger"
def test_nested_section_reference_works():
"""The nested workaround ${section.key} should work as before."""
config_str = """
[logger]
name = "test_logger"
[training]
batch_size = 32
logger_name = ${logger.name}
"""
cfg = Config().from_str(config_str)
assert cfg["training"]["logger_name"] == "test_logger"
explosion-confection-a98f9d6/tests/test_pydantic_shim.py 0000664 0000000 0000000 00000013572 15160554312 0023733 0 ustar 00root root 0000000 0000000 """Tests for backward compatibility with pydantic BaseModel schemas.
These tests verify that downstream libraries (spaCy, thinc, etc.) can
continue passing pydantic BaseModel subclasses to registry.resolve()
and registry.fill() even though confection no longer depends on pydantic.
"""
import sys
import pytest
if sys.version_info >= (3, 14):
pytest.skip(
"pydantic v1 is not compatible with Python 3.14+", allow_module_level=True
)
pydantic = pytest.importorskip("pydantic")
try:
from pydantic.v1 import (
BaseModel,
Field,
StrictFloat,
StrictInt,
StrictStr,
validator,
)
except ImportError:
from pydantic import ( # type: ignore # noqa: F401
BaseModel,
Field,
StrictFloat,
StrictInt,
StrictStr,
validator,
)
from confection.validation import Schema, ValidationError, ensure_schema # noqa: E402
# --- ensure_schema conversion ---
class SimpleSchema(BaseModel):
name: StrictStr = Field(..., title="Name")
value: StrictInt = Field(10, title="Value")
class Config:
extra = "forbid"
class InnerSchema(BaseModel):
x: StrictInt
class Config:
extra = "forbid"
class OuterSchema(BaseModel):
inner: InnerSchema
label: StrictStr = "default"
class Config:
extra = "forbid"
def test_converts_to_schema_subclass():
converted = ensure_schema(SimpleSchema)
assert issubclass(converted, Schema)
def test_extracts_fields():
converted = ensure_schema(SimpleSchema)
assert "name" in converted.model_fields
assert "value" in converted.model_fields
assert converted.model_fields["name"].is_required()
assert not converted.model_fields["value"].is_required()
assert converted.model_fields["value"].default == 10
def test_extracts_config():
converted = ensure_schema(SimpleSchema)
assert converted.model_config["extra"] == "forbid"
def test_schema_passthrough():
class MySchema(Schema):
x: int
assert ensure_schema(MySchema) is MySchema
def test_caching():
a = ensure_schema(SimpleSchema)
b = ensure_schema(SimpleSchema)
assert a is b
def test_nested_conversion():
converted = ensure_schema(OuterSchema)
inner_type = converted.model_fields["inner"].annotation
assert issubclass(inner_type, Schema)
assert "x" in inner_type.model_fields
# --- Validation delegates to pydantic ---
def test_validate_correct_data():
converted = ensure_schema(SimpleSchema)
result = converted.model_validate({"name": "test", "value": 5})
assert result.name == "test"
assert result.value == 5
def test_validate_fills_defaults():
converted = ensure_schema(SimpleSchema)
result = converted.model_validate({"name": "test"})
assert result.value == 10
def test_strict_str_rejects_int():
converted = ensure_schema(SimpleSchema)
with pytest.raises(ValidationError):
converted.model_validate({"name": 123})
def test_extra_fields_rejected():
converted = ensure_schema(SimpleSchema)
with pytest.raises(ValidationError):
converted.model_validate({"name": "x", "extra": 1})
def test_pydantic_validator_works():
class ValidatedModel(BaseModel):
name: StrictStr
class Config:
extra = "forbid"
@validator("name")
def name_must_be_upper(cls, v):
if v != v.upper():
raise ValueError("must be uppercase")
return v
converted = ensure_schema(ValidatedModel)
with pytest.raises(ValidationError):
converted.model_validate({"name": "hello"})
converted.model_validate({"name": "HELLO"})
# --- Pydantic v2 native models ---
import pydantic as _pydantic_v2 # noqa: E402
class V2SimpleSchema(_pydantic_v2.BaseModel):
model_config = _pydantic_v2.ConfigDict(extra="forbid")
name: str
value: int = 10
class V2InnerSchema(_pydantic_v2.BaseModel):
model_config = _pydantic_v2.ConfigDict(extra="forbid")
x: int
class V2OuterSchema(_pydantic_v2.BaseModel):
model_config = _pydantic_v2.ConfigDict(extra="forbid")
inner: V2InnerSchema
label: str = "default"
def test_v2_converts_to_schema():
converted = ensure_schema(V2SimpleSchema)
assert issubclass(converted, Schema)
def test_v2_extracts_fields():
converted = ensure_schema(V2SimpleSchema)
assert "name" in converted.model_fields
assert "value" in converted.model_fields
assert converted.model_fields["name"].is_required()
assert converted.model_fields["value"].default == 10
def test_v2_extracts_config():
converted = ensure_schema(V2SimpleSchema)
assert converted.model_config["extra"] == "forbid"
def test_v2_nested_conversion():
converted = ensure_schema(V2OuterSchema)
inner_type = converted.model_fields["inner"].annotation
assert issubclass(inner_type, Schema)
assert "x" in inner_type.model_fields
def test_v2_validate_correct():
converted = ensure_schema(V2SimpleSchema)
result = converted.model_validate({"name": "test"})
assert result.name == "test"
assert result.value == 10
def test_v2_validate_rejects_extra():
converted = ensure_schema(V2SimpleSchema)
with pytest.raises(ValidationError):
converted.model_validate({"name": "x", "extra": 1})
# --- Config integration with pydantic schema ---
def test_config_from_str_with_pydantic_schema():
"""Config.from_str works with a pydantic schema for validation and defaults."""
from confection import Config
class MyPydanticSchema(BaseModel):
name: StrictStr
value: StrictInt = 10
class Config:
extra = "forbid"
class TopSchema(BaseModel):
section: MyPydanticSchema
config = Config().from_str(
"""
[section]
name = "test"
""",
interpolate=False,
schema=TopSchema,
)
assert config["section"]["name"] == "test"
assert config["section"]["value"] == 10
explosion-confection-a98f9d6/tests/test_registry_fill.py 0000664 0000000 0000000 00000021366 15160554312 0023756 0 ustar 00root root 0000000 0000000 """Test registry.fill() default-filling from function signatures."""
import catalogue
import pytest
from confection import Config, registry
from confection._errors import ConfigValidationError
# --- Test registry setup ---
class _test_registry(registry):
optimizers = catalogue.create("test_fill", "optimizers", entry_points=False)
schedules = catalogue.create("test_fill", "schedules", entry_points=False)
models = catalogue.create("test_fill", "models", entry_points=False)
@_test_registry.optimizers.register("Adam.v1")
def adam(learn_rate: float = 0.001, beta1: float = 0.9, beta2: float = 0.999):
return {"learn_rate": learn_rate, "beta1": beta1, "beta2": beta2}
@_test_registry.optimizers.register("SGD.v1")
def sgd(learn_rate: float = 0.01, momentum: float = 0.9):
return {"learn_rate": learn_rate, "momentum": momentum}
@_test_registry.schedules.register("linear.v1")
def linear(start: float = 0.0, end: float = 1.0, steps: int = 100):
return [start + (end - start) * i / steps for i in range(steps)]
@_test_registry.models.register("cnn.v1")
def cnn(width: int = 128, depth: int = 3, dropout: float = 0.1):
return {"width": width, "depth": depth, "dropout": dropout}
@_test_registry.models.register("no_defaults.v1")
def no_defaults(width: int, depth: int):
return {"width": width, "depth": depth}
# --- Tests ---
def test_fill_basic():
"""Fill adds missing defaults from the registered function."""
config = Config().from_str(
"""
[optimizer]
@optimizers = "Adam.v1"
learn_rate = 0.01
""",
interpolate=False,
)
filled = _test_registry.fill(config)
assert filled["optimizer"]["learn_rate"] == 0.01
assert filled["optimizer"]["beta1"] == 0.9
assert filled["optimizer"]["beta2"] == 0.999
def test_fill_preserves_provided():
"""Values explicitly provided should not be overwritten."""
config = Config().from_str(
"""
[optimizer]
@optimizers = "Adam.v1"
learn_rate = 0.05
beta1 = 0.8
beta2 = 0.99
""",
interpolate=False,
)
filled = _test_registry.fill(config)
assert filled["optimizer"]["learn_rate"] == 0.05
assert filled["optimizer"]["beta1"] == 0.8
assert filled["optimizer"]["beta2"] == 0.99
def test_fill_all_defaults():
"""When no args are provided, all defaults should be filled."""
config = Config().from_str(
"""
[optimizer]
@optimizers = "Adam.v1"
""",
interpolate=False,
)
filled = _test_registry.fill(config)
assert filled["optimizer"]["learn_rate"] == 0.001
assert filled["optimizer"]["beta1"] == 0.9
assert filled["optimizer"]["beta2"] == 0.999
def test_fill_no_defaults():
"""Function with no defaults should not add anything."""
config = Config().from_str(
"""
[model]
@models = "no_defaults.v1"
width = 64
depth = 2
""",
interpolate=False,
)
filled = _test_registry.fill(config)
assert filled["model"]["width"] == 64
assert filled["model"]["depth"] == 2
def test_fill_nested_promise():
"""Defaults are filled recursively into nested promises."""
config = Config().from_str(
"""
[optimizer]
@optimizers = "SGD.v1"
[optimizer.learn_rate]
@schedules = "linear.v1"
end = 0.5
""",
interpolate=False,
)
filled = _test_registry.fill(config)
# Outer promise filled
assert filled["optimizer"]["momentum"] == 0.9
# Inner promise filled
lr = filled["optimizer"]["learn_rate"]
assert lr["start"] == 0.0
assert lr["end"] == 0.5 # provided, not overwritten
assert lr["steps"] == 100
def test_fill_non_promise_sections():
"""Non-promise sections are passed through unchanged."""
config = Config().from_str(
"""
[training]
epochs = 10
[optimizer]
@optimizers = "Adam.v1"
""",
interpolate=False,
)
filled = _test_registry.fill(config)
assert filled["training"]["epochs"] == 10
assert filled["optimizer"]["beta1"] == 0.9
def test_fill_preserves_registry_key():
"""The @registry key should be preserved in the filled config."""
config = Config().from_str(
"""
[optimizer]
@optimizers = "Adam.v1"
""",
interpolate=False,
)
filled = _test_registry.fill(config)
assert filled["optimizer"]["@optimizers"] == "Adam.v1"
def test_fill_returns_config():
"""fill() should return a Config object."""
config = Config().from_str(
"""
[optimizer]
@optimizers = "Adam.v1"
""",
interpolate=False,
)
filled = _test_registry.fill(config)
assert isinstance(filled, Config)
def test_fill_top_level_promise_raises():
"""Top-level config can't be a promise."""
with pytest.raises(ConfigValidationError):
_test_registry.fill({"@optimizers": "Adam.v1"})
def test_fill_with_overrides():
"""Overrides should be applied before filling defaults."""
config = Config().from_str(
"""
[optimizer]
@optimizers = "Adam.v1"
learn_rate = 0.01
""",
interpolate=False,
)
filled = _test_registry.fill(
config, overrides={"optimizer.learn_rate": 0.1}, interpolate=True
)
assert filled["optimizer"]["learn_rate"] == 0.1
assert filled["optimizer"]["beta1"] == 0.9
def test_fill_promise_rejects_extra_args():
"""Extra args not in the function signature should be rejected."""
config = Config(
{
"optimizer": {
"@optimizers": "Adam.v1",
"learn_rate": 0.01,
"metadata": {"nested_model": {"@models": "cnn.v1"}},
}
}
)
with pytest.raises(ConfigValidationError):
_test_registry.fill(config)
def test_fill_nested_non_promise_dict():
"""Non-promise dicts nested under a section with promises should recurse."""
config = Config().from_str(
"""
[section]
[section.sub]
x = 1
[section.model]
@models = "cnn.v1"
width = 64
""",
interpolate=False,
)
filled = _test_registry.fill(config)
assert filled["section"]["sub"] == {"x": 1}
assert filled["section"]["model"]["depth"] == 3
def test_fill_skips_unknown_functions():
"""fill() should skip promises referencing unknown functions, not crash."""
config = Config(
{
"section": {
"known": {"@optimizers": "Adam.v1"},
"unknown": {"@optimizers": "nonexistent.v1"},
}
}
)
filled = _test_registry.fill(config)
# Known function gets defaults filled
assert filled["section"]["known"]["beta1"] == 0.9
# Unknown function passes through unchanged
assert filled["section"]["unknown"] == {"@optimizers": "nonexistent.v1"}
def test_fill_strips_extra_fields_with_schema():
"""fill() with a schema that has extra='forbid' should strip unknown keys."""
from confection.validation import Schema
class TrainingSchema(Schema):
model_config = {"extra": "forbid"}
patience: int = 10
dropout: float = 0.2
class MySchema(Schema):
model_config = {"extra": "forbid"}
training: TrainingSchema
config = Config({"training": {"patience": 5, "extra_field": "hello"}})
filled = _test_registry.fill(config, schema=MySchema)
assert filled["training"]["patience"] == 5
assert filled["training"]["dropout"] == 0.2
assert "extra_field" not in filled["training"]
def test_fill_positional_args():
"""Functions with *args should have positional args filled under '*'."""
@_test_registry.models.register("chain.v1")
def chain(*layers):
return list(layers)
config = Config(
{
"model": {
"@models": "chain.v1",
"*": {
"layer1": {"@models": "cnn.v1", "width": 64},
"layer2": {"@models": "cnn.v1"},
},
}
}
)
filled = _test_registry.fill(config)
# Positional args should pass through, nested promise defaults filled
assert filled["model"]["*"]["layer2"]["depth"] == 3
assert filled["model"]["*"]["layer1"]["width"] == 64
def test_fill_positional_args_not_flagged_as_unexpected():
"""The '*' key should not be flagged as an unexpected argument."""
config = Config(
{
"model": {
"@models": "chain.v1",
"*": {"a": {"@models": "cnn.v1"}},
}
}
)
# Should not raise ConfigValidationError about unexpected '*'
filled = _test_registry.fill(config)
assert "*" in filled["model"]
def test_fill_with_interpolation():
"""fill() with interpolate=True should resolve variables."""
config = Config().from_str(
"""
[hyper]
lr = 0.01
[optimizer]
@optimizers = "Adam.v1"
learn_rate = ${hyper.lr}
""",
interpolate=False,
)
filled = _test_registry.fill(config, interpolate=True)
assert filled["optimizer"]["learn_rate"] == 0.01
assert filled["optimizer"]["beta1"] == 0.9
explosion-confection-a98f9d6/tests/test_registry_resolve.py 0000664 0000000 0000000 00000021741 15160554312 0024504 0 ustar 00root root 0000000 0000000 """Test registry.resolve() and the Promise lifecycle."""
from typing import Callable, List
import catalogue
import pytest
from confection import Config, registry
from confection._constants import ARGS_FIELD_ALIAS, RESERVED_FIELDS_REVERSE
from confection._errors import ConfigValidationError
from confection._registry import (
Promise,
_deep_copy_with_uncopyable,
_is_config_section,
alias_generator,
fix_positionals,
insert_promises,
resolve_promises,
)
# --- Test registry setup ---
class _test_registry(registry):
namespace = "test_resolve"
cats = catalogue.create(namespace, "cats", entry_points=False)
optimizers = catalogue.create(namespace, "optimizers", entry_points=False)
schedules = catalogue.create(namespace, "schedules", entry_points=False)
layers = catalogue.create(namespace, "layers", entry_points=False)
@_test_registry.cats.register("catsie.v1")
def catsie(evil: bool, cute: bool = True) -> str:
if evil:
return "scratch!"
return "meow"
@_test_registry.optimizers.register("Adam.v1")
def adam(learn_rate: float = 0.001, beta1: float = 0.9, beta2: float = 0.999):
return {"learn_rate": learn_rate, "beta1": beta1, "beta2": beta2}
@_test_registry.schedules.register("decay.v1")
def decay(base_rate: float, repeat: int) -> List[float]:
return [base_rate] * repeat
@_test_registry.optimizers.register("cool.v1")
def cool_optimizer(learn_rate, beta1: float = 0.9):
return {"learn_rate": learn_rate, "beta1": beta1}
@_test_registry.layers.register("linear.v1")
def linear_layer(width: int, init: Callable = lambda: None):
return {"width": width, "init": init}
# --- resolve basic ---
def test_resolve_simple():
config = Config({"cat": {"@cats": "catsie.v1", "evil": True}})
result = _test_registry.resolve(config)
assert result["cat"] == "scratch!"
def test_resolve_with_defaults():
config = Config({"cat": {"@cats": "catsie.v1", "evil": False}})
result = _test_registry.resolve(config)
assert result["cat"] == "meow"
def test_resolve_nested_promise():
"""A promise arg can be another promise."""
config = Config(
{
"optimizer": {
"@optimizers": "cool.v1",
"learn_rate": {
"@schedules": "decay.v1",
"base_rate": 0.001,
"repeat": 4,
},
}
}
)
result = _test_registry.resolve(config)
assert result["optimizer"]["learn_rate"] == [0.001] * 4
assert result["optimizer"]["beta1"] == 0.9
def test_resolve_non_promise_passthrough():
config = Config(
{"training": {"epochs": 10}, "cat": {"@cats": "catsie.v1", "evil": True}}
)
result = _test_registry.resolve(config)
assert result["training"] == {"epochs": 10}
assert result["cat"] == "scratch!"
def test_resolve_from_str():
config = Config().from_str("""
[optimizer]
@optimizers = "Adam.v1"
learn_rate = 0.01
""")
result = _test_registry.resolve(config)
assert result["optimizer"]["learn_rate"] == 0.01
assert result["optimizer"]["beta1"] == 0.9
# --- Promise class ---
def test_promise_from_dict():
values = {"@cats": "catsie.v1", "evil": True, "cute": False}
promise = Promise.from_dict(_test_registry, values)
assert promise.registry == "cats"
assert promise.name == "catsie.v1"
assert promise.kwargs == {"evil": True, "cute": False}
def test_promise_resolve():
values = {"@cats": "catsie.v1", "evil": True}
promise = Promise.from_dict(_test_registry, values)
assert promise.resolve() == "scratch!"
def test_promise_return_type():
values = {"@cats": "catsie.v1", "evil": True}
promise = Promise.from_dict(_test_registry, values)
assert promise.return_type is str
# --- insert_promises ---
def test_insert_promises():
config = {"cat": {"@cats": "catsie.v1", "evil": True}, "x": 1}
result = insert_promises(_test_registry, config, resolve=True)
assert isinstance(result["cat"], Promise)
assert result["x"] == 1
def test_insert_promises_nested():
config = {
"section": {
"cat": {"@cats": "catsie.v1", "evil": False},
"val": 42,
}
}
result = insert_promises(_test_registry, config, resolve=True)
assert isinstance(result["section"]["cat"], Promise)
assert result["section"]["val"] == 42
# --- resolve_promises ---
def test_resolve_promises():
config = {"cat": {"@cats": "catsie.v1", "evil": True}, "x": 1}
promised = insert_promises(_test_registry, config, resolve=True)
resolved = resolve_promises(promised)
assert resolved["cat"] == "scratch!"
assert resolved["x"] == 1
# --- fix_positionals ---
def test_fix_positionals_dict_to_tuple():
config = {"*": {"0": "a", "1": "b"}, "x": 1}
result = fix_positionals(config)
assert result["*"] == ("a", "b")
assert result["x"] == 1
def test_fix_positionals_nested():
config = {"section": {"*": {"0": "a"}, "val": 1}}
result = fix_positionals(config)
assert result["section"]["*"] == ("a",)
def test_fix_positionals_list():
config = [{"*": {"0": 1}}, {"x": 2}]
result = fix_positionals(config)
assert result[0]["*"] == (1,)
assert result[1] == {"x": 2}
def test_fix_positionals_tuple():
config = ({"*": {"0": 1}},)
result = fix_positionals(config)
assert result[0]["*"] == (1,)
assert isinstance(result, tuple)
def test_fix_positionals_scalar():
assert fix_positionals(42) == 42
assert fix_positionals("hello") == "hello"
# --- _is_config_section ---
def test_is_config_section():
assert _is_config_section({"a": 1, "b": 2})
assert not _is_config_section([1, 2])
assert not _is_config_section("hello")
assert _is_config_section({})
# --- alias_generator ---
def test_alias_generator_args_field():
assert alias_generator(ARGS_FIELD_ALIAS) == "*"
def test_alias_generator_reserved():
for alias, original in RESERVED_FIELDS_REVERSE.items():
assert alias_generator(alias) == original
def test_alias_generator_normal():
assert alias_generator("some_field") == "some_field"
# --- _deep_copy_with_uncopyable ---
def test_deep_copy_dict():
original = {"a": [1, 2], "b": {"c": 3}}
copied = _deep_copy_with_uncopyable(original)
assert copied == original
copied["a"].append(3)
assert original["a"] == [1, 2]
def test_deep_copy_list():
original = [{"a": 1}, {"b": 2}]
copied = _deep_copy_with_uncopyable(original)
assert copied == original
copied[0]["a"] = 99
assert original[0]["a"] == 1
def test_deep_copy_tuple():
original = ({"a": 1},)
copied = _deep_copy_with_uncopyable(original)
assert copied == original
assert isinstance(copied, tuple)
def test_deep_copy_scalar():
assert _deep_copy_with_uncopyable(42) == 42
assert _deep_copy_with_uncopyable("hello") == "hello"
def test_deep_copy_generator():
"""Generators can't be deepcopied — should pass through."""
def gen():
yield 1
g = gen()
copied = _deep_copy_with_uncopyable(g)
assert copied is g # same object, not copied
def test_deep_copy_memo():
"""Shared references should be preserved."""
shared = [1, 2, 3]
original = {"a": shared, "b": shared}
copied = _deep_copy_with_uncopyable(original)
assert copied["a"] is copied["b"] # still shared
assert copied["a"] is not original["a"] # but different from original
# --- registry.has / registry.get ---
def test_registry_has():
assert _test_registry.has("cats", "catsie.v1")
assert not _test_registry.has("cats", "nonexistent")
assert not _test_registry.has("nonexistent_registry", "anything")
def test_registry_get():
func = _test_registry.get("cats", "catsie.v1")
assert func is catsie
def test_registry_get_unknown_registry():
with pytest.raises(ValueError, match="Unknown registry"):
_test_registry.get("nonexistent", "anything")
def test_registry_get_unknown_func():
with pytest.raises((ValueError, catalogue.RegistryError)):
_test_registry.get("cats", "nonexistent")
# --- get_constructor / parse_args ---
def test_get_constructor():
assert _test_registry.get_constructor({"@cats": "catsie.v1", "evil": True}) == (
"cats",
"catsie.v1",
)
def test_get_constructor_multiple_refs():
with pytest.raises(ConfigValidationError):
_test_registry.get_constructor({"@cats": "catsie.v1", "@dogs": "doggo.v1"})
def test_parse_args():
args, kwargs = _test_registry.parse_args(
{"@cats": "catsie.v1", "evil": True, "cute": False}
)
assert args == []
assert kwargs == {"evil": True, "cute": False}
def test_parse_args_with_positionals():
args, kwargs = _test_registry.parse_args(
{"@cats": "catsie.v1", "*": [1, 2], "evil": True}
)
assert args == [1, 2]
assert kwargs == {"evil": True}
# --- resolve top-level promise raises ---
def test_resolve_top_level_promise():
with pytest.raises(ConfigValidationError):
_test_registry.resolve({"@cats": "catsie.v1", "evil": True})
explosion-confection-a98f9d6/tests/test_schema.py 0000664 0000000 0000000 00000007670 15160554312 0022342 0 ustar 00root root 0000000 0000000 """Test schema validation and default filling at the Config layer."""
from typing import List, Optional
import pytest
from confection import Config
from confection._errors import ConfigValidationError
from confection.validation import Schema
class TrainingSchema(Schema):
patience: int
dropout: float = 0.2
use_vectors: bool = False
class NlpSchema(Schema):
lang: str
pipeline: List[str] = []
class FullSchema(Schema):
training: TrainingSchema
nlp: NlpSchema
# -- fill_defaults --
def test_fill_defaults_top_level():
config = Config({"training": {"patience": 10}})
config.fill_defaults(FullSchema)
assert config["training"]["dropout"] == 0.2
assert config["training"]["use_vectors"] is False
def test_fill_defaults_preserves_existing():
config = Config({"training": {"patience": 10, "dropout": 0.5}})
config.fill_defaults(FullSchema)
assert config["training"]["dropout"] == 0.5
def test_fill_defaults_adds_missing_sections():
config = Config({"training": {"patience": 10}})
config.fill_defaults(FullSchema)
# nlp has all defaults so it should be filled
assert "nlp" not in config # nlp section itself is required, not defaulted
def test_fill_defaults_returns_self():
config = Config({"training": {"patience": 10}})
result = config.fill_defaults(FullSchema)
assert result is config
# -- validate --
def test_validate_passes():
config = Config(
{"training": {"patience": 10, "dropout": 0.2}, "nlp": {"lang": "en"}}
)
config.validate(FullSchema) # should not raise
def test_validate_missing_required():
config = Config({"training": {"dropout": 0.5}, "nlp": {"lang": "en"}})
with pytest.raises(ConfigValidationError):
config.validate(FullSchema)
def test_validate_wrong_type():
config = Config(
{"training": {"patience": "nope", "dropout": 0.2}, "nlp": {"lang": "en"}}
)
with pytest.raises(ConfigValidationError):
config.validate(FullSchema)
# -- from_str with schema --
def test_from_str_with_schema():
config = Config().from_str(
"""
[training]
patience = 10
[nlp]
lang = "en"
""",
interpolate=False,
schema=FullSchema,
)
assert config["training"]["patience"] == 10
assert config["training"]["dropout"] == 0.2
assert config["training"]["use_vectors"] is False
def test_from_str_schema_validates():
with pytest.raises(ConfigValidationError):
Config().from_str(
"""
[training]
dropout = 0.5
[nlp]
lang = "en"
""",
interpolate=False,
schema=FullSchema,
)
def test_from_str_schema_with_interpolation():
config = Config().from_str(
"""
[training]
patience = 10
[nlp]
lang = "en"
""",
interpolate=True,
schema=FullSchema,
)
assert config["training"]["dropout"] == 0.2
# -- Schema with extra="forbid" --
class StrictSchema(Schema):
model_config = {"extra": "forbid"}
x: int
y: str = "default"
def test_validate_extra_forbidden():
config = Config({"x": 1, "y": "hello", "z": "extra"})
with pytest.raises(ConfigValidationError):
config.validate(StrictSchema)
# -- Schema with Optional fields --
class OptionalSchema(Schema):
name: str
description: Optional[str] = None
def test_optional_field_defaults_to_none():
config = Config({"name": "test"})
config.fill_defaults(OptionalSchema)
assert config["description"] is None
def test_optional_field_accepts_value():
config = Config({"name": "test", "description": "hello"})
config.validate(OptionalSchema) # should not raise
# -- Flat schema (no nesting) --
class FlatSchema(Schema):
x: int
y: float = 3.14
z: str = "hello"
def test_flat_schema_from_str():
"""Schema works for flat configs too (top-level keys are leaves, not sections)."""
config = Config({"x": 1})
config.fill_defaults(FlatSchema)
assert config["y"] == 3.14
assert config["z"] == "hello"
explosion-confection-a98f9d6/tests/test_typechecker_edges.py 0000664 0000000 0000000 00000014645 15160554312 0024557 0 ustar 00root root 0000000 0000000 """Tests for typechecker edge cases."""
from dataclasses import dataclass
from enum import Enum
from typing import Annotated, Any
from confection.typechecker import check_type
from confection.validation import Field, Schema
# --- custom_handlers ---
def test_custom_handler():
def handle_str(value, annotation, handlers, ctx):
return value == "magic"
assert check_type("magic", int, custom_handlers={str: handle_str})
assert not check_type("other", int, custom_handlers={str: handle_str})
# --- Annotated with Strict() ---
try:
from pydantic import Strict
except ImportError:
class Strict: # type: ignore
strict = True
def test_strict_int():
assert check_type(42, Annotated[int, Strict()])
assert not check_type(True, Annotated[int, Strict()])
assert not check_type("42", Annotated[int, Strict()])
def test_strict_float():
assert check_type(3.14, Annotated[float, Strict()])
assert not check_type(42, Annotated[float, Strict()])
def test_strict_str():
assert check_type("hi", Annotated[str, Strict()])
assert not check_type(42, Annotated[str, Strict()])
def test_strict_bool():
assert check_type(True, Annotated[bool, Strict()])
assert not check_type(1, Annotated[bool, Strict()])
def test_strict_other():
assert check_type([1], Annotated[list, Strict()])
# --- String forward reference ---
def test_string_annotation_accepts_anything():
assert check_type(42, "SomeForwardRef")
assert check_type("hi", "SomeForwardRef")
# --- String enum ---
class Color(str, Enum):
RED = "red"
GREEN = "green"
def test_string_enum():
assert check_type("red", Color)
assert not check_type("blue", Color)
# --- float with non-numeric ---
def test_float_rejects_list():
assert not check_type([], float)
# --- Type[X] with non-class generic arg ---
def test_type_non_class_arg():
"""Type[X] where X causes TypeError in issubclass should accept."""
from typing import Type
# This would raise TypeError in issubclass
assert check_type(int, Type[Any])
# --- Dataclass with dict ---
@dataclass
class Point:
x: int
y: int
def test_dataclass_accepts_dict():
assert check_type({"x": 1, "y": 2}, Point)
def test_dataclass_dict_validates_fields():
assert not check_type({"x": "bad", "y": 2}, Point)
def test_dataclass_with_dataclass_value():
p = Point(x=1, y=2)
assert check_type(p, Point)
def test_dataclass_value_validates_fields():
p = Point(x=1, y=2)
# Manually set bad type to test field validation
p.x = "bad" # type: ignore
assert not check_type(p, Point)
# --- Schema with aliased field in decompose ---
class AliasedSchema(Schema):
x: int = Field(default=..., alias="x_alias")
def test_schema_decompose_alias():
assert check_type({"x_alias": 42}, AliasedSchema)
assert not check_type({"x_alias": "bad"}, AliasedSchema)
# --- Fall-through return False ---
def test_unknown_annotation():
"""An annotation that doesn't match any branch returns False."""
# A module object isn't a type, not a TypeVar, not a string, etc.
import os
assert not check_type(42, os)
# --- Pydantic v1 validator edges ---
def test_pydantic_v1_validator_many_params():
"""Validators with >2 params are skipped."""
class MyType:
@classmethod
def __get_validators__(cls):
def three_params(v, field, config):
return v
yield three_params
# Should still pass (validator is skipped)
assert check_type(42, MyType)
def test_pydantic_v1_validator_sig_error():
"""Validators with un-inspectable signatures default to 1 param."""
class MyType:
@classmethod
def __get_validators__(cls):
yield len # len's signature can be inspected, but let's test the path
# len(42) will raise TypeError
assert not check_type(42, MyType)
# --- Pydantic v2 match edges ---
def test_pydantic_v2_no_function_key():
"""Schema without 'function' key in result."""
class MyType:
@classmethod
def __get_pydantic_core_schema__(cls, source, handler):
return {"type": "any"} # no function key
assert not check_type(42, MyType)
def test_pydantic_v2_isinstance_shortcut():
"""If value is already an instance, skip validator."""
class MyType:
@classmethod
def __get_pydantic_core_schema__(cls, source, handler):
return {"type": "any"}
assert check_type(MyType(), MyType)
# --- Parameter.empty ---
def test_parameter_empty():
import inspect
assert check_type(42, inspect.Parameter.empty)
assert check_type("anything", inspect.Parameter.empty)
# --- Type[Union[...]] ---
def test_type_union_arg():
"""Type[Union[int, str]] — issubclass raises TypeError, should accept."""
from typing import Type, Union
assert check_type(int, Type[Union[int, str]])
# --- _pydantic_v1_match isinstance shortcut ---
def test_pydantic_v1_isinstance_shortcut():
class MyType:
@classmethod
def __get_validators__(cls):
yield lambda v: v
assert check_type(MyType(), MyType)
# --- _pydantic_v1_match uninspectable signature ---
def test_pydantic_v1_uninspectable_sig():
"""Validator with uninspectable signature defaults to 1 param."""
class MyType:
@classmethod
def __get_validators__(cls):
# A builtin with no inspectable sig in some Python versions
class Uninspectable:
def __call__(self, v):
if not isinstance(v, int):
raise ValueError
return v
# Make signature() raise
__signature__ = property(lambda self: (_ for _ in ()).throw(ValueError))
yield Uninspectable()
assert check_type(42, MyType)
assert not check_type("hi", MyType)
# --- TypeVar bound that fails isinstance ---
def test_typevar_bound_isinstance_typeerror():
"""TypeVar bound with ForwardRef that fails isinstance falls back to outer_match."""
from typing import ForwardRef, TypeVar
T = TypeVar("T", bound=ForwardRef("NonExistent"))
# ForwardRef can't be used with isinstance — falls back to outer_match
# which accepts ForwardRefs
assert check_type(42, T)
# --- ctx default ---
def test_check_type_creates_default_ctx():
"""check_type works without explicit ctx."""
assert check_type(42, int)
assert not check_type("hi", int)
explosion-confection-a98f9d6/tests/test_util.py 0000664 0000000 0000000 00000006007 15160554312 0022050 0 ustar 00root root 0000000 0000000 """Test util.py: frozen collections, partial, try_dump_json."""
import copy
import pytest
from confection._errors import ConfigValidationError
from confection.util import (
SimpleFrozenDict,
SimpleFrozenList,
partial,
try_dump_json,
)
# --- SimpleFrozenDict ---
def test_frozen_dict_init():
d = SimpleFrozenDict({"a": 1, "b": 2})
assert d["a"] == 1
def test_frozen_dict_setitem():
d = SimpleFrozenDict()
with pytest.raises(NotImplementedError):
d["x"] = 1
def test_frozen_dict_pop():
d = SimpleFrozenDict({"a": 1})
with pytest.raises(NotImplementedError):
d.pop("a")
def test_frozen_dict_update():
d = SimpleFrozenDict()
with pytest.raises(NotImplementedError):
d.update({"x": 1})
def test_frozen_dict_deepcopy():
d = SimpleFrozenDict({"a": [1, 2]})
d2 = copy.deepcopy(d)
assert isinstance(d2, SimpleFrozenDict)
assert d2["a"] == [1, 2]
assert d2["a"] is not d["a"]
def test_frozen_dict_custom_error():
d = SimpleFrozenDict(error="custom error")
with pytest.raises(NotImplementedError, match="custom error"):
d["x"] = 1
# --- SimpleFrozenList ---
def test_frozen_list_init():
lst = SimpleFrozenList([1, 2, 3])
assert lst[0] == 1
def test_frozen_list_append():
lst = SimpleFrozenList()
with pytest.raises(NotImplementedError):
lst.append(1)
def test_frozen_list_clear():
lst = SimpleFrozenList([1])
with pytest.raises(NotImplementedError):
lst.clear()
def test_frozen_list_extend():
lst = SimpleFrozenList()
with pytest.raises(NotImplementedError):
lst.extend([1])
def test_frozen_list_insert():
lst = SimpleFrozenList()
with pytest.raises(NotImplementedError):
lst.insert(0, 1)
def test_frozen_list_pop():
lst = SimpleFrozenList([1])
with pytest.raises(NotImplementedError):
lst.pop()
def test_frozen_list_remove():
lst = SimpleFrozenList([1])
with pytest.raises(NotImplementedError):
lst.remove(1)
def test_frozen_list_reverse():
lst = SimpleFrozenList([1, 2])
with pytest.raises(NotImplementedError):
lst.reverse()
def test_frozen_list_sort():
lst = SimpleFrozenList([2, 1])
with pytest.raises(NotImplementedError):
lst.sort()
def test_frozen_list_deepcopy():
lst = SimpleFrozenList([[1, 2], [3]])
lst2 = copy.deepcopy(lst)
assert isinstance(lst2, SimpleFrozenList)
assert lst2 == [[1, 2], [3]]
assert lst2[0] is not lst[0]
def test_frozen_list_custom_error():
lst = SimpleFrozenList(error="nope")
with pytest.raises(NotImplementedError, match="nope"):
lst.append(1)
# --- partial ---
def test_partial_basic():
def add(a, b):
"""Add two numbers."""
return a + b
p = partial(add, 1)
assert p(2) == 3
assert p.__doc__ == "Add two numbers."
# --- try_dump_json error ---
def test_try_dump_json_unserializable():
with pytest.raises(ConfigValidationError, match="Couldn't serialize"):
try_dump_json(object())
explosion-confection-a98f9d6/tests/test_validate_type.py 0000664 0000000 0000000 00000027614 15160554312 0023734 0 ustar 00root root 0000000 0000000 """Tests for validate_type covering all type branches."""
from pathlib import Path, PurePath
from typing import (
Any,
Callable,
Dict,
FrozenSet,
Iterable,
Iterator,
List,
Literal,
Mapping,
NewType,
Optional,
Sequence,
Set,
Tuple,
Type,
TypeVar,
Union,
)
import pytest
from confection.validation import (
Field,
FieldInfo,
Schema,
_validate_schema,
create_schema,
ensure_schema,
validate_type,
)
# === None ===
def test_none():
assert validate_type(None, type(None)) is None
assert validate_type(42, type(None)) is not None
# === Annotated ===
try:
from typing import Annotated
except ImportError:
Annotated = None # type: ignore
@pytest.mark.skipif(Annotated is None, reason="Annotated not available")
def test_annotated():
assert validate_type(42, Annotated[int, "metadata"]) is None
assert validate_type("hi", Annotated[int, "metadata"]) is not None
# === Union / Optional ===
def test_union():
assert validate_type(42, Union[int, str]) is None
assert validate_type("hi", Union[int, str]) is None
assert validate_type(3.14, Union[int, str]) is not None
def test_optional():
assert validate_type(None, Optional[int]) is None
assert validate_type(42, Optional[int]) is None
assert validate_type("hi", Optional[int]) is not None
def test_union_pipe_syntax():
assert validate_type(42, int | str) is None
assert validate_type("hi", int | str) is None
assert validate_type(3.14, int | str) is not None
# === Literal ===
def test_literal():
assert validate_type("a", Literal["a", "b"]) is None
assert validate_type("b", Literal["a", "b"]) is None
assert validate_type("c", Literal["a", "b"]) is not None
assert validate_type(1, Literal[1, 2]) is None
assert validate_type(3, Literal[1, 2]) is not None
# === NewType ===
def test_newtype():
UserId = NewType("UserId", int)
assert validate_type(42, UserId) is None
assert validate_type("hi", UserId) is not None
# === TypeVar ===
def test_typevar_unbound():
T = TypeVar("T")
assert validate_type("anything", T) is None
def test_typevar_bound():
T = TypeVar("T", bound=int)
assert validate_type(42, T) is None
assert validate_type("hi", T) is not None
def test_typevar_bound_union():
T = TypeVar("T", bound=Union[str, int])
assert validate_type("hello", T) is None
assert validate_type(42, T) is None
assert validate_type(3.14, T) is not None
def test_typevar_bound_union_negative():
T = TypeVar("T", bound=Union[None, float])
assert validate_type(None, T) is None
assert validate_type(3.14, T) is None
assert validate_type("hello", T) is not None
def test_typevar_constraints():
T = TypeVar("T", int, str)
assert validate_type(42, T) is None
assert validate_type("hi", T) is None
assert validate_type(3.14, T) is not None
# === Plain types ===
def test_bool():
assert validate_type(True, bool) is None
assert validate_type(1, bool) is not None
def test_int():
assert validate_type(42, int) is None
assert validate_type(True, int) is not None # bool is not accepted as int
assert validate_type("123", int) is None # string coercion
assert validate_type("abc", int) is not None
def test_float():
assert validate_type(3.14, float) is None
assert validate_type(42, float) is None # int accepted for float
assert validate_type(True, float) is not None
assert validate_type("3.14", float) is None # string coercion
assert validate_type("abc", float) is not None
def test_str():
assert validate_type("hello", str) is None
assert validate_type(42, str) is not None
def test_path():
assert validate_type(Path("/tmp"), Path) is None
assert validate_type("/tmp/foo", Path) is None # string coercion
assert validate_type(PurePath("/tmp"), PurePath) is None
assert validate_type("/tmp", PurePath) is None
assert validate_type(42, Path) is not None
# === Callable ===
def test_callable():
assert validate_type(lambda: None, Callable) is None
assert validate_type(len, Callable) is None
assert validate_type(42, Callable) is not None
# === List ===
def test_list():
assert validate_type([1, 2, 3], list) is None
assert validate_type([1, 2, 3], List[int]) is None
assert validate_type([1, "a"], List[int]) is not None
assert validate_type("not a list", list) is not None
# === Dict ===
def test_dict():
assert validate_type({"a": 1}, dict) is None
assert validate_type({"a": 1}, Dict[str, int]) is None
assert validate_type({"a": "b"}, Dict[str, int]) is not None
assert validate_type({1: "a"}, Dict[str, int]) is not None
# === Tuple ===
def test_tuple_bare():
assert validate_type((1, 2), tuple) is None
assert validate_type("hi", tuple) is not None
def test_tuple_fixed():
assert validate_type((1, "a"), Tuple[int, str]) is None
assert validate_type((1, 2), Tuple[int, str]) is not None
assert validate_type((1,), Tuple[int, str]) is not None # wrong length
assert validate_type((1, "a", 3), Tuple[int, str]) is not None
def test_tuple_variable():
assert validate_type((1, 2, 3), Tuple[int, ...]) is None
assert validate_type((), Tuple[int, ...]) is None
assert validate_type((1, "a"), Tuple[int, ...]) is not None
# === Set / FrozenSet ===
def test_set():
assert validate_type({1, 2}, set) is None
assert validate_type({1, 2}, Set[int]) is None
assert validate_type({1, "a"}, Set[int]) is not None
assert validate_type([1, 2], set) is not None
def test_frozenset():
assert validate_type(frozenset([1, 2]), frozenset) is None
assert validate_type(frozenset([1, 2]), FrozenSet[int]) is None
assert validate_type(frozenset([1, "a"]), FrozenSet[int]) is not None
assert validate_type({1, 2}, FrozenSet[int]) is not None
# === Sequence ===
def test_sequence():
assert validate_type([1, 2], Sequence[int]) is None
assert validate_type((1, 2), Sequence[int]) is None
assert validate_type("hello", Sequence) is None # str is a Sequence
assert validate_type([1, "a"], Sequence[int]) is not None
assert validate_type(42, Sequence) is not None
# === Iterable ===
def test_iterable():
assert validate_type([1, 2], Iterable) is None
assert validate_type("hi", Iterable) is None
assert validate_type(42, Iterable) is not None
# === Mapping ===
def test_mapping():
assert validate_type({"a": 1}, Mapping[str, int]) is None
assert validate_type(42, Mapping) is not None
# === Iterator ===
def test_iterator():
assert validate_type(iter([1, 2]), Iterator) is None
assert validate_type(42, Iterator) is not None
# === Type[X] ===
def test_type():
assert validate_type(int, Type[int]) is None
assert validate_type(bool, Type[int]) is None # subclass
assert validate_type(str, Type[int]) is not None
assert validate_type(42, Type[int]) is not None
assert validate_type(int, Type[Any]) is None
# === Schema-as-dict validation ===
def test_schema_as_dict():
MySchema = create_schema(
"MySchema",
__config__={"extra": "forbid"},
x=(int, Field(...)),
y=(str, Field("default")),
)
assert validate_type({"x": 1}, MySchema) is None
assert validate_type({"x": 1, "y": "hi"}, MySchema) is None
assert validate_type(42, MySchema) is not None # not a dict
# === Pydantic hooks ===
def test_pydantic_core_schema_hook():
"""Types with __get_pydantic_core_schema__ get their validator called."""
class MyType:
@classmethod
def __get_pydantic_core_schema__(cls, source_type, handler):
def validate(v):
if not isinstance(v, int):
raise ValueError("expected int")
return v
return {
"type": "function-plain",
"function": {"type": "no-info", "function": validate},
}
assert validate_type(42, MyType) is None
assert validate_type("hi", MyType) is not None
def test_pydantic_v1_validators_hook():
"""Types with __get_validators__ get their validators called."""
class MyType:
@classmethod
def __get_validators__(cls):
def check_positive(v):
if not isinstance(v, int) or v <= 0:
raise ValueError("must be positive int")
return v
yield check_positive
assert validate_type(5, MyType) is None
assert validate_type(-1, MyType) is not None
assert validate_type("hi", MyType) is not None
# === Generator passthrough ===
def test_generator_passthrough():
def gen():
yield 1
g = gen()
assert validate_type(g, int) is None # generators always pass
assert next(g) == 1 # not consumed
# === _validate_schema ===
def test_validate_schema_extra_forbid():
fields = {"x": FieldInfo(default=...)}
fields["x"].annotation = int
config = {"extra": "forbid"}
errors = _validate_schema({"x": 1, "extra_key": 2}, fields, config, None)
assert any("Extra inputs" in e["msg"] for e in errors)
def test_validate_schema_extra_allow():
fields = {"x": FieldInfo(default=...)}
fields["x"].annotation = int
config = {"extra": "allow"}
errors = _validate_schema({"x": 1, "extra_key": 2}, fields, config, None)
assert not errors
def test_validate_schema_missing_required():
fields = {"x": FieldInfo(default=...)}
fields["x"].annotation = int
config = {"extra": "forbid"}
errors = _validate_schema({}, fields, config, None)
assert any("required" in e["msg"].lower() for e in errors)
def test_validate_schema_alias():
f = FieldInfo(default=..., alias="x_alias")
f.annotation = int
fields = {"x": f}
config = {"extra": "forbid"}
errors = _validate_schema({"x_alias": 1}, fields, config, None)
assert not errors
def test_validate_schema_alias_generator():
f = FieldInfo(default=...)
f.annotation = int
fields = {"my_field": f}
config = {"extra": "forbid"}
errors = _validate_schema(
{"MY_FIELD": 1}, fields, config, lambda name: name.upper()
)
assert not errors
# === ensure_schema ===
def test_ensure_schema_passthrough():
"""Schema subclass passes through unchanged."""
class MySchema(Schema):
model_config = {"extra": "forbid"}
assert ensure_schema(MySchema) is MySchema
def test_ensure_schema_from_our_schema():
"""create_schema output passes through."""
s = create_schema("Test", __config__={"extra": "forbid"}, x=(int, Field(...)))
assert ensure_schema(s) is s
# === model_dump ===
def test_model_dump():
MySchema = create_schema(
"Test",
__config__={"extra": "forbid"},
x=(int, Field(...)),
y=(str, Field("default")),
)
instance = MySchema(x=1)
assert instance.model_dump() == {"x": 1, "y": "default"}
# === Schema.from_function ===
def test_from_function_basic():
def my_func(x: int, y: str = "hello"):
pass
schema = Schema.from_function(my_func)
assert "x" in schema.model_fields
assert "y" in schema.model_fields
assert schema.model_fields["x"].annotation is int
assert schema.model_fields["y"].default == "hello"
def test_from_function_var_positional():
def my_func(*args: int):
pass
schema = Schema.from_function(my_func)
assert "args" in schema.model_fields
# Should be Sequence[int]
ann = schema.model_fields["args"].annotation
assert hasattr(ann, "__origin__") # is generic
def test_from_function_var_keyword_skipped():
def my_func(x: int, **kwargs):
pass
schema = Schema.from_function(my_func)
assert "x" in schema.model_fields
assert "kwargs" not in schema.model_fields
def test_from_function_no_annotations():
def my_func(x, y=10):
pass
schema = Schema.from_function(my_func)
assert schema.model_fields["x"].annotation is Any
assert schema.model_fields["y"].default == 10
explosion-confection-a98f9d6/tests/test_validation_edges.py 0000664 0000000 0000000 00000020114 15160554312 0024367 0 ustar 00root root 0000000 0000000 """Tests for edge cases in validation.py."""
import sys
from typing import Optional
import pytest
from confection.validation import (
Field,
FieldInfo,
Schema,
_is_pydantic_model,
_pydantic_instance_to_dict,
_validate_schema,
create_schema,
ensure_schema,
validate_type,
)
# --- Schema with FieldInfo as class default ---
class SchemaWithFieldInfo(Schema):
x: int = Field(42)
y: str = Field("hello")
def test_field_info_as_class_default():
assert SchemaWithFieldInfo.model_fields["x"].default == 42
assert SchemaWithFieldInfo.model_fields["y"].default == "hello"
def test_field_info_required():
class RequiredSchema(Schema):
x: int = Field(...)
assert RequiredSchema.model_fields["x"].is_required()
# --- Schema field aliases ---
class AliasedSchema(Schema):
model_config = {"extra": "forbid"}
x: int = Field(default=..., alias="x_alias")
def test_model_validate_with_alias():
result = AliasedSchema.model_validate({"x_alias": 42})
assert result.x_alias == 42
def test_model_validate_alias_fills_default():
class AliasDefault(Schema):
x: int = Field(default=10, alias="x_alias")
result = AliasDefault.model_validate({})
assert result.x_alias == 10
# --- Schema with alias_generator ---
class AliasGenSchema(Schema):
model_config = {"extra": "forbid", "alias_generator": lambda name: name.upper()}
my_field: int
def test_model_validate_with_alias_generator():
result = AliasGenSchema.model_validate({"MY_FIELD": 42})
assert result.MY_FIELD == 42
def test_model_validate_alias_generator_fills_default():
class AliasGenDefault(Schema):
model_config = {"alias_generator": lambda name: name.upper()}
x: int = 99
result = AliasGenDefault.model_validate({})
assert result.X == 99
# --- model_dump with nested Schema ---
class Inner(Schema):
a: int
class Outer(Schema):
inner: Inner
b: str = "hi"
def test_model_dump_nested():
instance = Outer(inner=Inner(a=1))
dumped = instance.model_dump()
assert dumped == {"inner": {"a": 1}, "b": "hi"}
# --- create_schema edge cases ---
def test_create_schema_no_config():
s = create_schema("NoConfig", x=(int, Field(...)))
assert s.model_config == {"extra": "allow"}
def test_create_schema_bad_field():
with pytest.raises(ValueError, match="must be"):
create_schema("Bad", x="not a tuple")
def test_create_schema_with_alias_generator():
s = create_schema(
"AliasGen",
__config__={"extra": "forbid", "alias_generator": lambda n: n.upper()},
my_field=(int, Field(...)),
)
assert s.model_fields["my_field"].alias == "MY_FIELD"
def test_create_schema_plain_default():
"""Non-FieldInfo second element in tuple gets wrapped."""
s = create_schema("PlainDefault", x=(int, 42))
assert s.model_fields["x"].default == 42
# --- resolve_type_hints fallback ---
def test_from_function_unresolvable_forward_ref():
"""Forward refs that can't be resolved fall back to raw annotations."""
# Create a function with an annotation that can't be resolved
def func(x: "NonExistentType") -> None: # noqa: F821
pass
schema = Schema.from_function(func)
# Should still create the schema, just with Any or the raw annotation
assert "x" in schema.model_fields
# --- _error_type_for branches ---
def test_validate_type_str_error():
err = validate_type(42, str)
assert err is not None
def test_validate_type_float_error():
err = validate_type("abc", float)
assert err is not None
def test_validate_type_bool_error():
err = validate_type(1, bool)
assert err is not None
def test_validate_type_complex_error():
"""Non-primitive type should give value_error."""
from typing import List
err = validate_type("not a list", List[int])
assert err is not None
# --- _validate_schema error types ---
def test_validate_schema_str_error_type():
f = FieldInfo(default=...)
f.annotation = str
errors = _validate_schema({"x": 42}, {"x": f}, {"extra": "allow"}, None)
assert errors[0]["type"] == "string_type"
def test_validate_schema_float_error_type():
f = FieldInfo(default=...)
f.annotation = float
errors = _validate_schema({"x": "abc"}, {"x": f}, {"extra": "allow"}, None)
assert errors[0]["type"] == "float_parsing"
def test_validate_schema_bool_error_type():
f = FieldInfo(default=...)
f.annotation = bool
errors = _validate_schema({"x": 1}, {"x": f}, {"extra": "allow"}, None)
assert errors[0]["type"] == "bool_type"
def test_validate_schema_positive_int_error_type():
from confection.validation import PositiveInt
f = FieldInfo(default=...)
f.annotation = PositiveInt
errors = _validate_schema({"x": -1}, {"x": f}, {"extra": "allow"}, None)
assert errors[0]["type"] == "int_parsing"
def test_validate_schema_strict_float_error_type():
from confection.validation import StrictFloat
f = FieldInfo(default=...)
f.annotation = StrictFloat
errors = _validate_schema({"x": "abc"}, {"x": f}, {"extra": "allow"}, None)
assert errors[0]["type"] == "float_parsing"
def test_validate_schema_generic_error_type():
from typing import List
f = FieldInfo(default=...)
f.annotation = List[int]
errors = _validate_schema({"x": "nope"}, {"x": f}, {"extra": "allow"}, None)
assert errors[0]["type"] == "value_error"
# --- _is_pydantic_model edge cases ---
def test_is_pydantic_model_non_type():
assert _is_pydantic_model("not a type") is False
def test_is_pydantic_model_schema_subclass():
class MySchema(Schema):
x: int
assert _is_pydantic_model(MySchema) is False
# --- _pydantic_instance_to_dict ---
def test_pydantic_instance_to_dict_v2():
import pydantic
class M(pydantic.BaseModel):
x: int = 1
assert _pydantic_instance_to_dict(M()) == {"x": 1}
@pytest.mark.skipif(
sys.version_info >= (3, 14), reason="pydantic v1 not compatible with 3.14+"
)
def test_pydantic_instance_to_dict_v1():
from pydantic.v1 import BaseModel
class M(BaseModel):
x: int = 1
assert _pydantic_instance_to_dict(M()) == {"x": 1}
def test_pydantic_instance_to_dict_plain():
assert _pydantic_instance_to_dict(42) == 42
# --- ensure_schema passthrough for non-pydantic ---
def test_ensure_schema_plain_class():
class NotAModel:
pass
assert ensure_schema(NotAModel) is NotAModel
# --- v1 allow_none (Optional) ---
@pytest.mark.skipif(
sys.version_info >= (3, 14), reason="pydantic v1 not compatible with 3.14+"
)
def test_v1_optional_field():
from pydantic.v1 import BaseModel as V1Model
class M(V1Model):
x: Optional[int] = None
converted = ensure_schema(M)
# Should accept None
result = converted.model_validate({"x": None})
assert result.x is None
# --- v2 model with pydantic instance default ---
def test_v2_pydantic_instance_default():
import pydantic
class Inner(pydantic.BaseModel):
x: int = 1
class Outer(pydantic.BaseModel):
inner: Inner = Inner()
converted = ensure_schema(Outer)
# Default should be converted from pydantic instance to dict
assert converted.model_fields["inner"].default == {"x": 1}
# --- v1 model with pydantic instance default ---
@pytest.mark.skipif(
sys.version_info >= (3, 14), reason="pydantic v1 not compatible with 3.14+"
)
def test_v1_pydantic_instance_default():
from pydantic.v1 import BaseModel as V1Model
class Inner(V1Model):
x: int = 1
class Outer(V1Model):
inner: Inner = Inner()
converted = ensure_schema(Outer)
assert converted.model_fields["inner"].default == {"x": 1}
# --- Schema inheritance ---
class BaseSchema(Schema):
x: int = 1
class DerivedSchema(BaseSchema):
y: str = "hello"
def test_schema_inheritance():
assert "x" in DerivedSchema.model_fields
assert "y" in DerivedSchema.model_fields
assert DerivedSchema.model_fields["x"].default == 1
def test_schema_inheritance_override():
class Override(BaseSchema):
x: int = 99
assert Override.model_fields["x"].default == 99