././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1785733024.9521163 condense_json-1.1/0000755000175100017510000000000015234017641013653 5ustar00runnerrunner././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785733020.0 condense_json-1.1/LICENSE0000644000175100017510000002613515234017634014671 0ustar00runnerrunner Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1785733024.952081 condense_json-1.1/PKG-INFO0000644000175100017510000003106215234017641014752 0ustar00runnerrunnerMetadata-Version: 2.4 Name: condense-json Version: 1.1 Summary: Python function for condensing JSON using replacement strings Author: Simon Willison License-Expression: Apache-2.0 Project-URL: Homepage, https://github.com/simonw/condense-json Project-URL: Changelog, https://github.com/simonw/condense-json/releases Project-URL: Issues, https://github.com/simonw/condense-json/issues Project-URL: CI, https://github.com/simonw/condense-json/actions Classifier: Typing :: Typed Requires-Python: >=3.10 Description-Content-Type: text/markdown License-File: LICENSE Dynamic: license-file # condense-json [![PyPI](https://img.shields.io/pypi/v/condense-json.svg)](https://pypi.org/project/condense-json/) [![Tests](https://github.com/simonw/condense-json/actions/workflows/test.yml/badge.svg)](https://github.com/simonw/condense-json/actions/workflows/test.yml) [![Changelog](https://img.shields.io/github/v/release/simonw/condense-json?include_prereleases&label=changelog)](https://github.com/simonw/condense-json/releases) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/condense-json/blob/main/LICENSE) Python function for condensing JSON using replacement strings and values ## Installation Install this library using `pip`: ```bash pip install condense-json ``` ## Usage The `condense_json` function searches a JSON-like object for content that matches entries in a replacements mapping: strings that contain replacement substrings, subtrees that are structurally equal to replacement dicts or lists, and dicts that are *close* to a replacement dict, which are stored as that base plus a patch. It replaces each with a compact reference, making the JSON more concise. The `uncondense_json` function reverses this process. **`condense_json(obj: JSONInput, replacements: Mapping[str, Any]) -> Any`** * **`obj`**: The JSON value to condense - any nesting of dictionaries, lists, strings, numbers, booleans and `None`. Top-level lists and strings work too, not just dictionaries. * **`replacements`**: A mapping where keys are replacement IDs (e.g., "1", "2") and values are the content they represent - strings, which match as substrings, or dicts and lists, which match whole subtrees (see [Structural replacements](#structural-replacements-for-dicts-and-lists)); dict values additionally serve as bases for near-matches (see [Merge references](#merge-references-a-base-object-plus-a-patch)). Entries with empty values (`None`, `""`, `{}`, `[]`) or non-string scalar values are ignored. `JSONInput` is a recursive type alias covering anything representable in JSON, built from covariant container types so that narrowly typed values such as `dict[str, str]` are accepted without any extra annotation. Results are typed `Any`, so they can be indexed, iterated and serialized without narrowing. ```python JSONInput = Union[ str, int, float, bool, None, "Sequence[JSONInput]", "Mapping[str, JSONInput]" ] ``` The function returns a modified version of the input `obj` with matches replaced by references. Three reference forms appear in the output: - `{"$": replacement_id}` - a whole value that matched: a string consisting *entirely* of a replacement string, or a dict or list structurally equal to a replacement value. - `{"$r": [ ...segments... ]}` - a string that *contains* one or more replacement strings, broken into literal segments and `{"$": id}` references. - `{"$": {"m": base_id, "u": {...}, "d": [...]}}` - a dict stored as a replacement dict plus a patch (see [Merge references](#merge-references-a-base-object-plus-a-patch)). Matches are found scanning left to right. Where replacement substrings overlap - for example `"quick"` and `"quick brown fox"` - the longest match wins, regardless of the order of the `replacements` dictionary, so output is deterministic for equivalent inputs. **Example:** ```python from condense_json import condense_json input_json = { "foo": { "bar": { "string": "This is a string with foxes in it", "nested": { "more": ["Here is a string", "another with foxes in it too"] }, } } } replacements = {"1": "with foxes in it"} condensed_output = condense_json(input_json, replacements) print(condensed_output) # Expected output: # { # "foo": { # "bar": { # "string": {"$r": ["This is a string ", {"$": "1"}]}, # "nested": { # "more": [ # "Here is a string", # {"$r": ["another ", {"$": "1"}, " too"]} # ] # } # } # } # } ``` **`uncondense_json(obj: JSONInput, replacements: Mapping[str, Any]) -> Any`** * **`obj`**: The condensed JSON value. * **`replacements`**: The same `replacements` mapping used for condensing. This function reverses the `condense_json` operation. It finds the reference forms listed above and substitutes the original content from the `replacements` mapping: strings for string references, deep copies for dict and list references, and base-plus-patch reconstruction for merge references. **Example:** ```python from condense_json import uncondense_json, condense_json # Import both original = { "sentence": "The quick brown fox jumps over the lazy dog", "nested": {"list": ["fast fox", "lazy dog", "just some text"]}, } replacements = {"1": "quick brown fox", "2": "lazy dog"} condensed = condense_json(original, replacements) uncondensed = uncondense_json(condensed, replacements) assert uncondensed == original ``` If the input `obj` to `uncondense_json` doesn't contain any condensed structures, it returns the input unchanged. `uncondense_json` is strict: it raises `condense_json.UncondenseError` (a subclass of `ValueError`) if the condensed input is malformed rather than silently producing corrupted output. This covers markers referencing a replacement ID that is missing from `replacements` (or one with an empty value, which `condense_json` never emits markers for), a `$r` value that is not a list, `$r` segments that are not strings or `{"$": id}` dictionaries, `$r` segments referencing a dict or list replacement - a reference inside a string must resolve to a string - and malformed [merge references](#merge-references-a-base-object-plus-a-patch). ```python from condense_json import uncondense_json, UncondenseError try: uncondense_json({"query": {"$": "gt"}}, {"1": "with foxes in it"}) except UncondenseError as ex: print(ex) # Unknown replacement id: 'gt' ``` ### Structural replacements for dicts and lists A replacement value can also be a dict or a list. These match **structurally**: any subtree of the input that is equal to the value - compared in canonical JSON form, so key order and formatting never matter - is replaced whole with `{"$": replacement_id}`. This is useful when a payload embeds a known blob of JSON, such as an API response that echoes back the JSON schema or tool definitions it was called with: ```python from condense_json import condense_json, uncondense_json schema = { "type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"], } response = { "output": {"name": "Cleo"}, "format": { # Same schema, different key order - still matches "schema": { "required": ["name"], "type": "object", "properties": {"name": {"type": "string"}}, } }, } condensed = condense_json(response, {"s": schema}) # {'output': {'name': 'Cleo'}, 'format': {'schema': {'$': 's'}}} assert uncondense_json(condensed, {"s": schema}) == response ``` The rules: - **Matching is outermost-wins.** Once a subtree matches, its interior is not searched further. Inner replacements still match anywhere an outer one does not. - **Matching is strictly structural.** A *string* that happens to contain the JSON serialization of a replacement value is never matched - a reference in string context must resolve to a string. String and structural replacements compose freely in a single call. - **Resolution substitutes an independent copy.** Each `{"$": id}` for a dict or list resolves to a deep copy, so mutating the result never aliases the `replacements` mapping or the value behind another marker. - **Round-trips are structural, not byte-identical.** `uncondense_json(condense_json(obj, r), r) == obj` always holds, but a matched subtree comes back with the *replacement's* key order, since the original ordering is not recorded. If you re-serialize the result, bytes may differ even though the value is equal. - **Non-string scalars never participate.** A replacement of `42` or `True` is ignored rather than riddling the output with markers, as are empty dicts and lists. ### Merge references: a base object plus a patch Dict replacement values also act as **merge bases**. A dict in the input that is *mostly* equal to a base - some keys added, changed or missing - can be stored as a reference to the base plus a patch, using a dict-valued `$` marker: ```json {"$": {"m": "base_id", "u": {"added or changed": "keys"}, "d": ["keys the input lacks"]}} ``` `m` names the base entry in `replacements`, `u` holds keys to apply on top of it, and `d` lists base keys to remove first. Uncondensing deep-copies the base, deletes the `d` keys, then applies the `u` entries - which are themselves uncondensed, so they may contain markers of their own. This suits data with a large static envelope and a few varying fields, such as API response metadata: ```python from condense_json import condense_json, uncondense_json env = { "object": "response", "status": "completed", "service_tier": "default", "truncation": "disabled", "store": False, "tools": [], } response = dict(env, id="resp_123", usage={"total_tokens": 27}) condensed = condense_json(response, {"env": env}) # {'$': {'m': 'env', 'u': {'id': 'resp_123', 'usage': {'total_tokens': 27}}}} assert uncondense_json(condensed, {"env": env}) == response ``` How condensing decides: - **A byte-cost comparison, not a similarity heuristic.** For each base, the patch is computed and both encodings are measured; the merge reference is emitted only when it is smaller than writing the dict out. An unrelated base produces a patch bigger than the dict itself, so it prices itself out. Among competing bases the smallest encoding wins, with ties going to the earlier entry in the mapping. - **An exact match takes priority.** A dict equal to a base condenses to the plain `{"$": id}` form, never a merge reference. - **Per-key equality is canonical**, like structural matching - key order inside values never matters, and `True`/`1`/`1.0` are always distinct. - **Deletion is the explicit `d` list, never a `null` sentinel** (as in JSON Merge Patch), because `null` is a legitimate value in real payloads. - **Patches are flat.** If a nested value differs from the base's version at all, the whole value travels in `u` - though it is condensed recursively on the way, so a nested dict may itself become a reference or merge against another base. - The structural caveats apply here too: resolution substitutes independent deep copies, and round-trips are structurally equal rather than byte-identical. `uncondense_json` validates merge references strictly: an unknown or non-dict base, unexpected fields, a non-list `d`, deletion of a key the base does not have, or a non-dict `u` all raise `UncondenseError`. ### Escaping of `$`, `$r` and `$raw` keys The condensed format gives special meaning to single-key dictionaries with a `$` or `$r` key. If your input data already contains dictionaries of that shape - for example `{"price": {"$": "100"}}` - they could be misinterpreted when uncondensing. To prevent this, `condense_json` escapes any single-key dictionary whose sole key is `$`, `$r` or `$raw` by wrapping it in `{"$raw": ...}`: ```python from condense_json import condense_json, uncondense_json original = {"price": {"$": "100"}} condensed = condense_json(original, {"1": "with foxes"}) # {'price': {'$raw': {'$': '100'}}} assert uncondense_json(condensed, {"1": "with foxes"}) == original ``` `uncondense_json` removes exactly one `$raw` wrapper layer and restores the contents without interpreting them as a marker. Because `$raw` itself is escaped in the same way, this works even if your data already contains `$raw` keys, and round-trips of `condense_json` followed by `uncondense_json` are always lossless - including when applied more than once. A marker-shaped dict that structurally matches a replacement value is referenced rather than escaped, and restores identically either way. ## Development To contribute to this library, checkout the code and run the tests with `uv run pytest`: ```bash cd condense-json uv run pytest ``` ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785733020.0 condense_json-1.1/README.md0000644000175100017510000002775015234017634015147 0ustar00runnerrunner# condense-json [![PyPI](https://img.shields.io/pypi/v/condense-json.svg)](https://pypi.org/project/condense-json/) [![Tests](https://github.com/simonw/condense-json/actions/workflows/test.yml/badge.svg)](https://github.com/simonw/condense-json/actions/workflows/test.yml) [![Changelog](https://img.shields.io/github/v/release/simonw/condense-json?include_prereleases&label=changelog)](https://github.com/simonw/condense-json/releases) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/condense-json/blob/main/LICENSE) Python function for condensing JSON using replacement strings and values ## Installation Install this library using `pip`: ```bash pip install condense-json ``` ## Usage The `condense_json` function searches a JSON-like object for content that matches entries in a replacements mapping: strings that contain replacement substrings, subtrees that are structurally equal to replacement dicts or lists, and dicts that are *close* to a replacement dict, which are stored as that base plus a patch. It replaces each with a compact reference, making the JSON more concise. The `uncondense_json` function reverses this process. **`condense_json(obj: JSONInput, replacements: Mapping[str, Any]) -> Any`** * **`obj`**: The JSON value to condense - any nesting of dictionaries, lists, strings, numbers, booleans and `None`. Top-level lists and strings work too, not just dictionaries. * **`replacements`**: A mapping where keys are replacement IDs (e.g., "1", "2") and values are the content they represent - strings, which match as substrings, or dicts and lists, which match whole subtrees (see [Structural replacements](#structural-replacements-for-dicts-and-lists)); dict values additionally serve as bases for near-matches (see [Merge references](#merge-references-a-base-object-plus-a-patch)). Entries with empty values (`None`, `""`, `{}`, `[]`) or non-string scalar values are ignored. `JSONInput` is a recursive type alias covering anything representable in JSON, built from covariant container types so that narrowly typed values such as `dict[str, str]` are accepted without any extra annotation. Results are typed `Any`, so they can be indexed, iterated and serialized without narrowing. ```python JSONInput = Union[ str, int, float, bool, None, "Sequence[JSONInput]", "Mapping[str, JSONInput]" ] ``` The function returns a modified version of the input `obj` with matches replaced by references. Three reference forms appear in the output: - `{"$": replacement_id}` - a whole value that matched: a string consisting *entirely* of a replacement string, or a dict or list structurally equal to a replacement value. - `{"$r": [ ...segments... ]}` - a string that *contains* one or more replacement strings, broken into literal segments and `{"$": id}` references. - `{"$": {"m": base_id, "u": {...}, "d": [...]}}` - a dict stored as a replacement dict plus a patch (see [Merge references](#merge-references-a-base-object-plus-a-patch)). Matches are found scanning left to right. Where replacement substrings overlap - for example `"quick"` and `"quick brown fox"` - the longest match wins, regardless of the order of the `replacements` dictionary, so output is deterministic for equivalent inputs. **Example:** ```python from condense_json import condense_json input_json = { "foo": { "bar": { "string": "This is a string with foxes in it", "nested": { "more": ["Here is a string", "another with foxes in it too"] }, } } } replacements = {"1": "with foxes in it"} condensed_output = condense_json(input_json, replacements) print(condensed_output) # Expected output: # { # "foo": { # "bar": { # "string": {"$r": ["This is a string ", {"$": "1"}]}, # "nested": { # "more": [ # "Here is a string", # {"$r": ["another ", {"$": "1"}, " too"]} # ] # } # } # } # } ``` **`uncondense_json(obj: JSONInput, replacements: Mapping[str, Any]) -> Any`** * **`obj`**: The condensed JSON value. * **`replacements`**: The same `replacements` mapping used for condensing. This function reverses the `condense_json` operation. It finds the reference forms listed above and substitutes the original content from the `replacements` mapping: strings for string references, deep copies for dict and list references, and base-plus-patch reconstruction for merge references. **Example:** ```python from condense_json import uncondense_json, condense_json # Import both original = { "sentence": "The quick brown fox jumps over the lazy dog", "nested": {"list": ["fast fox", "lazy dog", "just some text"]}, } replacements = {"1": "quick brown fox", "2": "lazy dog"} condensed = condense_json(original, replacements) uncondensed = uncondense_json(condensed, replacements) assert uncondensed == original ``` If the input `obj` to `uncondense_json` doesn't contain any condensed structures, it returns the input unchanged. `uncondense_json` is strict: it raises `condense_json.UncondenseError` (a subclass of `ValueError`) if the condensed input is malformed rather than silently producing corrupted output. This covers markers referencing a replacement ID that is missing from `replacements` (or one with an empty value, which `condense_json` never emits markers for), a `$r` value that is not a list, `$r` segments that are not strings or `{"$": id}` dictionaries, `$r` segments referencing a dict or list replacement - a reference inside a string must resolve to a string - and malformed [merge references](#merge-references-a-base-object-plus-a-patch). ```python from condense_json import uncondense_json, UncondenseError try: uncondense_json({"query": {"$": "gt"}}, {"1": "with foxes in it"}) except UncondenseError as ex: print(ex) # Unknown replacement id: 'gt' ``` ### Structural replacements for dicts and lists A replacement value can also be a dict or a list. These match **structurally**: any subtree of the input that is equal to the value - compared in canonical JSON form, so key order and formatting never matter - is replaced whole with `{"$": replacement_id}`. This is useful when a payload embeds a known blob of JSON, such as an API response that echoes back the JSON schema or tool definitions it was called with: ```python from condense_json import condense_json, uncondense_json schema = { "type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"], } response = { "output": {"name": "Cleo"}, "format": { # Same schema, different key order - still matches "schema": { "required": ["name"], "type": "object", "properties": {"name": {"type": "string"}}, } }, } condensed = condense_json(response, {"s": schema}) # {'output': {'name': 'Cleo'}, 'format': {'schema': {'$': 's'}}} assert uncondense_json(condensed, {"s": schema}) == response ``` The rules: - **Matching is outermost-wins.** Once a subtree matches, its interior is not searched further. Inner replacements still match anywhere an outer one does not. - **Matching is strictly structural.** A *string* that happens to contain the JSON serialization of a replacement value is never matched - a reference in string context must resolve to a string. String and structural replacements compose freely in a single call. - **Resolution substitutes an independent copy.** Each `{"$": id}` for a dict or list resolves to a deep copy, so mutating the result never aliases the `replacements` mapping or the value behind another marker. - **Round-trips are structural, not byte-identical.** `uncondense_json(condense_json(obj, r), r) == obj` always holds, but a matched subtree comes back with the *replacement's* key order, since the original ordering is not recorded. If you re-serialize the result, bytes may differ even though the value is equal. - **Non-string scalars never participate.** A replacement of `42` or `True` is ignored rather than riddling the output with markers, as are empty dicts and lists. ### Merge references: a base object plus a patch Dict replacement values also act as **merge bases**. A dict in the input that is *mostly* equal to a base - some keys added, changed or missing - can be stored as a reference to the base plus a patch, using a dict-valued `$` marker: ```json {"$": {"m": "base_id", "u": {"added or changed": "keys"}, "d": ["keys the input lacks"]}} ``` `m` names the base entry in `replacements`, `u` holds keys to apply on top of it, and `d` lists base keys to remove first. Uncondensing deep-copies the base, deletes the `d` keys, then applies the `u` entries - which are themselves uncondensed, so they may contain markers of their own. This suits data with a large static envelope and a few varying fields, such as API response metadata: ```python from condense_json import condense_json, uncondense_json env = { "object": "response", "status": "completed", "service_tier": "default", "truncation": "disabled", "store": False, "tools": [], } response = dict(env, id="resp_123", usage={"total_tokens": 27}) condensed = condense_json(response, {"env": env}) # {'$': {'m': 'env', 'u': {'id': 'resp_123', 'usage': {'total_tokens': 27}}}} assert uncondense_json(condensed, {"env": env}) == response ``` How condensing decides: - **A byte-cost comparison, not a similarity heuristic.** For each base, the patch is computed and both encodings are measured; the merge reference is emitted only when it is smaller than writing the dict out. An unrelated base produces a patch bigger than the dict itself, so it prices itself out. Among competing bases the smallest encoding wins, with ties going to the earlier entry in the mapping. - **An exact match takes priority.** A dict equal to a base condenses to the plain `{"$": id}` form, never a merge reference. - **Per-key equality is canonical**, like structural matching - key order inside values never matters, and `True`/`1`/`1.0` are always distinct. - **Deletion is the explicit `d` list, never a `null` sentinel** (as in JSON Merge Patch), because `null` is a legitimate value in real payloads. - **Patches are flat.** If a nested value differs from the base's version at all, the whole value travels in `u` - though it is condensed recursively on the way, so a nested dict may itself become a reference or merge against another base. - The structural caveats apply here too: resolution substitutes independent deep copies, and round-trips are structurally equal rather than byte-identical. `uncondense_json` validates merge references strictly: an unknown or non-dict base, unexpected fields, a non-list `d`, deletion of a key the base does not have, or a non-dict `u` all raise `UncondenseError`. ### Escaping of `$`, `$r` and `$raw` keys The condensed format gives special meaning to single-key dictionaries with a `$` or `$r` key. If your input data already contains dictionaries of that shape - for example `{"price": {"$": "100"}}` - they could be misinterpreted when uncondensing. To prevent this, `condense_json` escapes any single-key dictionary whose sole key is `$`, `$r` or `$raw` by wrapping it in `{"$raw": ...}`: ```python from condense_json import condense_json, uncondense_json original = {"price": {"$": "100"}} condensed = condense_json(original, {"1": "with foxes"}) # {'price': {'$raw': {'$': '100'}}} assert uncondense_json(condensed, {"1": "with foxes"}) == original ``` `uncondense_json` removes exactly one `$raw` wrapper layer and restores the contents without interpreting them as a marker. Because `$raw` itself is escaped in the same way, this works even if your data already contains `$raw` keys, and round-trips of `condense_json` followed by `uncondense_json` are always lossless - including when applied more than once. A marker-shaped dict that structurally matches a replacement value is referenced rather than escaped, and restores identically either way. ## Development To contribute to this library, checkout the code and run the tests with `uv run pytest`: ```bash cd condense-json uv run pytest ``` ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1785733024.9496229 condense_json-1.1/condense_json/0000755000175100017510000000000015234017641016502 5ustar00runnerrunner././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785733020.0 condense_json-1.1/condense_json/__init__.py0000644000175100017510000004470515234017634020627 0ustar00runnerrunnerimport copy import json import re from typing import Any, Callable, Mapping, Optional, Sequence, Union # Any value that can be represented in JSON. Uses covariant container # types, so narrowly typed values such as dict[str, str] are accepted # without needing a broader annotation JSONInput = Union[ str, int, float, bool, None, "Sequence[JSONInput]", "Mapping[str, JSONInput]" ] # Single-key dicts using one of these keys have special meaning in the # condensed format, so any such dict found in the input must be escaped. _MARKER_KEYS = ("$", "$r", "$raw") class UncondenseError(ValueError): """ Raised by uncondense_json when the condensed object is malformed or references a replacement ID not present in the replacements dict. """ def _canonical(value: JSONInput) -> Optional[str]: """ The canonical JSON form used for structural equality: keys sorted, compact separators, non-ASCII left as-is. Returns None for values that cannot be serialized as JSON, which simply never match. """ try: return json.dumps( value, sort_keys=True, separators=(",", ":"), ensure_ascii=False ) except (TypeError, ValueError): return None def _split_replacements( replacements: Mapping[str, Any], ) -> tuple[dict[str, str], dict[str, str], set[tuple[type, int]]]: """ Split a replacements mapping into its two participating kinds. Returns (string_replacements, canonical_to_id, container_shapes): non-empty strings, which match as substrings; and non-empty dicts and lists, keyed here by canonical form for structural lookup, with the (type, length) shapes present so callers can skip canonicalizing nodes that cannot possibly match. Everything else - None, empty strings, empty containers, numbers, booleans - is filtered out, the same silent treatment blank strings have always had. """ strings: dict[str, str] = {} canonical_to_id: dict[str, str] = {} shapes: set[tuple[type, int]] = set() for rep_id, value in replacements.items(): if isinstance(value, str): if value: strings[rep_id] = value elif isinstance(value, (dict, list)): if value: canonical = _canonical(value) if canonical is not None: canonical_to_id.setdefault(canonical, rep_id) shapes.add((type(value), len(value))) return strings, canonical_to_id, shapes def condense_json(obj: JSONInput, replacements: Mapping[str, Any]) -> Any: """ Recursively search through the JSON-like object `obj`, replacing content that matches an entry in `replacements` with a reference to its ID. String replacement values match as substrings. For any string in `obj` that contains one or more of them, the string is broken into segments and each found occurrence becomes a dict of the form {"$": replacement_id}. The overall string becomes: {"$r": [ "text before", {"$": replacement_id}, "text after", ... ]} For example, with: obj = { "foo": { "bar": { "string": "This is a string with foxes in it", "nested": { "more": ["Here is a string", "another with foxes in it too"] } } } } and replacements = {"1": "with foxes in it"} The result will be: { "foo": { "bar": { "string": {"$r": ["This is a string ", {"$": "1"}]}, "nested": { "more": [ "Here is a string", {"$r": ["another ", {"$": "1"}, " too"]} ] } } } } Matches are found scanning left to right; where replacement substrings overlap, the longest match wins regardless of the order of the `replacements` dict. Dict and list replacement values match *structurally*: any subtree of `obj` that is equal to the replacement value - compared in canonical JSON form, so key order never matters - is replaced whole with {"$": replacement_id}. Matching is outermost-wins: once a subtree matches, its interior is not searched further. With: replacements = {"schema": {"type": "object", "properties": {...}}} any occurrence of that schema as a subtree condenses to {"$": "schema"}. Structural matching is strictly structural - a string that happens to contain the JSON serialization of the value is not matched, because a reference inside a string must resolve to a string. Dict replacements also serve as *merge bases*: a dict node that is mostly equal to a base is encoded as the base plus a patch, {"$": {"m": base_id, "u": {added or changed keys}, "d": [absent keys]}} whenever that form is smaller than writing the node out - a cost comparison decides, so unrelated bases price themselves out. Patch values in "u" are condensed recursively. Deletion is the explicit "d" key list, never a null sentinel, because null is a legitimate payload value. If multiple IDs share the same value, the first one wins. Values that are None, empty, or non-string scalars are ignored. Any single-key dict in the input whose sole key is "$", "$r" or "$raw" would be misinterpreted by uncondense_json, so it is escaped by wrapping it in {"$raw": ...}. uncondense_json removes exactly one wrapper layer, guaranteeing a lossless round-trip. A subtree that matches a structural replacement is referenced rather than escaped, whatever its shape. """ filtered, canonical_to_id, container_shapes = _split_replacements(replacements) # If multiple IDs share the same substring, the first one wins substr_to_id: dict[str, str] = {} for rep_id, substr in filtered.items(): substr_to_id.setdefault(substr, rep_id) # Longer substrings first, so overlapping replacements prefer the # longest match regardless of dict insertion order pattern: Optional[re.Pattern[str]] = ( re.compile( "|".join( re.escape(substr) for substr in sorted(filtered.values(), key=len, reverse=True) ) ) if filtered else None ) # Dict replacements double as merge bases: a dict node that is # *mostly* equal to a base can be encoded as the base plus a patch. # Kept in mapping order with equivalent values deduplicated, so the # first ID wins - the same rule every other replacement kind follows. merge_bases: dict[str, dict] = {} _seen_base_canonicals: set[str] = set() for rep_id, val in replacements.items(): if isinstance(val, dict) and val: base_canonical = _canonical(val) if base_canonical is None or base_canonical in _seen_base_canonicals: continue _seen_base_canonicals.add(base_canonical) merge_bases[rep_id] = val # Canonical forms of every base value, computed once per call - the # bases are fixed, so per-node recomputation would be pure waste. base_canonicals: dict[str, dict[str, Optional[str]]] = { rep_id: {key: _canonical(val) for key, val in base.items()} for rep_id, base in merge_bases.items() } # Canonical forms of input subtrees, memoized by object identity for # the duration of this call (the document keeps every node alive, so # ids are stable). A node's canonical form is otherwise recomputed # once per base that considers it. canonical_memo: dict[int, Optional[str]] = {} def canonical_of(value: JSONInput) -> Optional[str]: key = id(value) if key not in canonical_memo: canonical_memo[key] = _canonical(value) return canonical_memo[key] def structural_id(value: JSONInput) -> Optional[str]: # The shape check keeps this cheap: most nodes are not even the # same type and length as any replacement, so their canonical # form is never computed. if (type(value), len(value)) not in container_shapes: # type: ignore[arg-type] return None canonical = canonical_of(value) if canonical is None: return None return canonical_to_id.get(canonical) def _size(value: Any) -> Optional[int]: # None for values that cannot be serialized - condense has # always tolerated JSON-ish input, so sizing must not introduce # a crash; an unsizable form simply never wins. try: return len(json.dumps(value, separators=(",", ":"), ensure_ascii=False)) except (TypeError, ValueError): return None def best_merge(raw: dict, processed: dict, plain: Any) -> Optional[dict]: """The cheapest {"$": {"m": id, "u": ..., "d": ...}} encoding of ``raw``, or None when writing it out plainly is smaller. ``u`` holds keys added or changed relative to the base (their already-condensed forms), ``d`` lists base keys the node lacks. A base that shares little with the node produces a patch bigger than the node itself, so it prices itself out - no applicability heuristic needed beyond a small overlap prefilter. """ # Cheap overlap check first: most nodes share keys with no base # at all, and skipping them avoids ever serializing their plain # form for sizing - that serialization at every level is what # would otherwise make deeply nested documents quadratic. viable = [ (rep_id, base) for rep_id, base in merge_bases.items() if len(raw.keys() & base.keys()) >= 2 ] if not viable: return None plain_cost = _size(plain) if plain_cost is None: return None best: Optional[dict] = None best_cost = plain_cost for rep_id, base in viable: canonicals = base_canonicals[rep_id] u = { key: processed[key] for key in raw if key not in base or canonicals[key] is None or canonicals[key] != canonical_of(raw[key]) } d = [key for key in base if key not in raw] ref: dict[str, Any] = {"m": rep_id} if u: ref["u"] = u if d: ref["d"] = d candidate = {"$": ref} cost = _size(candidate) if cost is not None and cost < best_cost: best, best_cost = candidate, cost return best def process(value: JSONInput) -> Any: if isinstance(value, (dict, list)) and container_shapes: # Outermost-wins: a matched subtree is referenced whole, and # its interior is not searched. Checked before marker # escaping, on the raw value the caller supplied. rep_id = structural_id(value) if rep_id is not None: return {"$": rep_id} if isinstance(value, dict): processed = {key: process(val) for key, val in value.items()} plain: Any = processed if len(value) == 1 and next(iter(value)) in _MARKER_KEYS: plain = {"$raw": processed} if merge_bases: merged = best_merge(value, processed, plain) if merged is not None: return merged return plain elif isinstance(value, list): return [process(item) for item in value] elif isinstance(value, str): if pattern is None or not pattern.search(value): return value segments: list[Any] = [] last_index: int = 0 for match in pattern.finditer(value): start, end = match.start(), match.end() if start > last_index: segments.append(value[last_index:start]) matched_text: str = match.group(0) replacement_id: str = substr_to_id[matched_text] segments.append({"$": replacement_id}) last_index = end if last_index < len(value): segments.append(value[last_index:]) # If the entire string was replaced with just {"$": id}, return it directly if len(segments) == 1 and isinstance(segments[0], dict): return segments[0] return {"$r": segments} else: return value return process(obj) def uncondense_json(obj: JSONInput, replacements: Mapping[str, Any]) -> Any: """ Recursively reverses the transformation made by condense_json. It looks for objects of the form: - {"$": replacement_id} -> replaced entirely, so substitute with replacements[replacement_id]. String replacements substitute as strings; dict and list replacements substitute as an independent deep copy of the value, so mutating the result never aliases the replacements mapping or other markers. - {"$": {"m": base_id, "u": {...}, "d": [...]}} -> a merge against a dict replacement: the base is deep-copied, "d" keys removed, "u" entries applied on top (processed recursively, so they may contain markers). - {"$r": [ ... segments ... ]} -> rebuild the string by replacing any {"$": rep_id} segments with the actual replacement text. Only string replacements may appear here: a reference inside a string must resolve to a string. - {"$raw": ...} -> an escaped marker-shaped dict from the original input; one wrapper layer is removed and the contents are restored without being interpreted as a marker. Other types (lists, dicts without "$r", or regular strings) are left intact. Raises UncondenseError if a marker references an unknown (or blank) replacement ID, if a "$r" segment references a non-string replacement, or if a "$r" structure is malformed. """ # Values that condense_json filters out can never be referenced by a # valid marker - treat them as unknown IDs here strings, _, _ = _split_replacements(replacements) containers: dict[str, Any] = { rep_id: value for rep_id, value in replacements.items() if isinstance(value, (dict, list)) and value } def lookup(rep_id: JSONInput) -> Any: if isinstance(rep_id, str): if rep_id in strings: return strings[rep_id] if rep_id in containers: return copy.deepcopy(containers[rep_id]) raise UncondenseError("Unknown replacement id: {!r}".format(rep_id)) def lookup_string(rep_id: JSONInput) -> str: if isinstance(rep_id, dict): raise UncondenseError( 'Merge reference is not valid in a "$r" segment: {!r}'.format(rep_id) ) value = lookup(rep_id) if not isinstance(value, str): raise UncondenseError( 'Non-string replacement id in "$r" segment: {!r}'.format(rep_id) ) return value def apply_merge(ref: dict, process: "Callable[[JSONInput], Any]") -> dict: """Rebuild a dict from {"m": base_id, "u": updates, "d": deletions}. The base is deep-copied, ``d`` keys removed, ``u`` entries processed (they may contain markers of their own) and applied on top. Deletion is an explicit key list, never a null sentinel - payloads contain legitimate nulls. """ extra = set(ref) - {"m", "u", "d"} if extra or "m" not in ref: raise UncondenseError("Invalid merge reference: {!r}".format(ref)) base_id = ref["m"] base = containers.get(base_id) if isinstance(base_id, str) else None if not isinstance(base, dict): raise UncondenseError( "Merge base must be a dict replacement: {!r}".format(base_id) ) result = copy.deepcopy(base) deletions = ref.get("d", []) if not isinstance(deletions, list): raise UncondenseError('"d" must be a list of keys: {!r}'.format(deletions)) for key in deletions: if not isinstance(key, str) or key not in result: raise UncondenseError( "Merge deletion of a key the base does not have: {!r}".format(key) ) del result[key] updates = ref.get("u", {}) if not isinstance(updates, dict): raise UncondenseError('"u" must be a dict of updates: {!r}'.format(updates)) for key, val in updates.items(): result[key] = process(val) return result def process(value: JSONInput) -> Any: if isinstance(value, dict): # Check if this dict represents a condensed string: if "$raw" in value and len(value) == 1: # Escaped dict: unwrap one layer, and do not treat the # top level of the unwrapped value as a marker. raw = value["$raw"] if isinstance(raw, dict): return {k: process(v) for k, v in raw.items()} return process(raw) elif "$" in value and len(value) == 1: # Short form: the entire value was replaced - or, when # the reference is a dict, a merge against a base object. if isinstance(value["$"], dict): return apply_merge(value["$"], process) return lookup(value["$"]) elif "$r" in value and len(value) == 1: # Long form: a list of segments. segments = value["$r"] if not isinstance(segments, list): raise UncondenseError( '"$r" value must be a list of segments, got: {!r}'.format( segments ) ) rebuilt = "" for seg in segments: if isinstance(seg, str): rebuilt += seg elif isinstance(seg, dict) and len(seg) == 1 and "$" in seg: rebuilt += lookup_string(seg["$"]) else: raise UncondenseError('Invalid "$r" segment: {!r}'.format(seg)) return rebuilt else: # Not a condensed string; process the dict normally. return {k: process(v) for k, v in value.items()} elif isinstance(value, list): return [process(item) for item in value] else: return value return process(obj) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785733020.0 condense_json-1.1/condense_json/py.typed0000644000175100017510000000000015234017634020171 0ustar00runnerrunner././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1785733024.951705 condense_json-1.1/condense_json.egg-info/0000755000175100017510000000000015234017641020174 5ustar00runnerrunner././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785733024.0 condense_json-1.1/condense_json.egg-info/PKG-INFO0000644000175100017510000003106215234017640021272 0ustar00runnerrunnerMetadata-Version: 2.4 Name: condense-json Version: 1.1 Summary: Python function for condensing JSON using replacement strings Author: Simon Willison License-Expression: Apache-2.0 Project-URL: Homepage, https://github.com/simonw/condense-json Project-URL: Changelog, https://github.com/simonw/condense-json/releases Project-URL: Issues, https://github.com/simonw/condense-json/issues Project-URL: CI, https://github.com/simonw/condense-json/actions Classifier: Typing :: Typed Requires-Python: >=3.10 Description-Content-Type: text/markdown License-File: LICENSE Dynamic: license-file # condense-json [![PyPI](https://img.shields.io/pypi/v/condense-json.svg)](https://pypi.org/project/condense-json/) [![Tests](https://github.com/simonw/condense-json/actions/workflows/test.yml/badge.svg)](https://github.com/simonw/condense-json/actions/workflows/test.yml) [![Changelog](https://img.shields.io/github/v/release/simonw/condense-json?include_prereleases&label=changelog)](https://github.com/simonw/condense-json/releases) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/condense-json/blob/main/LICENSE) Python function for condensing JSON using replacement strings and values ## Installation Install this library using `pip`: ```bash pip install condense-json ``` ## Usage The `condense_json` function searches a JSON-like object for content that matches entries in a replacements mapping: strings that contain replacement substrings, subtrees that are structurally equal to replacement dicts or lists, and dicts that are *close* to a replacement dict, which are stored as that base plus a patch. It replaces each with a compact reference, making the JSON more concise. The `uncondense_json` function reverses this process. **`condense_json(obj: JSONInput, replacements: Mapping[str, Any]) -> Any`** * **`obj`**: The JSON value to condense - any nesting of dictionaries, lists, strings, numbers, booleans and `None`. Top-level lists and strings work too, not just dictionaries. * **`replacements`**: A mapping where keys are replacement IDs (e.g., "1", "2") and values are the content they represent - strings, which match as substrings, or dicts and lists, which match whole subtrees (see [Structural replacements](#structural-replacements-for-dicts-and-lists)); dict values additionally serve as bases for near-matches (see [Merge references](#merge-references-a-base-object-plus-a-patch)). Entries with empty values (`None`, `""`, `{}`, `[]`) or non-string scalar values are ignored. `JSONInput` is a recursive type alias covering anything representable in JSON, built from covariant container types so that narrowly typed values such as `dict[str, str]` are accepted without any extra annotation. Results are typed `Any`, so they can be indexed, iterated and serialized without narrowing. ```python JSONInput = Union[ str, int, float, bool, None, "Sequence[JSONInput]", "Mapping[str, JSONInput]" ] ``` The function returns a modified version of the input `obj` with matches replaced by references. Three reference forms appear in the output: - `{"$": replacement_id}` - a whole value that matched: a string consisting *entirely* of a replacement string, or a dict or list structurally equal to a replacement value. - `{"$r": [ ...segments... ]}` - a string that *contains* one or more replacement strings, broken into literal segments and `{"$": id}` references. - `{"$": {"m": base_id, "u": {...}, "d": [...]}}` - a dict stored as a replacement dict plus a patch (see [Merge references](#merge-references-a-base-object-plus-a-patch)). Matches are found scanning left to right. Where replacement substrings overlap - for example `"quick"` and `"quick brown fox"` - the longest match wins, regardless of the order of the `replacements` dictionary, so output is deterministic for equivalent inputs. **Example:** ```python from condense_json import condense_json input_json = { "foo": { "bar": { "string": "This is a string with foxes in it", "nested": { "more": ["Here is a string", "another with foxes in it too"] }, } } } replacements = {"1": "with foxes in it"} condensed_output = condense_json(input_json, replacements) print(condensed_output) # Expected output: # { # "foo": { # "bar": { # "string": {"$r": ["This is a string ", {"$": "1"}]}, # "nested": { # "more": [ # "Here is a string", # {"$r": ["another ", {"$": "1"}, " too"]} # ] # } # } # } # } ``` **`uncondense_json(obj: JSONInput, replacements: Mapping[str, Any]) -> Any`** * **`obj`**: The condensed JSON value. * **`replacements`**: The same `replacements` mapping used for condensing. This function reverses the `condense_json` operation. It finds the reference forms listed above and substitutes the original content from the `replacements` mapping: strings for string references, deep copies for dict and list references, and base-plus-patch reconstruction for merge references. **Example:** ```python from condense_json import uncondense_json, condense_json # Import both original = { "sentence": "The quick brown fox jumps over the lazy dog", "nested": {"list": ["fast fox", "lazy dog", "just some text"]}, } replacements = {"1": "quick brown fox", "2": "lazy dog"} condensed = condense_json(original, replacements) uncondensed = uncondense_json(condensed, replacements) assert uncondensed == original ``` If the input `obj` to `uncondense_json` doesn't contain any condensed structures, it returns the input unchanged. `uncondense_json` is strict: it raises `condense_json.UncondenseError` (a subclass of `ValueError`) if the condensed input is malformed rather than silently producing corrupted output. This covers markers referencing a replacement ID that is missing from `replacements` (or one with an empty value, which `condense_json` never emits markers for), a `$r` value that is not a list, `$r` segments that are not strings or `{"$": id}` dictionaries, `$r` segments referencing a dict or list replacement - a reference inside a string must resolve to a string - and malformed [merge references](#merge-references-a-base-object-plus-a-patch). ```python from condense_json import uncondense_json, UncondenseError try: uncondense_json({"query": {"$": "gt"}}, {"1": "with foxes in it"}) except UncondenseError as ex: print(ex) # Unknown replacement id: 'gt' ``` ### Structural replacements for dicts and lists A replacement value can also be a dict or a list. These match **structurally**: any subtree of the input that is equal to the value - compared in canonical JSON form, so key order and formatting never matter - is replaced whole with `{"$": replacement_id}`. This is useful when a payload embeds a known blob of JSON, such as an API response that echoes back the JSON schema or tool definitions it was called with: ```python from condense_json import condense_json, uncondense_json schema = { "type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"], } response = { "output": {"name": "Cleo"}, "format": { # Same schema, different key order - still matches "schema": { "required": ["name"], "type": "object", "properties": {"name": {"type": "string"}}, } }, } condensed = condense_json(response, {"s": schema}) # {'output': {'name': 'Cleo'}, 'format': {'schema': {'$': 's'}}} assert uncondense_json(condensed, {"s": schema}) == response ``` The rules: - **Matching is outermost-wins.** Once a subtree matches, its interior is not searched further. Inner replacements still match anywhere an outer one does not. - **Matching is strictly structural.** A *string* that happens to contain the JSON serialization of a replacement value is never matched - a reference in string context must resolve to a string. String and structural replacements compose freely in a single call. - **Resolution substitutes an independent copy.** Each `{"$": id}` for a dict or list resolves to a deep copy, so mutating the result never aliases the `replacements` mapping or the value behind another marker. - **Round-trips are structural, not byte-identical.** `uncondense_json(condense_json(obj, r), r) == obj` always holds, but a matched subtree comes back with the *replacement's* key order, since the original ordering is not recorded. If you re-serialize the result, bytes may differ even though the value is equal. - **Non-string scalars never participate.** A replacement of `42` or `True` is ignored rather than riddling the output with markers, as are empty dicts and lists. ### Merge references: a base object plus a patch Dict replacement values also act as **merge bases**. A dict in the input that is *mostly* equal to a base - some keys added, changed or missing - can be stored as a reference to the base plus a patch, using a dict-valued `$` marker: ```json {"$": {"m": "base_id", "u": {"added or changed": "keys"}, "d": ["keys the input lacks"]}} ``` `m` names the base entry in `replacements`, `u` holds keys to apply on top of it, and `d` lists base keys to remove first. Uncondensing deep-copies the base, deletes the `d` keys, then applies the `u` entries - which are themselves uncondensed, so they may contain markers of their own. This suits data with a large static envelope and a few varying fields, such as API response metadata: ```python from condense_json import condense_json, uncondense_json env = { "object": "response", "status": "completed", "service_tier": "default", "truncation": "disabled", "store": False, "tools": [], } response = dict(env, id="resp_123", usage={"total_tokens": 27}) condensed = condense_json(response, {"env": env}) # {'$': {'m': 'env', 'u': {'id': 'resp_123', 'usage': {'total_tokens': 27}}}} assert uncondense_json(condensed, {"env": env}) == response ``` How condensing decides: - **A byte-cost comparison, not a similarity heuristic.** For each base, the patch is computed and both encodings are measured; the merge reference is emitted only when it is smaller than writing the dict out. An unrelated base produces a patch bigger than the dict itself, so it prices itself out. Among competing bases the smallest encoding wins, with ties going to the earlier entry in the mapping. - **An exact match takes priority.** A dict equal to a base condenses to the plain `{"$": id}` form, never a merge reference. - **Per-key equality is canonical**, like structural matching - key order inside values never matters, and `True`/`1`/`1.0` are always distinct. - **Deletion is the explicit `d` list, never a `null` sentinel** (as in JSON Merge Patch), because `null` is a legitimate value in real payloads. - **Patches are flat.** If a nested value differs from the base's version at all, the whole value travels in `u` - though it is condensed recursively on the way, so a nested dict may itself become a reference or merge against another base. - The structural caveats apply here too: resolution substitutes independent deep copies, and round-trips are structurally equal rather than byte-identical. `uncondense_json` validates merge references strictly: an unknown or non-dict base, unexpected fields, a non-list `d`, deletion of a key the base does not have, or a non-dict `u` all raise `UncondenseError`. ### Escaping of `$`, `$r` and `$raw` keys The condensed format gives special meaning to single-key dictionaries with a `$` or `$r` key. If your input data already contains dictionaries of that shape - for example `{"price": {"$": "100"}}` - they could be misinterpreted when uncondensing. To prevent this, `condense_json` escapes any single-key dictionary whose sole key is `$`, `$r` or `$raw` by wrapping it in `{"$raw": ...}`: ```python from condense_json import condense_json, uncondense_json original = {"price": {"$": "100"}} condensed = condense_json(original, {"1": "with foxes"}) # {'price': {'$raw': {'$': '100'}}} assert uncondense_json(condensed, {"1": "with foxes"}) == original ``` `uncondense_json` removes exactly one `$raw` wrapper layer and restores the contents without interpreting them as a marker. Because `$raw` itself is escaped in the same way, this works even if your data already contains `$raw` keys, and round-trips of `condense_json` followed by `uncondense_json` are always lossless - including when applied more than once. A marker-shaped dict that structurally matches a replacement value is referenced rather than escaped, and restores identically either way. ## Development To contribute to this library, checkout the code and run the tests with `uv run pytest`: ```bash cd condense-json uv run pytest ``` ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785733024.0 condense_json-1.1/condense_json.egg-info/SOURCES.txt0000644000175100017510000000060015234017640022053 0ustar00runnerrunnerLICENSE README.md pyproject.toml condense_json/__init__.py condense_json/py.typed condense_json.egg-info/PKG-INFO condense_json.egg-info/SOURCES.txt condense_json.egg-info/dependency_links.txt condense_json.egg-info/top_level.txt tests/test_condense_json.py tests/test_merge.py tests/test_properties.py tests/test_raw_escaping.py tests/test_structural.py tests/test_uncondense_json.py././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785733024.0 condense_json-1.1/condense_json.egg-info/dependency_links.txt0000644000175100017510000000000115234017640024241 0ustar00runnerrunner ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785733024.0 condense_json-1.1/condense_json.egg-info/top_level.txt0000644000175100017510000000001615234017640022722 0ustar00runnerrunnercondense_json ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785733020.0 condense_json-1.1/pyproject.toml0000644000175100017510000000153615234017634016576 0ustar00runnerrunner[project] name = "condense-json" version = "1.1" description = "Python function for condensing JSON using replacement strings" readme = "README.md" requires-python = ">=3.10" authors = [{name = "Simon Willison"}] license = "Apache-2.0" license-files = ["LICENSE"] classifiers = [ "Typing :: Typed" ] dependencies = [] [build-system] requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [project.urls] Homepage = "https://github.com/simonw/condense-json" Changelog = "https://github.com/simonw/condense-json/releases" Issues = "https://github.com/simonw/condense-json/issues" CI = "https://github.com/simonw/condense-json/actions" [dependency-groups] dev = ["pytest", "mypy", "hypothesis"] [tool.mypy] python_version = "3.10" warn_return_any = true warn_unused_configs = true disallow_untyped_defs = true disallow_incomplete_defs = true ././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1785733024.952423 condense_json-1.1/setup.cfg0000644000175100017510000000004615234017641015474 0ustar00runnerrunner[egg_info] tag_build = tag_date = 0 ././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1785733024.951538 condense_json-1.1/tests/0000755000175100017510000000000015234017641015015 5ustar00runnerrunner././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785733020.0 condense_json-1.1/tests/test_condense_json.py0000644000175100017510000001140015234017634021253 0ustar00runnerrunnerfrom condense_json import condense_json from typing import Dict, Any def test_condense_json() -> None: input_json: Dict[str, Any] = { "foo": { "bar": { "string": "This is a string with foxes in it", "nested": { "more": ["Here is a string", "another with foxes in it too"] }, } } } replacements: Dict[str, str] = {"1": "with foxes in it"} expected_output: Dict[str, Any] = { "foo": { "bar": { "string": {"$r": ["This is a string ", {"$": "1"}]}, "nested": { "more": [ "Here is a string", {"$r": ["another ", {"$": "1"}, " too"]}, ] }, } } } assert condense_json(input_json, replacements) == expected_output def test_no_replacements() -> None: input_json: Dict[str, str] = {"text": "This is a normal string"} replacements: Dict[str, str] = {"1": "not in the text"} expected_output: Dict[str, str] = {"text": "This is a normal string"} assert condense_json(input_json, replacements) == expected_output def test_replacement_not_used() -> None: input = {"messages": [{"role": "user", "content": "What is 1231 * 2331?"}]} replacements = {"r:01jv577ycee7re8wqdebbvygys": ""} output = condense_json(input, replacements) assert output == input def test_empty_json() -> None: input_json: Dict[str, Any] = {} replacements: Dict[str, str] = {"1": "anything"} expected_output: Dict[str, Any] = {} assert condense_json(input_json, replacements) == expected_output def test_multiple_replacements() -> None: input_json: Dict[str, Any] = { "sentence": "The quick brown fox jumps over the lazy dog", "nested": {"list": ["fast fox", "lazy dog", "just some text"]}, } replacements: Dict[str, str] = {"1": "quick brown fox", "2": "lazy dog"} expected_output: Dict[str, Any] = { "sentence": {"$r": ["The ", {"$": "1"}, " jumps over the ", {"$": "2"}]}, "nested": {"list": ["fast fox", {"$": "2"}, "just some text"]}, } assert condense_json(input_json, replacements) == expected_output def test_nested_replacements() -> None: input_json: Dict[str, Any] = { "outer": {"inner": {"deep": "something deep inside with foxes in it"}} } replacements: Dict[str, str] = {"1": "with foxes in it"} expected_output: Dict[str, Any] = { "outer": {"inner": {"deep": {"$r": ["something deep inside ", {"$": "1"}]}}} } assert condense_json(input_json, replacements) == expected_output def test_longest_replacement_wins() -> None: input_json: Dict[str, str] = {"s": "the quick brown fox"} expected: Dict[str, Any] = {"s": {"$r": ["the ", {"$": "2"}]}} # The longer replacement must win no matter the dict insertion order shorter_first: Dict[str, str] = {"1": "quick", "2": "quick brown fox"} longer_first: Dict[str, str] = {"2": "quick brown fox", "1": "quick"} assert condense_json(input_json, shorter_first) == expected assert condense_json(input_json, longer_first) == expected def test_shorter_replacement_still_used_where_longer_does_not_match() -> None: input_json: Dict[str, str] = {"s": "a quick step by the quick brown fox"} replacements: Dict[str, str] = {"1": "quick", "2": "quick brown fox"} expected: Dict[str, Any] = { "s": {"$r": ["a ", {"$": "1"}, " step by the ", {"$": "2"}]} } assert condense_json(input_json, replacements) == expected def test_overlapping_matches_are_leftmost_longest() -> None: # "abc" starts before "bcd" so it wins the overlap, then "bcd" # can no longer match input_json: Dict[str, str] = {"s": "abcd"} replacements: Dict[str, str] = {"1": "bcd", "2": "abc"} expected: Dict[str, Any] = {"s": {"$r": [{"$": "2"}, "d"]}} assert condense_json(input_json, replacements) == expected def test_duplicate_replacement_values_first_wins() -> None: input_json: Dict[str, str] = {"s": "a fox ran"} replacements: Dict[str, str] = {"1": "fox", "2": "fox"} expected: Dict[str, Any] = {"s": {"$r": ["a ", {"$": "1"}, " ran"]}} assert condense_json(input_json, replacements) == expected def test_blank_or_none_replacements() -> None: input_json: Dict[str, Any] = { "outer": {"inner": {"deep": "something deep inside with foxes in it"}} } replacements: Dict[str, Any] = {"1": "deep", "2": None, "3": ""} expected_output: Dict[str, Any] = { "outer": { "inner": { "deep": {"$r": ["something ", {"$": "1"}, " inside with foxes in it"]} } } } assert condense_json(input_json, replacements) == expected_output ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785733020.0 condense_json-1.1/tests/test_merge.py0000644000175100017510000001437215234017634017536 0ustar00runnerrunner"""Merge references: encoding a dict as a base object plus a patch.""" import json from typing import Any import pytest from condense_json import UncondenseError, condense_json, uncondense_json ENV: dict[str, Any] = { "object": "response", "service_tier": "default", "status": "completed", "truncation": "disabled", "frequency_penalty": 0.0, "presence_penalty": 0.0, "store": False, "top_logprobs": 0, "tools": [], } def test_mostly_matching_dict_becomes_base_plus_patch() -> None: obj = dict(ENV, id="resp_123", usage={"input_tokens": 9}) condensed = condense_json(obj, {"env": ENV}) assert condensed == { "$": {"m": "env", "u": {"id": "resp_123", "usage": {"input_tokens": 9}}} } assert uncondense_json(condensed, {"env": ENV}) == obj def test_changed_key_lands_in_the_patch() -> None: obj = dict(ENV, status="in_progress") condensed = condense_json(obj, {"env": ENV}) assert condensed == {"$": {"m": "env", "u": {"status": "in_progress"}}} assert uncondense_json(condensed, {"env": ENV}) == obj def test_missing_base_keys_become_deletions() -> None: obj = {k: v for k, v in ENV.items() if k != "tools"} condensed = condense_json(obj, {"env": ENV}) assert condensed == {"$": {"m": "env", "d": ["tools"]}} assert uncondense_json(condensed, {"env": ENV}) == obj def test_exact_match_still_uses_the_short_form() -> None: condensed = condense_json(dict(ENV), {"env": ENV}) assert condensed == {"$": "env"} def test_patch_values_are_condensed_recursively() -> None: details = {"cached_tokens": 0, "cache_write_tokens": 0, "padding": "p" * 40} obj = dict(ENV, usage={"input_tokens": 4, "input_tokens_details": details}) replacements = {"env": ENV, "details": details} condensed = condense_json(obj, replacements) assert condensed["$"]["u"]["usage"]["input_tokens_details"] == {"$": "details"} assert uncondense_json(condensed, replacements) == obj def test_unrelated_dict_prices_itself_out() -> None: obj = {"name": "Cleo", "species": "dog"} assert condense_json(obj, {"env": ENV}) == obj def test_plain_form_wins_when_patch_would_be_bigger() -> None: # Only two short keys shared; the deletion list alone outweighs # writing the dict out plainly obj: dict[str, Any] = {"status": "completed", "store": False} assert condense_json(obj, {"env": ENV}) == obj def test_best_of_multiple_bases_is_chosen() -> None: near = dict(ENV, extra="value") obj = dict(ENV, extra="value", id="x") condensed = condense_json(obj, {"far": {"status": "completed"}, "near": near}) assert condensed == {"$": {"m": "near", "u": {"id": "x"}}} assert ( uncondense_json(condensed, {"far": {"status": "completed"}, "near": near}) == obj ) def test_key_order_never_matters() -> None: reordered = dict(reversed(list(ENV.items()))) condensed = condense_json(dict(reordered, id="x"), {"env": ENV}) assert condensed == {"$": {"m": "env", "u": {"id": "x"}}} def test_scalar_type_confusion_is_not_equality() -> None: # store False in the base, 0 in the object: different in canonical # form, so the key must travel in the patch obj = dict(ENV, store=0) condensed = condense_json(obj, {"env": ENV}) assert condensed["$"]["u"] == {"store": 0} result = uncondense_json(condensed, {"env": ENV}) assert result["store"] == 0 and result["store"] is not False def test_input_containing_a_merge_shaped_marker_round_trips() -> None: obj: dict[str, Any] = {"payload": {"$": {"m": "env", "u": {"id": "fake"}}}} replacements: dict[str, Any] = {"other": {"a": 1, "b": 2, "c": 3}} condensed = condense_json(obj, replacements) assert condensed == {"payload": {"$raw": {"$": {"m": "env", "u": {"id": "fake"}}}}} assert uncondense_json(condensed, replacements) == obj def test_merged_results_are_independent_copies() -> None: condensed: dict[str, Any] = { "a": {"$": {"m": "env", "u": {"id": "x"}}}, "b": {"$": {"m": "env"}}, } replacements = {"env": ENV} result = uncondense_json(condensed, replacements) result["a"]["tools"].append("mutated") assert result["b"]["tools"] == [] assert ENV["tools"] == [] def test_uncondense_rejects_malformed_merge_references() -> None: replacements: dict[str, Any] = {"env": ENV, "s": "some string value here"} bad_refs: list[dict[str, Any]] = [ {"$": {"u": {"a": 1}}}, # no base {"$": {"m": "missing"}}, # unknown base {"$": {"m": "s"}}, # base is a string replacement {"$": {"m": "env", "x": 1}}, # unknown field {"$": {"m": "env", "d": "tools"}}, # d not a list {"$": {"m": "env", "d": ["nope"]}}, # deleting a key the base lacks {"$": {"m": "env", "u": [1]}}, # u not a dict ] for bad in bad_refs: with pytest.raises(UncondenseError): uncondense_json(bad, replacements) def test_merge_reference_inside_r_segments_is_rejected() -> None: condensed: dict[str, Any] = {"text": {"$r": ["x", {"$": {"m": "env"}}]}} with pytest.raises(UncondenseError): uncondense_json(condensed, {"env": ENV}) def test_round_trip_on_a_payload_shaped_document() -> None: payload = dict( ENV, id="resp_abc", model="gpt-5-mini", usage={"input_tokens": 256, "output_tokens": 19}, output=[ {"type": "message", "content": [{"type": "output_text", "text": "hi"}]} ], ) replacements = {"env": ENV} condensed = condense_json(payload, replacements) assert json.dumps(condensed) != json.dumps(payload) # something happened assert uncondense_json(condensed, replacements) == payload def test_first_id_wins_for_duplicate_merge_bases() -> None: # Two equivalent bases: the one that appears first in the mapping # is the one a partial match references, matching the documented # first-ID-wins rule - not the one that sorts first. # Equal-length IDs, so the byte-cost comparison cannot secretly # break the tie in the test's favour obj = dict(ENV, id="x") replacements: dict[str, Any] = {"zz": dict(ENV), "aa": dict(ENV)} condensed = condense_json(obj, replacements) assert condensed == {"$": {"m": "zz", "u": {"id": "x"}}} assert uncondense_json(condensed, replacements) == obj ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785733020.0 condense_json-1.1/tests/test_properties.py0000644000175100017510000001472515234017634020635 0ustar00runnerrunner"""Property-based tests: the invariants that must hold for every input. The central contract is uncondense_json(condense_json(obj, r), r) == obj for any JSON-safe obj and any replacements mapping. Hypothesis generates documents with adversarial shapes (marker-like keys, merge-reference lookalikes) and replacements derived from the documents themselves so that string, structural and merge matching all actually fire. """ import copy import json from typing import Any from hypothesis import given, settings, strategies as st from condense_json import UncondenseError, condense_json, uncondense_json def assert_equivalent(a: Any, b: Any) -> None: """Equality that Python == cannot fake. == alone would let a bool/int swap slip through (True == 1), so also compare canonical JSON forms, where they serialize differently. sort_keys neutralizes the legitimate key-order changes that structural and merge substitution are documented to make. """ assert a == b assert json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True) # Keys biased toward the marker vocabulary so escaping and merge-shaped # data get exercised far more often than random text would manage keys = st.one_of( st.sampled_from(["$", "$r", "$raw", "m", "u", "d", "a", "b", "key"]), st.text(max_size=8), ) # Values that Python == conflates but canonical JSON distinguishes - # drawn often, so equality-semantics bugs cannot hide in rarity confusables = st.sampled_from([True, False, 0, 1, 0.0, 1.0, "", "0", "1"]) scalars = st.one_of( confusables, st.none(), st.booleans(), st.integers(min_value=-(10**9), max_value=10**9), st.floats(allow_nan=False, allow_infinity=False), st.text(max_size=30), ) json_values = st.recursive( scalars, lambda children: st.one_of( st.lists(children, max_size=5), st.dictionaries(keys, children, max_size=5), ), max_leaves=25, ) replacement_values = st.one_of( st.text(max_size=20), st.dictionaries(keys, scalars, max_size=6), st.lists(scalars, max_size=6), ) replacements_strategy = st.dictionaries( st.text(min_size=1, max_size=6), replacement_values, max_size=5 ) def _subtrees(obj: Any, containers: list, strings: list) -> None: if isinstance(obj, dict): if obj: containers.append(obj) for value in obj.values(): _subtrees(value, containers, strings) elif isinstance(obj, list): if obj: containers.append(obj) for value in obj: _subtrees(value, containers, strings) elif isinstance(obj, str) and obj: strings.append(obj) @st.composite def doc_with_derived_replacements(draw: st.DrawFn) -> tuple: """A document plus replacements sampled from its own content. Containers are deep-copied and dicts key-reversed, so matching runs on structural equality rather than object identity or key order. """ doc = draw(json_values) containers: list[Any] = [] strings: list[str] = [] _subtrees(doc, containers, strings) replacements = {} if containers: picks = draw(st.lists(st.sampled_from(containers), max_size=3)) for i, node in enumerate(picks): node = copy.deepcopy(node) if isinstance(node, dict): node = dict(reversed(list(node.items()))) replacements[f"c{i}"] = node if strings: picks = draw(st.lists(st.sampled_from(strings), max_size=2)) for i, text in enumerate(picks): replacements[f"s{i}"] = text # A few unrelated entries mixed in, which must never affect anything replacements.update(draw(replacements_strategy)) return doc, replacements @st.composite def doc_with_merge_bases(draw: st.DrawFn) -> tuple: """Documents built from near-copies of a base dict, to force the merge path: keys removed, values changed, keys added.""" base = draw( st.dictionaries(keys, scalars, min_size=3, max_size=8).filter( lambda d: not (len(d) == 1 and next(iter(d)) in ("$", "$r", "$raw")) ) ) variants = [] for _ in range(draw(st.integers(min_value=1, max_value=4))): variant = copy.deepcopy(base) for key in draw(st.lists(st.sampled_from(sorted(base)), max_size=2)): if draw(st.booleans()): variant.pop(key, None) else: variant[key] = draw(scalars) for _ in range(draw(st.integers(min_value=0, max_value=2))): variant[draw(keys)] = draw(json_values) variants.append(variant) doc = {"results": variants, "extra": draw(json_values)} return doc, {"base": base} @settings(deadline=None) @given(json_values, replacements_strategy) def test_round_trip_arbitrary(doc: Any, replacements: dict) -> None: condensed = condense_json(doc, replacements) assert_equivalent(uncondense_json(condensed, replacements), doc) @settings(deadline=None) @given(doc_with_derived_replacements()) def test_round_trip_with_matching_replacements(pair: tuple) -> None: doc, replacements = pair condensed = condense_json(doc, replacements) assert_equivalent(uncondense_json(condensed, replacements), doc) @settings(deadline=None) @given(doc_with_merge_bases()) def test_round_trip_through_merge_encodings(pair: tuple) -> None: doc, replacements = pair condensed = condense_json(doc, replacements) assert_equivalent(uncondense_json(condensed, replacements), doc) @settings(deadline=None) @given(doc_with_derived_replacements()) def test_condensed_output_is_json_serializable(pair: tuple) -> None: doc, replacements = pair json.dumps(condense_json(doc, replacements)) @settings(deadline=None) @given(doc_with_derived_replacements()) def test_repeated_application_still_round_trips(pair: tuple) -> None: # The README promises losslessness "including when applied more # than once" - double condense must unwind under double uncondense doc, replacements = pair twice = condense_json(condense_json(doc, replacements), replacements) once = uncondense_json(twice, replacements) assert_equivalent(uncondense_json(once, replacements), doc) @settings(deadline=None) @given(json_values, replacements_strategy) def test_uncondense_never_raises_anything_but_uncondense_error( doc: Any, replacements: dict ) -> None: # Arbitrary input treated as a condensed document: either it # resolves or it fails with the typed error, never anything else try: uncondense_json(doc, replacements) except UncondenseError: pass ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785733020.0 condense_json-1.1/tests/test_raw_escaping.py0000644000175100017510000000645415234017634021103 0ustar00runnerrunnerfrom condense_json import condense_json, uncondense_json from typing import Dict, Any def test_raw_escape_roundtrip() -> None: """Test that $raw content remains intact through a complete roundtrip.""" original: Dict[str, Any] = { "text": "This is a string with foxes in it", "special_markers": {"$raw": {"$": "1"}}, } replacements = {"1": "with foxes in it"} condensed = condense_json(original, replacements) uncondensed = uncondense_json(condensed, replacements) assert uncondensed == original def test_dollar_key_is_escaped() -> None: original: Dict[str, Any] = {"price": {"$": "100"}} replacements: Dict[str, str] = {"1": "with foxes"} condensed = condense_json(original, replacements) assert condensed == {"price": {"$raw": {"$": "100"}}} assert uncondense_json(condensed, replacements) == original def test_dollar_r_key_is_escaped() -> None: original: Dict[str, Any] = {"note": {"$r": ["hello", "world"]}} replacements: Dict[str, str] = {"1": "with foxes"} condensed = condense_json(original, replacements) assert condensed == {"note": {"$raw": {"$r": ["hello", "world"]}}} assert uncondense_json(condensed, replacements) == original def test_strings_inside_escaped_marker_are_still_condensed() -> None: original: Dict[str, Any] = {"$": "a string with foxes in it"} replacements: Dict[str, str] = {"1": "with foxes in it"} condensed = condense_json(original, replacements) assert condensed == {"$raw": {"$": {"$r": ["a string ", {"$": "1"}]}}} assert uncondense_json(condensed, replacements) == original def test_nested_marker_shaped_dicts() -> None: original: Dict[str, Any] = {"$": {"$": "a"}} replacements: Dict[str, str] = {"1": "with foxes"} condensed = condense_json(original, replacements) uncondensed = uncondense_json(condensed, replacements) assert uncondensed == original def test_double_condense_roundtrip() -> None: original: Dict[str, Any] = { "text": "This is a string with foxes in it", "price": {"$": "100"}, } replacements: Dict[str, str] = {"1": "with foxes in it"} condensed_twice = condense_json(condense_json(original, replacements), replacements) uncondensed_twice = uncondense_json( uncondense_json(condensed_twice, replacements), replacements ) assert uncondensed_twice == original def test_escaping_applies_even_with_no_replacements() -> None: original: Dict[str, Any] = {"query": {"$": "gt"}} condensed = condense_json(original, {}) assert condensed == {"query": {"$raw": {"$": "gt"}}} assert uncondense_json(condensed, {}) == original def test_multi_key_dicts_are_not_escaped() -> None: original: Dict[str, Any] = {"data": {"$": "gt", "other": "value"}} replacements: Dict[str, str] = {"1": "with foxes"} condensed = condense_json(original, replacements) assert condensed == original assert uncondense_json(condensed, replacements) == original def test_escaped_markers_inside_lists() -> None: original: Dict[str, Any] = { "items": [{"$": "a"}, {"$r": "b"}, {"$raw": "c"}, "plain string"] } replacements: Dict[str, str] = {"1": "with foxes"} condensed = condense_json(original, replacements) uncondensed = uncondense_json(condensed, replacements) assert uncondensed == original ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785733020.0 condense_json-1.1/tests/test_structural.py0000644000175100017510000001351315234017634020643 0ustar00runnerrunner"""Structural (whole-subtree) replacement values: dicts and lists.""" from typing import Any import pytest from condense_json import UncondenseError, condense_json, uncondense_json SCHEMA: dict[str, Any] = { "type": "object", "properties": { "name": {"type": "string", "description": "Full name of the person"}, "age": {"type": "integer"}, }, "required": ["name", "age"], } def test_whole_dict_subtree_is_replaced() -> None: obj: dict[str, Any] = {"text": {"format": {"type": "json_schema", "schema": SCHEMA}}} condensed = condense_json(obj, {"s": SCHEMA}) assert condensed == { "text": {"format": {"type": "json_schema", "schema": {"$": "s"}}} } assert uncondense_json(condensed, {"s": SCHEMA}) == obj def test_key_order_does_not_matter() -> None: reordered: dict[str, Any] = { "required": ["name", "age"], "properties": { "age": {"type": "integer"}, "name": {"type": "string", "description": "Full name of the person"}, }, "type": "object", } condensed = condense_json({"schema": reordered}, {"s": SCHEMA}) assert condensed == {"schema": {"$": "s"}} # Resolution substitutes the replacement's form, structurally equal # to what was condensed assert uncondense_json(condensed, {"s": SCHEMA}) == {"schema": SCHEMA} def test_list_replacement() -> None: steps: list[Any] = ["wash", "rinse", {"repeat": True}] obj: dict[str, Any] = {"a": ["wash", "rinse", {"repeat": True}], "b": "unrelated"} condensed = condense_json(obj, {"steps": steps}) assert condensed == {"a": {"$": "steps"}, "b": "unrelated"} assert uncondense_json(condensed, {"steps": steps}) == obj def test_outermost_match_wins() -> None: inner = {"type": "integer"} outer = {"age": {"type": "integer"}} obj = {"properties": {"age": {"type": "integer"}}} condensed = condense_json(obj, {"outer": outer, "inner": inner}) assert condensed == {"properties": {"$": "outer"}} def test_inner_matches_still_found_when_outer_does_not_match() -> None: inner = {"type": "integer"} obj: dict[str, Any] = {"properties": {"age": {"type": "integer"}, "extra": True}} condensed = condense_json(obj, {"inner": inner}) assert condensed == {"properties": {"age": {"$": "inner"}, "extra": True}} def test_string_form_of_a_container_is_not_matched() -> None: # A string that happens to contain the JSON serialization of the # value must survive untouched - a reference inside a string context # must resolve to a string. obj: dict[str, Any] = {"as_string": '{"repeat":true}', "as_value": {"repeat": True}} replacements: dict[str, Any] = {"r": {"repeat": True}} condensed = condense_json(obj, replacements) assert condensed == {"as_string": '{"repeat":true}', "as_value": {"$": "r"}} assert uncondense_json(condensed, replacements) == obj def test_first_id_wins_for_duplicate_values() -> None: obj = {"a": {"x": 1}} condensed = condense_json(obj, {"first": {"x": 1}, "second": {"x": 1}}) assert condensed == {"a": {"$": "first"}} def test_scalars_and_empties_are_ignored() -> None: obj: dict[str, Any] = {"n": 42, "flag": True, "nothing": None, "d": {}, "l": []} replacements: dict[str, Any] = {"a": 42, "b": True, "c": None, "d": {}, "e": []} assert condense_json(obj, replacements) == obj def test_marker_shaped_subtree_matching_a_replacement_round_trips() -> None: weird = {"$": "i-am-data"} obj = {"payload": {"$": "i-am-data"}} replacements = {"w": weird} condensed = condense_json(obj, replacements) assert condensed == {"payload": {"$": "w"}} assert uncondense_json(condensed, replacements) == obj def test_marker_shaped_subtree_not_matching_is_still_escaped() -> None: obj = {"payload": {"$": "i-am-data"}} condensed = condense_json(obj, {"unrelated": {"x": 1}}) assert condensed == {"payload": {"$raw": {"$": "i-am-data"}}} assert uncondense_json(condensed, {"unrelated": {"x": 1}}) == obj def test_structural_and_substring_replacements_compose() -> None: novel = "It was the best of times, it was the worst of times" obj: dict[str, Any] = { "quote": f"Opening: {novel}", "schema": SCHEMA, } replacements = {"n": novel, "s": SCHEMA} condensed = condense_json(obj, replacements) assert condensed == { "quote": {"$r": ["Opening: ", {"$": "n"}]}, "schema": {"$": "s"}, } assert uncondense_json(condensed, replacements) == obj def test_uncondensed_containers_are_independent_copies() -> None: replacements = {"s": {"x": [1, 2]}} condensed = {"a": {"$": "s"}, "b": {"$": "s"}} result = uncondense_json(condensed, replacements) result["a"]["x"].append(3) # Sibling markers and the replacements mapping are unaffected assert result["b"] == {"x": [1, 2]} assert replacements["s"] == {"x": [1, 2]} def test_container_id_inside_r_segments_raises() -> None: condensed: dict[str, Any] = {"text": {"$r": ["prefix ", {"$": "s"}]}} with pytest.raises(UncondenseError): uncondense_json(condensed, {"s": {"x": 1}}) def test_unknown_id_still_raises() -> None: with pytest.raises(UncondenseError): uncondense_json({"a": {"$": "missing"}}, {"s": {"x": 1}}) def test_numbers_are_not_conflated_by_canonical_form() -> None: # 1 and 1.0 and True serialize differently in canonical form, so a # replacement containing one never matches a subtree containing # another assert condense_json({"a": [1.0]}, {"r": [1]}) == {"a": [1.0]} assert condense_json({"a": [True]}, {"r": [1]}) == {"a": [True]} assert condense_json({"a": [1]}, {"r": [1]}) == {"a": {"$": "r"}} def test_top_level_value_can_be_replaced() -> None: condensed = condense_json(SCHEMA, {"s": SCHEMA}) assert condensed == {"$": "s"} assert uncondense_json(condensed, {"s": SCHEMA}) == SCHEMA ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785733020.0 condense_json-1.1/tests/test_uncondense_json.py0000644000175100017510000001011015234017634021613 0ustar00runnerrunnerimport pytest from condense_json import condense_json, uncondense_json, UncondenseError from typing import Dict, Any def test_uncondense_basic() -> None: original: Dict[str, Any] = { "foo": { "bar": { "string": "This is a string with foxes in it", "nested": { "more": ["Here is a string", "another with foxes in it too"] }, } } } replacements: Dict[str, str] = {"1": "with foxes in it"} condensed = condense_json(original, replacements) # Now uncondense should recover the original uncondensed = uncondense_json(condensed, replacements) assert uncondensed == original def test_uncondense_non_condensed() -> None: # If the object is not condensed (no markers), it should remain unchanged. original: Dict[str, str] = {"text": "This is a normal string without any changes."} replacements: Dict[str, str] = {"1": "not in the text"} uncondensed = uncondense_json(original, replacements) assert uncondensed == original def test_uncondense_multiple_replacements() -> None: original: Dict[str, Any] = { "sentence": "The quick brown fox jumps over the lazy dog", "nested": {"list": ["fast fox", "lazy dog", "just some text"]}, } replacements: Dict[str, str] = {"1": "quick brown fox", "2": "lazy dog"} condensed = condense_json(original, replacements) uncondensed = uncondense_json(condensed, replacements) assert uncondensed == original def test_uncondense_mixed() -> None: # Mixed object where only some strings were condensed. original: Dict[str, Any] = { "a": "Hello world!", "b": "Greetings from the quick brown fox", "c": {"d": ["No change here", "Another quick brown fox example"]}, } replacements: Dict[str, str] = {"1": "quick brown fox"} condensed = condense_json(original, replacements) uncondensed = uncondense_json(condensed, replacements) assert uncondensed == original def test_uncondense_error_is_a_value_error() -> None: assert issubclass(UncondenseError, ValueError) def test_unknown_replacement_id_raises() -> None: with pytest.raises(UncondenseError) as excinfo: uncondense_json({"query": {"$": "gt"}}, {"1": "with foxes"}) assert "gt" in str(excinfo.value) def test_unknown_replacement_id_in_segments_raises() -> None: with pytest.raises(UncondenseError) as excinfo: uncondense_json( {"s": {"$r": ["hello ", {"$": "missing"}]}}, {"1": "with foxes"} ) assert "missing" in str(excinfo.value) def test_non_string_replacement_id_raises() -> None: with pytest.raises(UncondenseError): uncondense_json({"x": {"$": 100}}, {"1": "with foxes"}) def test_non_list_segments_raises() -> None: with pytest.raises(UncondenseError): uncondense_json({"x": {"$r": "hello"}}, {"1": "with foxes"}) def test_invalid_segment_type_raises() -> None: with pytest.raises(UncondenseError) as excinfo: uncondense_json({"x": {"$r": ["a", ["b", "c"]]}}, {"1": "with foxes"}) assert "segment" in str(excinfo.value) def test_multi_key_dict_segment_raises() -> None: with pytest.raises(UncondenseError): uncondense_json( {"x": {"$r": [{"$": "1", "extra": "key"}]}}, {"1": "with foxes"} ) def test_blank_replacement_id_raises() -> None: # condense_json never emits markers for blank replacements, so a marker # referencing one is malformed with pytest.raises(UncondenseError): uncondense_json({"x": {"$": "2"}}, {"1": "with foxes", "2": None}) with pytest.raises(UncondenseError): uncondense_json({"x": {"$": "3"}}, {"1": "with foxes", "3": ""}) def test_escaped_markers_do_not_raise() -> None: # A $raw-escaped marker-shaped dict is data, not a marker, so its # contents must not be validated as markers original = {"query": {"$": "gt"}, "weird": {"$r": "not a list"}} replacements: Dict[str, str] = {"1": "with foxes"} condensed = condense_json(original, replacements) assert uncondense_json(condensed, replacements) == original