pax_global_header00006660000000000000000000000064152244422170014515gustar00rootroot0000000000000052 comment=638fddbbd2dc644ac69b14ee33a92153f3762140 TkTech-mutf8-638fddb/000077500000000000000000000000001522444221700144425ustar00rootroot00000000000000TkTech-mutf8-638fddb/.clang-format000066400000000000000000000010021522444221700170060ustar00rootroot00000000000000# Does a reasonable job of following PEP7 linting conventions. # # Yoinked from: # https://gist.github.com/pganssle/0e3a5f828b4d07d79447f6ced8e7e4db BasedOnStyle: Google AlwaysBreakAfterReturnType: All AllowShortIfStatementsOnASingleLine: false AlignAfterOpenBracket: Align BreakBeforeBraces: Stroustrup ColumnLimit: 79 DerivePointerAlignment: false IndentWidth: 4 Language: Cpp PointerAlignment: Right ReflowComments: true SpaceBeforeParens: ControlStatements SpacesInParentheses: false TabWidth: 4 UseTab: Never TkTech-mutf8-638fddb/.github/000077500000000000000000000000001522444221700160025ustar00rootroot00000000000000TkTech-mutf8-638fddb/.github/FUNDING.yml000066400000000000000000000010761522444221700176230ustar00rootroot00000000000000# These are supported funding model platforms github: TkTech patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry liberapay: # issuehunt: # Replace with a single IssueHunt username otechie: # Replace with a single Otechie username custom: # Replace with a single custom sponsorship URL TkTech-mutf8-638fddb/.github/workflows/000077500000000000000000000000001522444221700200375ustar00rootroot00000000000000TkTech-mutf8-638fddb/.github/workflows/release.yml000066400000000000000000000031301522444221700221770ustar00rootroot00000000000000name: CI on: push: branches: [master] pull_request: release: types: [published] workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: build_wheels: name: Wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, windows-11-arm, macos-14, macos-14-large] steps: - uses: actions/checkout@v7 - name: Build and test wheels uses: pypa/cibuildwheel@v4.1.0 - uses: actions/upload-artifact@v7 with: name: wheels-${{ matrix.os }} path: ./wheelhouse/*.whl build_sdist: name: Source distribution runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - name: Install uv uses: astral-sh/setup-uv@v8.3.2 with: enable-cache: true - name: Build sdist run: uv build --sdist - uses: actions/upload-artifact@v7 with: name: sdist path: ./dist/*.tar.gz publish: name: Publish to PyPI needs: [build_wheels, build_sdist] runs-on: ubuntu-latest if: github.event_name == 'release' && github.repository == 'TkTech/mutf8' environment: name: pypi url: https://pypi.org/p/mutf8 permissions: id-token: write steps: - uses: actions/download-artifact@v8 with: path: dist merge-multiple: true - name: Publish uses: pypa/gh-action-pypi-publish@release/v1 TkTech-mutf8-638fddb/.gitignore000066400000000000000000000001551522444221700164330ustar00rootroot00000000000000*.pyc *.pyd venv .venv/ .eggs cython_debug/* build/ _build/ dist/ wheelhouse/ *.egg-info *.so uv.lock *.idea TkTech-mutf8-638fddb/LICENCE000066400000000000000000000021241522444221700154260ustar00rootroot00000000000000Copyright (c) 2012-2015 Tyler Kennedy . All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. TkTech-mutf8-638fddb/README.md000066400000000000000000000063001522444221700157200ustar00rootroot00000000000000![CI](https://github.com/TkTech/mutf8/actions/workflows/release.yml/badge.svg) # mutf-8 This package contains simple pure-python as well as C encoders and decoders for the MUTF-8 character encoding. These days, you'll most likely encounter MUTF-8 when working on files or protocols related to the JVM. Strings in a Java `.class` file are encoded using MUTF-8, strings passed by the JNI, as well as strings exported by the object serializer. This library was extracted from [Lawu][], a Python library for working with JVM class files. ## Installation Prebuilt binary packages are available for most platforms and Python versions. Install the package from PyPi: ``` pip install mutf8 ``` ## Usage Encoding and decoding is simple: ```python from mutf8 import encode_modified_utf8, decode_modified_utf8 unicode = decode_modified_utf8(byte_like_object) bytes = encode_modified_utf8(unicode) ``` This module *does not* register itself globally as a codec, since importing should be side-effect-free. ## Benchmarks The C extension is dramatically faster than the pure-Python fallback — from ~10× on tiny strings (where per-call overhead dominates) to several hundred times on realistic payloads, and over 1000× when encoding large ASCII. ### MUTF-8 Decoding | Input | C | Python | Speedup | |-------|---|--------|--------:| | ASCII (16 B) | 81.5 ± 26 ns | 1.04 ± 0.18 µs | 13× | | ASCII (1 KB) | 185 ± 38 ns | 65.7 ± 8.5 µs | 356× | | ASCII (64 KB) | 5.58 ± 0.66 µs | 4.35 ± 0.23 ms | 780× | | CJK (16 B) | 127 ± 15 ns | 1.33 ± 0.15 µs | 10× | | CJK (1 KB) | 681 ± 117 ns | 99.3 ± 9.4 µs | 146× | | CJK (64 KB) | 31.8 ± 3.3 µs | 6.28 ± 0.063 ms | 198× | | Emoji (16 B) | 100 ± 21 ns | 882 ± 122 ns | 9× | | Emoji (1 KB) | 538 ± 119 ns | 79.6 ± 6.9 µs | 148× | | Emoji (64 KB) | 26.2 ± 4.1 µs | 5.32 ± 0.13 ms | 203× | | Mixed (16 B) | 118 ± 27 ns | 2.17 ± 0.16 µs | 18× | | Mixed (1 KB) | 860 ± 162 ns | 76.7 ± 9.2 µs | 89× | | Mixed (64 KB) | 38 ± 5 µs | 5.3 ± 0.48 ms | 139× | ### MUTF-8 Encoding | Input | C | Python | Speedup | |-------|---|--------|--------:| | ASCII (16 B) | 70.5 ± 5.8 ns | 1.23 ± 0.15 µs | 18× | | ASCII (1 KB) | 109 ± 16 ns | 49 ± 6.2 µs | 449× | | ASCII (64 KB) | 1.55 ± 0.27 µs | 2.93 ± 0.19 ms | 1889× | | CJK (16 B) | 81.4 ± 12 ns | 2.02 ± 0.26 µs | 25× | | CJK (1 KB) | 624 ± 127 ns | 108 ± 16 µs | 173× | | CJK (64 KB) | 32.5 ± 2.9 µs | 6.8 ± 0.24 ms | 209× | | Emoji (16 B) | 73 ± 10 ns | 1.17 ± 0.12 µs | 16× | | Emoji (1 KB) | 621 ± 76 ns | 67 ± 8.5 µs | 108× | | Emoji (64 KB) | 33.3 ± 6 µs | 3.98 ± 0.12 ms | 119× | | Mixed (16 B) | 108 ± 97 ns | 2.48 ± 0.35 µs | 23× | | Mixed (1 KB) | 1.39 ± 0.29 µs | 63.3 ± 3.5 µs | 45× | | Mixed (64 KB) | 81.2 ± 14 µs | 3.91 ± 0.093 ms | 48× | ## C Extension The C extension is optional. If a binary package is not available, or a C compiler is not present, the pure-python version will be used instead. If you want to ensure you're using the C version, import it directly: ```python from mutf8.cmutf8 import decode_modified_utf8 decode_modified_utf(b'\xED\xA1\x80\xED\xB0\x80') ``` [Lawu]: https://github.com/tktech/lawu TkTech-mutf8-638fddb/mutf8/000077500000000000000000000000001522444221700155055ustar00rootroot00000000000000TkTech-mutf8-638fddb/mutf8/__init__.py000066400000000000000000000011711522444221700176160ustar00rootroot00000000000000""" Utility methods for handling oddities in character encoding encountered when parsing and writing JVM ClassFiles or object serialization archives. MUTF-8 is the same as CESU-8, but with different encoding for 0x00 bytes. .. note:: http://bugs.python.org/issue2857 was an attempt in 2008 to get support for MUTF-8/CESU-8 into the python core. """ __version__ = "1.1.0" try: from mutf8.cmutf8 import decode_modified_utf8, encode_modified_utf8 except ImportError: from mutf8.mutf8 import decode_modified_utf8, encode_modified_utf8 # Shut up linters. ALL_IMPORTS = [decode_modified_utf8, encode_modified_utf8] TkTech-mutf8-638fddb/mutf8/cmutf8.c000066400000000000000000000232561522444221700170670ustar00rootroot00000000000000#define PY_SSIZE_T_CLEAN #include #include #include PyDoc_STRVAR(decode_doc, "Decodes a bytestring containing MUTF-8 as defined in section\n" "4.4.7 of the JVM specification.\n\n" ":param s: A byte/buffer-like to be converted.\n" ":returns: A unicode representation of the original string."); static PyObject * decode_modified_utf8(PyObject *self, PyObject *args) { #define return_err(_msg) \ do { \ PyObject *exc = PyObject_CallFunction(PyExc_UnicodeDecodeError, \ "sy#nns", "mutf-8", view.buf, \ view.len, ix, ix + 1, _msg); \ if (exc != NULL) { \ PyCodec_StrictErrors(exc); \ Py_DECREF(exc); \ } \ PyMem_Free(cp_out); \ PyBuffer_Release(&view); \ return NULL; \ } while (0) Py_buffer view; if (!PyArg_ParseTuple(args, "y*", &view)) { return NULL; } // MUTF-8 input. uint8_t *buf = (uint8_t *)view.buf; // Fast path: without 0xED (leads every surrogate/6-byte form), 0xC0 (the // NUL escape) or a raw 0x00 (illegal), the input is plain UTF-8. Offload // to CPython's optimized decoder; on any error fall back to the strict // loop below so the exception is still reported as mutf-8. if (memchr(buf, 0xED, view.len) == NULL && memchr(buf, 0xC0, view.len) == NULL && memchr(buf, 0x00, view.len) == NULL) { PyObject *fast = PyUnicode_DecodeUTF8((const char *)buf, view.len, NULL); if (fast != NULL) { PyBuffer_Release(&view); return fast; } PyErr_Clear(); } // Array of temporary UCS-4 codepoints. // There's no point using PyUnicode_new and _WriteChar, because // it requires us to have iterated the string to get the maximum unicode // codepoint and count anyways. Py_UCS4 *cp_out = PyMem_Calloc(view.len, sizeof(Py_UCS4)); if (!cp_out) { return PyErr_NoMemory(); } // # of codepoints we found & current index into cp_out. Py_ssize_t cp_count = 0; for (Py_ssize_t ix = 0; ix < view.len; ix++) { Py_UCS4 x = buf[ix]; if (x == 0) { return_err("Embedded NULL byte in input."); } else if (x < 0x80) { // ASCII/one-byte codepoint. x &= 0x7F; } else if ((x & 0xE0) == 0xC0) { // Two-byte codepoint. if (ix + 1 >= view.len) { return_err( "2-byte codepoint started, but input too short" " to finish."); } x = ((x & 0x1F) << 0x06 | (buf[ix + 1] & 0x3F)); ix++; } else if ((x & 0xF0) == 0xE0) { // Three-byte codepoint. if (ix + 2 >= view.len) { return_err( "3-byte or 6-byte codepoint started, but input too short" " to finish."); } uint8_t b2 = buf[ix + 1]; uint8_t b3 = buf[ix + 2]; if (x == 0xED && (b2 & 0xF0) == 0xA0 && ix + 5 < view.len && buf[ix + 3] == 0xED && (buf[ix + 4] & 0xF0) == 0xB0) { // Six-byte codepoint: a supplementary character written as a // UTF-16 surrogate pair, each half encoded as its own 3-byte // sequence. We only take this branch once a low surrogate is // confirmed to follow; otherwise `x b2 b3` is a lone // (unpaired) surrogate and is decoded as a normal 3-byte // codepoint below. uint8_t b5 = buf[ix + 4]; uint8_t b6 = buf[ix + 5]; x = ( 0x10000 + ((b2 & 0x0F) << 0x10 | (b3 & 0x3F) << 0x0A | (b5 & 0x0F) << 0x06 | (b6 & 0x3F)) ); ix += 5; cp_out[cp_count++] = x; continue; } // Regular three-byte codepoint. This also covers lone/unpaired // surrogates, each of which is encoded as a single 3-byte sequence. x = ( (x & 0x0F) << 0x0C | (b2 & 0x3F) << 0x06 | (b3 & 0x3F) ); ix += 2; } cp_out[cp_count++] = x; } PyObject *out = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, cp_out, cp_count); PyMem_Free(cp_out); PyBuffer_Release(&view); return out; #undef return_err } inline Py_ssize_t _encoded_size(void *data, Py_ssize_t length, int kind) { Py_ssize_t byte_count = 0; for (Py_ssize_t i = 0; i < length; i++) { Py_UCS4 cp = PyUnicode_READ(kind, data, i); if (cp == 0x00) { // NULLs will get encoded as C0 80. byte_count += 2; } else if (cp <= 0x7F) { byte_count++; } else if (cp <= 0x7FF) { byte_count += 2; } else if (cp <= 0xFFFF) { byte_count += 3; } else { byte_count += 6; } } return byte_count; } PyDoc_STRVAR(encoded_size_doc, "Returns the number of bytes required to store the given\n" "unicode string when encoded as MUTF-8.\n\n" ":param u: Unicode string to be converted.\n" ":returns: The number of bytes required."); static PyObject * encoded_size(PyObject *self, PyObject *args) { PyObject *src = NULL; if (!PyArg_ParseTuple(args, "U", &src)) { return NULL; } return PyLong_FromSsize_t( _encoded_size( PyUnicode_DATA(src), PyUnicode_GET_LENGTH(src), PyUnicode_KIND(src) ) ); } PyDoc_STRVAR(encode_doc, "Encodes a unicode string as MUTF-8 as defined in section\n" "4.4.7 of the JVM specification.\n\n" ":param u: Unicode string to be converted.\n" ":returns: The encoded string as a `bytes` object."); static PyObject * encode_modified_utf8(PyObject *self, PyObject *args) { PyObject *src = NULL; if (!PyArg_ParseTuple(args, "U", &src)) { return NULL; } void *data = PyUnicode_DATA(src); Py_ssize_t length = PyUnicode_GET_LENGTH(src); int kind = PyUnicode_KIND(src); // Fast path: with no supplementary char (kind < 4-byte) and no embedded // NUL, MUTF-8 == UTF-8 with "surrogatepass" (same 3-byte form for lone // surrogates). Offload to CPython's optimized encoder. if (kind != PyUnicode_4BYTE_KIND && PyUnicode_FindChar(src, 0, 0, length, 1) == -1) { return PyUnicode_AsEncodedString(src, "utf-8", "surrogatepass"); } // Allocate the result up front at its exact size and write straight into // it, avoiding a temporary buffer, its zeroing, and a full-output copy. Py_ssize_t size = _encoded_size(data, length, kind); PyObject *out = PyBytes_FromStringAndSize(NULL, size); if (out == NULL) { return NULL; } char *byte_out = PyBytes_AS_STRING(out); Py_ssize_t byte_count = 0; for (Py_ssize_t i = 0; i < length; i++) { Py_UCS4 cp = PyUnicode_READ(kind, data, i); if (cp == 0x00) { // NULL byte encoding shortcircuit. byte_out[byte_count++] = 0xC0; byte_out[byte_count++] = 0x80; } else if (cp <= 0x7F) { // ASCII byte_out[byte_count++] = cp; } else if (cp <= 0x7FF) { // Two-byte codepoint. byte_out[byte_count++] = (0xC0 | (0x1F & (cp >> 0x06))); byte_out[byte_count++] = (0x80 | (0x3F & cp)); } else if (cp <= 0xFFFF) { // Three-byte codepoint byte_out[byte_count++] = (0xE0 | (0x0F & (cp >> 0x0C))); byte_out[byte_count++] = (0x80 | (0x3F & (cp >> 0x06))); byte_out[byte_count++] = (0x80 | (0x3F & cp)); } else { // Six-byte codepoint: a supplementary character written as a // UTF-16 surrogate pair, each half encoded as its own 3-byte // sequence. The codepoint is first offset by 0x10000 as required // by the surrogate-pair algorithm. cp -= 0x10000; byte_out[byte_count++] = 0xED; byte_out[byte_count++] = 0xA0 | ((cp >> 0x10) & 0x0F); byte_out[byte_count++] = 0x80 | ((cp >> 0x0A) & 0x3F); byte_out[byte_count++] = 0xED; byte_out[byte_count++] = 0xB0 | ((cp >> 0x06) & 0x0F); byte_out[byte_count++] = 0x80 | (cp & 0x3F); } } return out; } static PyMethodDef module_methods[] = { {"decode_modified_utf8", decode_modified_utf8, METH_VARARGS, decode_doc}, {"encode_modified_utf8", encode_modified_utf8, METH_VARARGS, encode_doc}, {"encoded_size", encoded_size, METH_VARARGS, encoded_size_doc}, {NULL, NULL, 0, NULL}}; static struct PyModuleDef cmutf8_module = { PyModuleDef_HEAD_INIT, "mutf8.cmutf8", PyDoc_STR("Encoders and decoders for the MUTF-8 encoding."), -1, module_methods, }; PyMODINIT_FUNC PyInit_cmutf8(void) { PyObject *m; m = PyModule_Create(&cmutf8_module); if (m == NULL) return NULL; return m; } TkTech-mutf8-638fddb/mutf8/mutf8.py000066400000000000000000000111701522444221700171220ustar00rootroot00000000000000def decode_modified_utf8(s: bytes) -> str: """ Decodes a bytestring containing modified UTF-8 as defined in section 4.4.7 of the JVM specification. :param s: bytestring to be converted. :returns: A unicode representation of the original string. """ s_out = [] s_len = len(s) s_ix = 0 while s_ix < s_len: b1 = s[s_ix] s_ix += 1 if b1 == 0: raise UnicodeDecodeError( 'mutf-8', s, s_ix - 1, s_ix, 'Embedded NULL byte in input.' ) if b1 < 0x80: # ASCII/one-byte codepoint. s_out.append(chr(b1)) elif (b1 & 0xE0) == 0xC0: # Two-byte codepoint. if s_ix >= s_len: raise UnicodeDecodeError( 'mutf-8', s, s_ix - 1, s_ix, '2-byte codepoint started, but input too short to' ' finish.' ) s_out.append( chr( (b1 & 0x1F) << 0x06 | (s[s_ix] & 0x3F) ) ) s_ix += 1 elif (b1 & 0xF0) == 0xE0: # Three-byte codepoint. if s_ix + 1 >= s_len: raise UnicodeDecodeError( 'mutf-8', s, s_ix - 1, s_ix, '3-byte or 6-byte codepoint started, but input too' ' short to finish.' ) b2 = s[s_ix] b3 = s[s_ix + 1] if (b1 == 0xED and (b2 & 0xF0) == 0xA0 and s_ix + 4 < s_len and s[s_ix + 2] == 0xED and (s[s_ix + 3] & 0xF0) == 0xB0): # Six-byte codepoint: a supplementary character written as a # UTF-16 surrogate pair, each half encoded as its own 3-byte # sequence. Only take this branch once we've confirmed a low # surrogate actually follows; otherwise `b1 b2 b3` is a lone # (unpaired) surrogate and is decoded as a normal 3-byte # codepoint below. b5 = s[s_ix + 3] b6 = s[s_ix + 4] s_out.append( chr( 0x10000 + ((b2 & 0x0F) << 0x10 | (b3 & 0x3F) << 0x0A | (b5 & 0x0F) << 0x06 | (b6 & 0x3F)) ) ) s_ix += 5 continue # Regular three-byte codepoint. This also covers lone/unpaired # surrogates, each of which is encoded as a single 3-byte sequence. s_out.append( chr( (b1 & 0x0F) << 0x0C | (b2 & 0x3F) << 0x06 | (b3 & 0x3F) ) ) s_ix += 2 else: raise RuntimeError return u''.join(s_out) def encode_modified_utf8(u: str) -> bytes: """ Encodes a unicode string as modified UTF-8 as defined in section 4.4.7 of the JVM specification. :param u: unicode string to be converted. :returns: A decoded bytearray. """ final_string = bytearray() for c in (ord(char) for char in u): if c == 0x00: # NULL byte encoding shortcircuit. final_string.extend([0xC0, 0x80]) elif c <= 0x7F: # ASCII final_string.append(c) elif c <= 0x7FF: # Two-byte codepoint. final_string.extend([ (0xC0 | (0x1F & (c >> 0x06))), (0x80 | (0x3F & c)) ]) elif c <= 0xFFFF: # Three-byte codepoint. final_string.extend([ (0xE0 | (0x0F & (c >> 0x0C))), (0x80 | (0x3F & (c >> 0x06))), (0x80 | (0x3F & c)) ]) else: # Six-byte codepoint: a supplementary character written as a # UTF-16 surrogate pair, each half encoded as its own 3-byte # sequence. The codepoint is first offset by 0x10000 as required # by the surrogate-pair algorithm. c -= 0x10000 final_string.extend([ 0xED, 0xA0 | ((c >> 0x10) & 0x0F), 0x80 | ((c >> 0x0A) & 0x3F), 0xED, 0xB0 | ((c >> 0x06) & 0x0F), 0x80 | (c & 0x3F) ]) return bytes(final_string) TkTech-mutf8-638fddb/pyproject.toml000066400000000000000000000035631522444221700173650ustar00rootroot00000000000000[build-system] requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [project] name = "mutf8" dynamic = ["version"] description = "Fast MUTF-8 encoder & decoder" readme = "README.md" requires-python = ">=3.9" license = "MIT" license-files = ["LICENCE"] authors = [{ name = "Tyler Kennedy", email = "tk@tkte.ch" }] keywords = ["mutf-8", "cesu-8", "jvm"] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Operating System :: OS Independent", "Programming Language :: C", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Software Development :: Libraries :: Python Modules", ] [project.optional-dependencies] test = ["pytest", "pytest-benchmark"] [project.urls] Homepage = "https://github.com/TkTech/mutf8" Source = "https://github.com/TkTech/mutf8" Issues = "https://github.com/TkTech/mutf8/issues" [tool.setuptools] packages = ["mutf8"] # optional=true so a failed compile falls back to the pure-Python mutf8/mutf8.py. ext-modules = [ { name = "mutf8.cmutf8", sources = ["mutf8/cmutf8.c"], language = "c", optional = true }, ] [tool.setuptools.dynamic] version = { attr = "mutf8.__version__" } [tool.pytest.ini_options] testpaths = ["tests"] markers = ["slow: mark test as slow to run"] [tool.cibuildwheel] skip = ["*_i686"] test-extras = ["test"] test-command = "pytest {project}/tests" [tool.bumpversion] current_version = "1.1.0" commit = true tag = true tag_name = "v{new_version}" allow_dirty = false [[tool.bumpversion.files]] filename = "mutf8/__init__.py" TkTech-mutf8-638fddb/tests/000077500000000000000000000000001522444221700156045ustar00rootroot00000000000000TkTech-mutf8-638fddb/tests/conftest.py000066400000000000000000000016331522444221700200060ustar00rootroot00000000000000import pytest import mutf8.mutf8 as pymutf8 import mutf8.cmutf8 as cmutf8 def pytest_addoption(parser): parser.addoption( '--runslow', action='store_true', default=False, help='run slow tests' ) def pytest_configure(config): config.addinivalue_line('markers', 'slow: mark test as slow to run') def pytest_collection_modifyitems(config, items): if config.getoption('--runslow'): # --runslow given in cli: do not skip slow tests return skip_slow = pytest.mark.skip(reason='need --runslow option to run') for item in items: if 'slow' in item.keywords: item.add_marker(skip_slow) @pytest.fixture(params=[pymutf8, cmutf8]) def module(request): yield request.param @pytest.fixture() def decoder(module): yield module.decode_modified_utf8 @pytest.fixture() def encoder(module): yield module.encode_modified_utf8 TkTech-mutf8-638fddb/tests/test_benchmark.py000066400000000000000000000046111522444221700211510ustar00rootroot00000000000000""" Benchmarks comparing the pure-Python and C encoders/decoders across realistic inputs (ASCII, CJK, emoji, mixed) and payload sizes. Marked ``slow``; run with ``pytest --runslow tests/test_benchmark.py``. """ import pytest import mutf8.mutf8 as pymutf8 import mutf8.cmutf8 as cmutf8 # Short representative units, repeated to hit each target size. Ordered from # the most common real-world case (ASCII JVM symbols) to the rarest. _UNITS = { "ascii": "java/lang/Object;", # 1-byte codepoints "cjk": "編碼測試字", # 3-byte codepoints "emoji": "🚀🎉", # 6-byte (surrogate pair) codepoints "mixed": "café ☕ item#42 shipped 🚚 ", # 1/2/3/6-byte mix } # Target *encoded* sizes: 16b exposes per-call overhead, the rest throughput. _SIZES = {"16b": 16, "1kb": 1024, "64kb": 64 * 1024} def _scale(unit, target_bytes): reps = max(1, round(target_bytes / len(cmutf8.encode_modified_utf8(unit)))) return unit * reps # (content, size) -> decoded str and its MUTF-8 encoding. STR_CORPORA = { (content, size): _scale(unit, nbytes) for content, unit in _UNITS.items() for size, nbytes in _SIZES.items() } BYTES_CORPORA = { key: cmutf8.encode_modified_utf8(text) for key, text in STR_CORPORA.items() } CORPORA = list(STR_CORPORA) _CORPUS_IDS = [f"{content}-{size}" for content, size in CORPORA] _IMPLS = (("py", pymutf8), ("c", cmutf8)) _IMPL_IDS = [name for name, _ in _IMPLS] @pytest.mark.slow @pytest.mark.parametrize("corpus", CORPORA, ids=_CORPUS_IDS) @pytest.mark.parametrize("impl,module", _IMPLS, ids=_IMPL_IDS) def test_decode(corpus, impl, module, benchmark): content, size = corpus data = BYTES_CORPORA[corpus] benchmark.group = f"decode {content}-{size}" benchmark.extra_info.update(impl=impl, encoded_bytes=len(data)) result = benchmark(module.decode_modified_utf8, data) assert result == STR_CORPORA[corpus] @pytest.mark.slow @pytest.mark.parametrize("corpus", CORPORA, ids=_CORPUS_IDS) @pytest.mark.parametrize("impl,module", _IMPLS, ids=_IMPL_IDS) def test_encode(corpus, impl, module, benchmark): content, size = corpus text = STR_CORPORA[corpus] benchmark.group = f"encode {content}-{size}" benchmark.extra_info.update(impl=impl, encoded_bytes=len(BYTES_CORPORA[corpus])) result = benchmark(module.encode_modified_utf8, text) assert result == BYTES_CORPORA[corpus] TkTech-mutf8-638fddb/tests/test_bugs.py000066400000000000000000000074531522444221700201660ustar00rootroot00000000000000def test_issue_1(encoder, decoder): """ Ensure we do not regress on https://github.com/TkTech/mutf8/issues/1. Two issues found here: 1. Python-based decoder could hit an infinite loop, since it didn't *always* increment s_ix on an iteration. 2. C & Python decoders would incorrectly decode the `sample` below, because the logic for surrogate pair decoding made the incorrect assumption that we could short-circuit when b1 == 0xED. """ # b'[\xea\xb0\x80 \xeb\x82\x98 \xeb\x8b\xa4 \xeb\x9d\xbc \xeb\xa7\x88 # \xeb\xb0\x94 \xec\x82\xac \xec\x95\x84\xec\x9e\x90 \xec\xb0\xa8 # \xec\xb9\xb4 \xed\x83\x80 \xed\x8c\x8c \xed\x95\x98]' sample = ( u'[\uAC00 \uB098 \uB2E4 \uB77C \uB9C8 \uBC14 \uC0AC \uC544' u'\uC790 \uCC28 \uCE74 \uD0C0 \uD30C \uD558]' ) encoded = encoder(sample) decoded = decoder(encoded) assert sample == decoded def test_issue_3(encoder, decoder): """ Underallocation due to an incorrect assumption on the maximum expansion of an encoded string. """ str = '黑人抬棺組裝包' assert decoder(encoder(str)) == str # Supplementary characters and their *correct* MUTF-8 encoding, cross-checked # against Java's DataOutputStream.writeUTF, Android/DEX and Rust's # residua-mutf8. See https://github.com/TkTech/mutf8/issues/5 and /issues/6. ISSUE_5_6_VECTORS = ( ('\U00010000', b'\xed\xa0\x80\xed\xb0\x80'), ('\U00020000', b'\xed\xa1\x80\xed\xb0\x80'), ('\U00040000', b'\xed\xa3\x80\xed\xb0\x80'), ('\U00080000', b'\xed\xa7\x80\xed\xb0\x80'), ('\U00100000', b'\xed\xaf\x80\xed\xb0\x80'), ('\U0010FFFF', b'\xed\xaf\xbf\xed\xbf\xbf'), ('\U00010401', b'\xed\xa0\x81\xed\xb0\x81'), # residua-mutf8 example ('\U0001F4CB', b'\xed\xa0\xbd\xed\xb3\x8b'), # 📋 ('\U0001F3F9', b'\xed\xa0\xbc\xed\xbf\xb9'), # 🏹 ('\U0001F41F', b'\xed\xa0\xbd\xed\xb0\x9f'), # 🐟 ) def test_issue_5_and_6(encoder, decoder): """ Supplementary characters (> U+FFFF) must be encoded as a UTF-16 surrogate pair after subtracting 0x10000, matching Java/Android/CESU-8. Previously the 0x10000 offset was omitted on encode (and compensated with a bitwise OR on decode), producing non-standard bytes and losing data for plane 16. https://github.com/TkTech/mutf8/issues/5 https://github.com/TkTech/mutf8/issues/6 """ for text, expected in ISSUE_5_6_VECTORS: assert encoder(text) == expected, hex(ord(text)) # Decoding standard MUTF-8 must recover the original codepoint. assert decoder(expected) == text, hex(ord(text)) # And a full round-trip must be lossless across the whole range. assert decoder(encoder(text)) == text, hex(ord(text)) def test_issue_4(decoder, encoder): """ A lone (unpaired) high surrogate is legal in a Java string and is encoded as a single 3-byte sequence (0xED 0xA0-0xAF ...). The decoder must not assume every 0xED 0xAx sequence begins a six-byte pair; when no low surrogate follows (including at end-of-input) it must decode a 3-byte lone surrogate instead of raising "input too short". https://github.com/TkTech/mutf8/issues/4 """ # A lone high surrogate at end-of-input (previously raised spuriously). assert decoder(b'\xed\xa1\x80') == '\ud840' # A lone low surrogate, likewise. assert decoder(b'\xed\xb0\x80') == '\udc00' # A high surrogate followed by non-surrogate data, not a pair. assert decoder(b'\xed\xa1\x80AB') == '\ud840AB' # Two high surrogates in a row: each is its own 3-byte codepoint. assert decoder(b'\xed\xa1\x80\xed\xa1\x80') == '\ud840\ud840' # Lone surrogates must survive a round-trip. for cp in (0xD800, 0xD840, 0xDBFF, 0xDC00, 0xDFFF): assert decoder(encoder(chr(cp))) == chr(cp), hex(cp) TkTech-mutf8-638fddb/tests/test_modified_utf8.py000066400000000000000000000137521522444221700217530ustar00rootroot00000000000000import pytest def _pairs(*args): for a, b in args: yield chr(a), bytearray(b) def test_decode_bad_mutf8(decoder): """Ensure we do the right thing when we encounter invalid MUTF-8.""" # There should never be a null byte in a MUTF-8 string. It's the # entire point of using MUTF-8. with pytest.raises(UnicodeDecodeError) as excinfo: decoder(b'\x00') assert excinfo.value.encoding == 'mutf-8' assert 'Embedded NULL' in excinfo.value.reason # Start of a two-byte codepoint without the sibling. with pytest.raises(UnicodeDecodeError) as excinfo: decoder(b'\xC2') assert excinfo.value.encoding == 'mutf-8' assert '2-byte' in excinfo.value.reason # Start of a six-byte codepoint without the sibling. with pytest.raises(UnicodeDecodeError) as excinfo: decoder(b'\xED') assert excinfo.value.encoding == 'mutf-8' assert '6-byte' in excinfo.value.reason # Start of a three-byte codepoint without the sibling. with pytest.raises(UnicodeDecodeError) as excinfo: decoder(b'\xE2') assert excinfo.value.encoding == 'mutf-8' assert '3-byte' in excinfo.value.reason def test_two_byte(module): """ Test two-byte encoding and decoding. Test data taken from py2jdbc. """ pairs = _pairs( (0x0080, (0xc2, 0x80)), (0x0081, (0xc2, 0x81)), (0x0082, (0xc2, 0x82)), (0x0084, (0xc2, 0x84)), (0x0088, (0xc2, 0x88)), (0x0090, (0xc2, 0x90)), (0x00a0, (0xc2, 0xa0)), (0x00c0, (0xc3, 0x80)), (0x0180, (0xc6, 0x80)), (0x0280, (0xca, 0x80)), (0x0480, (0xd2, 0x80)), (0x0481, (0xd2, 0x81)), (0x0483, (0xd2, 0x83)), (0x0487, (0xd2, 0x87)), (0x048f, (0xd2, 0x8f)), (0x049f, (0xd2, 0x9f)), (0x04af, (0xd2, 0xaf)), (0x04bf, (0xd2, 0xbf)), (0x04ff, (0xd3, 0xbf)), (0x05ff, (0xd7, 0xbf)), (0x05ff, (0xd7, 0xbf)), (0x07ff, (0xdf, 0xbf)) ) for decoded, original in pairs: assert module.decode_modified_utf8(original) == decoded assert module.encode_modified_utf8(decoded) == original def test_three_byte(module): """ Test three-byte encoding and decoding. Test data taken from py2jdbc. """ pairs = _pairs( (0x0800, (0xe0, 0xa0, 0x80)), (0x0801, (0xe0, 0xa0, 0x81)), (0x0802, (0xe0, 0xa0, 0x82)), (0x0804, (0xe0, 0xa0, 0x84)), (0x0808, (0xe0, 0xa0, 0x88)), (0x0810, (0xe0, 0xa0, 0x90)), (0x0820, (0xe0, 0xa0, 0xa0)), (0x0840, (0xe0, 0xa1, 0x80)), (0x0880, (0xe0, 0xa2, 0x80)), (0x0900, (0xe0, 0xa4, 0x80)), (0x0a00, (0xe0, 0xa8, 0x80)), (0x0c00, (0xe0, 0xb0, 0x80)), (0x1800, (0xe1, 0xa0, 0x80)), (0x2800, (0xe2, 0xa0, 0x80)), (0x4800, (0xe4, 0xa0, 0x80)), (0x8800, (0xe8, 0xa0, 0x80)), (0x8801, (0xe8, 0xa0, 0x81)), (0x8803, (0xe8, 0xa0, 0x83)), (0x8807, (0xe8, 0xa0, 0x87)), (0x880f, (0xe8, 0xa0, 0x8f)), (0x881f, (0xe8, 0xa0, 0x9f)), (0x883f, (0xe8, 0xa0, 0xbf)), (0x887f, (0xe8, 0xa1, 0xbf)), (0x88ff, (0xe8, 0xa3, 0xbf)), (0x89ff, (0xe8, 0xa7, 0xbf)), (0x8bff, (0xe8, 0xaf, 0xbf)), (0x8fff, (0xe8, 0xbf, 0xbf)), (0x9fff, (0xe9, 0xbf, 0xbf)), (0xbfff, (0xeb, 0xbf, 0xbf)), (0xffff, (0xef, 0xbf, 0xbf)) ) for decoded, original in pairs: assert module.decode_modified_utf8(original) == decoded assert module.encode_modified_utf8(decoded) == original def test_six_byte(module): """ Test six-byte encoding and decoding. Test data taken from py2jdbc. """ pairs = _pairs( (0x10000, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80)), (0x10001, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x81)), (0x10002, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x82)), (0x10004, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x84)), (0x10008, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x88)), (0x10010, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x90)), (0x10020, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0xa0)), (0x10040, (0xed, 0xa0, 0x80, 0xed, 0xb1, 0x80)), (0x10080, (0xed, 0xa0, 0x80, 0xed, 0xb2, 0x80)), (0x10100, (0xed, 0xa0, 0x80, 0xed, 0xb4, 0x80)), (0x10200, (0xed, 0xa0, 0x80, 0xed, 0xb8, 0x80)), (0x10400, (0xed, 0xa0, 0x81, 0xed, 0xb0, 0x80)), (0x10800, (0xed, 0xa0, 0x82, 0xed, 0xb0, 0x80)), (0x11000, (0xed, 0xa0, 0x84, 0xed, 0xb0, 0x80)), (0x12000, (0xed, 0xa0, 0x88, 0xed, 0xb0, 0x80)), (0x14000, (0xed, 0xa0, 0x90, 0xed, 0xb0, 0x80)), (0x18000, (0xed, 0xa0, 0xa0, 0xed, 0xb0, 0x80)), (0x30000, (0xed, 0xa2, 0x80, 0xed, 0xb0, 0x80)), (0x50000, (0xed, 0xa4, 0x80, 0xed, 0xb0, 0x80)), (0x90000, (0xed, 0xa8, 0x80, 0xed, 0xb0, 0x80)), (0x10003, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x83)), (0x10007, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x87)), (0x1000f, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x8f)), (0x1001f, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x9f)), (0x1003f, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0xbf)), (0x1007f, (0xed, 0xa0, 0x80, 0xed, 0xb1, 0xbf)), (0x100ff, (0xed, 0xa0, 0x80, 0xed, 0xb3, 0xbf)), (0x101ff, (0xed, 0xa0, 0x80, 0xed, 0xb7, 0xbf)), (0x103ff, (0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf)), (0x107ff, (0xed, 0xa0, 0x81, 0xed, 0xbf, 0xbf)), (0x10fff, (0xed, 0xa0, 0x83, 0xed, 0xbf, 0xbf)), (0x11fff, (0xed, 0xa0, 0x87, 0xed, 0xbf, 0xbf)), (0x13fff, (0xed, 0xa0, 0x8f, 0xed, 0xbf, 0xbf)), (0x17fff, (0xed, 0xa0, 0x9f, 0xed, 0xbf, 0xbf)), (0x1ffff, (0xed, 0xa0, 0xbf, 0xed, 0xbf, 0xbf)), (0x3ffff, (0xed, 0xa2, 0xbf, 0xed, 0xbf, 0xbf)), (0x7ffff, (0xed, 0xa6, 0xbf, 0xed, 0xbf, 0xbf)), (0xfffff, (0xed, 0xae, 0xbf, 0xed, 0xbf, 0xbf)) ) for decoded, original in pairs: assert module.decode_modified_utf8(original) == decoded assert module.encode_modified_utf8(decoded) == original