aiofile-3.12.3/aiofile/__init__.py0000644000000000000000000000124013615410400013646 0ustar00from .aio import AIOFile from .utils import ( BinaryFileWrapper, FileIOCloner, FileIOWrapperBase, LineReader, Reader, TextFileWrapper, Writer, async_open, clone, ) from .version import ( __author__, __version__, author_info, package_info, package_license, project_home, team_email, version_info, ) __all__ = ( "AIOFile", "BinaryFileWrapper", "FileIOCloner", "FileIOWrapperBase", "LineReader", "Reader", "TextFileWrapper", "Writer", "__author__", "__version__", "async_open", "author_info", "clone", "package_info", "package_license", "project_home", "team_email", "version_info", ) aiofile-3.12.3/aiofile/aio.py0000644000000000000000000002516413615410400012672 0ustar00import asyncio import os from collections import namedtuple from concurrent.futures import Executor from functools import partial, wraps from os import strerror from pathlib import Path from typing import ( Any, Awaitable, BinaryIO, Callable, Dict, Generator, Optional, TextIO, TypeVar, Union, cast, ) from weakref import finalize import caio from caio.asyncio_base import AsyncioContextBase _T = TypeVar("_T") AIO_FILE_NOT_OPENED = -1 AIO_FILE_CLOSED = -2 FileIOType = Union[TextIO, BinaryIO] FileMode = namedtuple( "FileMode", ( "readable", "writable", "plus", "appending", "created", "flags", "binary", ), ) def parse_mode(mode: str) -> FileMode: # noqa: C901 """ Rewritten from `cpython fileno`_ .. _cpython fileio: https://bit.ly/2JY2cnp """ flags = os.O_RDONLY rwa = False writable = False readable = False plus = False appending = False created = False binary = False for m in mode: if m == "x": rwa = True created = True writable = True flags |= os.O_EXCL | os.O_CREAT if m == "r": if rwa: raise Exception("Bad mode") rwa = True readable = True if m == "w": if rwa: raise Exception("Bad mode") rwa = True writable = True flags |= os.O_CREAT | os.O_TRUNC if m == "a": if rwa: raise Exception("Bad mode") rwa = True writable = True appending = True flags |= os.O_CREAT | os.O_APPEND if m == "+": if plus: raise Exception("Bad mode") readable = True writable = True plus = True if m == "b": binary = True if hasattr(os, "O_BINARY"): flags |= os.O_BINARY if readable and writable: flags |= os.O_RDWR elif readable: flags |= os.O_RDONLY else: flags |= os.O_WRONLY return FileMode( readable=readable, writable=writable, plus=plus, appending=appending, created=created, flags=flags, binary=binary, ) class AIOFile: _file_obj: Optional[FileIOType] _file_obj_owner: bool _encoding: str _executor: Optional[Executor] mode: FileMode __open_result: "Optional[asyncio.Future[FileIOType]]" def __init__( self, filename: Union[str, Path], mode: str = "r", encoding: str = "utf-8", context: Optional[AsyncioContextBase] = None, executor: Optional[Executor] = None, ): self.__context = context or get_default_context() self.__open_result = None self._fname = str(filename) self._open_mode = mode self.mode = parse_mode(mode) self._file_obj = None self._file_obj_owner = True self._encoding = encoding self._executor = executor self._clone_lock = asyncio.Lock() self._clones = 0 @classmethod def from_fp(cls, fp: FileIOType, **kwargs: Any) -> "AIOFile": afp = cls(fp.name, fp.mode, **kwargs) afp._file_obj = fp afp._open_mode = fp.mode afp._file_obj_owner = False return afp def _run_in_thread( self, func: "Callable[..., _T]", *args: Any, **kwargs: Any, ) -> "asyncio.Future[_T]": return self.__context.loop.run_in_executor( self._executor, partial(func, *args, **kwargs), ) @property def name(self) -> str: return self._fname @property def loop(self) -> asyncio.AbstractEventLoop: return self.__context.loop @property def encoding(self) -> str: return self._encoding @property def closed(self) -> bool: return self._file_obj is None or self._file_obj.closed async def open(self) -> Optional[int]: if self._file_obj is not None: if self._file_obj.closed: raise asyncio.InvalidStateError("AIOFile closed") return None if self.__open_result is None: self.__open_result = cast( "asyncio.Future[FileIOType]", self._run_in_thread(open, self._fname, self._open_mode), ) self._file_obj = await self.__open_result self.__open_result = None return self._file_obj.fileno() await self.__open_result return None def __repr__(self) -> str: return "" % self._fname async def clone(self) -> "AIOFile": """Returns self with a ref-count bump; close() is deferred until all clones are released.""" async with self._clone_lock: self._clones += 1 return self async def close(self) -> None: if ( self._file_obj is None or not self._file_obj_owner or self._file_obj.closed ): return async with self._clone_lock: if self._clones > 0: self._clones -= 1 return if self.mode.writable: await self.fdsync() await self._run_in_thread(self._file_obj.close) def fileno(self) -> int: if self._file_obj is None: raise asyncio.InvalidStateError("AIOFile closed") return self._file_obj.fileno() def __await__(self) -> Generator[None, Any, "AIOFile"]: yield from self.open().__await__() return self async def __aenter__(self) -> "AIOFile": await self.open() return self def __aexit__(self, *args: Any) -> Awaitable[Any]: return asyncio.get_event_loop().create_task(self.close()) async def read(self, size: int = -1, offset: int = 0) -> Union[bytes, str]: data = await self.read_bytes(size, offset) return data if self.mode.binary else self.decode_bytes(data) async def read_bytes(self, size: int = -1, offset: int = 0) -> bytes: if size < -1: raise ValueError("Unsupported value %d for size" % size) if size == -1: size = ( await self._run_in_thread( os.stat, self.fileno(), ) ).st_size return await self.__context.read(size, self.fileno(), offset) async def write(self, data: Union[str, bytes], offset: int = 0) -> int: if self.mode.binary: if not isinstance(data, bytes): raise ValueError("Data must be bytes in binary mode") bytes_data = data else: if not isinstance(data, str): raise ValueError("Data must be str in text mode") bytes_data = self.encode_bytes(data) return await self.write_bytes(bytes_data, offset) def encode_bytes(self, data: str) -> bytes: return data.encode(self._encoding) def decode_bytes(self, data: bytes) -> str: return data.decode(self._encoding) async def write_bytes(self, data: bytes, offset: int = 0) -> int: data_size = len(data) if data_size == 0: return 0 # data can be written partially, see write(2) # (https://www.man7.org/linux/man-pages/man2/write.2.html) # for example, it can happen when a disk quota or a resource limit # is exceeded (in that case subsequent call will return a # corresponding error) or write has been interrupted by # an incoming signal # behaviour here in regard to continue trying to write remaining data # corresponds to the behaviour of io.BufferedIOBase # (https://docs.python.org/3/library/io.html#io.BufferedIOBase.write) # which used by object returned open() with `buffering` argument >= 1 # (effectively the default) written = 0 while written < data_size: res = await self.__context.write( data[written:], self.fileno(), offset + written, ) if res == 0: raise RuntimeError( "Write operation returned 0", self, offset, written, ) elif res < 0: # fix for linux_aio implementation bug in caio<=0.6.1 # (https://github.com/mosquito/caio/pull/7) # and safeguard against future similar issues errno = -res raise OSError(errno, strerror(errno), self._fname) written += res return written async def fsync(self) -> None: return await self.__context.fsync(self.fileno()) async def fdsync(self) -> None: return await self.__context.fdsync(self.fileno()) def truncate(self, length: int = 0) -> Awaitable[None]: return self._run_in_thread( os.ftruncate, self.fileno(), length, ) # Keyed by id(loop) so the keys themselves do not retain event loops. Values # still retain their loops through caio contexts, so create_context() also # installs synchronous cleanup into loop.close(). This is important for # short-lived loops: waiting for garbage collection would keep native AIO # resources (and io_uring SQPOLL threads) alive between loop instances. ContextStoreType = Dict[int, caio.AsyncioContext] DEFAULT_CONTEXT_STORE: ContextStoreType = {} def _release_context(loop_id: int) -> None: context = DEFAULT_CONTEXT_STORE.pop(loop_id, None) if context is not None: context.close() def _install_context_cleanup( loop: asyncio.AbstractEventLoop, ) -> None: original_close = loop.close loop_id = id(loop) @wraps(original_close) def close() -> None: if loop.is_running(): original_close() return try: _release_context(loop_id) finally: original_close() try: setattr(loop, "close", close) except (AttributeError, TypeError): # Some third-party event loops do not allow instance attributes. # The finalizer below still provides interpreter-shutdown cleanup. pass def create_context( max_requests: int = caio.AsyncioContext.MAX_REQUESTS_DEFAULT, ) -> caio.AsyncioContext: loop = asyncio.get_event_loop() context = caio.AsyncioContext(max_requests, loop=loop) _install_context_cleanup(loop) finalize(loop, _release_context, id(loop)) DEFAULT_CONTEXT_STORE[id(loop)] = context return context def get_default_context() -> caio.AsyncioContext: loop = asyncio.get_event_loop() context = DEFAULT_CONTEXT_STORE.get(id(loop)) if context is not None: return context return create_context() aiofile-3.12.3/aiofile/py.typed0000644000000000000000000000000113615410400013226 0ustar00 aiofile-3.12.3/aiofile/utils.py0000644000000000000000000002663313615410400013264 0ustar00import asyncio import collections.abc import io import os from abc import ABC, abstractmethod from pathlib import Path from types import MappingProxyType from typing import Any, Generator, Generic, Optional, Tuple, TypeVar, Union from .aio import AIOFile, FileIOType ENCODING_MAP = MappingProxyType({ "utf-8": 4, "utf-16": 8, "UTF-8": 4, "UTF-16": 8, }) async def unicode_reader( afp: AIOFile, chunk_size: int, offset: int, encoding: str = "utf-8", ) -> Tuple[int, str]: if chunk_size < 0: chunk_bytes = await afp.read_bytes(-1, offset) return len(chunk_bytes), chunk_bytes.decode(encoding=encoding) last_error = None for retry in range(ENCODING_MAP.get(encoding, 4)): chunk_bytes = await afp.read_bytes(chunk_size + retry, offset) try: chunk = chunk_bytes.decode(encoding=encoding) break except UnicodeDecodeError as e: last_error = e else: raise last_error # type: ignore chunk_size = len(chunk_bytes) return chunk_size, chunk class Reader(collections.abc.AsyncIterable): __slots__ = "_chunk_size", "__offset", "file", "__lock", "encoding" CHUNK_SIZE = 32 * 1024 def __init__( self, aio_file: AIOFile, offset: int = 0, chunk_size: int = CHUNK_SIZE, ): self.__lock = asyncio.Lock() self.__offset = int(offset) self._chunk_size = int(chunk_size) self.file = aio_file self.encoding = self.file.encoding async def read_chunk(self) -> Union[str, bytes]: async with self.__lock: if self.file.mode.binary: chunk = await self.file.read_bytes( self._chunk_size, self.__offset, ) # type: Union[str, bytes] chunk_size = len(chunk) else: chunk_size, chunk = await unicode_reader( self.file, self._chunk_size, self.__offset, encoding=self.encoding, ) self.__offset += chunk_size return chunk async def __anext__(self) -> Union[str, bytes]: chunk = await self.read_chunk() if not chunk: raise StopAsyncIteration(chunk) return chunk def __aiter__(self) -> "Reader": return self class Writer: __slots__ = "__chunk_size", "__offset", "__aio_file", "__lock" def __init__(self, aio_file: AIOFile, offset: int = 0): self.__offset = int(offset) self.__aio_file = aio_file self.__lock = asyncio.Lock() async def __call__(self, data: Union[str, bytes]) -> None: async with self.__lock: if isinstance(data, str): data = self.__aio_file.encode_bytes(data) await self.__aio_file.write_bytes(data, self.__offset) self.__offset += len(data) class LineReader(collections.abc.AsyncIterable): CHUNK_SIZE = 4192 def __init__( self, aio_file: AIOFile, offset: int = 0, chunk_size: int = CHUNK_SIZE, line_sep: str = "\n", ): self.__reader = Reader(aio_file, chunk_size=chunk_size, offset=offset) self._buffer: Any = ( io.BytesIO() if aio_file.mode.binary else io.StringIO() ) self.linesep: Any = ( aio_file.encode_bytes(line_sep) if aio_file.mode.binary else line_sep ) async def readline(self) -> Union[str, bytes]: while True: line = self._buffer.readline() if line and line.endswith(self.linesep): return line buffer_remainder = line + self._buffer.read() self._buffer.truncate(0) self._buffer.seek(0) # No line in buffer, read more data chunk = await self.__reader.read_chunk() if not chunk: # No more data, return any remaining content in the buffer return buffer_remainder # Write remaining + new data back to buffer for next iteration self._buffer.write(buffer_remainder) self._buffer.write(chunk) self._buffer.seek(0) async def __anext__(self) -> Union[bytes, str]: line = await self.readline() if not line: # We are finished, close the buffer and raise StopAsyncIteration self._buffer.close() raise StopAsyncIteration(line) return line def __aiter__(self) -> "LineReader": return self class FileIOWrapperBase(ABC): _READLINE_CHUNK_SIZE = 4192 def __init__(self, afp: AIOFile, *, offset: int = 0): self._offset = offset self._lock = asyncio.Lock() self.file = afp if self.file.mode.appending: try: self._offset = os.stat(afp.name).st_size except FileNotFoundError: self._offset = 0 @abstractmethod async def read(self, length: int = -1) -> Any: raise NotImplementedError @abstractmethod async def write(self, data: Any) -> int: raise NotImplementedError @abstractmethod async def readline( self, size: int = -1, newline: Any = ..., ) -> Union[str, bytes]: raise NotImplementedError def seek(self, offset: int) -> None: self._offset = offset def tell(self) -> int: return self._offset @property def closed(self) -> bool: return self.file.closed async def flush(self, sync_metadata: bool = False) -> None: if sync_metadata: await self.file.fsync() else: await self.file.fdsync() async def close(self) -> None: await self.file.close() def __await__(self) -> Generator[None, None, "FileIOWrapperBase"]: yield from self.file.__await__() return self async def __aenter__(self) -> "FileIOWrapperBase": await self.file.open() return self async def __aexit__(self, *_: Any) -> None: await self.close() def __aiter__(self) -> LineReader: return LineReader(self.file) def iter_chunked(self, chunk_size: int = Reader.CHUNK_SIZE) -> Reader: return Reader(self.file, chunk_size=chunk_size, offset=self._offset) class BinaryFileWrapper(FileIOWrapperBase): def __init__(self, afp: AIOFile): if not afp.mode.binary: raise ValueError("Expected file in binary mode") super().__init__(afp) async def __read(self, length: int) -> bytes: data = await self.file.read_bytes(length, self._offset) self._offset += len(data) return data async def read(self, length: int = -1) -> bytes: async with self._lock: return await self.__read(length) async def write(self, data: bytes) -> int: async with self._lock: operation = self.file.write_bytes(data, self._offset) self._offset += len(data) await operation return len(data) async def readline(self, size: int = -1, newline: bytes = b"\n") -> bytes: async with self._lock: offset = self._offset with io.BytesIO() as fp: while True: chunk = await self.__read(self._READLINE_CHUNK_SIZE) if chunk: if newline not in chunk: fp.write(chunk) continue fp.write(chunk) if 0 < size <= fp.tell(): fp.seek(size) fp.truncate(size) return fp.getvalue() fp.seek(0) line = fp.readline() self._offset = offset + fp.tell() return line class TextFileWrapper(FileIOWrapperBase): def __init__(self, afp: AIOFile): if afp.mode.binary: raise ValueError("Expected file in text mode") super().__init__(afp) self.encoding = self.file.encoding async def __read(self, length: int) -> str: offset = self._offset chunk = "" while length < 0 or length > len(chunk): part_offset, part = await unicode_reader( self.file, length, offset, self.encoding, ) if not part: break chunk += part offset += part_offset if 0 < length < len(chunk): extra = chunk[length:] chunk = chunk[:length] offset -= len(extra.encode(self.encoding)) self._offset = offset return chunk async def read(self, length: int = -1) -> str: async with self._lock: return await self.__read(length) async def write(self, data: str) -> int: async with self._lock: data_bytes = data.encode(self.encoding) operation = self.file.write_bytes(data_bytes, self._offset) self._offset += len(data_bytes) await operation return len(data_bytes) async def readline(self, size: int = -1, newline: str = "\n") -> str: async with self._lock: offset = self._offset with io.StringIO() as fp: while True: chunk = await self.__read(self._READLINE_CHUNK_SIZE) if chunk: if newline not in chunk: fp.write(chunk) continue fp.write(chunk) if 0 < size <= fp.tell(): fp.seek(size) fp.truncate(size) return fp.getvalue() fp.seek(0) line = fp.readline() self._offset = offset + len( line.encode(encoding=self.encoding), ) return line def async_open( file_specifier: Union[str, Path, FileIOType], mode: str = "r", *args: Any, **kwargs: Any, ) -> Union[BinaryFileWrapper, TextFileWrapper]: if isinstance(file_specifier, (str, Path)): afp = AIOFile(str(file_specifier), mode, *args, **kwargs) else: if args: raise ValueError("Arguments denied when IO[Any] opening.") afp = AIOFile.from_fp(file_specifier, **kwargs) if not afp.mode.binary: return TextFileWrapper(afp) return BinaryFileWrapper(afp) T = TypeVar("T", bound=FileIOWrapperBase) class FileIOCloner(Generic[T]): def __init__(self, file: T): self.source_afp = file self.cloned_afp: Optional[T] = None self._lock = asyncio.Lock() async def __clone(self) -> T: async with self._lock: if self.cloned_afp is not None: return self.cloned_afp self.cloned_afp = self.source_afp.__class__( await self.source_afp.file.clone(), ) return self.cloned_afp def __await__(self) -> Generator[Any, None, T]: return self.__clone().__await__() async def __aenter__(self) -> T: return await self.__clone() async def __aexit__(self, *_: Any) -> None: if self.cloned_afp is not None: await self.cloned_afp.close() def clone(afp: FileIOWrapperBase) -> "FileIOCloner[FileIOWrapperBase]": return FileIOCloner(afp) __all__ = ( "BinaryFileWrapper", "FileIOCloner", "FileIOWrapperBase", "LineReader", "Reader", "TextFileWrapper", "Writer", "async_open", "clone", "unicode_reader", ) aiofile-3.12.3/aiofile/version.py0000644000000000000000000000164413615410400013604 0ustar00import importlib.metadata from email.message import Message from email.utils import parseaddr from typing import cast package_metadata = cast(Message, importlib.metadata.metadata("aiofile")) _author_email_raw = package_metadata.get("Author-email", "") _author_name, _author_email_addr = parseaddr(_author_email_raw) __author__ = package_metadata.get("Author", _author_name) __version__ = package_metadata["Version"] author_info = [(__author__, _author_email_addr or _author_email_raw)] package_info = package_metadata.get("Summary", "") package_license = package_metadata.get( "License-Expression", package_metadata.get("License", ""), ) project_home = next( ( url.split(",")[1].strip() for url in package_metadata.get_all("Project-URL", []) if "homepage" in url.lower() ), "", ) team_email = _author_email_addr or _author_email_raw version_info = tuple(map(int, __version__.split("."))) aiofile-3.12.3/tests/__init__.py0000644000000000000000000000000013615410400013371 0ustar00aiofile-3.12.3/tests/conftest.py0000644000000000000000000000264013615410400013473 0ustar00from functools import partial from types import ModuleType from typing import List, Union import pytest from caio import python_aio_asyncio from aiofile import AIOFile, async_open try: from caio import thread_aio_asyncio except ImportError: thread_aio_asyncio = None # type: ignore try: from caio import linux_aio_asyncio except ImportError: linux_aio_asyncio = None # type: ignore try: from caio import linux_uring_asyncio except ImportError: linux_uring_asyncio = None # type: ignore IMPLEMENTATIONS: List[Union[ModuleType, None]] = list( filter( None, [ linux_uring_asyncio, linux_aio_asyncio, thread_aio_asyncio, python_aio_asyncio, ], ), ) IMPLEMENTATIONS.insert(0, None) IMPLEMENTATION_NAMES: List[str] = list( map( lambda x: x.__name__ if x is not None else "default", IMPLEMENTATIONS ) ) @pytest.fixture(params=IMPLEMENTATIONS, ids=IMPLEMENTATION_NAMES) async def aio_context(request, event_loop): if request.param is None: yield None return async with request.param.AsyncioContext(loop=event_loop) as context: yield context @pytest.fixture def aio_file_maker(aio_context): return partial(AIOFile, context=aio_context) @pytest.fixture(name="async_open") def async_open_maker(aio_context): return partial(async_open, context=aio_context) aiofile-3.12.3/tests/impl.py0000644000000000000000000000013713615410400012606 0ustar00def split_by(seq, n): seq = seq while seq: yield seq[:n] seq = seq[n:] aiofile-3.12.3/tests/test_aio.py0000644000000000000000000005121613615410400013460 0ustar00import asyncio import hashlib import json import os from base64 import b64encode from io import BytesIO from pathlib import Path from random import shuffle from typing import cast from unittest.mock import Mock, call from uuid import uuid4 import caio import pytest from aiofile import AIOFile, clone from aiofile.aio import DEFAULT_CONTEXT_STORE, create_context from aiofile.utils import ( BinaryFileWrapper, LineReader, Reader, TextFileWrapper, Writer, ) from .impl import split_by class CoroutineMock(Mock): async def __call__(self, *args, **kwargs): return super().__call__(*args, **kwargs) @pytest.fixture def temp_file(tmp_path): path = str(tmp_path / "file.bin") with open(path, "wb"): pass return path def test_default_context_closed_with_event_loop(monkeypatch): loop = asyncio.new_event_loop() context = Mock() monkeypatch.setattr(asyncio, "get_event_loop", lambda: loop) monkeypatch.setattr(caio, "AsyncioContext", Mock(return_value=context)) assert create_context() is context assert DEFAULT_CONTEXT_STORE[id(loop)] is context loop.close() context.close.assert_called_once_with() assert id(loop) not in DEFAULT_CONTEXT_STORE assert loop.is_closed() @pytest.fixture def uuid(tmp_path): return str(uuid4()) async def test_read(aio_file_maker, temp_file, uuid): with open(temp_file, "w") as f: f.write(uuid) aio_file = await aio_file_maker(temp_file, "r") data = await aio_file.read() assert data == uuid async def test_read_write(aio_file_maker, temp_file, uuid): r_file = await aio_file_maker(temp_file, "r") w_file = await aio_file_maker(temp_file, "w") await w_file.write(uuid) await w_file.fsync() data = await r_file.read() assert data == uuid async def test_read_write_pathlib(aio_file_maker, temp_file, uuid): r_file = await aio_file_maker(Path(temp_file), "r") w_file = await aio_file_maker(Path(temp_file), "w") await w_file.write(uuid) await w_file.fsync() data = await r_file.read() assert data == uuid @pytest.mark.parametrize("count", [2, 3, 5, 10, 20, 100]) async def test_read_offset(count, aio_file_maker, temp_file, uuid): with open(temp_file, "w") as f: for _ in range(count): f.write(uuid) aio_file = await aio_file_maker(temp_file, "r") data = await aio_file.read( offset=len(uuid), size=len(uuid), ) assert data == uuid @pytest.mark.parametrize("count", [2, 3, 5, 10, 20, 100]) async def test_read_write_offset(count, aio_file_maker, temp_file, uuid): r_file = await aio_file_maker(temp_file, "r") w_file = await aio_file_maker(temp_file, "w") for i in range(count): await w_file.write(uuid, offset=i * len(uuid)) await w_file.fsync() data = await r_file.read( offset=len(uuid), size=len(uuid), ) assert data == uuid @pytest.mark.parametrize("count", [1, 2, 3, 5, 10, 20, 100]) async def test_reader_writer(count, aio_file_maker, temp_file, uuid): r_file = await aio_file_maker(temp_file, "rb") w_file = await aio_file_maker(temp_file, "wb") chunk_size = 16 payload = os.urandom(chunk_size * count) writer = Writer(w_file) await writer(payload) await w_file.fsync() buff = BytesIO(payload) async for chunk in Reader(r_file, chunk_size=chunk_size): if not chunk: break assert chunk == buff.read(chunk_size) assert not buff.read() @pytest.mark.parametrize("count", [1, 2, 3, 5, 10, 20, 100, 1000]) async def test_reader_writer2(count, aio_file_maker, temp_file, uuid): r_file = await aio_file_maker(temp_file, "r") w_file = await aio_file_maker(temp_file, "w") writer = Writer(w_file) for _ in range(count): await writer(uuid) await w_file.fsync() async for chunk in Reader(r_file, chunk_size=len(uuid)): assert chunk == uuid async def test_parallel_writer(aio_file_maker, temp_file, uuid): w_file = await aio_file_maker(temp_file, "w") r_file = await aio_file_maker(temp_file, "r") futures = list() for i in range(1000): futures.append(w_file.write(uuid, i * len(uuid))) shuffle(futures) await asyncio.gather(*futures) await w_file.fsync() count = 0 async for chunk in Reader(r_file, chunk_size=len(uuid)): assert chunk == uuid count += 1 assert count == 1000 @pytest.mark.parametrize("count", [1, 2, 3, 5, 10, 20, 100, 1000]) async def test_parallel_writer_ordering( count, aio_file_maker, temp_file, uuid, ): w_file = await aio_file_maker(temp_file, "wb") r_file = await aio_file_maker(temp_file, "rb") chunk_size = 1024 data = os.urandom(chunk_size * count) futures = list() for idx, chunk in enumerate(split_by(data, chunk_size)): futures.append(w_file.write(chunk, idx * chunk_size)) shuffle(futures) await asyncio.gather(*futures) await w_file.fsync() result: bytes = b"" async for chunk in Reader(r_file, chunk_size=chunk_size): result += cast(bytes, chunk) assert data == result async def test_non_existent_file_ctx(aio_file_maker): with pytest.raises(FileNotFoundError): async with aio_file_maker("/c/windows/NonExistent.file", "r"): pass async def test_sequential_open(aio_file_maker, temp_file): file = aio_file_maker(temp_file, "r") try: assert isinstance(await file.open(), int) assert await file.open() is None finally: await file.close() with pytest.raises(asyncio.InvalidStateError): await file.open() async def test_double_close(aio_file_maker, temp_file): file = aio_file_maker(temp_file, "w") await file.open() await file.close() await file.close() async def test_closed_attribute(aio_file_maker, temp_file): file = aio_file_maker(temp_file, "w") assert file.closed await file.open() assert not file.closed await file.close() assert file.closed async def test_closed_attribute_wrapper(async_open, temp_file): fp = async_open(temp_file, "w") assert fp.closed await fp assert not fp.closed await fp.close() assert fp.closed async def test_parallel_open(aio_file_maker, temp_file): file = aio_file_maker(temp_file, "r") try: # open() returns a file descriptor if it has opened a file, otherwise # None. Only one of multiple parallel calls should actually open a file. opens = await asyncio.gather(*(file.open() for _ in range(3))) assert len([fd for fd in opens if fd is not None]) == 1 finally: await file.close() async def test_line_reader(aio_file_maker, temp_file, uuid): afp = await aio_file_maker(temp_file, "w+") writer = Writer(afp) max_length = 1000 chunk = b64encode(os.urandom(max_length)).decode() lines = [chunk[:i] for i in range(max_length)] line: str | bytes for line in lines: await writer(line) await writer("\n") await afp.fsync() read_lines = [] async for line in LineReader(afp): read_lines.append(line[:-1]) def hash_data(data_lines): return hashlib.md5("\n".join(data_lines).encode()).hexdigest() assert hash_data(read_lines) == hash_data(lines) @pytest.mark.parametrize("size", [1, 2, 3, 5, 10, 20, 100, 1000, 2000, 5000]) async def test_line_reader_one_line(size, aio_file_maker, temp_file): afp = await aio_file_maker(temp_file, "w+") writer = Writer(afp) payload = " ".join(uuid4().hex for _ in range(size)) await writer(payload) read_lines = [] async for line in LineReader(afp): read_lines.append(line) assert payload == read_lines[0] async def test_truncate(aio_file_maker, temp_file): afp = await aio_file_maker(temp_file, "w+") await afp.write("hello") await afp.fsync() assert (await afp.read()) == "hello" await afp.truncate(0) assert (await afp.read()) == "" @pytest.mark.parametrize("size", [1, 2, 3, 5, 10, 20, 100, 1000, 2000, 5000]) async def test_modes(size, aio_file_maker, tmpdir): tmpfile = tmpdir.join("test.txt") async with aio_file_maker(tmpfile, "w") as afp: await afp.write("foo") await afp.fsync() async with aio_file_maker(tmpfile, "r") as afp: assert await afp.read() == "foo" async with aio_file_maker(tmpfile, "a+") as afp: assert await afp.read() == "foo" async with aio_file_maker(tmpfile, "r+") as afp: assert await afp.read() == "foo" data = dict((str(i), i)for i in range(size)) tmpfile = tmpdir.join("test.json") async with aio_file_maker(tmpfile, "w") as afp: await afp.write(json.dumps(data, indent=1)) async with aio_file_maker(tmpfile, "r") as afp: result = json.loads(await afp.read()) assert result == data async def test_unicode_reader(aio_file_maker, temp_file): async with aio_file_maker(temp_file, "w+") as afp: await afp.write("한글") async with aio_file_maker(temp_file, "r") as afp: reader = Reader(afp, chunk_size=1) assert await reader.read_chunk() == "한" assert await reader.read_chunk() == "글" async def test_unicode_writer(aio_file_maker, temp_file): async with aio_file_maker(temp_file, "w+") as afp: writer = Writer(afp) await writer("한") await writer("글") async with aio_file_maker(temp_file, "r") as afp: reader = Reader(afp, chunk_size=1) assert await reader.read_chunk() == "한" assert await reader.read_chunk() == "글" @pytest.mark.parametrize(("mode", "data"), [("w+", ""), ("wb+", b"")]) async def test_write_read_nothing(aio_file_maker, temp_file, mode, data): async with aio_file_maker(temp_file, mode) as afp: assert await afp.write(data) == 0 assert await afp.read() == data async def test_partial_writes(temp_file, event_loop): ctx = Mock(caio.AbstractContext) ctx.loop = event_loop ctx.fdsync = CoroutineMock(return_value=None) ctx.write = CoroutineMock(side_effect=asyncio.InvalidStateError) async with AIOFile(temp_file, "w", context=ctx) as afp: # 1. writing first three bytes then last four return_iter = iter((3, 4)) ctx.write.side_effect = lambda *_, **__: next(return_iter) await afp.write("aiofile", offset=0) # 2. then writing 12 bytes then one and the last 6. return_iter = iter((12, 1, 6)) ctx.write.side_effect = lambda *_, **__: next(return_iter) await afp.write("test_partial_writes", offset=8) # compare all chunks against expected assert ctx.write.call_args_list == [ # 1 call(b"aiofile", afp.fileno(), 0), call(b"file", afp.fileno(), 3), # 2 call(b"test_partial_writes", afp.fileno(), 8), call(b"_writes", afp.fileno(), 20), call(b"writes", afp.fileno(), 21), ] async def test_write_returned_negative(temp_file, event_loop): ctx = Mock(caio.AbstractContext) ctx.loop = event_loop ctx.fdsync = CoroutineMock(return_value=None) ctx.write = CoroutineMock(side_effect=asyncio.InvalidStateError) async with AIOFile(temp_file, "w", context=ctx) as afp: return_iter = iter((3, -27)) ctx.write.side_effect = lambda *_, **__: next(return_iter) with pytest.raises(OSError) as raises: await afp.write("aiofile") assert raises.value.errno == 27 assert raises.value.filename == temp_file ctx.write.reset_mock(side_effect=True) ctx.write.return_value = -122 with pytest.raises(OSError) as raises: await afp.write("aiofile") assert raises.value.errno == 122 assert raises.value.filename == temp_file async def test_write_returned_zero(temp_file, event_loop): ctx = Mock(caio.AbstractContext) ctx.loop = event_loop ctx.fdsync = CoroutineMock(return_value=None) ctx.write = CoroutineMock(side_effect=asyncio.InvalidStateError) async with AIOFile(temp_file, "w", context=ctx) as afp: return_iter = iter((3, 0)) ctx.write.side_effect = lambda *_, **__: next(return_iter) with pytest.raises(RuntimeError, match="Write operation returned 0"): await afp.write("aiofile") ctx.write.reset_mock(side_effect=True) ctx.write.return_value = 0 with pytest.raises(RuntimeError, match="Write operation returned 0"): await afp.write("aiofile") async def test_text_io_wrapper(aio_file_maker, temp_file): async with aio_file_maker(temp_file, "w+") as afp: data = "💾💀" await afp.write(data * 5) with open(temp_file, "a+", encoding="utf-8") as fp: assert not fp.read(1) fp.seek(0) assert fp.read() == data * 5 fp.seek(0) assert fp.read(1) == "💾" assert fp.tell() == 4 async with TextFileWrapper(aio_file_maker(temp_file, "a+")) as fp: assert not await fp.read(1) fp.seek(0) chunk = await fp.read() assert chunk == data * 5 fp.seek(0) assert await fp.read(1) == "💾" assert fp.tell() == 4 async def test_text_io_wrapper_read_length_multibyte(aio_file_maker, temp_file): data = ( "世界人権宣言前文人類社会のすべての構成員の固有の尊厳と平等で" "譲ることのできない権利とを承認する" ) async with aio_file_maker(temp_file, "w+") as afp: await afp.write(data) collected = "" async with TextFileWrapper(aio_file_maker(temp_file, "r")) as fp: while chunk := await fp.read(12): assert len(chunk) <= 12 collected += chunk assert collected == data async def test_binary_io_wrapper(aio_file_maker, temp_file): async with aio_file_maker(temp_file, "wb+") as afp: data = b"\x01\x02\x03" await afp.write(data * 32) with open(temp_file, "ab+") as fp: assert not fp.read(1) fp.seek(0) assert fp.read() == data * 32 fp.seek(0) assert fp.read(1) == b"\x01" assert fp.tell() == 1 async with BinaryFileWrapper(aio_file_maker(temp_file, "ab+")) as fp: assert not await fp.read(1) fp.seek(0) assert await fp.read() == data * 32 fp.seek(0) assert await fp.read(1) == b"\x01" assert fp.tell() == 1 async def test_async_open(async_open, temp_file): async with async_open(temp_file, "wb+") as afp: data = b"\x01\x02\x03" + "🦠📱".encode() await afp.write(data * 32) assert isinstance(async_open(temp_file, "ab+"), BinaryFileWrapper) async with async_open(temp_file, "ab+") as fp: assert not await fp.read(1) fp.seek(3) assert await fp.read(8) == "🦠📱".encode() assert isinstance(async_open(temp_file, "a+"), TextFileWrapper) async with async_open(temp_file, "a+") as fp: assert not await fp.read(1) fp.seek(3) assert await fp.read(2) == "🦠📱" async def test_async_open_unicode(aio_file_maker, async_open, temp_file): async with aio_file_maker(temp_file, "w+") as afp: data = "🏁💾🏴‍☠️" await afp.write(data) async with async_open(temp_file, "a+") as afp: with open(temp_file, "a+", encoding="utf-8") as fp: assert not await afp.read(1) assert not fp.read(1) afp.seek(0) fp.seek(0) assert await afp.read(1) == fp.read(1) assert await afp.read(1) == fp.read(1) assert await afp.read(1) == fp.read(1) assert await afp.read(1) == fp.read(1) afp.seek(0) fp.seek(0) assert await afp.read(3) == fp.read(3) assert afp.tell() == fp.tell() async def test_async_open_readline(aio_file_maker, async_open, temp_file): async with aio_file_maker(temp_file, "w+") as afp: data = "Hello\nworld\n" + ("h" * 10000) await afp.write(data) async with async_open(temp_file, "a+") as afp: with open(temp_file, "a+", encoding="utf-8") as fp: afp.seek(0) fp.seek(0) assert await afp.readline() == fp.readline() assert await afp.readline() == fp.readline() assert await afp.readline() == fp.readline() async with async_open(temp_file, "ab+") as afp: with open(temp_file, "ab+") as fp: assert not await afp.read(1) assert not fp.read(1) afp.seek(0) fp.seek(0) assert await afp.readline() == fp.readline() assert await afp.readline() == fp.readline() assert await afp.readline() == fp.readline() async def test_async_open_fp(async_open, tmp_path: Path): data = "Hello\nworld\n" with open(tmp_path / "file.txt", "w+") as fp: async with async_open(fp) as afp: await afp.write(data) afp.seek(0) assert await afp.read() == data assert not afp.file.mode.binary assert not fp.closed with open(tmp_path / "file.txt", "rb") as fp: async with async_open(fp) as afp: assert afp.file.mode.binary assert fp.read().decode() == data @pytest.mark.parametrize("sizes", [[1, 2], [2, 10], [10, 20], [100, 500]]) async def test_async_open_line_iter(sizes, async_open, tmp_path: Path): async with async_open(tmp_path / "file.txt", "w+") as afp: for i in range(*sizes): await afp.write(str(i)) await afp.write("\n") afp.seek(0) idx = sizes[0] async for line in afp: assert line.endswith("\n") assert int(line.strip()) == idx idx += 1 async with async_open(tmp_path / "file.bin", "wb+") as afp: for i in range(*sizes): await afp.write(str(i).encode()) await afp.write(b"\n") afp.seek(0) idx = sizes[0] async for line in afp: assert line.endswith(b"\n") assert int(line.decode().strip()) == idx idx += 1 @pytest.mark.parametrize("size", [1, 2, 3, 5, 10, 20, 100, 1000, 2000, 5000]) async def test_async_open_iter_chunked(size, async_open, tmp_path: Path): src_path = tmp_path / "src.txt" dst_path = tmp_path / "dst.txt" async with async_open(src_path, "w") as afp: for i in range(0, size): await afp.write(str(i)) await afp.write("\n") async with async_open(src_path, "r") as src, \ async_open(dst_path, "w") as dest: async for chunk in src.iter_chunked(): await dest.write(chunk) assert src_path.stat().st_size == dst_path.stat().st_size def hash_file(path): with open(path, "rb") as fp: hasher = hashlib.md5() chunk = fp.read(65535) while chunk: hasher.update(chunk) chunk = fp.read(65535) return hasher.hexdigest() assert hash_file(src_path) == hash_file(dst_path) async def test_open_non_existent_file_with_append( async_open, tmp_path: Path, ): tmp_fpath = tmp_path / "numbers.txt" async with async_open(tmp_fpath, "a+") as afp: for i in range(10): await afp.write(str(i)) await afp.write("\n") async with async_open(tmp_fpath, "a+") as afp: for i in range(10, 20): await afp.write(str(i)) await afp.write("\n") numbers = [] async with async_open(tmp_fpath, "r") as afp: async for line in afp: numbers.append(int(line.strip())) assert numbers == list(range(20)) async def test_clone( async_open, tmp_path: Path, ): tmp_fpath = tmp_path / "test.txt" async with async_open(tmp_fpath, "w") as afp: for i in range(1000, 1003): await afp.write(str(i)) await afp.write("\n") async with async_open(tmp_fpath, "r") as afp: assert await afp.read(5) == "1000\n" async with clone(afp) as cloned: assert await cloned.read(5) == "1000\n" async with clone(afp) as cloned2: assert await cloned2.read(5) == "1000\n" assert await cloned.read(5) == await afp.read(5) == "1001\n" async def test_clone_close( async_open, tmp_path: Path, ): tmp_fpath = tmp_path / "test.txt" async with async_open(tmp_fpath, "w") as afp: for i in range(10): await afp.write(str(i)) await afp.write("\n") async with async_open(tmp_fpath, "r") as afp: await afp.read(10) afp_clone = await clone(afp) await afp_clone.close() assert await afp.read() async with async_open(tmp_fpath, "r") as afp: await afp.read(10) async with clone(afp): assert await afp.read() aiofile-3.12.3/.gitignore0000644000000000000000000000375013615410400012125 0ustar00# Created by .ignore support plugin (hsz.mobi) ### Python template # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *,cover .hypothesis/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # dotenv .env # virtualenv .venv/ venv/ ENV/ # Spyder project settings .spyderproject # Rope project settings .ropeproject ### JetBrains template # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff: .idea/workspace.xml .idea/tasks.xml # Sensitive or high-churn files: .idea/dataSources/ .idea/dataSources.ids .idea/dataSources.xml .idea/dataSources.local.xml .idea/sqlDataSources.xml .idea/dynamic.xml .idea/uiDesigner.xml # Gradle: .idea/gradle.xml .idea/libraries # Mongo Explorer plugin: .idea/mongoSettings.xml ## File-based project format: *.iws ## Plugin-specific files: # IntelliJ /out/ # mpeltonen/sbt-idea plugin .idea_modules/ # JIRA plugin atlassian-ide-plugin.xml # Crashlytics plugin (for Android Studio and IntelliJ) com_crashlytics_export_strings.xml crashlytics.properties crashlytics-build.properties fabric.properties /env* __pycache__ *.c aiofile-3.12.3/LICENCE0000755000000000000000000000000013615410400012472 2LICENCE.mdustar00aiofile-3.12.3/LICENCE.md0000644000000000000000000002437713615410400011531 0ustar00Apache License ============== _Version 2.0, January 2004_ _<>_ ### Terms and Conditions for use, reproduction, and distribution #### 1. Definitions “License” shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. “Licensor” shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. “Legal Entity” shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, “control” means **(i)** the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or **(ii)** ownership of fifty percent (50%) or more of the outstanding shares, or **(iii)** beneficial ownership of such entity. “You” (or “Your”) shall mean an individual or Legal Entity exercising permissions granted by this License. “Source” form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. “Object” form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. “Work” shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). “Derivative Works” shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. “Contribution” shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, “submitted” means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as “Not a Contribution.” “Contributor” shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. #### 2. Grant of Copyright License Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. #### 3. Grant of Patent License Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. #### 4. Redistribution You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: * **(a)** You must give any other recipients of the Work or Derivative Works a copy of this License; and * **(b)** You must cause any modified files to carry prominent notices stating that You changed the files; and * **(c)** You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and * **(d)** If the Work includes a “NOTICE” text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. #### 5. Submission of Contributions Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. #### 6. Trademarks This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. #### 7. Disclaimer of Warranty Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. #### 8. Limitation of Liability In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. #### 9. Accepting Warranty or Additional Liability While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. _END OF TERMS AND CONDITIONS_ ### APPENDIX: How to apply the Apache License to your work To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets `[]` replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same “printed page” as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. aiofile-3.12.3/README.md0000644000000000000000000004020013615410400011403 0ustar00# AIOFile [![Github Actions](https://github.com/mosquito/aiofile/workflows/ci/badge.svg)](https://github.com/mosquito/aiofile/actions?query=workflow%3Aci) [![Latest Version](https://img.shields.io/pypi/v/aiofile.svg)](https://pypi.python.org/pypi/aiofile/) [![Python Versions](https://img.shields.io/pypi/pyversions/aiofile.svg)](https://pypi.python.org/pypi/aiofile/) [![License](https://img.shields.io/pypi/l/aiofile.svg)](https://pypi.python.org/pypi/aiofile/) [![Coverage Status](https://coveralls.io/repos/github/mosquito/aiofile/badge.svg?branch=master)](https://coveralls.io/github/mosquito/aiofile?branch=master) Real asynchronous file operations with asyncio support. ## Features * Since version 2.0.0, uses [caio](https://pypi.org/project/caio), which provides multiple async I/O backends: * **Linux io_uring** — the modern, high-performance Linux kernel AIO interface. * **Linux libaio** — the classic kernel AIO mechanism via [libaio](https://pagure.io/libaio). * **Thread-based (C)** — a [threadpool](https://github.com/mbrossard/threadpool/)-backed implementation for POSIX systems (macOS, Linux). * **Pure Python** — a thread-based fallback for any platform. * The best available backend is chosen automatically based on system compatibility. * `AIOFile` has no internal file pointer. Pass `offset` and `chunk_size` to each operation, or use the `Reader`/`Writer` helpers. For a file-like interface, use `async_open`. ## Limitations * The Linux native AIO and io_uring backends cannot open special files. Asynchronous operations against special filesystems such as `/proc/` or `/sys/` are not supported by the kernel — this is neither an aiofile nor a caio issue. In such cases, switch to a thread-based implementation (see the [Troubleshooting](#troubleshooting) section). ## Code examples All code examples require Python 3.11+. ### High-level API #### `async_open` helper This helper mimics Python's file-like objects, returning an object with equivalent asynchronous methods. Supported methods: * `async def read(length=-1)` — reads a chunk from the file; `-1` reads to the end. * `async def write(data)` — writes a chunk to the file. * `def seek(offset)` — sets the file pointer position. * `def tell()` — returns the current file pointer position. * `async def readline(size=-1, newline="\n")` — reads until a newline or EOF. Since version 3.7.0, `__aiter__` returns a `LineReader`. This method is suboptimal for small lines because it does not reuse the read buffer — prefer `LineReader` when reading line by line. * `def __aiter__() -> LineReader` — iterator over lines. * `def iter_chunked(chunk_size: int = 32768) -> Reader` — iterator over chunks. * `.file` — the underlying `AIOFile` object. Basic example: ```python import asyncio from pathlib import Path from tempfile import gettempdir from aiofile import async_open tmp_filename = Path(gettempdir()) / "hello.txt" async def main(): async with async_open(tmp_filename, 'w+') as afp: await afp.write("Hello ") await afp.write("world") afp.seek(0) print(await afp.read()) await afp.write("Hello from\nasync world") print(await afp.readline()) print(await afp.readline()) asyncio.run(main()) ``` Example without context manager: ```python import asyncio import atexit import os from tempfile import mktemp from aiofile import async_open TMP_NAME = mktemp() atexit.register(os.unlink, TMP_NAME) async def main(): afp = await async_open(TMP_NAME, "w") await afp.write("Hello") await afp.close() asyncio.run(main()) assert open(TMP_NAME, "r").read() == "Hello" ``` Concatenate example program (`cat`): ```python import asyncio import sys from argparse import ArgumentParser from pathlib import Path from aiofile import async_open parser = ArgumentParser( description="Read files line by line using asynchronous io API" ) parser.add_argument("file_name", nargs="+", type=Path) async def main(arguments): for src in arguments.file_name: async with async_open(src, "r") as afp: async for line in afp: sys.stdout.write(line) asyncio.run(main(parser.parse_args())) ``` Copy file example program (`cp`): ```python import asyncio from argparse import ArgumentParser from pathlib import Path from aiofile import async_open parser = ArgumentParser( description="Copying files using asynchronous io API" ) parser.add_argument("source", type=Path) parser.add_argument("dest", type=Path) parser.add_argument("--chunk-size", type=int, default=65535) async def main(arguments): async with async_open(arguments.source, "rb") as src, \ async_open(arguments.dest, "wb") as dest: async for chunk in src.iter_chunked(arguments.chunk_size): await dest.write(chunk) asyncio.run(main(parser.parse_args())) ``` Example with opening an already-open file pointer: ```python import asyncio from typing import IO, Any from aiofile import async_open async def main(fp: IO[Any]): async with async_open(fp) as afp: await afp.write("Hello from\nasync world") print(await afp.readline()) with open("test.txt", "w+") as fp: asyncio.run(main(fp)) ``` Linux native AIO and io_uring do not support reading or writing special files (procfs, sysfs, Unix pipes, etc.), so operations on these files require a compatible context object. ```python import asyncio from aiofile import async_open from caio import thread_aio_asyncio from contextlib import AsyncExitStack async def main(): async with AsyncExitStack() as stack: # Custom context should be reused ctx = await stack.enter_async_context( thread_aio_asyncio.AsyncioContext() ) # Open special file with custom context src = await stack.enter_async_context( async_open("/proc/cpuinfo", "r", context=ctx) ) # Open regular file with default context dest = await stack.enter_async_context( async_open("/tmp/cpuinfo", "w") ) # Copying file content line by line async for line in src: await dest.write(line) asyncio.run(main()) ``` ### `clone` helper An asynchronous context supports a limited number of concurrent operations at the low level, regardless of how many file descriptors are open. `clone` lets you create a second file-like object with its own independent offset from a single descriptor, without opening the file multiple times. ```python """ This example counts multiple hash functions from the file passed as the first argument. The hash functions are counted competitively, and the results are printed in the order of hashing completion. """ import asyncio import hashlib import sys import aiofile async def hasher(name, hash_func, afp): loop = asyncio.get_running_loop() async for chunk in afp.iter_chunked(2 ** 20): await loop.run_in_executor(None, hash_func.update, chunk) print(name, hash_func.hexdigest()) async def main(): async with aiofile.async_open(sys.argv[1], "rb") as source: hashers = [ ("MD5", hashlib.md5()), ("SHA1", hashlib.sha1()), ("SHA256", hashlib.sha256()), ("SHA512", hashlib.sha512()), ] await asyncio.gather(*[ hasher(name, hash_func, await aiofile.clone(source)) for name, hash_func in hashers ]) asyncio.run(main()) ``` > **Note:** This will likely perform poorly on Windows, so if that is > your target platform, this optimization may not be worth it. ### Low-level API The `AIOFile` class is a low-level interface for asynchronous file operations. Its `read` and `write` methods accept an `offset=0` (in bytes) at which the operation is performed. This allows many independent I/O operations on a single open file without an internal pointer. For sequential reading and writing, use `Writer`, `Reader`, and `LineReader`. Note that `async_open` is not the same as `AIOFile`: it wraps it to provide a file-like interface similar to the built-in `open`. ```python import asyncio from aiofile import AIOFile async def main(): async with AIOFile("hello.txt", 'w+') as afp: payload = "Hello world\n" await asyncio.gather( *[afp.write(payload, offset=i * len(payload)) for i in range(10)] ) await afp.fsync() assert await afp.read(len(payload) * 10) == payload * 10 asyncio.run(main()) ``` The low-level API is essentially a lightly sugared `caio` API. ```python import asyncio from aiofile import AIOFile async def main(): async with AIOFile("/tmp/hello.txt", 'w+') as afp: await afp.write("Hello ") await afp.write("world", offset=7) await afp.fsync() print(await afp.read()) asyncio.run(main()) ``` #### `Reader` and `Writer` To read or write a file linearly, the following example may be helpful. ```python import asyncio from aiofile import AIOFile, Reader, Writer async def main(): async with AIOFile("/tmp/hello.txt", 'w+') as afp: writer = Writer(afp) reader = Reader(afp, chunk_size=8) await writer("Hello") await writer(" ") await writer("World") await afp.fsync() async for chunk in reader: print(chunk) asyncio.run(main()) ``` #### `LineReader` - read file line by line `LineReader` is a helper for reading a file linearly, line by line. It maintains a buffer and reads file fragments chunk by chunk, searching for line boundaries. The default chunk size is 4KB. ```python import asyncio from aiofile import AIOFile, LineReader, Writer async def main(): async with AIOFile("/tmp/hello.txt", 'w+') as afp: writer = Writer(afp) await writer("Hello") await writer(" ") await writer("World") await writer("\n") await writer("\n") await writer("From async world") await afp.fsync() async for line in LineReader(afp): print(line) asyncio.run(main()) ``` To read a file line by line, prefer `LineReader` over `async_open`. ## More examples ### Async CSV Dict Reader ```python import asyncio import io from csv import DictReader from aiofile import AIOFile, LineReader class AsyncDictReader: def __init__(self, afp, **kwargs): self.buffer = io.BytesIO() self.file_reader = LineReader( afp, line_sep=kwargs.pop('line_sep', '\n'), chunk_size=kwargs.pop('chunk_size', 4096), offset=kwargs.pop('offset', 0), ) self.reader = DictReader( io.TextIOWrapper( self.buffer, encoding=kwargs.pop('encoding', 'utf-8'), errors=kwargs.pop('errors', 'replace'), ), **kwargs, ) self.line_num = 0 def __aiter__(self): return self async def __anext__(self): if self.line_num == 0: header = await self.file_reader.readline() self.buffer.write(header) line = await self.file_reader.readline() if not line: raise StopAsyncIteration self.buffer.write(line) self.buffer.seek(0) try: result = next(self.reader) except StopIteration as e: raise StopAsyncIteration from e self.buffer.seek(0) self.buffer.truncate(0) self.line_num = self.reader.line_num return result async def main(): async with AIOFile('sample.csv', 'rb') as afp: async for item in AsyncDictReader(afp): print(item) asyncio.run(main()) ``` ## Troubleshooting The caio Linux backends (io_uring and libaio) work well on modern kernels and filesystems. Problems are usually environment-specific and are not bugs. There are several ways to resolve them: 1. Upgrade the kernel. 2. Use a compatible filesystem. 3. Switch to a thread-based or pure-Python backend. Since version 0.7.0, caio offers several ways to select the backend: 1. Set the `CAIO_IMPL` environment variable at runtime: * `uring` — Linux io_uring (requires kernel ≥ 5.1) * `linux` — Linux libaio * `thread` — C-based thread pool implementation * `python` — pure-Python thread-based fallback 2. The `default_implementation` file located next to `__init__.py` in the caio installation directory. Useful for distribution package maintainers. The file may contain comments (lines starting with `#`); the first non-comment line should be one of the values above. 3. Manage contexts manually: ```python import asyncio from aiofile import async_open from caio import linux_aio_asyncio, thread_aio_asyncio async def main(): linux_ctx = linux_aio_asyncio.AsyncioContext() threads_ctx = thread_aio_asyncio.AsyncioContext() async with async_open("/tmp/test.txt", "w", context=linux_ctx) as afp: await afp.write("Hello") async with async_open("/tmp/test.txt", "r", context=threads_ctx) as afp: print(await afp.read()) asyncio.run(main()) ``` ## Benchmark stdlib, `aiofile` (once per `caio` backend, each in its own subprocess), `aiofiles`, `aiomisc.io` — linear/random block order, read/write, buffered/`O_DIRECT`, sequential/concurrent, fixed op count per cell (never a partial/cancelled measurement). Tool, methodology and raw data: [`benchmark/`](benchmark/); a third report at 4096B lives in `benchmark/results/`. ![aiofile benchmark, 65536B blocks](https://media.githubusercontent.com/media/mosquito/aiofile/master/benchmark/results/aiofile-bench-report-bs65536.png) Linear write, one Linux/ext4 run, 65536B blocks, 2000 ops/cell, per-op latency in ms (p50 / p95, ±stdev): | participant | x1 | x8 | |-----------------------|------------------------|------------------------| | stdlib (buffered) | 0.081 / 0.128 (±0.030) | 0.167 / 0.298 (±0.157) | | stdlib (O_DIRECT) | 0.148 / 0.398 (±0.097) | 0.266 / 0.650 (±0.192) | | aiofile (linux_uring) | 0.050 / 0.108 (±0.022) | 0.040 / 0.248 (±0.293) | | aiofile (linux_aio) | 0.009 / 0.024 (±0.020) | 0.040 / 0.089 (±0.020) | | aiofile (thread_aio) | 0.150 / 0.461 (±0.128) | 0.664 / 2.930 (±0.943) | | aiofile (python_aio) | 0.109 / 0.173 (±0.042) | 0.189 / 0.314 (±0.072) | | aiofiles | 0.155 / 0.257 (±0.175) | 1.166 / 2.238 (±0.570) | | aiomisc | 0.156 / 0.256 (±0.051) | 1.118 / 1.607 (±1.750) | `linux_aio` posts the lowest, tightest per-op latency at both concurrency levels here -- `io_submit()` is a lean, purpose-built syscall, while `linux_uring`'s ring/SQPOLL machinery is built to amortize *over batched submissions*, not to minimize one op's latency, so it only pulls ahead once there's a batch to amortize over. `aiofiles`/`aiomisc` latency *grows* at x8 (one seek cursor, serialized by a lock) instead of shrinking like the positional-I/O libraries. ![aiofile benchmark, 256B blocks](https://media.githubusercontent.com/media/mosquito/aiofile/master/benchmark/results/aiofile-bench-report-bs256.png) Linear write, 256B blocks, 2000 ops/cell, per-op latency in ms (p50 / p95, ±stdev): | participant | x1 | x8 | |-----------------------|-------------------------------|------------------------| | stdlib (buffered) | 0.066 / 0.104 (±0.017) | 0.152 / 0.299 (±0.125) | | stdlib (O_DIRECT) | n/a (256B isn't 4096-aligned) | n/a | | aiofile (linux_uring) | 0.086 / 0.159 (±0.043) | 0.043 / 0.070 (±0.021) | | aiofile (linux_aio) | 0.005 / 0.008 (±0.004) | 0.017 / 0.027 (±0.008) | | aiofile (thread_aio) | 0.128 / 0.290 (±0.090) | 0.464 / 1.503 (±0.470) | | aiofile (python_aio) | 0.113 / 0.201 (±0.037) | 0.179 / 0.354 (±0.124) | | aiofiles | 0.136 / 0.169 (±0.030) | 1.002 / 1.516 (±0.372) | | aiomisc | 0.144 / 0.223 (±0.040) | 1.046 / 1.999 (±0.415) | Same shape at a tiny block size: `linux_aio` still has the lowest latency by a wide margin (sub-10μs median at x1), `linux_uring` still needs concurrency to close the gap (and still doesn't fully close it), and the lock-serialized libraries still get *worse*, not better, under concurrency. aiofile-3.12.3/pyproject.toml0000644000000000000000000000325613615410400013052 0ustar00[project] name = "aiofile" version = "3.12.3" description = "Asynchronous file operations." license = "Apache-2.0" authors = [ {name = "Dmitry Orlov", email = "me@mosquito.su"}, ] readme = "README.md" keywords = ["aio", "python", "asyncio", "fileio", "io"] classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Console", "Intended Audience :: Developers", "Intended Audience :: Education", "Intended Audience :: End Users/Desktop", "License :: OSI Approved :: Apache Software License", "Natural Language :: English", "Natural Language :: Russian", "Operating System :: POSIX :: Linux", "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Software Development :: Libraries", "Topic :: System", "Topic :: System :: Operating System", ] requires-python = ">=3.11" dependencies = [ "caio~=0.12.0", ] [project.urls] Homepage = "https://github.com/mosquito/aiofile" [dependency-groups] dev = [ "markdown-pytest>=0.6.5,<0.7", "pytest>=8.4", "aiomisc-pytest>=2.0.0,<3", "pytest-cov>=5.0.0,<6", "coveralls<4", "ruff>=0.15", "ty>=0.0.37", ] [build-system] requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build.targets.sdist] include = [ "aiofile/", "tests/", ] [tool.ruff] line-length = 80 [tool.ruff.lint] select = ["E", "F", "W"] aiofile-3.12.3/PKG-INFO0000644000000000000000000004274213615410400011236 0ustar00Metadata-Version: 2.4 Name: aiofile Version: 3.12.3 Summary: Asynchronous file operations. Project-URL: Homepage, https://github.com/mosquito/aiofile Author-email: Dmitry Orlov License-Expression: Apache-2.0 License-File: LICENCE License-File: LICENCE.md Keywords: aio,asyncio,fileio,io,python Classifier: Development Status :: 5 - Production/Stable Classifier: Environment :: Console Classifier: Intended Audience :: Developers Classifier: Intended Audience :: Education Classifier: Intended Audience :: End Users/Desktop Classifier: License :: OSI Approved :: Apache Software License Classifier: Natural Language :: English Classifier: Natural Language :: Russian Classifier: Operating System :: MacOS :: MacOS X Classifier: Operating System :: POSIX :: Linux Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 :: Only Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: 3.14 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Topic :: Software Development :: Libraries Classifier: Topic :: System Classifier: Topic :: System :: Operating System Requires-Python: >=3.11 Requires-Dist: caio~=0.12.0 Description-Content-Type: text/markdown # AIOFile [![Github Actions](https://github.com/mosquito/aiofile/workflows/ci/badge.svg)](https://github.com/mosquito/aiofile/actions?query=workflow%3Aci) [![Latest Version](https://img.shields.io/pypi/v/aiofile.svg)](https://pypi.python.org/pypi/aiofile/) [![Python Versions](https://img.shields.io/pypi/pyversions/aiofile.svg)](https://pypi.python.org/pypi/aiofile/) [![License](https://img.shields.io/pypi/l/aiofile.svg)](https://pypi.python.org/pypi/aiofile/) [![Coverage Status](https://coveralls.io/repos/github/mosquito/aiofile/badge.svg?branch=master)](https://coveralls.io/github/mosquito/aiofile?branch=master) Real asynchronous file operations with asyncio support. ## Features * Since version 2.0.0, uses [caio](https://pypi.org/project/caio), which provides multiple async I/O backends: * **Linux io_uring** — the modern, high-performance Linux kernel AIO interface. * **Linux libaio** — the classic kernel AIO mechanism via [libaio](https://pagure.io/libaio). * **Thread-based (C)** — a [threadpool](https://github.com/mbrossard/threadpool/)-backed implementation for POSIX systems (macOS, Linux). * **Pure Python** — a thread-based fallback for any platform. * The best available backend is chosen automatically based on system compatibility. * `AIOFile` has no internal file pointer. Pass `offset` and `chunk_size` to each operation, or use the `Reader`/`Writer` helpers. For a file-like interface, use `async_open`. ## Limitations * The Linux native AIO and io_uring backends cannot open special files. Asynchronous operations against special filesystems such as `/proc/` or `/sys/` are not supported by the kernel — this is neither an aiofile nor a caio issue. In such cases, switch to a thread-based implementation (see the [Troubleshooting](#troubleshooting) section). ## Code examples All code examples require Python 3.11+. ### High-level API #### `async_open` helper This helper mimics Python's file-like objects, returning an object with equivalent asynchronous methods. Supported methods: * `async def read(length=-1)` — reads a chunk from the file; `-1` reads to the end. * `async def write(data)` — writes a chunk to the file. * `def seek(offset)` — sets the file pointer position. * `def tell()` — returns the current file pointer position. * `async def readline(size=-1, newline="\n")` — reads until a newline or EOF. Since version 3.7.0, `__aiter__` returns a `LineReader`. This method is suboptimal for small lines because it does not reuse the read buffer — prefer `LineReader` when reading line by line. * `def __aiter__() -> LineReader` — iterator over lines. * `def iter_chunked(chunk_size: int = 32768) -> Reader` — iterator over chunks. * `.file` — the underlying `AIOFile` object. Basic example: ```python import asyncio from pathlib import Path from tempfile import gettempdir from aiofile import async_open tmp_filename = Path(gettempdir()) / "hello.txt" async def main(): async with async_open(tmp_filename, 'w+') as afp: await afp.write("Hello ") await afp.write("world") afp.seek(0) print(await afp.read()) await afp.write("Hello from\nasync world") print(await afp.readline()) print(await afp.readline()) asyncio.run(main()) ``` Example without context manager: ```python import asyncio import atexit import os from tempfile import mktemp from aiofile import async_open TMP_NAME = mktemp() atexit.register(os.unlink, TMP_NAME) async def main(): afp = await async_open(TMP_NAME, "w") await afp.write("Hello") await afp.close() asyncio.run(main()) assert open(TMP_NAME, "r").read() == "Hello" ``` Concatenate example program (`cat`): ```python import asyncio import sys from argparse import ArgumentParser from pathlib import Path from aiofile import async_open parser = ArgumentParser( description="Read files line by line using asynchronous io API" ) parser.add_argument("file_name", nargs="+", type=Path) async def main(arguments): for src in arguments.file_name: async with async_open(src, "r") as afp: async for line in afp: sys.stdout.write(line) asyncio.run(main(parser.parse_args())) ``` Copy file example program (`cp`): ```python import asyncio from argparse import ArgumentParser from pathlib import Path from aiofile import async_open parser = ArgumentParser( description="Copying files using asynchronous io API" ) parser.add_argument("source", type=Path) parser.add_argument("dest", type=Path) parser.add_argument("--chunk-size", type=int, default=65535) async def main(arguments): async with async_open(arguments.source, "rb") as src, \ async_open(arguments.dest, "wb") as dest: async for chunk in src.iter_chunked(arguments.chunk_size): await dest.write(chunk) asyncio.run(main(parser.parse_args())) ``` Example with opening an already-open file pointer: ```python import asyncio from typing import IO, Any from aiofile import async_open async def main(fp: IO[Any]): async with async_open(fp) as afp: await afp.write("Hello from\nasync world") print(await afp.readline()) with open("test.txt", "w+") as fp: asyncio.run(main(fp)) ``` Linux native AIO and io_uring do not support reading or writing special files (procfs, sysfs, Unix pipes, etc.), so operations on these files require a compatible context object. ```python import asyncio from aiofile import async_open from caio import thread_aio_asyncio from contextlib import AsyncExitStack async def main(): async with AsyncExitStack() as stack: # Custom context should be reused ctx = await stack.enter_async_context( thread_aio_asyncio.AsyncioContext() ) # Open special file with custom context src = await stack.enter_async_context( async_open("/proc/cpuinfo", "r", context=ctx) ) # Open regular file with default context dest = await stack.enter_async_context( async_open("/tmp/cpuinfo", "w") ) # Copying file content line by line async for line in src: await dest.write(line) asyncio.run(main()) ``` ### `clone` helper An asynchronous context supports a limited number of concurrent operations at the low level, regardless of how many file descriptors are open. `clone` lets you create a second file-like object with its own independent offset from a single descriptor, without opening the file multiple times. ```python """ This example counts multiple hash functions from the file passed as the first argument. The hash functions are counted competitively, and the results are printed in the order of hashing completion. """ import asyncio import hashlib import sys import aiofile async def hasher(name, hash_func, afp): loop = asyncio.get_running_loop() async for chunk in afp.iter_chunked(2 ** 20): await loop.run_in_executor(None, hash_func.update, chunk) print(name, hash_func.hexdigest()) async def main(): async with aiofile.async_open(sys.argv[1], "rb") as source: hashers = [ ("MD5", hashlib.md5()), ("SHA1", hashlib.sha1()), ("SHA256", hashlib.sha256()), ("SHA512", hashlib.sha512()), ] await asyncio.gather(*[ hasher(name, hash_func, await aiofile.clone(source)) for name, hash_func in hashers ]) asyncio.run(main()) ``` > **Note:** This will likely perform poorly on Windows, so if that is > your target platform, this optimization may not be worth it. ### Low-level API The `AIOFile` class is a low-level interface for asynchronous file operations. Its `read` and `write` methods accept an `offset=0` (in bytes) at which the operation is performed. This allows many independent I/O operations on a single open file without an internal pointer. For sequential reading and writing, use `Writer`, `Reader`, and `LineReader`. Note that `async_open` is not the same as `AIOFile`: it wraps it to provide a file-like interface similar to the built-in `open`. ```python import asyncio from aiofile import AIOFile async def main(): async with AIOFile("hello.txt", 'w+') as afp: payload = "Hello world\n" await asyncio.gather( *[afp.write(payload, offset=i * len(payload)) for i in range(10)] ) await afp.fsync() assert await afp.read(len(payload) * 10) == payload * 10 asyncio.run(main()) ``` The low-level API is essentially a lightly sugared `caio` API. ```python import asyncio from aiofile import AIOFile async def main(): async with AIOFile("/tmp/hello.txt", 'w+') as afp: await afp.write("Hello ") await afp.write("world", offset=7) await afp.fsync() print(await afp.read()) asyncio.run(main()) ``` #### `Reader` and `Writer` To read or write a file linearly, the following example may be helpful. ```python import asyncio from aiofile import AIOFile, Reader, Writer async def main(): async with AIOFile("/tmp/hello.txt", 'w+') as afp: writer = Writer(afp) reader = Reader(afp, chunk_size=8) await writer("Hello") await writer(" ") await writer("World") await afp.fsync() async for chunk in reader: print(chunk) asyncio.run(main()) ``` #### `LineReader` - read file line by line `LineReader` is a helper for reading a file linearly, line by line. It maintains a buffer and reads file fragments chunk by chunk, searching for line boundaries. The default chunk size is 4KB. ```python import asyncio from aiofile import AIOFile, LineReader, Writer async def main(): async with AIOFile("/tmp/hello.txt", 'w+') as afp: writer = Writer(afp) await writer("Hello") await writer(" ") await writer("World") await writer("\n") await writer("\n") await writer("From async world") await afp.fsync() async for line in LineReader(afp): print(line) asyncio.run(main()) ``` To read a file line by line, prefer `LineReader` over `async_open`. ## More examples ### Async CSV Dict Reader ```python import asyncio import io from csv import DictReader from aiofile import AIOFile, LineReader class AsyncDictReader: def __init__(self, afp, **kwargs): self.buffer = io.BytesIO() self.file_reader = LineReader( afp, line_sep=kwargs.pop('line_sep', '\n'), chunk_size=kwargs.pop('chunk_size', 4096), offset=kwargs.pop('offset', 0), ) self.reader = DictReader( io.TextIOWrapper( self.buffer, encoding=kwargs.pop('encoding', 'utf-8'), errors=kwargs.pop('errors', 'replace'), ), **kwargs, ) self.line_num = 0 def __aiter__(self): return self async def __anext__(self): if self.line_num == 0: header = await self.file_reader.readline() self.buffer.write(header) line = await self.file_reader.readline() if not line: raise StopAsyncIteration self.buffer.write(line) self.buffer.seek(0) try: result = next(self.reader) except StopIteration as e: raise StopAsyncIteration from e self.buffer.seek(0) self.buffer.truncate(0) self.line_num = self.reader.line_num return result async def main(): async with AIOFile('sample.csv', 'rb') as afp: async for item in AsyncDictReader(afp): print(item) asyncio.run(main()) ``` ## Troubleshooting The caio Linux backends (io_uring and libaio) work well on modern kernels and filesystems. Problems are usually environment-specific and are not bugs. There are several ways to resolve them: 1. Upgrade the kernel. 2. Use a compatible filesystem. 3. Switch to a thread-based or pure-Python backend. Since version 0.7.0, caio offers several ways to select the backend: 1. Set the `CAIO_IMPL` environment variable at runtime: * `uring` — Linux io_uring (requires kernel ≥ 5.1) * `linux` — Linux libaio * `thread` — C-based thread pool implementation * `python` — pure-Python thread-based fallback 2. The `default_implementation` file located next to `__init__.py` in the caio installation directory. Useful for distribution package maintainers. The file may contain comments (lines starting with `#`); the first non-comment line should be one of the values above. 3. Manage contexts manually: ```python import asyncio from aiofile import async_open from caio import linux_aio_asyncio, thread_aio_asyncio async def main(): linux_ctx = linux_aio_asyncio.AsyncioContext() threads_ctx = thread_aio_asyncio.AsyncioContext() async with async_open("/tmp/test.txt", "w", context=linux_ctx) as afp: await afp.write("Hello") async with async_open("/tmp/test.txt", "r", context=threads_ctx) as afp: print(await afp.read()) asyncio.run(main()) ``` ## Benchmark stdlib, `aiofile` (once per `caio` backend, each in its own subprocess), `aiofiles`, `aiomisc.io` — linear/random block order, read/write, buffered/`O_DIRECT`, sequential/concurrent, fixed op count per cell (never a partial/cancelled measurement). Tool, methodology and raw data: [`benchmark/`](benchmark/); a third report at 4096B lives in `benchmark/results/`. ![aiofile benchmark, 65536B blocks](https://media.githubusercontent.com/media/mosquito/aiofile/master/benchmark/results/aiofile-bench-report-bs65536.png) Linear write, one Linux/ext4 run, 65536B blocks, 2000 ops/cell, per-op latency in ms (p50 / p95, ±stdev): | participant | x1 | x8 | |-----------------------|------------------------|------------------------| | stdlib (buffered) | 0.081 / 0.128 (±0.030) | 0.167 / 0.298 (±0.157) | | stdlib (O_DIRECT) | 0.148 / 0.398 (±0.097) | 0.266 / 0.650 (±0.192) | | aiofile (linux_uring) | 0.050 / 0.108 (±0.022) | 0.040 / 0.248 (±0.293) | | aiofile (linux_aio) | 0.009 / 0.024 (±0.020) | 0.040 / 0.089 (±0.020) | | aiofile (thread_aio) | 0.150 / 0.461 (±0.128) | 0.664 / 2.930 (±0.943) | | aiofile (python_aio) | 0.109 / 0.173 (±0.042) | 0.189 / 0.314 (±0.072) | | aiofiles | 0.155 / 0.257 (±0.175) | 1.166 / 2.238 (±0.570) | | aiomisc | 0.156 / 0.256 (±0.051) | 1.118 / 1.607 (±1.750) | `linux_aio` posts the lowest, tightest per-op latency at both concurrency levels here -- `io_submit()` is a lean, purpose-built syscall, while `linux_uring`'s ring/SQPOLL machinery is built to amortize *over batched submissions*, not to minimize one op's latency, so it only pulls ahead once there's a batch to amortize over. `aiofiles`/`aiomisc` latency *grows* at x8 (one seek cursor, serialized by a lock) instead of shrinking like the positional-I/O libraries. ![aiofile benchmark, 256B blocks](https://media.githubusercontent.com/media/mosquito/aiofile/master/benchmark/results/aiofile-bench-report-bs256.png) Linear write, 256B blocks, 2000 ops/cell, per-op latency in ms (p50 / p95, ±stdev): | participant | x1 | x8 | |-----------------------|-------------------------------|------------------------| | stdlib (buffered) | 0.066 / 0.104 (±0.017) | 0.152 / 0.299 (±0.125) | | stdlib (O_DIRECT) | n/a (256B isn't 4096-aligned) | n/a | | aiofile (linux_uring) | 0.086 / 0.159 (±0.043) | 0.043 / 0.070 (±0.021) | | aiofile (linux_aio) | 0.005 / 0.008 (±0.004) | 0.017 / 0.027 (±0.008) | | aiofile (thread_aio) | 0.128 / 0.290 (±0.090) | 0.464 / 1.503 (±0.470) | | aiofile (python_aio) | 0.113 / 0.201 (±0.037) | 0.179 / 0.354 (±0.124) | | aiofiles | 0.136 / 0.169 (±0.030) | 1.002 / 1.516 (±0.372) | | aiomisc | 0.144 / 0.223 (±0.040) | 1.046 / 1.999 (±0.415) | Same shape at a tiny block size: `linux_aio` still has the lowest latency by a wide margin (sub-10μs median at x1), `linux_uring` still needs concurrency to close the gap (and still doesn't fully close it), and the lock-serialized libraries still get *worse*, not better, under concurrency.