first commit

This commit is contained in:
2020-11-03 18:30:14 -08:00
commit 31d8522470
1881 changed files with 345408 additions and 0 deletions

View File

@@ -0,0 +1,6 @@
"""Utilities for asyncio-friendly file handling."""
from .threadpool import open
__version__ = "0.4.0"
__all__ = (open,)

View File

@@ -0,0 +1,8 @@
import sys
try:
from functools import singledispatch
except ImportError: # pragma: nocover
from singledispatch import singledispatch
PY_35 = sys.version_info >= (3, 5)

View File

@@ -0,0 +1,93 @@
"""Various base classes."""
import asyncio
from collections.abc import Coroutine
class AsyncBase:
def __init__(self, file, loop, executor):
self._file = file
self._loop = loop
self._executor = executor
def __aiter__(self):
"""We are our own iterator."""
return self
@asyncio.coroutine
def __anext__(self):
"""Simulate normal file iteration."""
line = yield from self.readline()
if line:
return line
else:
raise StopAsyncIteration
class _ContextManager(Coroutine):
__slots__ = ('_coro', '_obj')
def __init__(self, coro):
self._coro = coro
self._obj = None
def send(self, value):
return self._coro.send(value)
def throw(self, typ, val=None, tb=None):
if val is None:
return self._coro.throw(typ)
elif tb is None:
return self._coro.throw(typ, val)
else:
return self._coro.throw(typ, val, tb)
def close(self):
return self._coro.close()
@property
def gi_frame(self):
return self._coro.gi_frame
@property
def gi_running(self):
return self._coro.gi_running
@property
def gi_code(self):
return self._coro.gi_code
def __next__(self):
return self.send(None)
@asyncio.coroutine
def __iter__(self):
resp = yield from self._coro
return resp
def __await__(self):
resp = yield from self._coro
return resp
@asyncio.coroutine
def __anext__(self):
resp = yield from self._coro
return resp
@asyncio.coroutine
def __aenter__(self):
self._obj = yield from self._coro
return self._obj
@asyncio.coroutine
def __aexit__(self, exc_type, exc, tb):
self._obj.close()
self._obj = None
class AiofilesContextManager(_ContextManager):
"""An adjusted async context manager for aiofiles."""
@asyncio.coroutine
def __aexit__(self, exc_type, exc_val, exc_tb):
yield from self._obj.close()
self._obj = None

View File

@@ -0,0 +1,22 @@
"""Async executor versions of file functions from the os module."""
import asyncio
from functools import partial, wraps
import os
def wrap(func):
@asyncio.coroutine
@wraps(func)
def run(*args, loop=None, executor=None, **kwargs):
if loop is None:
loop = asyncio.get_event_loop()
pfunc = partial(func, *args, **kwargs)
return loop.run_in_executor(executor, pfunc)
return run
stat = wrap(os.stat)
if hasattr(os, "sendfile"):
sendfile = wrap(os.sendfile)

View File

@@ -0,0 +1,63 @@
"""Handle files using a thread pool executor."""
import asyncio
from io import (FileIO, TextIOBase, BufferedReader, BufferedWriter,
BufferedRandom)
from functools import partial, singledispatch
from .binary import AsyncBufferedIOBase, AsyncBufferedReader, AsyncFileIO
from .text import AsyncTextIOWrapper
from ..base import AiofilesContextManager
sync_open = open
__all__ = ('open', )
def open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None,
closefd=True, opener=None, *, loop=None, executor=None):
return AiofilesContextManager(_open(file, mode=mode, buffering=buffering,
encoding=encoding, errors=errors,
newline=newline, closefd=closefd,
opener=opener, loop=loop,
executor=executor))
@asyncio.coroutine
def _open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None,
closefd=True, opener=None, *, loop=None, executor=None):
"""Open an asyncio file."""
if loop is None:
loop = asyncio.get_event_loop()
cb = partial(sync_open, file, mode=mode, buffering=buffering,
encoding=encoding, errors=errors, newline=newline,
closefd=closefd, opener=opener)
f = yield from loop.run_in_executor(executor, cb)
return wrap(f, loop=loop, executor=executor)
@singledispatch
def wrap(file, *, loop=None, executor=None):
raise TypeError('Unsupported io type: {}.'.format(file))
@wrap.register(TextIOBase)
def _(file, *, loop=None, executor=None):
return AsyncTextIOWrapper(file, loop=loop, executor=executor)
@wrap.register(BufferedWriter)
def _(file, *, loop=None, executor=None):
return AsyncBufferedIOBase(file, loop=loop, executor=executor)
@wrap.register(BufferedReader)
@wrap.register(BufferedRandom)
def _(file, *, loop=None, executor=None):
return AsyncBufferedReader(file, loop=loop, executor=executor)
@wrap.register(FileIO)
def _(file, *, loop=None, executor=None):
return AsyncFileIO(file, loop, executor)

View File

@@ -0,0 +1,26 @@
from ..base import AsyncBase
from .utils import (delegate_to_executor, proxy_property_directly,
proxy_method_directly)
@delegate_to_executor('close', 'flush', 'isatty', 'read', 'read1', 'readinto',
'readline', 'readlines', 'seek', 'seekable', 'tell',
'truncate', 'writable', 'write', 'writelines')
@proxy_method_directly('detach', 'fileno', 'readable')
@proxy_property_directly('closed', 'raw')
class AsyncBufferedIOBase(AsyncBase):
"""The asyncio executor version of io.BufferedWriter."""
@delegate_to_executor('peek')
class AsyncBufferedReader(AsyncBufferedIOBase):
"""The asyncio executor version of io.BufferedReader and Random."""
@delegate_to_executor('close', 'flush', 'isatty', 'read', 'readall', 'readinto',
'readline', 'readlines', 'seek', 'seekable', 'tell',
'truncate', 'writable', 'write', 'writelines')
@proxy_method_directly('fileno', 'readable')
@proxy_property_directly('closed')
class AsyncFileIO(AsyncBase):
"""The asyncio executor version of io.FileIO."""

View File

@@ -0,0 +1,13 @@
from .utils import (delegate_to_executor, proxy_property_directly,
proxy_method_directly)
from ..base import AsyncBase
@delegate_to_executor('close', 'flush', 'isatty', 'read', 'readable',
'readline', 'readlines', 'seek', 'seekable', 'tell',
'truncate', 'write', 'writable', 'writelines')
@proxy_method_directly('detach', 'fileno', 'readable')
@proxy_property_directly('buffer', 'closed', 'encoding', 'errors',
'line_buffering', 'newlines')
class AsyncTextIOWrapper(AsyncBase):
"""The asyncio executor version of io.TextIOWrapper."""

View File

@@ -0,0 +1,49 @@
import asyncio
import functools
def delegate_to_executor(*attrs):
def cls_builder(cls):
for attr_name in attrs:
setattr(cls, attr_name, _make_delegate_method(attr_name))
return cls
return cls_builder
def proxy_method_directly(*attrs):
def cls_builder(cls):
for attr_name in attrs:
setattr(cls, attr_name, _make_proxy_method(attr_name))
return cls
return cls_builder
def proxy_property_directly(*attrs):
def cls_builder(cls):
for attr_name in attrs:
setattr(cls, attr_name, _make_proxy_property(attr_name))
return cls
return cls_builder
def _make_delegate_method(attr_name):
@asyncio.coroutine
def method(self, *args, **kwargs):
cb = functools.partial(getattr(self._file, attr_name),
*args, **kwargs)
return (yield from self._loop.run_in_executor(self._executor, cb))
return method
def _make_proxy_method(attr_name):
def method(self, *args, **kwargs):
return getattr(self._file, attr_name)(*args, **kwargs)
return method
def _make_proxy_property(attr_name):
def proxy_property(self):
return getattr(self._file, attr_name)
return property(proxy_property)