feat(assessment): auto comprehensive assessment - rules engine from college scoring standard, per-semester stats, manual overrides, word export
This commit is contained in:
root
@@ -0,0 +1 @@
|
||||
# This directory is a Python package.
|
||||
@@ -0,0 +1,51 @@
|
||||
# Copyright 2009 Brian Quinlan. All Rights Reserved.
|
||||
# Licensed to PSF under a Contributor Agreement.
|
||||
|
||||
"""Execute computations asynchronously using threads or processes."""
|
||||
|
||||
__author__ = 'Brian Quinlan (brian@sweetapp.com)'
|
||||
|
||||
from concurrent.futures._base import (FIRST_COMPLETED,
|
||||
FIRST_EXCEPTION,
|
||||
ALL_COMPLETED,
|
||||
CancelledError,
|
||||
TimeoutError,
|
||||
InvalidStateError,
|
||||
BrokenExecutor,
|
||||
Future,
|
||||
Executor,
|
||||
wait,
|
||||
as_completed)
|
||||
|
||||
lazy from .process import ProcessPoolExecutor
|
||||
lazy from .thread import ThreadPoolExecutor
|
||||
|
||||
__all__ = [
|
||||
'FIRST_COMPLETED',
|
||||
'FIRST_EXCEPTION',
|
||||
'ALL_COMPLETED',
|
||||
'CancelledError',
|
||||
'TimeoutError',
|
||||
'InvalidStateError',
|
||||
'BrokenExecutor',
|
||||
'Future',
|
||||
'Executor',
|
||||
'wait',
|
||||
'as_completed',
|
||||
'ProcessPoolExecutor',
|
||||
'ThreadPoolExecutor',
|
||||
]
|
||||
|
||||
|
||||
try:
|
||||
import _interpreters
|
||||
except ImportError:
|
||||
_interpreters = None
|
||||
|
||||
if _interpreters:
|
||||
lazy from .interpreter import InterpreterPoolExecutor # noqa: F401
|
||||
__all__.append('InterpreterPoolExecutor')
|
||||
|
||||
|
||||
def __dir__():
|
||||
return __all__ + ['__author__', '__doc__']
|
||||
@@ -0,0 +1,707 @@
|
||||
# Copyright 2009 Brian Quinlan. All Rights Reserved.
|
||||
# Licensed to PSF under a Contributor Agreement.
|
||||
|
||||
__author__ = 'Brian Quinlan (brian@sweetapp.com)'
|
||||
|
||||
import collections
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import types
|
||||
import weakref
|
||||
from itertools import islice
|
||||
|
||||
FIRST_COMPLETED = 'FIRST_COMPLETED'
|
||||
FIRST_EXCEPTION = 'FIRST_EXCEPTION'
|
||||
ALL_COMPLETED = 'ALL_COMPLETED'
|
||||
_AS_COMPLETED = '_AS_COMPLETED'
|
||||
|
||||
# Possible future states (for internal use by the futures package).
|
||||
PENDING = 'PENDING'
|
||||
RUNNING = 'RUNNING'
|
||||
# The future was cancelled by the user...
|
||||
CANCELLED = 'CANCELLED'
|
||||
# ...and _Waiter.add_cancelled() was called by a worker.
|
||||
CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED'
|
||||
FINISHED = 'FINISHED'
|
||||
|
||||
_STATE_TO_DESCRIPTION_MAP = {
|
||||
PENDING: "pending",
|
||||
RUNNING: "running",
|
||||
CANCELLED: "cancelled",
|
||||
CANCELLED_AND_NOTIFIED: "cancelled",
|
||||
FINISHED: "finished"
|
||||
}
|
||||
|
||||
# Logger for internal use by the futures package.
|
||||
LOGGER = logging.getLogger("concurrent.futures")
|
||||
|
||||
class Error(Exception):
|
||||
"""Base class for all future-related exceptions."""
|
||||
pass
|
||||
|
||||
class CancelledError(Error):
|
||||
"""The Future was cancelled."""
|
||||
pass
|
||||
|
||||
TimeoutError = TimeoutError # make local alias for the standard exception
|
||||
|
||||
class InvalidStateError(Error):
|
||||
"""The operation is not allowed in this state."""
|
||||
pass
|
||||
|
||||
class _Waiter(object):
|
||||
"""Provides the event that wait() and as_completed() block on."""
|
||||
def __init__(self):
|
||||
self.event = threading.Event()
|
||||
self.finished_futures = []
|
||||
|
||||
def add_result(self, future):
|
||||
self.finished_futures.append(future)
|
||||
|
||||
def add_exception(self, future):
|
||||
self.finished_futures.append(future)
|
||||
|
||||
def add_cancelled(self, future):
|
||||
self.finished_futures.append(future)
|
||||
|
||||
class _AsCompletedWaiter(_Waiter):
|
||||
"""Used by as_completed()."""
|
||||
|
||||
def __init__(self):
|
||||
super(_AsCompletedWaiter, self).__init__()
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def add_result(self, future):
|
||||
with self.lock:
|
||||
super(_AsCompletedWaiter, self).add_result(future)
|
||||
self.event.set()
|
||||
|
||||
def add_exception(self, future):
|
||||
with self.lock:
|
||||
super(_AsCompletedWaiter, self).add_exception(future)
|
||||
self.event.set()
|
||||
|
||||
def add_cancelled(self, future):
|
||||
with self.lock:
|
||||
super(_AsCompletedWaiter, self).add_cancelled(future)
|
||||
self.event.set()
|
||||
|
||||
class _FirstCompletedWaiter(_Waiter):
|
||||
"""Used by wait(return_when=FIRST_COMPLETED)."""
|
||||
|
||||
def add_result(self, future):
|
||||
super().add_result(future)
|
||||
self.event.set()
|
||||
|
||||
def add_exception(self, future):
|
||||
super().add_exception(future)
|
||||
self.event.set()
|
||||
|
||||
def add_cancelled(self, future):
|
||||
super().add_cancelled(future)
|
||||
self.event.set()
|
||||
|
||||
class _AllCompletedWaiter(_Waiter):
|
||||
"""Used by wait(return_when=FIRST_EXCEPTION and ALL_COMPLETED)."""
|
||||
|
||||
def __init__(self, num_pending_calls, stop_on_exception):
|
||||
self.num_pending_calls = num_pending_calls
|
||||
self.stop_on_exception = stop_on_exception
|
||||
self.lock = threading.Lock()
|
||||
super().__init__()
|
||||
|
||||
def _decrement_pending_calls(self):
|
||||
with self.lock:
|
||||
self.num_pending_calls -= 1
|
||||
if not self.num_pending_calls:
|
||||
self.event.set()
|
||||
|
||||
def add_result(self, future):
|
||||
super().add_result(future)
|
||||
self._decrement_pending_calls()
|
||||
|
||||
def add_exception(self, future):
|
||||
super().add_exception(future)
|
||||
if self.stop_on_exception:
|
||||
self.event.set()
|
||||
else:
|
||||
self._decrement_pending_calls()
|
||||
|
||||
def add_cancelled(self, future):
|
||||
super().add_cancelled(future)
|
||||
self._decrement_pending_calls()
|
||||
|
||||
class _AcquireFutures(object):
|
||||
"""A context manager that does an ordered acquire of Future conditions."""
|
||||
|
||||
def __init__(self, futures):
|
||||
self.futures = sorted(futures, key=id)
|
||||
|
||||
def __enter__(self):
|
||||
for future in self.futures:
|
||||
future._condition.acquire()
|
||||
|
||||
def __exit__(self, *args):
|
||||
for future in self.futures:
|
||||
future._condition.release()
|
||||
|
||||
def _create_and_install_waiters(fs, return_when):
|
||||
if return_when == _AS_COMPLETED:
|
||||
waiter = _AsCompletedWaiter()
|
||||
elif return_when == FIRST_COMPLETED:
|
||||
waiter = _FirstCompletedWaiter()
|
||||
else:
|
||||
pending_count = sum(
|
||||
f._state not in [CANCELLED_AND_NOTIFIED, FINISHED] for f in fs)
|
||||
|
||||
if return_when == FIRST_EXCEPTION:
|
||||
waiter = _AllCompletedWaiter(pending_count, stop_on_exception=True)
|
||||
elif return_when == ALL_COMPLETED:
|
||||
waiter = _AllCompletedWaiter(pending_count, stop_on_exception=False)
|
||||
else:
|
||||
raise ValueError("Invalid return condition: %r" % return_when)
|
||||
|
||||
for f in fs:
|
||||
f._waiters.append(waiter)
|
||||
|
||||
return waiter
|
||||
|
||||
|
||||
def _yield_finished_futures(fs, waiter, ref_collect):
|
||||
"""
|
||||
Iterate on the list *fs*, yielding finished futures one by one in
|
||||
reverse order.
|
||||
Before yielding a future, *waiter* is removed from its waiters
|
||||
and the future is removed from each set in the collection of sets
|
||||
*ref_collect*.
|
||||
|
||||
The aim of this function is to avoid keeping stale references after
|
||||
the future is yielded and before the iterator resumes.
|
||||
"""
|
||||
while fs:
|
||||
f = fs[-1]
|
||||
for futures_set in ref_collect:
|
||||
futures_set.remove(f)
|
||||
with f._condition:
|
||||
f._waiters.remove(waiter)
|
||||
del f
|
||||
# Careful not to keep a reference to the popped value
|
||||
yield fs.pop()
|
||||
|
||||
|
||||
def as_completed(fs, timeout=None):
|
||||
"""An iterator over the given futures that yields each as it completes.
|
||||
|
||||
Args:
|
||||
fs: The sequence of Futures (possibly created by different
|
||||
Executors) to iterate over.
|
||||
timeout: The maximum number of seconds to wait. If None, then
|
||||
there is no limit on the wait time.
|
||||
|
||||
Returns:
|
||||
An iterator that yields the given Futures as they complete
|
||||
(finished or cancelled). If any given Futures are duplicated,
|
||||
they will be returned once.
|
||||
|
||||
Raises:
|
||||
TimeoutError: If the entire result iterator could not be generated
|
||||
before the given timeout.
|
||||
"""
|
||||
if timeout is not None:
|
||||
end_time = timeout + time.monotonic()
|
||||
|
||||
fs = set(fs)
|
||||
total_futures = len(fs)
|
||||
with _AcquireFutures(fs):
|
||||
finished = set(
|
||||
f for f in fs
|
||||
if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
|
||||
pending = fs - finished
|
||||
waiter = _create_and_install_waiters(fs, _AS_COMPLETED)
|
||||
finished = list(finished)
|
||||
try:
|
||||
yield from _yield_finished_futures(finished, waiter,
|
||||
ref_collect=(fs,))
|
||||
|
||||
while pending:
|
||||
if timeout is None:
|
||||
wait_timeout = None
|
||||
else:
|
||||
wait_timeout = end_time - time.monotonic()
|
||||
if wait_timeout < 0:
|
||||
raise TimeoutError(
|
||||
'%d (of %d) futures unfinished' % (
|
||||
len(pending), total_futures))
|
||||
|
||||
waiter.event.wait(wait_timeout)
|
||||
|
||||
with waiter.lock:
|
||||
finished = waiter.finished_futures
|
||||
waiter.finished_futures = []
|
||||
waiter.event.clear()
|
||||
|
||||
# reverse to keep finishing order
|
||||
finished.reverse()
|
||||
yield from _yield_finished_futures(finished, waiter,
|
||||
ref_collect=(fs, pending))
|
||||
|
||||
finally:
|
||||
# Remove waiter from unfinished futures
|
||||
for f in fs:
|
||||
with f._condition:
|
||||
f._waiters.remove(waiter)
|
||||
|
||||
DoneAndNotDoneFutures = collections.namedtuple(
|
||||
'DoneAndNotDoneFutures', 'done not_done')
|
||||
def wait(fs, timeout=None, return_when=ALL_COMPLETED):
|
||||
"""Wait for the futures in the given sequence to complete.
|
||||
|
||||
Args:
|
||||
fs: The sequence of Futures (possibly created by different
|
||||
Executors) to wait upon.
|
||||
timeout: The maximum number of seconds to wait. If None, then
|
||||
there is no limit on the wait time.
|
||||
return_when: Indicates when this function should return.
|
||||
The options are:
|
||||
|
||||
FIRST_COMPLETED - Return when any future finishes or is
|
||||
cancelled.
|
||||
FIRST_EXCEPTION - Return when any future finishes by raising an
|
||||
exception. If no future raises an exception
|
||||
then it is equivalent to ALL_COMPLETED.
|
||||
ALL_COMPLETED - Return when all futures finish or are
|
||||
cancelled.
|
||||
|
||||
Returns:
|
||||
A named 2-tuple of sets. The first set, named 'done', contains the
|
||||
futures that completed (is finished or cancelled) before the wait
|
||||
completed. The second set, named 'not_done', contains uncompleted
|
||||
futures. Duplicate futures given to *fs* are removed and will be
|
||||
returned only once.
|
||||
"""
|
||||
fs = set(fs)
|
||||
with _AcquireFutures(fs):
|
||||
done = {f for f in fs
|
||||
if f._state in [CANCELLED_AND_NOTIFIED, FINISHED]}
|
||||
not_done = fs - done
|
||||
if (return_when == FIRST_COMPLETED) and done:
|
||||
return DoneAndNotDoneFutures(done, not_done)
|
||||
elif (return_when == FIRST_EXCEPTION) and done:
|
||||
if any(f for f in done
|
||||
if not f.cancelled() and f.exception() is not None):
|
||||
return DoneAndNotDoneFutures(done, not_done)
|
||||
|
||||
if len(done) == len(fs):
|
||||
return DoneAndNotDoneFutures(done, not_done)
|
||||
|
||||
waiter = _create_and_install_waiters(fs, return_when)
|
||||
|
||||
waiter.event.wait(timeout)
|
||||
for f in fs:
|
||||
with f._condition:
|
||||
f._waiters.remove(waiter)
|
||||
|
||||
done.update(waiter.finished_futures)
|
||||
return DoneAndNotDoneFutures(done, fs - done)
|
||||
|
||||
|
||||
def _result_or_cancel(fut, timeout=None):
|
||||
try:
|
||||
try:
|
||||
return fut.result(timeout)
|
||||
finally:
|
||||
fut.cancel()
|
||||
finally:
|
||||
# Break a reference cycle with the exception in self._exception
|
||||
del fut
|
||||
|
||||
|
||||
class Future(object):
|
||||
"""Represents the result of an asynchronous computation."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initializes the future. Should not be called by clients."""
|
||||
self._condition = threading.Condition()
|
||||
self._state = PENDING
|
||||
self._result = None
|
||||
self._exception = None
|
||||
self._waiters = []
|
||||
self._done_callbacks = []
|
||||
|
||||
def _invoke_callbacks(self):
|
||||
for callback in self._done_callbacks:
|
||||
try:
|
||||
callback(self)
|
||||
except Exception:
|
||||
LOGGER.exception('exception calling callback for %r', self)
|
||||
|
||||
def __repr__(self):
|
||||
with self._condition:
|
||||
if self._state == FINISHED:
|
||||
if self._exception:
|
||||
return '<%s at %#x state=%s raised %s>' % (
|
||||
self.__class__.__name__,
|
||||
id(self),
|
||||
_STATE_TO_DESCRIPTION_MAP[self._state],
|
||||
self._exception.__class__.__name__)
|
||||
else:
|
||||
return '<%s at %#x state=%s returned %s>' % (
|
||||
self.__class__.__name__,
|
||||
id(self),
|
||||
_STATE_TO_DESCRIPTION_MAP[self._state],
|
||||
self._result.__class__.__name__)
|
||||
return '<%s at %#x state=%s>' % (
|
||||
self.__class__.__name__,
|
||||
id(self),
|
||||
_STATE_TO_DESCRIPTION_MAP[self._state])
|
||||
|
||||
def cancel(self):
|
||||
"""Cancel the future if possible.
|
||||
|
||||
Returns True if the future was cancelled, False otherwise. A future
|
||||
cannot be cancelled if it is running or has already completed.
|
||||
"""
|
||||
with self._condition:
|
||||
if self._state in [RUNNING, FINISHED]:
|
||||
return False
|
||||
|
||||
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
|
||||
return True
|
||||
|
||||
self._state = CANCELLED
|
||||
self._condition.notify_all()
|
||||
|
||||
self._invoke_callbacks()
|
||||
return True
|
||||
|
||||
def cancelled(self):
|
||||
"""Return True if the future was cancelled."""
|
||||
with self._condition:
|
||||
return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]
|
||||
|
||||
def running(self):
|
||||
"""Return True if the future is currently executing."""
|
||||
with self._condition:
|
||||
return self._state == RUNNING
|
||||
|
||||
def done(self):
|
||||
"""Return True if the future was cancelled or finished executing."""
|
||||
with self._condition:
|
||||
return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]
|
||||
|
||||
def __get_result(self):
|
||||
if self._exception is not None:
|
||||
try:
|
||||
raise self._exception
|
||||
finally:
|
||||
# Break a reference cycle with the exception in self._exception
|
||||
self = None
|
||||
else:
|
||||
return self._result
|
||||
|
||||
def add_done_callback(self, fn):
|
||||
"""Attaches a callable that will be called when the future finishes.
|
||||
|
||||
Args:
|
||||
fn: A callable that will be called with this future as its only
|
||||
argument when the future completes or is cancelled. The
|
||||
callable will always be called by a thread in the same
|
||||
process in which it was added. If the future has already
|
||||
completed or been cancelled then the callable will be
|
||||
called immediately. These callables are called in the
|
||||
order that they were added.
|
||||
"""
|
||||
with self._condition:
|
||||
if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]:
|
||||
self._done_callbacks.append(fn)
|
||||
return
|
||||
try:
|
||||
fn(self)
|
||||
except Exception:
|
||||
LOGGER.exception('exception calling callback for %r', self)
|
||||
|
||||
def result(self, timeout=None):
|
||||
"""Return the result of the call that the future represents.
|
||||
|
||||
Args:
|
||||
timeout: The number of seconds to wait for the result if the
|
||||
future isn't done. If None, then there is no limit on the
|
||||
wait time.
|
||||
|
||||
Returns:
|
||||
The result of the call that the future represents.
|
||||
|
||||
Raises:
|
||||
CancelledError: If the future was cancelled.
|
||||
TimeoutError: If the future didn't finish executing before the
|
||||
given timeout.
|
||||
Exception: If the call raised then that exception will be
|
||||
raised.
|
||||
"""
|
||||
try:
|
||||
with self._condition:
|
||||
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
|
||||
raise CancelledError()
|
||||
elif self._state == FINISHED:
|
||||
return self.__get_result()
|
||||
|
||||
self._condition.wait(timeout)
|
||||
|
||||
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
|
||||
raise CancelledError()
|
||||
elif self._state == FINISHED:
|
||||
return self.__get_result()
|
||||
else:
|
||||
raise TimeoutError()
|
||||
finally:
|
||||
# Break a reference cycle with the exception in self._exception
|
||||
self = None
|
||||
|
||||
def exception(self, timeout=None):
|
||||
"""Return the exception raised by the call that the future represents.
|
||||
|
||||
Args:
|
||||
timeout: The number of seconds to wait for the exception if the
|
||||
future isn't done. If None, then there is no limit on the
|
||||
wait time.
|
||||
|
||||
Returns:
|
||||
The exception raised by the call that the future represents or
|
||||
None if the call completed without raising.
|
||||
|
||||
Raises:
|
||||
CancelledError: If the future was cancelled.
|
||||
TimeoutError: If the future didn't finish executing before the
|
||||
given timeout.
|
||||
"""
|
||||
|
||||
with self._condition:
|
||||
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
|
||||
raise CancelledError()
|
||||
elif self._state == FINISHED:
|
||||
return self._exception
|
||||
|
||||
self._condition.wait(timeout)
|
||||
|
||||
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
|
||||
raise CancelledError()
|
||||
elif self._state == FINISHED:
|
||||
return self._exception
|
||||
else:
|
||||
raise TimeoutError()
|
||||
|
||||
# The following methods should only be used by Executors and in tests.
|
||||
def set_running_or_notify_cancel(self):
|
||||
"""Mark the future as running or process any cancel notifications.
|
||||
|
||||
Should only be used by Executor implementations and unit tests.
|
||||
|
||||
If the future has been cancelled (cancel() was called and returned
|
||||
True) then any threads waiting on the future completing (though
|
||||
calls to as_completed() or wait()) are notified and False is
|
||||
returned.
|
||||
|
||||
If the future was not cancelled then it is put in the running state
|
||||
(future calls to running() will return True) and True is returned.
|
||||
|
||||
This method should be called by Executor implementations before
|
||||
executing the work associated with this future. If this method
|
||||
returns False then the work should not be executed.
|
||||
|
||||
Returns:
|
||||
False if the Future was cancelled, True otherwise.
|
||||
|
||||
Raises:
|
||||
RuntimeError: if this method was already called or if
|
||||
set_result() or set_exception() was called.
|
||||
"""
|
||||
with self._condition:
|
||||
if self._state == CANCELLED:
|
||||
self._state = CANCELLED_AND_NOTIFIED
|
||||
for waiter in self._waiters:
|
||||
waiter.add_cancelled(self)
|
||||
# self._condition.notify_all() is not necessary because
|
||||
# self.cancel() triggers a notification.
|
||||
return False
|
||||
elif self._state == PENDING:
|
||||
self._state = RUNNING
|
||||
return True
|
||||
else:
|
||||
LOGGER.critical('Future %s in unexpected state: %s',
|
||||
id(self),
|
||||
self._state)
|
||||
raise RuntimeError('Future in unexpected state')
|
||||
|
||||
def set_result(self, result):
|
||||
"""Sets the return value of work associated with the future.
|
||||
|
||||
Should only be used by Executor implementations and unit tests.
|
||||
"""
|
||||
with self._condition:
|
||||
if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED}:
|
||||
raise InvalidStateError('{}: {!r}'.format(self._state, self))
|
||||
self._result = result
|
||||
self._state = FINISHED
|
||||
for waiter in self._waiters:
|
||||
waiter.add_result(self)
|
||||
self._condition.notify_all()
|
||||
self._invoke_callbacks()
|
||||
|
||||
def set_exception(self, exception):
|
||||
"""Sets the result of the future as being the given exception.
|
||||
|
||||
Should only be used by Executor implementations and unit tests.
|
||||
"""
|
||||
with self._condition:
|
||||
if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED}:
|
||||
raise InvalidStateError('{}: {!r}'.format(self._state, self))
|
||||
self._exception = exception
|
||||
self._state = FINISHED
|
||||
for waiter in self._waiters:
|
||||
waiter.add_exception(self)
|
||||
self._condition.notify_all()
|
||||
self._invoke_callbacks()
|
||||
|
||||
def _get_snapshot(self):
|
||||
"""Get a snapshot of the future's current state.
|
||||
|
||||
This method atomically retrieves the state in one lock acquisition,
|
||||
which is significantly faster than multiple method calls.
|
||||
|
||||
Returns:
|
||||
Tuple of (done, cancelled, result, exception)
|
||||
- done: True if the future is done (cancelled or finished)
|
||||
- cancelled: True if the future was cancelled
|
||||
- result: The result if available and not cancelled
|
||||
- exception: The exception if available and not cancelled
|
||||
"""
|
||||
# Fast path: check if already finished without lock
|
||||
if self._state == FINISHED:
|
||||
return True, False, self._result, self._exception
|
||||
|
||||
# Need lock for other states since they can change
|
||||
with self._condition:
|
||||
# We have to check the state again after acquiring the lock
|
||||
# because it may have changed in the meantime.
|
||||
if self._state == FINISHED:
|
||||
return True, False, self._result, self._exception
|
||||
if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED}:
|
||||
return True, True, None, None
|
||||
return False, False, None, None
|
||||
|
||||
__class_getitem__ = classmethod(types.GenericAlias)
|
||||
|
||||
class Executor(object):
|
||||
"""This is an abstract base class for concrete asynchronous executors."""
|
||||
|
||||
def submit(self, fn, /, *args, **kwargs):
|
||||
"""Submits a callable to be executed with the given arguments.
|
||||
|
||||
Schedules the callable to be executed as fn(*args, **kwargs) and
|
||||
returns a Future instance representing the execution of the
|
||||
callable.
|
||||
|
||||
Returns:
|
||||
A Future representing the given call.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None):
|
||||
"""Returns an iterator equivalent to map(fn, iter).
|
||||
|
||||
Args:
|
||||
fn: A callable that will take as many arguments as there are
|
||||
passed iterables.
|
||||
timeout: The maximum number of seconds to wait. If None, then
|
||||
there is no limit on the wait time.
|
||||
chunksize: The size of the chunks the iterable will be broken
|
||||
into before being passed to a child process. This argument
|
||||
is only used by ProcessPoolExecutor; it is ignored by
|
||||
ThreadPoolExecutor.
|
||||
buffersize: The number of submitted tasks whose results have not
|
||||
yet been yielded. If the buffer is full, iteration over the
|
||||
iterables pauses until a result is yielded from the buffer.
|
||||
If None, all input elements are eagerly collected, and
|
||||
a task is submitted for each.
|
||||
|
||||
Returns:
|
||||
An iterator equivalent to: map(func, *iterables) but the calls
|
||||
may be evaluated out-of-order.
|
||||
|
||||
Raises:
|
||||
TimeoutError: If the entire result iterator could not be
|
||||
generated before the given timeout.
|
||||
Exception: If fn(*args) raises for any values.
|
||||
"""
|
||||
if buffersize is not None and not isinstance(buffersize, int):
|
||||
raise TypeError("buffersize must be an integer or None")
|
||||
if buffersize is not None and buffersize < 1:
|
||||
raise ValueError("buffersize must be None or > 0")
|
||||
|
||||
if timeout is not None:
|
||||
end_time = timeout + time.monotonic()
|
||||
|
||||
zipped_iterables = zip(*iterables)
|
||||
if buffersize:
|
||||
fs = collections.deque(
|
||||
self.submit(fn, *args) for args in islice(zipped_iterables, buffersize)
|
||||
)
|
||||
else:
|
||||
fs = [self.submit(fn, *args) for args in zipped_iterables]
|
||||
|
||||
# Use a weak reference to ensure that the executor can be garbage
|
||||
# collected independently of the result_iterator closure.
|
||||
executor_weakref = weakref.ref(self)
|
||||
|
||||
# Yield must be hidden in closure so that the futures are submitted
|
||||
# before the first iterator value is required.
|
||||
def result_iterator():
|
||||
try:
|
||||
# reverse to keep finishing order
|
||||
fs.reverse()
|
||||
while fs:
|
||||
if (
|
||||
buffersize
|
||||
and (executor := executor_weakref())
|
||||
and (args := next(zipped_iterables, None))
|
||||
):
|
||||
fs.appendleft(executor.submit(fn, *args))
|
||||
# Careful not to keep a reference to the popped future
|
||||
if timeout is None:
|
||||
yield _result_or_cancel(fs.pop())
|
||||
else:
|
||||
yield _result_or_cancel(fs.pop(), end_time - time.monotonic())
|
||||
finally:
|
||||
for future in fs:
|
||||
future.cancel()
|
||||
return result_iterator()
|
||||
|
||||
def shutdown(self, wait=True, *, cancel_futures=False):
|
||||
"""Clean-up the resources associated with the Executor.
|
||||
|
||||
It is safe to call this method several times. Otherwise, no other
|
||||
methods can be called after this one.
|
||||
|
||||
Args:
|
||||
wait: If True then shutdown will not return until all running
|
||||
futures have finished executing and the resources used by
|
||||
the executor have been reclaimed.
|
||||
cancel_futures: If True then shutdown will cancel all pending
|
||||
futures. Futures that are completed or running will not be
|
||||
cancelled.
|
||||
"""
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.shutdown(wait=True)
|
||||
return False
|
||||
|
||||
|
||||
class BrokenExecutor(RuntimeError):
|
||||
"""
|
||||
Raised when an executor has become non-functional after a severe failure.
|
||||
"""
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Implements InterpreterPoolExecutor."""
|
||||
|
||||
from concurrent import interpreters
|
||||
import sys
|
||||
from . import thread as _thread
|
||||
import traceback
|
||||
|
||||
|
||||
def do_call(results, func, args, kwargs):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except BaseException as exc:
|
||||
# Send the captured exception out on the results queue,
|
||||
# but still leave it unhandled for the interpreter to handle.
|
||||
try:
|
||||
results.put(exc)
|
||||
except interpreters.NotShareableError:
|
||||
# The exception is not shareable.
|
||||
print('exception is not shareable:', file=sys.stderr)
|
||||
traceback.print_exception(exc)
|
||||
results.put(None)
|
||||
raise # re-raise
|
||||
|
||||
|
||||
class WorkerContext(_thread.WorkerContext):
|
||||
|
||||
@classmethod
|
||||
def prepare(cls, initializer, initargs):
|
||||
def resolve_task(fn, args, kwargs):
|
||||
if isinstance(fn, str):
|
||||
# XXX Circle back to this later.
|
||||
raise TypeError('scripts not supported')
|
||||
else:
|
||||
task = (fn, args, kwargs)
|
||||
return task
|
||||
|
||||
if initializer is not None:
|
||||
try:
|
||||
initdata = resolve_task(initializer, initargs, {})
|
||||
except ValueError:
|
||||
if isinstance(initializer, str) and initargs:
|
||||
raise ValueError(f'an initializer script does not take args, got {initargs!r}')
|
||||
raise # re-raise
|
||||
else:
|
||||
initdata = None
|
||||
def create_context():
|
||||
return cls(initdata)
|
||||
return create_context, resolve_task
|
||||
|
||||
def __init__(self, initdata):
|
||||
self.initdata = initdata
|
||||
self.interp = None
|
||||
self.results = None
|
||||
|
||||
def __del__(self):
|
||||
if self.interp is not None:
|
||||
self.finalize()
|
||||
|
||||
def initialize(self):
|
||||
assert self.interp is None, self.interp
|
||||
self.interp = interpreters.create()
|
||||
try:
|
||||
maxsize = 0
|
||||
self.results = interpreters.create_queue(maxsize)
|
||||
|
||||
if self.initdata:
|
||||
self.run(self.initdata)
|
||||
except BaseException:
|
||||
self.finalize()
|
||||
raise # re-raise
|
||||
|
||||
def finalize(self):
|
||||
interp = self.interp
|
||||
results = self.results
|
||||
self.results = None
|
||||
self.interp = None
|
||||
if results is not None:
|
||||
del results
|
||||
if interp is not None:
|
||||
interp.close()
|
||||
|
||||
def run(self, task):
|
||||
try:
|
||||
return self.interp.call(do_call, self.results, *task)
|
||||
except interpreters.ExecutionFailed as wrapper:
|
||||
# Wait for the exception data to show up.
|
||||
exc = self.results.get()
|
||||
if exc is None:
|
||||
# The exception must have been not shareable.
|
||||
raise # re-raise
|
||||
raise exc from wrapper
|
||||
|
||||
|
||||
class BrokenInterpreterPool(_thread.BrokenThreadPool):
|
||||
"""
|
||||
Raised when a worker thread in an InterpreterPoolExecutor failed initializing.
|
||||
"""
|
||||
|
||||
|
||||
class InterpreterPoolExecutor(_thread.ThreadPoolExecutor):
|
||||
|
||||
BROKEN = BrokenInterpreterPool
|
||||
|
||||
@classmethod
|
||||
def prepare_context(cls, initializer, initargs):
|
||||
return WorkerContext.prepare(initializer, initargs)
|
||||
|
||||
def __init__(self, max_workers=None, thread_name_prefix='',
|
||||
initializer=None, initargs=()):
|
||||
"""Initializes a new InterpreterPoolExecutor instance.
|
||||
|
||||
Args:
|
||||
max_workers: The maximum number of interpreters that can be used
|
||||
to execute the given calls.
|
||||
thread_name_prefix: An optional name prefix to give our threads.
|
||||
initializer: A callable or script used to initialize
|
||||
each worker interpreter.
|
||||
initargs: A tuple of arguments to pass to the initializer.
|
||||
"""
|
||||
thread_name_prefix = (thread_name_prefix or
|
||||
(f"InterpreterPoolExecutor-{self._counter()}"))
|
||||
super().__init__(max_workers, thread_name_prefix,
|
||||
initializer, initargs)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,274 @@
|
||||
# Copyright 2009 Brian Quinlan. All Rights Reserved.
|
||||
# Licensed to PSF under a Contributor Agreement.
|
||||
|
||||
"""Implements ThreadPoolExecutor."""
|
||||
|
||||
__author__ = 'Brian Quinlan (brian@sweetapp.com)'
|
||||
|
||||
from concurrent.futures import _base
|
||||
import itertools
|
||||
import queue
|
||||
import threading
|
||||
import types
|
||||
import weakref
|
||||
import os
|
||||
|
||||
|
||||
_threads_queues = weakref.WeakKeyDictionary()
|
||||
_shutdown = False
|
||||
# Lock that ensures that new workers are not created while the interpreter is
|
||||
# shutting down. Must be held while mutating _threads_queues and _shutdown.
|
||||
_global_shutdown_lock = threading.Lock()
|
||||
|
||||
def _python_exit():
|
||||
global _shutdown
|
||||
with _global_shutdown_lock:
|
||||
_shutdown = True
|
||||
items = list(_threads_queues.items())
|
||||
for t, q in items:
|
||||
q.put(None)
|
||||
for t, q in items:
|
||||
t.join()
|
||||
|
||||
# Register for `_python_exit()` to be called just before joining all
|
||||
# non-daemon threads. This is used instead of `atexit.register()` for
|
||||
# compatibility with subinterpreters, which no longer support daemon threads.
|
||||
# See bpo-39812 for context.
|
||||
threading._register_atexit(_python_exit)
|
||||
|
||||
# At fork, reinitialize the `_global_shutdown_lock` lock in the child process
|
||||
if hasattr(os, 'register_at_fork'):
|
||||
os.register_at_fork(before=_global_shutdown_lock.acquire,
|
||||
after_in_child=_global_shutdown_lock._at_fork_reinit,
|
||||
after_in_parent=_global_shutdown_lock.release)
|
||||
os.register_at_fork(after_in_child=_threads_queues.clear)
|
||||
|
||||
|
||||
class WorkerContext:
|
||||
|
||||
@classmethod
|
||||
def prepare(cls, initializer, initargs):
|
||||
if initializer is not None:
|
||||
if not callable(initializer):
|
||||
raise TypeError("initializer must be a callable")
|
||||
def create_context():
|
||||
return cls(initializer, initargs)
|
||||
def resolve_task(fn, args, kwargs):
|
||||
return (fn, args, kwargs)
|
||||
return create_context, resolve_task
|
||||
|
||||
def __init__(self, initializer, initargs):
|
||||
self.initializer = initializer
|
||||
self.initargs = initargs
|
||||
|
||||
def initialize(self):
|
||||
if self.initializer is not None:
|
||||
self.initializer(*self.initargs)
|
||||
|
||||
def finalize(self):
|
||||
pass
|
||||
|
||||
def run(self, task):
|
||||
fn, args, kwargs = task
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
|
||||
class _WorkItem:
|
||||
def __init__(self, future, task):
|
||||
self.future = future
|
||||
self.task = task
|
||||
|
||||
def run(self, ctx):
|
||||
if not self.future.set_running_or_notify_cancel():
|
||||
return
|
||||
|
||||
try:
|
||||
result = ctx.run(self.task)
|
||||
except BaseException as exc:
|
||||
self.future.set_exception(exc)
|
||||
# Break a reference cycle with the exception 'exc'
|
||||
self = None
|
||||
else:
|
||||
self.future.set_result(result)
|
||||
|
||||
__class_getitem__ = classmethod(types.GenericAlias)
|
||||
|
||||
|
||||
def _worker(executor_reference, ctx, work_queue):
|
||||
try:
|
||||
ctx.initialize()
|
||||
except BaseException:
|
||||
_base.LOGGER.critical('Exception in initializer:', exc_info=True)
|
||||
executor = executor_reference()
|
||||
if executor is not None:
|
||||
executor._initializer_failed()
|
||||
return
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
work_item = work_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
# attempt to increment idle count if queue is empty
|
||||
executor = executor_reference()
|
||||
if executor is not None:
|
||||
executor._idle_semaphore.release()
|
||||
del executor
|
||||
work_item = work_queue.get(block=True)
|
||||
|
||||
if work_item is not None:
|
||||
work_item.run(ctx)
|
||||
# Delete references to object. See GH-60488
|
||||
del work_item
|
||||
continue
|
||||
|
||||
executor = executor_reference()
|
||||
# Exit if:
|
||||
# - The interpreter is shutting down OR
|
||||
# - The executor that owns the worker has been collected OR
|
||||
# - The executor that owns the worker has been shutdown.
|
||||
if _shutdown or executor is None or executor._shutdown:
|
||||
# Flag the executor as shutting down as early as possible if it
|
||||
# is not gc-ed yet.
|
||||
if executor is not None:
|
||||
executor._shutdown = True
|
||||
# Notice other workers
|
||||
work_queue.put(None)
|
||||
return
|
||||
del executor
|
||||
except BaseException:
|
||||
_base.LOGGER.critical('Exception in worker', exc_info=True)
|
||||
finally:
|
||||
ctx.finalize()
|
||||
|
||||
|
||||
class BrokenThreadPool(_base.BrokenExecutor):
|
||||
"""
|
||||
Raised when a worker thread in a ThreadPoolExecutor failed initializing.
|
||||
"""
|
||||
|
||||
|
||||
class ThreadPoolExecutor(_base.Executor):
|
||||
|
||||
BROKEN = BrokenThreadPool
|
||||
|
||||
# Used to assign unique thread names when thread_name_prefix is not supplied.
|
||||
_counter = itertools.count().__next__
|
||||
|
||||
@classmethod
|
||||
def prepare_context(cls, initializer, initargs):
|
||||
return WorkerContext.prepare(initializer, initargs)
|
||||
|
||||
def __init__(self, max_workers=None, thread_name_prefix='',
|
||||
initializer=None, initargs=(), **ctxkwargs):
|
||||
"""Initializes a new ThreadPoolExecutor instance.
|
||||
|
||||
Args:
|
||||
max_workers: The maximum number of threads that can be used to
|
||||
execute the given calls.
|
||||
thread_name_prefix: An optional name prefix to give our threads.
|
||||
initializer: A callable used to initialize worker threads.
|
||||
initargs: A tuple of arguments to pass to the initializer.
|
||||
ctxkwargs: Additional arguments to cls.prepare_context().
|
||||
"""
|
||||
if max_workers is None:
|
||||
# ThreadPoolExecutor is often used to:
|
||||
# * CPU bound task which releases GIL
|
||||
# * I/O bound task (which releases GIL, of course)
|
||||
#
|
||||
# We use process_cpu_count + 4 for both types of tasks.
|
||||
# But we limit it to 32 to avoid consuming surprisingly large resource
|
||||
# on many core machine.
|
||||
max_workers = min(32, (os.process_cpu_count() or 1) + 4)
|
||||
if max_workers <= 0:
|
||||
raise ValueError("max_workers must be greater than 0")
|
||||
|
||||
(self._create_worker_context,
|
||||
self._resolve_work_item_task,
|
||||
) = type(self).prepare_context(initializer, initargs, **ctxkwargs)
|
||||
|
||||
self._max_workers = max_workers
|
||||
self._work_queue = queue.SimpleQueue()
|
||||
self._idle_semaphore = threading.Semaphore(0)
|
||||
self._threads = set()
|
||||
self._broken = False
|
||||
self._shutdown = False
|
||||
self._shutdown_lock = threading.Lock()
|
||||
self._thread_name_prefix = (thread_name_prefix or
|
||||
("ThreadPoolExecutor-%d" % self._counter()))
|
||||
|
||||
def submit(self, fn, /, *args, **kwargs):
|
||||
with self._shutdown_lock, _global_shutdown_lock:
|
||||
if self._broken:
|
||||
raise self.BROKEN(self._broken)
|
||||
|
||||
if self._shutdown:
|
||||
raise RuntimeError('cannot schedule new futures after shutdown')
|
||||
if _shutdown:
|
||||
raise RuntimeError('cannot schedule new futures after '
|
||||
'interpreter shutdown')
|
||||
|
||||
f = _base.Future()
|
||||
task = self._resolve_work_item_task(fn, args, kwargs)
|
||||
w = _WorkItem(f, task)
|
||||
|
||||
self._work_queue.put(w)
|
||||
self._adjust_thread_count()
|
||||
return f
|
||||
submit.__doc__ = _base.Executor.submit.__doc__
|
||||
|
||||
def _adjust_thread_count(self):
|
||||
# if idle threads are available, don't spin new threads
|
||||
if self._idle_semaphore.acquire(timeout=0):
|
||||
return
|
||||
|
||||
# When the executor gets lost, the weakref callback will wake up
|
||||
# the worker threads.
|
||||
def weakref_cb(_, q=self._work_queue):
|
||||
q.put(None)
|
||||
|
||||
num_threads = len(self._threads)
|
||||
if num_threads < self._max_workers:
|
||||
thread_name = '%s_%d' % (self._thread_name_prefix or self,
|
||||
num_threads)
|
||||
t = threading.Thread(name=thread_name, target=_worker,
|
||||
args=(weakref.ref(self, weakref_cb),
|
||||
self._create_worker_context(),
|
||||
self._work_queue))
|
||||
t.start()
|
||||
self._threads.add(t)
|
||||
_threads_queues[t] = self._work_queue
|
||||
|
||||
def _initializer_failed(self):
|
||||
with self._shutdown_lock:
|
||||
self._broken = ('A thread initializer failed, the thread pool '
|
||||
'is not usable anymore')
|
||||
# Drain work queue and mark pending futures failed
|
||||
while True:
|
||||
try:
|
||||
work_item = self._work_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
if work_item is not None:
|
||||
work_item.future.set_exception(self.BROKEN(self._broken))
|
||||
|
||||
def shutdown(self, wait=True, *, cancel_futures=False):
|
||||
with self._shutdown_lock:
|
||||
self._shutdown = True
|
||||
if cancel_futures:
|
||||
# Drain all work items from the queue, and then cancel their
|
||||
# associated futures.
|
||||
while True:
|
||||
try:
|
||||
work_item = self._work_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
if work_item is not None:
|
||||
work_item.future.cancel()
|
||||
|
||||
# Send a wake-up to prevent threads calling
|
||||
# _work_queue.get(block=True) from permanently blocking.
|
||||
self._work_queue.put(None)
|
||||
if wait:
|
||||
for t in self._threads:
|
||||
t.join()
|
||||
shutdown.__doc__ = _base.Executor.shutdown.__doc__
|
||||
@@ -0,0 +1,247 @@
|
||||
"""Subinterpreters High Level Module."""
|
||||
|
||||
import threading
|
||||
import weakref
|
||||
import _interpreters
|
||||
|
||||
# aliases:
|
||||
from _interpreters import (
|
||||
InterpreterError, InterpreterNotFoundError, NotShareableError,
|
||||
is_shareable,
|
||||
)
|
||||
from ._queues import (
|
||||
create as create_queue,
|
||||
Queue, QueueEmpty, QueueFull,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'get_current', 'get_main', 'create', 'list_all', 'is_shareable',
|
||||
'Interpreter',
|
||||
'InterpreterError', 'InterpreterNotFoundError', 'ExecutionFailed',
|
||||
'NotShareableError',
|
||||
'create_queue', 'Queue', 'QueueEmpty', 'QueueFull',
|
||||
]
|
||||
|
||||
|
||||
_EXEC_FAILURE_STR = """
|
||||
{superstr}
|
||||
|
||||
Uncaught in the interpreter:
|
||||
|
||||
{formatted}
|
||||
""".strip()
|
||||
|
||||
class ExecutionFailed(InterpreterError):
|
||||
"""An unhandled exception happened during execution.
|
||||
|
||||
This is raised from Interpreter.exec() and Interpreter.call().
|
||||
"""
|
||||
|
||||
def __init__(self, excinfo):
|
||||
msg = excinfo.formatted
|
||||
if not msg:
|
||||
if excinfo.type and excinfo.msg:
|
||||
msg = f'{excinfo.type.__name__}: {excinfo.msg}'
|
||||
else:
|
||||
msg = excinfo.type.__name__ or excinfo.msg
|
||||
super().__init__(msg)
|
||||
self.excinfo = excinfo
|
||||
|
||||
def __str__(self):
|
||||
try:
|
||||
formatted = self.excinfo.errdisplay
|
||||
except Exception:
|
||||
return super().__str__()
|
||||
else:
|
||||
return _EXEC_FAILURE_STR.format(
|
||||
superstr=super().__str__(),
|
||||
formatted=formatted,
|
||||
)
|
||||
|
||||
|
||||
def create():
|
||||
"""Return a new (idle) Python interpreter."""
|
||||
id = _interpreters.create(reqrefs=True)
|
||||
return Interpreter(id, _ownsref=True)
|
||||
|
||||
|
||||
def list_all():
|
||||
"""Return all existing interpreters."""
|
||||
return [Interpreter(id, _whence=whence)
|
||||
for id, whence in _interpreters.list_all(require_ready=True)]
|
||||
|
||||
|
||||
def get_current():
|
||||
"""Return the currently running interpreter."""
|
||||
id, whence = _interpreters.get_current()
|
||||
return Interpreter(id, _whence=whence)
|
||||
|
||||
|
||||
def get_main():
|
||||
"""Return the main interpreter."""
|
||||
id, whence = _interpreters.get_main()
|
||||
assert whence == _interpreters.WHENCE_RUNTIME, repr(whence)
|
||||
return Interpreter(id, _whence=whence)
|
||||
|
||||
|
||||
_known = weakref.WeakValueDictionary()
|
||||
|
||||
class Interpreter:
|
||||
"""A single Python interpreter.
|
||||
|
||||
Attributes:
|
||||
|
||||
"id" - the unique process-global ID number for the interpreter
|
||||
"whence" - indicates where the interpreter was created
|
||||
|
||||
If the interpreter wasn't created by this module
|
||||
then any method that modifies the interpreter will fail,
|
||||
i.e. .close(), .prepare_main(), .exec(), and .call()
|
||||
"""
|
||||
|
||||
_WHENCE_TO_STR = {
|
||||
_interpreters.WHENCE_UNKNOWN: 'unknown',
|
||||
_interpreters.WHENCE_RUNTIME: 'runtime init',
|
||||
_interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API',
|
||||
_interpreters.WHENCE_CAPI: 'C-API',
|
||||
_interpreters.WHENCE_XI: 'cross-interpreter C-API',
|
||||
_interpreters.WHENCE_STDLIB: '_interpreters module',
|
||||
}
|
||||
|
||||
def __new__(cls, id, /, _whence=None, _ownsref=None):
|
||||
# There is only one instance for any given ID.
|
||||
if not isinstance(id, int):
|
||||
raise TypeError(f'id must be an int, got {id!r}')
|
||||
id = int(id)
|
||||
if _whence is None:
|
||||
if _ownsref:
|
||||
_whence = _interpreters.WHENCE_STDLIB
|
||||
else:
|
||||
_whence = _interpreters.whence(id)
|
||||
assert _whence in cls._WHENCE_TO_STR, repr(_whence)
|
||||
if _ownsref is None:
|
||||
_ownsref = (_whence == _interpreters.WHENCE_STDLIB)
|
||||
try:
|
||||
self = _known[id]
|
||||
assert hasattr(self, '_ownsref')
|
||||
except KeyError:
|
||||
self = super().__new__(cls)
|
||||
_known[id] = self
|
||||
self._id = id
|
||||
self._whence = _whence
|
||||
self._ownsref = _ownsref
|
||||
if _ownsref:
|
||||
# This may raise InterpreterNotFoundError:
|
||||
_interpreters.incref(id)
|
||||
return self
|
||||
|
||||
def __repr__(self):
|
||||
return f'{type(self).__name__}({self.id})'
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self._id)
|
||||
|
||||
def __del__(self):
|
||||
self._decref()
|
||||
|
||||
# for pickling:
|
||||
def __reduce__(self):
|
||||
return (type(self), (self._id,))
|
||||
|
||||
# gh-135729: Globals might be destroyed by the time this is called, so we
|
||||
# need to keep references ourself
|
||||
def _decref(self, *,
|
||||
InterpreterNotFoundError=InterpreterNotFoundError,
|
||||
_interp_decref=_interpreters.decref,
|
||||
):
|
||||
if not self._ownsref:
|
||||
return
|
||||
self._ownsref = False
|
||||
try:
|
||||
_interp_decref(self._id)
|
||||
except InterpreterNotFoundError:
|
||||
pass
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return self._id
|
||||
|
||||
@property
|
||||
def whence(self):
|
||||
return self._WHENCE_TO_STR[self._whence]
|
||||
|
||||
def is_running(self):
|
||||
"""Return whether or not the identified interpreter is running."""
|
||||
return _interpreters.is_running(self._id)
|
||||
|
||||
# Everything past here is available only to interpreters created by
|
||||
# interpreters.create().
|
||||
|
||||
def close(self):
|
||||
"""Finalize and destroy the interpreter.
|
||||
|
||||
Attempting to destroy the current interpreter results
|
||||
in an InterpreterError.
|
||||
"""
|
||||
return _interpreters.destroy(self._id, restrict=True)
|
||||
|
||||
def prepare_main(self, ns=None, /, **kwargs):
|
||||
"""Bind the given values into the interpreter's __main__.
|
||||
|
||||
The values must be shareable.
|
||||
"""
|
||||
ns = dict(ns, **kwargs) if ns is not None else kwargs
|
||||
_interpreters.set___main___attrs(self._id, ns, restrict=True)
|
||||
|
||||
def exec(self, code, /):
|
||||
"""Run the given source code in the interpreter.
|
||||
|
||||
This is essentially the same as calling the builtin "exec"
|
||||
with this interpreter, using the __dict__ of its __main__
|
||||
module as both globals and locals.
|
||||
|
||||
There is no return value.
|
||||
|
||||
If the code raises an unhandled exception then an ExecutionFailed
|
||||
exception is raised, which summarizes the unhandled exception.
|
||||
The actual exception is discarded because objects cannot be
|
||||
shared between interpreters.
|
||||
|
||||
This blocks the current Python thread until done. During
|
||||
that time, the previous interpreter is allowed to run
|
||||
in other threads.
|
||||
"""
|
||||
excinfo = _interpreters.exec(self._id, code, restrict=True)
|
||||
if excinfo is not None:
|
||||
raise ExecutionFailed(excinfo)
|
||||
|
||||
def _call(self, callable, args, kwargs):
|
||||
res, excinfo = _interpreters.call(self._id, callable, args, kwargs, restrict=True)
|
||||
if excinfo is not None:
|
||||
raise ExecutionFailed(excinfo)
|
||||
return res
|
||||
|
||||
def call(self, callable, /, *args, **kwargs):
|
||||
"""Call the object in the interpreter with given args/kwargs.
|
||||
|
||||
Nearly all callables, args, kwargs, and return values are
|
||||
supported. All "shareable" objects are supported, as are
|
||||
"stateless" functions (meaning non-closures that do not use
|
||||
any globals). This method will fall back to pickle.
|
||||
|
||||
If the callable raises an exception then the error display
|
||||
(including full traceback) is sent back between the interpreters
|
||||
and an ExecutionFailed exception is raised, much like what
|
||||
happens with Interpreter.exec().
|
||||
"""
|
||||
return self._call(callable, args, kwargs)
|
||||
|
||||
def call_in_thread(self, callable, /, *args, **kwargs):
|
||||
"""Return a new thread that calls the object in the interpreter.
|
||||
|
||||
The return value and any raised exception are discarded.
|
||||
"""
|
||||
t = threading.Thread(target=self._call, args=(callable, args, kwargs))
|
||||
t.start()
|
||||
return t
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Common code between queues and channels."""
|
||||
|
||||
|
||||
class ItemInterpreterDestroyed(Exception):
|
||||
"""Raised when trying to get an item whose interpreter was destroyed."""
|
||||
|
||||
|
||||
class classonly:
|
||||
"""A non-data descriptor that makes a value only visible on the class.
|
||||
|
||||
This is like the "classmethod" builtin, but does not show up on
|
||||
instances of the class. It may be used as a decorator.
|
||||
"""
|
||||
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
self.getter = classmethod(value).__get__
|
||||
self.name = None
|
||||
|
||||
def __set_name__(self, cls, name):
|
||||
if self.name is not None:
|
||||
raise TypeError('already used')
|
||||
self.name = name
|
||||
|
||||
def __get__(self, obj, cls):
|
||||
if obj is not None:
|
||||
raise AttributeError(self.name)
|
||||
# called on the class
|
||||
return self.getter(None, cls)
|
||||
|
||||
|
||||
class UnboundItem:
|
||||
"""Represents a cross-interpreter item no longer bound to an interpreter.
|
||||
|
||||
An item is unbound when the interpreter that added it to the
|
||||
cross-interpreter container is destroyed.
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@classonly
|
||||
def singleton(cls, kind, module, name='UNBOUND'):
|
||||
doc = cls.__doc__
|
||||
if doc:
|
||||
doc = doc.replace(
|
||||
'cross-interpreter container', kind,
|
||||
).replace(
|
||||
'cross-interpreter', kind,
|
||||
)
|
||||
subclass = type(
|
||||
f'Unbound{kind.capitalize()}Item',
|
||||
(cls,),
|
||||
{
|
||||
"_MODULE": module,
|
||||
"_NAME": name,
|
||||
"__doc__": doc,
|
||||
},
|
||||
)
|
||||
return object.__new__(subclass)
|
||||
|
||||
_MODULE = __name__
|
||||
_NAME = 'UNBOUND'
|
||||
|
||||
def __new__(cls):
|
||||
raise Exception(f'use {cls._MODULE}.{cls._NAME}')
|
||||
|
||||
def __repr__(self):
|
||||
return f'{self._MODULE}.{self._NAME}'
|
||||
# return f'interpreters._queues.UNBOUND'
|
||||
|
||||
|
||||
UNBOUND = object.__new__(UnboundItem)
|
||||
UNBOUND_ERROR = object()
|
||||
UNBOUND_REMOVE = object()
|
||||
|
||||
_UNBOUND_CONSTANT_TO_FLAG = {
|
||||
UNBOUND_REMOVE: 1,
|
||||
UNBOUND_ERROR: 2,
|
||||
UNBOUND: 3,
|
||||
}
|
||||
_UNBOUND_FLAG_TO_CONSTANT = {v: k
|
||||
for k, v in _UNBOUND_CONSTANT_TO_FLAG.items()}
|
||||
|
||||
|
||||
def serialize_unbound(unbound):
|
||||
op = unbound
|
||||
try:
|
||||
flag = _UNBOUND_CONSTANT_TO_FLAG[op]
|
||||
except KeyError:
|
||||
raise NotImplementedError(f'unsupported unbound replacement op {op!r}')
|
||||
return flag,
|
||||
|
||||
|
||||
def resolve_unbound(flag, exctype_destroyed):
|
||||
try:
|
||||
op = _UNBOUND_FLAG_TO_CONSTANT[flag]
|
||||
except KeyError:
|
||||
raise NotImplementedError(f'unsupported unbound replacement op {flag!r}')
|
||||
if op is UNBOUND_REMOVE:
|
||||
# "remove" not possible here
|
||||
raise NotImplementedError
|
||||
elif op is UNBOUND_ERROR:
|
||||
raise exctype_destroyed("item's original interpreter destroyed")
|
||||
elif op is UNBOUND:
|
||||
return UNBOUND
|
||||
else:
|
||||
raise NotImplementedError(repr(op))
|
||||
@@ -0,0 +1,289 @@
|
||||
"""Cross-interpreter Queues High Level Module."""
|
||||
|
||||
import queue
|
||||
import time
|
||||
import weakref
|
||||
import _interpqueues as _queues
|
||||
from . import _crossinterp
|
||||
|
||||
# aliases:
|
||||
from _interpqueues import (
|
||||
QueueError, QueueNotFoundError,
|
||||
)
|
||||
from ._crossinterp import (
|
||||
UNBOUND_ERROR, UNBOUND_REMOVE,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE',
|
||||
'create', 'list_all',
|
||||
'Queue',
|
||||
'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull',
|
||||
'ItemInterpreterDestroyed',
|
||||
]
|
||||
|
||||
|
||||
class QueueEmpty(QueueError, queue.Empty):
|
||||
"""Raised from get_nowait() when the queue is empty.
|
||||
|
||||
It is also raised from get() if it times out.
|
||||
"""
|
||||
|
||||
|
||||
class QueueFull(QueueError, queue.Full):
|
||||
"""Raised from put_nowait() when the queue is full.
|
||||
|
||||
It is also raised from put() if it times out.
|
||||
"""
|
||||
|
||||
|
||||
class ItemInterpreterDestroyed(QueueError,
|
||||
_crossinterp.ItemInterpreterDestroyed):
|
||||
"""Raised from get() and get_nowait()."""
|
||||
|
||||
|
||||
_SHARED_ONLY = 0
|
||||
_PICKLED = 1
|
||||
|
||||
|
||||
UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__)
|
||||
|
||||
|
||||
def _serialize_unbound(unbound):
|
||||
if unbound is UNBOUND:
|
||||
unbound = _crossinterp.UNBOUND
|
||||
return _crossinterp.serialize_unbound(unbound)
|
||||
|
||||
|
||||
def _resolve_unbound(flag):
|
||||
resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed)
|
||||
if resolved is _crossinterp.UNBOUND:
|
||||
resolved = UNBOUND
|
||||
return resolved
|
||||
|
||||
|
||||
def create(maxsize=0, *, unbounditems=UNBOUND):
|
||||
"""Return a new cross-interpreter queue.
|
||||
|
||||
The queue may be used to pass data safely between interpreters.
|
||||
|
||||
"unbounditems" sets the default for Queue.put(); see that method for
|
||||
supported values. The default value is UNBOUND, which replaces
|
||||
the unbound item.
|
||||
"""
|
||||
unbound = _serialize_unbound(unbounditems)
|
||||
unboundop, = unbound
|
||||
qid = _queues.create(maxsize, unboundop, -1)
|
||||
self = Queue(qid)
|
||||
self._set_unbound(unboundop, unbounditems)
|
||||
return self
|
||||
|
||||
|
||||
def list_all():
|
||||
"""Return a list of all open queues."""
|
||||
queues = []
|
||||
for qid, unboundop, _ in _queues.list_all():
|
||||
self = Queue(qid)
|
||||
if not hasattr(self, '_unbound'):
|
||||
self._set_unbound(unboundop)
|
||||
else:
|
||||
assert self._unbound[0] == unboundop
|
||||
queues.append(self)
|
||||
return queues
|
||||
|
||||
|
||||
_known_queues = weakref.WeakValueDictionary()
|
||||
|
||||
class Queue:
|
||||
"""A cross-interpreter queue."""
|
||||
|
||||
def __new__(cls, id, /):
|
||||
# There is only one instance for any given ID.
|
||||
if isinstance(id, int):
|
||||
id = int(id)
|
||||
else:
|
||||
raise TypeError(f'id must be an int, got {id!r}')
|
||||
try:
|
||||
self = _known_queues[id]
|
||||
except KeyError:
|
||||
self = super().__new__(cls)
|
||||
self._id = id
|
||||
_known_queues[id] = self
|
||||
_queues.bind(id)
|
||||
return self
|
||||
|
||||
def __del__(self):
|
||||
try:
|
||||
_queues.release(self._id)
|
||||
except QueueNotFoundError:
|
||||
pass
|
||||
try:
|
||||
del _known_queues[self._id]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
def __repr__(self):
|
||||
return f'{type(self).__name__}({self.id})'
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self._id)
|
||||
|
||||
# for pickling:
|
||||
def __reduce__(self):
|
||||
return (type(self), (self._id,))
|
||||
|
||||
def _set_unbound(self, op, items=None):
|
||||
assert not hasattr(self, '_unbound')
|
||||
if items is None:
|
||||
items = _resolve_unbound(op)
|
||||
unbound = (op, items)
|
||||
self._unbound = unbound
|
||||
return unbound
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return self._id
|
||||
|
||||
@property
|
||||
def unbounditems(self):
|
||||
try:
|
||||
_, items = self._unbound
|
||||
except AttributeError:
|
||||
op, _ = _queues.get_queue_defaults(self._id)
|
||||
_, items = self._set_unbound(op)
|
||||
return items
|
||||
|
||||
@property
|
||||
def maxsize(self):
|
||||
try:
|
||||
return self._maxsize
|
||||
except AttributeError:
|
||||
self._maxsize = _queues.get_maxsize(self._id)
|
||||
return self._maxsize
|
||||
|
||||
def empty(self):
|
||||
return self.qsize() == 0
|
||||
|
||||
def full(self):
|
||||
return _queues.is_full(self._id)
|
||||
|
||||
def qsize(self):
|
||||
return _queues.get_count(self._id)
|
||||
|
||||
def put(self, obj, block=True, timeout=None, *,
|
||||
unbounditems=None,
|
||||
_delay=10 / 1000, # 10 milliseconds
|
||||
):
|
||||
"""Add the object to the queue.
|
||||
|
||||
If "block" is true, this blocks while the queue is full.
|
||||
|
||||
For most objects, the object received through Queue.get() will
|
||||
be a new one, equivalent to the original and not sharing any
|
||||
actual underlying data. The notable exceptions include
|
||||
cross-interpreter types (like Queue) and memoryview, where the
|
||||
underlying data is actually shared. Furthermore, some types
|
||||
can be sent through a queue more efficiently than others. This
|
||||
group includes various immutable types like int, str, bytes, and
|
||||
tuple (if the items are likewise efficiently shareable).
|
||||
See interpreters.is_shareable().
|
||||
|
||||
"unbounditems" controls the behavior of Queue.get() for the given
|
||||
object if the current interpreter (calling put()) is later
|
||||
destroyed.
|
||||
|
||||
If "unbounditems" is None (the default) then it uses the
|
||||
queue's default, set with create_queue(),
|
||||
which is usually UNBOUND.
|
||||
|
||||
If "unbounditems" is UNBOUND_ERROR then get() will raise an
|
||||
ItemInterpreterDestroyed exception if the original interpreter
|
||||
has been destroyed. This does not otherwise affect the queue;
|
||||
the next call to put() will work like normal, returning the next
|
||||
item in the queue.
|
||||
|
||||
If "unbounditems" is UNBOUND_REMOVE then the item will be removed
|
||||
from the queue as soon as the original interpreter is destroyed.
|
||||
Be aware that this will introduce an imbalance between put()
|
||||
and get() calls.
|
||||
|
||||
If "unbounditems" is UNBOUND then it is returned by get() in place
|
||||
of the unbound item.
|
||||
"""
|
||||
if not block:
|
||||
return self.put_nowait(obj, unbounditems=unbounditems)
|
||||
if unbounditems is None:
|
||||
unboundop = -1
|
||||
else:
|
||||
unboundop, = _serialize_unbound(unbounditems)
|
||||
if timeout is not None:
|
||||
timeout = int(timeout)
|
||||
if timeout < 0:
|
||||
raise ValueError(f'timeout value must be non-negative')
|
||||
end = time.time() + timeout
|
||||
while True:
|
||||
try:
|
||||
_queues.put(self._id, obj, unboundop)
|
||||
except QueueFull:
|
||||
if timeout is not None and time.time() >= end:
|
||||
raise # re-raise
|
||||
time.sleep(_delay)
|
||||
else:
|
||||
break
|
||||
|
||||
def put_nowait(self, obj, *, unbounditems=None):
|
||||
if unbounditems is None:
|
||||
unboundop = -1
|
||||
else:
|
||||
unboundop, = _serialize_unbound(unbounditems)
|
||||
_queues.put(self._id, obj, unboundop)
|
||||
|
||||
def get(self, block=True, timeout=None, *,
|
||||
_delay=10 / 1000, # 10 milliseconds
|
||||
):
|
||||
"""Return the next object from the queue.
|
||||
|
||||
If "block" is true, this blocks while the queue is empty.
|
||||
|
||||
If the next item's original interpreter has been destroyed
|
||||
then the "next object" is determined by the value of the
|
||||
"unbounditems" argument to put().
|
||||
"""
|
||||
if not block:
|
||||
return self.get_nowait()
|
||||
if timeout is not None:
|
||||
timeout = int(timeout)
|
||||
if timeout < 0:
|
||||
raise ValueError(f'timeout value must be non-negative')
|
||||
end = time.time() + timeout
|
||||
while True:
|
||||
try:
|
||||
obj, unboundop = _queues.get(self._id)
|
||||
except QueueEmpty:
|
||||
if timeout is not None and time.time() >= end:
|
||||
raise # re-raise
|
||||
time.sleep(_delay)
|
||||
else:
|
||||
break
|
||||
if unboundop is not None:
|
||||
assert obj is None, repr(obj)
|
||||
return _resolve_unbound(unboundop)
|
||||
return obj
|
||||
|
||||
def get_nowait(self):
|
||||
"""Return the next object from the channel.
|
||||
|
||||
If the queue is empty then raise QueueEmpty. Otherwise this
|
||||
is the same as get().
|
||||
"""
|
||||
try:
|
||||
obj, unboundop = _queues.get(self._id)
|
||||
except QueueEmpty:
|
||||
raise # re-raise
|
||||
if unboundop is not None:
|
||||
assert obj is None, repr(obj)
|
||||
return _resolve_unbound(unboundop)
|
||||
return obj
|
||||
|
||||
|
||||
_queues._register_heap_types(Queue, QueueEmpty, QueueFull)
|
||||
Reference in New Issue
Block a user