This commit is contained in:
2024-12-17 14:36:15 -08:00
parent b2dbf46d28
commit 06d106de53
17731 changed files with 3037186 additions and 144 deletions

View File

@@ -0,0 +1,12 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Twisted Internet: Asynchronous I/O and Events.
Twisted Internet is a collection of compatible event-loops for Python. It contains
the code to dispatch events to interested observers and a portable API so that
observers need not care about which event loop is running. Thus, it is possible
to use the same code for different loops, from Twisted's basic, yet portable,
select-based loop to the loops of various GUI toolkits like GTK+ or Tk.
"""

View File

@@ -0,0 +1,66 @@
# -*- test-case-name: twisted.test.test_process -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Cross-platform process-related functionality used by different
L{IReactorProcess} implementations.
"""
from typing import Optional
from twisted.logger import Logger
from twisted.python.deprecate import getWarningMethod
from twisted.python.failure import Failure
from twisted.python.reflect import qual
_log = Logger()
_missingProcessExited = (
"Since Twisted 8.2, IProcessProtocol.processExited "
"is required. %s must implement it."
)
class BaseProcess:
pid: Optional[int] = None
status: Optional[int] = None
lostProcess = 0
proto = None
def __init__(self, protocol):
self.proto = protocol
def _callProcessExited(self, reason):
default = object()
processExited = getattr(self.proto, "processExited", default)
if processExited is default:
getWarningMethod()(
_missingProcessExited % (qual(self.proto.__class__),),
DeprecationWarning,
stacklevel=0,
)
else:
with _log.failuresHandled("while calling processExited:"):
processExited(Failure(reason))
def processEnded(self, status):
"""
This is called when the child terminates.
"""
self.status = status
self.lostProcess += 1
self.pid = None
self._callProcessExited(self._getReason(status))
self.maybeCallProcessEnded()
def maybeCallProcessEnded(self):
"""
Call processEnded on protocol after final cleanup.
"""
if self.proto is not None:
reason = self._getReason(self.status)
proto = self.proto
self.proto = None
with _log.failuresHandled("while calling processEnded:"):
proto.processEnded(Failure(reason))

View File

@@ -0,0 +1,25 @@
"""
Support similar deprecation of several reactors.
"""
import warnings
from incremental import Version, getVersionString
from twisted.python.deprecate import DEPRECATION_WARNING_FORMAT
def deprecatedGnomeReactor(name: str, version: Version) -> None:
"""
Emit a deprecation warning about a gnome-related reactor.
@param name: The name of the reactor. For example, C{"gtk2reactor"}.
@param version: The version in which the deprecation was introduced.
"""
stem = DEPRECATION_WARNING_FORMAT % {
"fqpn": "twisted.internet." + name,
"version": getVersionString(version),
}
msg = stem + ". Please use twisted.internet.gireactor instead."
warnings.warn(msg, category=DeprecationWarning)

View File

@@ -0,0 +1,397 @@
# -*- test-case-name: twisted.test.test_process -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Windows Process Management, used with reactor.spawnProcess
"""
import os
import sys
from zope.interface import implementer
import pywintypes
# Win32 imports
import win32api
import win32con
import win32event
import win32file
import win32pipe
import win32process
import win32security
from twisted.internet import _pollingfile, error
from twisted.internet._baseprocess import BaseProcess
from twisted.internet.interfaces import IConsumer, IProcessTransport, IProducer
from twisted.python.win32 import quoteArguments
# Security attributes for pipes
PIPE_ATTRS_INHERITABLE = win32security.SECURITY_ATTRIBUTES()
PIPE_ATTRS_INHERITABLE.bInheritHandle = 1
def debug(msg):
print(msg)
sys.stdout.flush()
class _Reaper(_pollingfile._PollableResource):
def __init__(self, proc):
self.proc = proc
def checkWork(self):
if (
win32event.WaitForSingleObject(self.proc.hProcess, 0)
!= win32event.WAIT_OBJECT_0
):
return 0
exitCode = win32process.GetExitCodeProcess(self.proc.hProcess)
self.deactivate()
self.proc.processEnded(exitCode)
return 0
def _findShebang(filename):
"""
Look for a #! line, and return the value following the #! if one exists, or
None if this file is not a script.
I don't know if there are any conventions for quoting in Windows shebang
lines, so this doesn't support any; therefore, you may not pass any
arguments to scripts invoked as filters. That's probably wrong, so if
somebody knows more about the cultural expectations on Windows, please feel
free to fix.
This shebang line support was added in support of the CGI tests;
appropriately enough, I determined that shebang lines are culturally
accepted in the Windows world through this page::
http://www.cgi101.com/learn/connect/winxp.html
@param filename: str representing a filename
@return: a str representing another filename.
"""
with open(filename) as f:
if f.read(2) == "#!":
exe = f.readline(1024).strip("\n")
return exe
def _invalidWin32App(pywinerr):
"""
Determine if a pywintypes.error is telling us that the given process is
'not a valid win32 application', i.e. not a PE format executable.
@param pywinerr: a pywintypes.error instance raised by CreateProcess
@return: a boolean
"""
# Let's do this better in the future, but I have no idea what this error
# is; MSDN doesn't mention it, and there is no symbolic constant in
# win32process module that represents 193.
return pywinerr.args[0] == 193
@implementer(IProcessTransport, IConsumer, IProducer)
class Process(_pollingfile._PollingTimer, BaseProcess):
"""
A process that integrates with the Twisted event loop.
If your subprocess is a python program, you need to:
- Run python.exe with the '-u' command line option - this turns on
unbuffered I/O. Buffering stdout/err/in can cause problems, see e.g.
http://support.microsoft.com/default.aspx?scid=kb;EN-US;q1903
- If you don't want Windows messing with data passed over
stdin/out/err, set the pipes to be in binary mode::
import os, sys, mscvrt
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
"""
closedNotifies = 0
def __init__(self, reactor, protocol, command, args, environment, path):
"""
Create a new child process.
"""
_pollingfile._PollingTimer.__init__(self, reactor)
BaseProcess.__init__(self, protocol)
# security attributes for pipes
sAttrs = win32security.SECURITY_ATTRIBUTES()
sAttrs.bInheritHandle = 1
# create the pipes which will connect to the secondary process
self.hStdoutR, hStdoutW = win32pipe.CreatePipe(sAttrs, 0)
self.hStderrR, hStderrW = win32pipe.CreatePipe(sAttrs, 0)
hStdinR, self.hStdinW = win32pipe.CreatePipe(sAttrs, 0)
win32pipe.SetNamedPipeHandleState(
self.hStdinW, win32pipe.PIPE_NOWAIT, None, None
)
# set the info structure for the new process.
StartupInfo = win32process.STARTUPINFO()
StartupInfo.hStdOutput = hStdoutW
StartupInfo.hStdError = hStderrW
StartupInfo.hStdInput = hStdinR
StartupInfo.dwFlags = win32process.STARTF_USESTDHANDLES
# Create new handles whose inheritance property is false
currentPid = win32api.GetCurrentProcess()
tmp = win32api.DuplicateHandle(
currentPid, self.hStdoutR, currentPid, 0, 0, win32con.DUPLICATE_SAME_ACCESS
)
win32file.CloseHandle(self.hStdoutR)
self.hStdoutR = tmp
tmp = win32api.DuplicateHandle(
currentPid, self.hStderrR, currentPid, 0, 0, win32con.DUPLICATE_SAME_ACCESS
)
win32file.CloseHandle(self.hStderrR)
self.hStderrR = tmp
tmp = win32api.DuplicateHandle(
currentPid, self.hStdinW, currentPid, 0, 0, win32con.DUPLICATE_SAME_ACCESS
)
win32file.CloseHandle(self.hStdinW)
self.hStdinW = tmp
# Add the specified environment to the current environment - this is
# necessary because certain operations are only supported on Windows
# if certain environment variables are present.
env = os.environ.copy()
env.update(environment or {})
env = {os.fsdecode(key): os.fsdecode(value) for key, value in env.items()}
# Make sure all the arguments are Unicode.
args = [os.fsdecode(x) for x in args]
cmdline = quoteArguments(args)
# The command, too, needs to be Unicode, if it is a value.
command = os.fsdecode(command) if command else command
path = os.fsdecode(path) if path else path
# TODO: error detection here. See #2787 and #4184.
def doCreate():
flags = win32con.CREATE_NO_WINDOW
self.hProcess, self.hThread, self.pid, dwTid = win32process.CreateProcess(
command, cmdline, None, None, 1, flags, env, path, StartupInfo
)
try:
doCreate()
except pywintypes.error as pwte:
if not _invalidWin32App(pwte):
# This behavior isn't _really_ documented, but let's make it
# consistent with the behavior that is documented.
raise OSError(pwte)
else:
# look for a shebang line. Insert the original 'command'
# (actually a script) into the new arguments list.
sheb = _findShebang(command)
if sheb is None:
raise OSError(
"%r is neither a Windows executable, "
"nor a script with a shebang line" % command
)
else:
args = list(args)
args.insert(0, command)
cmdline = quoteArguments(args)
origcmd = command
command = sheb
try:
# Let's try again.
doCreate()
except pywintypes.error as pwte2:
# d'oh, failed again!
if _invalidWin32App(pwte2):
raise OSError(
"%r has an invalid shebang line: "
"%r is not a valid executable" % (origcmd, sheb)
)
raise OSError(pwte2)
# close handles which only the child will use
win32file.CloseHandle(hStderrW)
win32file.CloseHandle(hStdoutW)
win32file.CloseHandle(hStdinR)
# set up everything
self.stdout = _pollingfile._PollableReadPipe(
self.hStdoutR,
lambda data: self.proto.childDataReceived(1, data),
self.outConnectionLost,
)
self.stderr = _pollingfile._PollableReadPipe(
self.hStderrR,
lambda data: self.proto.childDataReceived(2, data),
self.errConnectionLost,
)
self.stdin = _pollingfile._PollableWritePipe(
self.hStdinW, self.inConnectionLost
)
for pipewatcher in self.stdout, self.stderr, self.stdin:
self._addPollableResource(pipewatcher)
# notify protocol
self.proto.makeConnection(self)
self._addPollableResource(_Reaper(self))
def signalProcess(self, signalID):
if self.pid is None:
raise error.ProcessExitedAlready()
if signalID in ("INT", "TERM", "KILL"):
win32process.TerminateProcess(self.hProcess, 1)
def _getReason(self, status):
if status == 0:
return error.ProcessDone(status)
return error.ProcessTerminated(status)
def write(self, data):
"""
Write data to the process' stdin.
@type data: C{bytes}
"""
self.stdin.write(data)
def writeSequence(self, seq):
"""
Write data to the process' stdin.
@type seq: C{list} of C{bytes}
"""
self.stdin.writeSequence(seq)
def writeToChild(self, fd, data):
"""
Similar to L{ITransport.write} but also allows the file descriptor in
the child process which will receive the bytes to be specified.
This implementation is limited to writing to the child's standard input.
@param fd: The file descriptor to which to write. Only stdin (C{0}) is
supported.
@type fd: C{int}
@param data: The bytes to write.
@type data: C{bytes}
@return: L{None}
@raise KeyError: If C{fd} is anything other than the stdin file
descriptor (C{0}).
"""
if fd == 0:
self.stdin.write(data)
else:
raise KeyError(fd)
def closeChildFD(self, fd):
if fd == 0:
self.closeStdin()
elif fd == 1:
self.closeStdout()
elif fd == 2:
self.closeStderr()
else:
raise NotImplementedError(
"Only standard-IO file descriptors available on win32"
)
def closeStdin(self):
"""Close the process' stdin."""
self.stdin.close()
def closeStderr(self):
self.stderr.close()
def closeStdout(self):
self.stdout.close()
def loseConnection(self):
"""
Close the process' stdout, in and err.
"""
self.closeStdin()
self.closeStdout()
self.closeStderr()
def outConnectionLost(self):
self.proto.childConnectionLost(1)
self.connectionLostNotify()
def errConnectionLost(self):
self.proto.childConnectionLost(2)
self.connectionLostNotify()
def inConnectionLost(self):
self.proto.childConnectionLost(0)
self.connectionLostNotify()
def connectionLostNotify(self):
"""
Will be called 3 times, by stdout/err threads and process handle.
"""
self.closedNotifies += 1
self.maybeCallProcessEnded()
def maybeCallProcessEnded(self):
if self.closedNotifies == 3 and self.lostProcess:
win32file.CloseHandle(self.hProcess)
win32file.CloseHandle(self.hThread)
self.hProcess = None
self.hThread = None
BaseProcess.maybeCallProcessEnded(self)
# IConsumer
def registerProducer(self, producer, streaming):
self.stdin.registerProducer(producer, streaming)
def unregisterProducer(self):
self.stdin.unregisterProducer()
# IProducer
def pauseProducing(self):
self._pause()
def resumeProducing(self):
self._unpause()
def stopProducing(self):
self.loseConnection()
def getHost(self):
# ITransport.getHost
raise NotImplementedError("Unimplemented: Process.getHost")
def getPeer(self):
# ITransport.getPeer
raise NotImplementedError("Unimplemented: Process.getPeer")
def __repr__(self) -> str:
"""
Return a string representation of the process.
"""
return f"<{self.__class__.__name__} pid={self.pid}>"

View File

@@ -0,0 +1,369 @@
# -*- test-case-name: twisted.internet.test -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
This module provides base support for Twisted to interact with the glib/gtk
mainloops.
The classes in this module should not be used directly, but rather you should
import gireactor or gtk3reactor for GObject Introspection based applications,
or glib2reactor or gtk2reactor for applications using legacy static bindings.
"""
import sys
from typing import Any, Callable, Dict, Set
from zope.interface import implementer
from twisted.internet import posixbase
from twisted.internet.abstract import FileDescriptor
from twisted.internet.interfaces import IReactorFDSet, IReadDescriptor, IWriteDescriptor
from twisted.python import log
from twisted.python.monkey import MonkeyPatcher
from ._signals import _IWaker, _UnixWaker
def ensureNotImported(moduleNames, errorMessage, preventImports=[]):
"""
Check whether the given modules were imported, and if requested, ensure
they will not be importable in the future.
@param moduleNames: A list of module names we make sure aren't imported.
@type moduleNames: C{list} of C{str}
@param preventImports: A list of module name whose future imports should
be prevented.
@type preventImports: C{list} of C{str}
@param errorMessage: Message to use when raising an C{ImportError}.
@type errorMessage: C{str}
@raise ImportError: with given error message if a given module name
has already been imported.
"""
for name in moduleNames:
if sys.modules.get(name) is not None:
raise ImportError(errorMessage)
# Disable module imports to avoid potential problems.
for name in preventImports:
sys.modules[name] = None
class GlibWaker(_UnixWaker):
"""
Run scheduled events after waking up.
"""
def __init__(self, reactor):
super().__init__()
self.reactor = reactor
def doRead(self) -> None:
super().doRead()
self.reactor._simulate()
def _signalGlue():
"""
Integrate glib's wakeup file descriptor usage and our own.
Python supports only one wakeup file descriptor at a time and both Twisted
and glib want to use it.
This is a context manager that can be wrapped around the whole glib
reactor main loop which makes our signal handling work with glib's signal
handling.
"""
from gi import _ossighelper as signalGlue
patcher = MonkeyPatcher()
patcher.addPatch(signalGlue, "_wakeup_fd_is_active", True)
return patcher
def _loopQuitter(
idleAdd: Callable[[Callable[[], None]], None], loopQuit: Callable[[], None]
) -> Callable[[], None]:
"""
Combine the C{glib.idle_add} and C{glib.MainLoop.quit} functions into a
function suitable for crashing the reactor.
"""
return lambda: idleAdd(loopQuit)
@implementer(IReactorFDSet)
class GlibReactorBase(posixbase.PosixReactorBase, posixbase._PollLikeMixin):
"""
Base class for GObject event loop reactors.
Notification for I/O events (reads and writes on file descriptors) is done
by the gobject-based event loop. File descriptors are registered with
gobject with the appropriate flags for read/write/disconnect notification.
Time-based events, the results of C{callLater} and C{callFromThread}, are
handled differently. Rather than registering each event with gobject, a
single gobject timeout is registered for the earliest scheduled event, the
output of C{reactor.timeout()}. For example, if there are timeouts in 1, 2
and 3.4 seconds, a single timeout is registered for 1 second in the
future. When this timeout is hit, C{_simulate} is called, which calls the
appropriate Twisted-level handlers, and a new timeout is added to gobject
by the C{_reschedule} method.
To handle C{callFromThread} events, we use a custom waker that calls
C{_simulate} whenever it wakes up.
@ivar _sources: A dictionary mapping L{FileDescriptor} instances to
GSource handles.
@ivar _reads: A set of L{FileDescriptor} instances currently monitored for
reading.
@ivar _writes: A set of L{FileDescriptor} instances currently monitored for
writing.
@ivar _simtag: A GSource handle for the next L{simulate} call.
"""
# Install a waker that knows it needs to call C{_simulate} in order to run
# callbacks queued from a thread:
def _wakerFactory(self) -> _IWaker:
return GlibWaker(self)
def __init__(self, glib_module: Any, gtk_module: Any, useGtk: bool = False) -> None:
self._simtag = None
self._reads: Set[IReadDescriptor] = set()
self._writes: Set[IWriteDescriptor] = set()
self._sources: Dict[FileDescriptor, int] = {}
self._glib = glib_module
self._POLL_DISCONNECTED = (
glib_module.IOCondition.HUP
| glib_module.IOCondition.ERR
| glib_module.IOCondition.NVAL
)
self._POLL_IN = glib_module.IOCondition.IN
self._POLL_OUT = glib_module.IOCondition.OUT
# glib's iochannel sources won't tell us about any events that we haven't
# asked for, even if those events aren't sensible inputs to the poll()
# call.
self.INFLAGS = self._POLL_IN | self._POLL_DISCONNECTED
self.OUTFLAGS = self._POLL_OUT | self._POLL_DISCONNECTED
super().__init__()
self._source_remove = self._glib.source_remove
self._timeout_add = self._glib.timeout_add
self.context = self._glib.main_context_default()
self._pending = self.context.pending
self._iteration = self.context.iteration
self.loop = self._glib.MainLoop()
self._crash = _loopQuitter(self._glib.idle_add, self.loop.quit)
self._run = self.loop.run
def _reallyStartRunning(self):
"""
Make sure the reactor's signal handlers are installed despite any
outside interference.
"""
# First, install SIGINT and friends:
super()._reallyStartRunning()
# Next, since certain versions of gtk will clobber our signal handler,
# set all signal handlers again after the event loop has started to
# ensure they're *really* set.
#
# We don't actually know which versions of gtk do this so this might
# be obsolete. If so, that would be great and this whole method can
# go away. Someone needs to find out, though.
#
# https://github.com/twisted/twisted/issues/11762
def reinitSignals():
self._signals.uninstall()
self._signals.install()
self.callLater(0, reinitSignals)
# The input_add function in pygtk1 checks for objects with a
# 'fileno' method and, if present, uses the result of that method
# as the input source. The pygtk2 input_add does not do this. The
# function below replicates the pygtk1 functionality.
# In addition, pygtk maps gtk.input_add to _gobject.io_add_watch, and
# g_io_add_watch() takes different condition bitfields than
# gtk_input_add(). We use g_io_add_watch() here in case pygtk fixes this
# bug.
def input_add(self, source, condition, callback):
if hasattr(source, "fileno"):
# handle python objects
def wrapper(ignored, condition):
return callback(source, condition)
fileno = source.fileno()
else:
fileno = source
wrapper = callback
return self._glib.io_add_watch(
fileno,
self._glib.PRIORITY_DEFAULT_IDLE,
condition,
wrapper,
)
def _ioEventCallback(self, source, condition):
"""
Called by event loop when an I/O event occurs.
"""
log.callWithLogger(source, self._doReadOrWrite, source, source, condition)
return True # True = don't auto-remove the source
def _add(self, source, primary, other, primaryFlag, otherFlag):
"""
Add the given L{FileDescriptor} for monitoring either for reading or
writing. If the file is already monitored for the other operation, we
delete the previous registration and re-register it for both reading
and writing.
"""
if source in primary:
return
flags = primaryFlag
if source in other:
self._source_remove(self._sources[source])
flags |= otherFlag
self._sources[source] = self.input_add(source, flags, self._ioEventCallback)
primary.add(source)
def addReader(self, reader):
"""
Add a L{FileDescriptor} for monitoring of data available to read.
"""
self._add(reader, self._reads, self._writes, self.INFLAGS, self.OUTFLAGS)
def addWriter(self, writer):
"""
Add a L{FileDescriptor} for monitoring ability to write data.
"""
self._add(writer, self._writes, self._reads, self.OUTFLAGS, self.INFLAGS)
def getReaders(self):
"""
Retrieve the list of current L{FileDescriptor} monitored for reading.
"""
return list(self._reads)
def getWriters(self):
"""
Retrieve the list of current L{FileDescriptor} monitored for writing.
"""
return list(self._writes)
def removeAll(self):
"""
Remove monitoring for all registered L{FileDescriptor}s.
"""
return self._removeAll(self._reads, self._writes)
def _remove(self, source, primary, other, flags):
"""
Remove monitoring the given L{FileDescriptor} for either reading or
writing. If it's still monitored for the other operation, we
re-register the L{FileDescriptor} for only that operation.
"""
if source not in primary:
return
self._source_remove(self._sources[source])
primary.remove(source)
if source in other:
self._sources[source] = self.input_add(source, flags, self._ioEventCallback)
else:
self._sources.pop(source)
def removeReader(self, reader):
"""
Stop monitoring the given L{FileDescriptor} for reading.
"""
self._remove(reader, self._reads, self._writes, self.OUTFLAGS)
def removeWriter(self, writer):
"""
Stop monitoring the given L{FileDescriptor} for writing.
"""
self._remove(writer, self._writes, self._reads, self.INFLAGS)
def iterate(self, delay=0):
"""
One iteration of the event loop, for trial's use.
This is not used for actual reactor runs.
"""
self.runUntilCurrent()
while self._pending():
self._iteration(0)
def crash(self):
"""
Crash the reactor.
"""
posixbase.PosixReactorBase.crash(self)
self._crash()
def stop(self):
"""
Stop the reactor.
"""
posixbase.PosixReactorBase.stop(self)
# The base implementation only sets a flag, to ensure shutting down is
# not reentrant. Unfortunately, this flag is not meaningful to the
# gobject event loop. We therefore call wakeUp() to ensure the event
# loop will call back into Twisted once this iteration is done. This
# will result in self.runUntilCurrent() being called, where the stop
# flag will trigger the actual shutdown process, eventually calling
# crash() which will do the actual gobject event loop shutdown.
self.wakeUp()
def run(self, installSignalHandlers=True):
"""
Run the reactor.
"""
with _signalGlue():
self.callWhenRunning(self._reschedule)
self.startRunning(installSignalHandlers=installSignalHandlers)
if self._started:
self._run()
def callLater(self, *args, **kwargs):
"""
Schedule a C{DelayedCall}.
"""
result = posixbase.PosixReactorBase.callLater(self, *args, **kwargs)
# Make sure we'll get woken up at correct time to handle this new
# scheduled call:
self._reschedule()
return result
def _reschedule(self):
"""
Schedule a glib timeout for C{_simulate}.
"""
if self._simtag is not None:
self._source_remove(self._simtag)
self._simtag = None
timeout = self.timeout()
if timeout is not None:
self._simtag = self._timeout_add(
int(timeout * 1000),
self._simulate,
priority=self._glib.PRIORITY_DEFAULT_IDLE,
)
def _simulate(self):
"""
Run timers, and then reschedule glib timeout for next scheduled event.
"""
self.runUntilCurrent()
self._reschedule()

View File

@@ -0,0 +1,51 @@
# -*- test-case-name: twisted.test.test_sslverify -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Shared interface to IDNA encoding and decoding, using the C{idna} PyPI package
if available, otherwise the stdlib implementation.
"""
def _idnaBytes(text: str) -> bytes:
"""
Convert some text typed by a human into some ASCII bytes.
This is provided to allow us to use the U{partially-broken IDNA
implementation in the standard library <http://bugs.python.org/issue17305>}
if the more-correct U{idna <https://pypi.python.org/pypi/idna>} package is
not available; C{service_identity} is somewhat stricter about this.
@param text: A domain name, hopefully.
@type text: L{unicode}
@return: The domain name's IDNA representation, encoded as bytes.
@rtype: L{bytes}
"""
try:
import idna
except ImportError:
return text.encode("idna")
else:
return idna.encode(text)
def _idnaText(octets: bytes) -> str:
"""
Convert some IDNA-encoded octets into some human-readable text.
Currently only used by the tests.
@param octets: Some bytes representing a hostname.
@type octets: L{bytes}
@return: A human-readable domain name.
@rtype: L{unicode}
"""
try:
import idna
except ImportError:
return octets.decode("idna")
else:
return idna.decode(octets)

View File

@@ -0,0 +1,161 @@
# -*- test-case-name: twisted.test.test_udp -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
from __future__ import annotations
import socket
import struct
from typing import Any
from twisted.internet.abstract import isIPAddress, isIPv6Address
from twisted.internet.defer import Deferred, succeed
from twisted.internet.error import MulticastJoinError
from twisted.internet.interfaces import IReactorCore
def _maybeResolve(reactor: IReactorCore, addr: str) -> Deferred[str]:
if isIPv6Address(addr) or isIPAddress(addr):
return succeed(addr)
return reactor.resolve(addr)
class MulticastMixin:
"""
Implement multicast functionality.
"""
addressFamily: socket.AddressFamily
reactor: Any
socket: socket.socket
def _addrpack(self, addr: str) -> bytes:
"""
Pack an IP address literal into bytes, according to the address family
of this transport.
"""
try:
return socket.inet_pton(self.addressFamily, addr)
except OSError:
raise MulticastJoinError(
f"invalid address literal for {socket.AddressFamily(self.addressFamily).name}: {addr!r}"
)
@property
def _ipproto(self) -> int:
return (
socket.IPPROTO_IP
if self.addressFamily == socket.AF_INET
else socket.IPPROTO_IPV6
)
@property
def _multiloop(self) -> int:
return (
socket.IP_MULTICAST_LOOP
if self.addressFamily == socket.AF_INET
else socket.IPV6_MULTICAST_LOOP
)
@property
def _multiif(self) -> int:
return (
socket.IP_MULTICAST_IF
if self.addressFamily == socket.AF_INET
else socket.IPV6_MULTICAST_IF
)
@property
def _joingroup(self) -> int:
return (
socket.IP_ADD_MEMBERSHIP
if self.addressFamily == socket.AF_INET
else socket.IPV6_JOIN_GROUP
)
@property
def _leavegroup(self) -> int:
return (
socket.IP_DROP_MEMBERSHIP
if self.addressFamily == socket.AF_INET
else socket.IPV6_LEAVE_GROUP
)
def getOutgoingInterface(self) -> str | int:
blen = 0x4 if self.addressFamily == socket.AF_INET else 0x10
ipproto = self._ipproto
multiif = self._multiif
i = self.socket.getsockopt(ipproto, multiif, blen)
from sys import byteorder
if self.addressFamily == socket.AF_INET6:
return int.from_bytes(i, byteorder)
return socket.inet_ntop(self.addressFamily, i)
def setOutgoingInterface(self, addr: str | int) -> Deferred[int]:
"""
@see: L{IMulticastTransport.setOutgoingInterface}
"""
async def asynchronously() -> int:
i: bytes | int
if self.addressFamily == socket.AF_INET:
assert isinstance(
addr, str
), "IPv4 interfaces are specified as addresses"
i = self._addrpack(await _maybeResolve(self.reactor, addr))
else:
assert isinstance(
addr, int
), "IPv6 interfaces are specified as integers"
i = addr
self.socket.setsockopt(self._ipproto, self._multiif, i)
return 1
return Deferred.fromCoroutine(asynchronously())
def getLoopbackMode(self) -> bool:
return bool(self.socket.getsockopt(self._ipproto, self._multiloop))
def setLoopbackMode(self, mode: int) -> None:
# mode = struct.pack("b", bool(mode))
a = self._ipproto
b = self._multiloop
self.socket.setsockopt(a, b, int(bool(mode)))
def getTTL(self) -> int:
return self.socket.getsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL)
def setTTL(self, ttl: int) -> None:
bttl = struct.pack("B", ttl)
self.socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, bttl)
def _joinleave(self, addr: str, interface: str, join: bool) -> Deferred[None]:
cmd = self._joingroup if join else self._leavegroup
if not interface:
interface = "0.0.0.0" if self.addressFamily == socket.AF_INET else "::"
async def impl() -> None:
resaddr = await _maybeResolve(self.reactor, addr)
resif = await _maybeResolve(self.reactor, interface)
packaddr = self._addrpack(resaddr)
packif = self._addrpack(resif)
try:
self.socket.setsockopt(self._ipproto, cmd, packaddr + packif)
except OSError as e:
raise MulticastJoinError(addr, interface, *e.args) from e
return Deferred.fromCoroutine(impl())
def joinGroup(self, addr: str, interface: str = "") -> Deferred[None]:
"""
@see: L{IMulticastTransport.joinGroup}
"""
return self._joinleave(addr, interface, True)
def leaveGroup(self, addr: str, interface: str = "") -> Deferred[None]:
"""
@see: L{IMulticastTransport.leaveGroup}
"""
return self._joinleave(addr, interface, False)

View File

@@ -0,0 +1,256 @@
# -*- test-case-name: twisted.test.test_ssl -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
This module implements memory BIO based TLS support. It is the preferred
implementation and will be used whenever pyOpenSSL 0.10 or newer is installed
(whenever L{twisted.protocols.tls} is importable).
@since: 11.1
"""
from zope.interface import directlyProvides
from twisted.internet.abstract import FileDescriptor
from twisted.internet.interfaces import ISSLTransport
from twisted.protocols.tls import TLSMemoryBIOFactory
class _BypassTLS:
"""
L{_BypassTLS} is used as the transport object for the TLS protocol object
used to implement C{startTLS}. Its methods skip any TLS logic which
C{startTLS} enables.
@ivar _base: A transport class L{_BypassTLS} has been mixed in with to which
methods will be forwarded. This class is only responsible for sending
bytes over the connection, not doing TLS.
@ivar _connection: A L{Connection} which TLS has been started on which will
be proxied to by this object. Any method which has its behavior
altered after C{startTLS} will be skipped in favor of the base class's
implementation. This allows the TLS protocol object to have direct
access to the transport, necessary to actually implement TLS.
"""
def __init__(self, base, connection):
self._base = base
self._connection = connection
def __getattr__(self, name):
"""
Forward any extra attribute access to the original transport object.
For example, this exposes C{getHost}, the behavior of which does not
change after TLS is enabled.
"""
return getattr(self._connection, name)
def write(self, data):
"""
Write some bytes directly to the connection.
"""
return self._base.write(self._connection, data)
def writeSequence(self, iovec):
"""
Write a some bytes directly to the connection.
"""
return self._base.writeSequence(self._connection, iovec)
def loseConnection(self, *args, **kwargs):
"""
Close the underlying connection.
"""
return self._base.loseConnection(self._connection, *args, **kwargs)
def registerProducer(self, producer, streaming):
"""
Register a producer with the underlying connection.
"""
return self._base.registerProducer(self._connection, producer, streaming)
def unregisterProducer(self):
"""
Unregister a producer with the underlying connection.
"""
return self._base.unregisterProducer(self._connection)
def startTLS(transport, contextFactory, normal, bypass):
"""
Add a layer of SSL to a transport.
@param transport: The transport which will be modified. This can either by
a L{FileDescriptor<twisted.internet.abstract.FileDescriptor>} or a
L{FileHandle<twisted.internet.iocpreactor.abstract.FileHandle>}. The
actual requirements of this instance are that it have:
- a C{_tlsClientDefault} attribute indicating whether the transport is
a client (C{True}) or a server (C{False})
- a settable C{TLS} attribute which can be used to mark the fact
that SSL has been started
- settable C{getHandle} and C{getPeerCertificate} attributes so
these L{ISSLTransport} methods can be added to it
- a C{protocol} attribute referring to the L{IProtocol} currently
connected to the transport, which can also be set to a new
L{IProtocol} for the transport to deliver data to
@param contextFactory: An SSL context factory defining SSL parameters for
the new SSL layer.
@type contextFactory: L{twisted.internet.interfaces.IOpenSSLContextFactory}
@param normal: A flag indicating whether SSL will go in the same direction
as the underlying transport goes. That is, if the SSL client will be
the underlying client and the SSL server will be the underlying server.
C{True} means it is the same, C{False} means they are switched.
@type normal: L{bool}
@param bypass: A transport base class to call methods on to bypass the new
SSL layer (so that the SSL layer itself can send its bytes).
@type bypass: L{type}
"""
# Figure out which direction the SSL goes in. If normal is True,
# we'll go in the direction indicated by the subclass. Otherwise,
# we'll go the other way (client = not normal ^ _tlsClientDefault,
# in other words).
if normal:
client = transport._tlsClientDefault
else:
client = not transport._tlsClientDefault
# If we have a producer, unregister it, and then re-register it below once
# we've switched to TLS mode, so it gets hooked up correctly:
producer, streaming = None, None
if transport.producer is not None:
producer, streaming = transport.producer, transport.streamingProducer
transport.unregisterProducer()
tlsFactory = TLSMemoryBIOFactory(contextFactory, client, None)
tlsProtocol = tlsFactory.protocol(tlsFactory, transport.protocol, False)
# Hook up the new TLS protocol to the transport:
transport.protocol = tlsProtocol
transport.getHandle = tlsProtocol.getHandle
transport.getPeerCertificate = tlsProtocol.getPeerCertificate
# Mark the transport as secure.
directlyProvides(transport, ISSLTransport)
# Remember we did this so that write and writeSequence can send the
# data to the right place.
transport.TLS = True
# Hook it up
transport.protocol.makeConnection(_BypassTLS(bypass, transport))
# Restore producer if necessary:
if producer:
transport.registerProducer(producer, streaming)
class ConnectionMixin:
"""
A mixin for L{twisted.internet.abstract.FileDescriptor} which adds an
L{ITLSTransport} implementation.
@ivar TLS: A flag indicating whether TLS is currently in use on this
transport. This is not a good way for applications to check for TLS,
instead use L{twisted.internet.interfaces.ISSLTransport}.
"""
TLS = False
def startTLS(self, ctx, normal=True):
"""
@see: L{ITLSTransport.startTLS}
"""
startTLS(self, ctx, normal, FileDescriptor)
def write(self, bytes):
"""
Write some bytes to this connection, passing them through a TLS layer if
necessary, or discarding them if the connection has already been lost.
"""
if self.TLS:
if self.connected:
self.protocol.write(bytes)
else:
FileDescriptor.write(self, bytes)
def writeSequence(self, iovec):
"""
Write some bytes to this connection, scatter/gather-style, passing them
through a TLS layer if necessary, or discarding them if the connection
has already been lost.
"""
if self.TLS:
if self.connected:
self.protocol.writeSequence(iovec)
else:
FileDescriptor.writeSequence(self, iovec)
def loseConnection(self):
"""
Close this connection after writing all pending data.
If TLS has been negotiated, perform a TLS shutdown.
"""
if self.TLS:
if self.connected and not self.disconnecting:
self.protocol.loseConnection()
else:
FileDescriptor.loseConnection(self)
def registerProducer(self, producer, streaming):
"""
Register a producer.
If TLS is enabled, the TLS connection handles this.
"""
if self.TLS:
# Registering a producer before we're connected shouldn't be a
# problem. If we end up with a write(), that's already handled in
# the write() code above, and there are no other potential
# side-effects.
self.protocol.registerProducer(producer, streaming)
else:
FileDescriptor.registerProducer(self, producer, streaming)
def unregisterProducer(self):
"""
Unregister a producer.
If TLS is enabled, the TLS connection handles this.
"""
if self.TLS:
self.protocol.unregisterProducer()
else:
FileDescriptor.unregisterProducer(self)
class ClientMixin:
"""
A mixin for L{twisted.internet.tcp.Client} which just marks it as a client
for the purposes of the default TLS handshake.
@ivar _tlsClientDefault: Always C{True}, indicating that this is a client
connection, and by default when TLS is negotiated this class will act as
a TLS client.
"""
_tlsClientDefault = True
class ServerMixin:
"""
A mixin for L{twisted.internet.tcp.Server} which just marks it as a server
for the purposes of the default TLS handshake.
@ivar _tlsClientDefault: Always C{False}, indicating that this is a server
connection, and by default when TLS is negotiated this class will act as
a TLS server.
"""
_tlsClientDefault = False

View File

@@ -0,0 +1,291 @@
# -*- test-case-name: twisted.internet.test.test_pollingfile -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Implements a simple polling interface for file descriptors that don't work with
select() - this is pretty much only useful on Windows.
"""
from zope.interface import implementer
from twisted.internet.interfaces import IConsumer, IPushProducer
MIN_TIMEOUT = 0.000000001
MAX_TIMEOUT = 0.1
class _PollableResource:
active = True
def activate(self):
self.active = True
def deactivate(self):
self.active = False
class _PollingTimer:
# Everything is private here because it is really an implementation detail.
def __init__(self, reactor):
self.reactor = reactor
self._resources = []
self._pollTimer = None
self._currentTimeout = MAX_TIMEOUT
self._paused = False
def _addPollableResource(self, res):
self._resources.append(res)
self._checkPollingState()
def _checkPollingState(self):
for resource in self._resources:
if resource.active:
self._startPolling()
break
else:
self._stopPolling()
def _startPolling(self):
if self._pollTimer is None:
self._pollTimer = self._reschedule()
def _stopPolling(self):
if self._pollTimer is not None:
self._pollTimer.cancel()
self._pollTimer = None
def _pause(self):
self._paused = True
def _unpause(self):
self._paused = False
self._checkPollingState()
def _reschedule(self):
if not self._paused:
return self.reactor.callLater(self._currentTimeout, self._pollEvent)
def _pollEvent(self):
workUnits = 0.0
anyActive = []
for resource in self._resources:
if resource.active:
workUnits += resource.checkWork()
# Check AFTER work has been done
if resource.active:
anyActive.append(resource)
newTimeout = self._currentTimeout
if workUnits:
newTimeout = self._currentTimeout / (workUnits + 1.0)
if newTimeout < MIN_TIMEOUT:
newTimeout = MIN_TIMEOUT
else:
newTimeout = self._currentTimeout * 2.0
if newTimeout > MAX_TIMEOUT:
newTimeout = MAX_TIMEOUT
self._currentTimeout = newTimeout
if anyActive:
self._pollTimer = self._reschedule()
# If we ever (let's hope not) need the above functionality on UNIX, this could
# be factored into a different module.
import pywintypes
import win32api
import win32file
import win32pipe
@implementer(IPushProducer)
class _PollableReadPipe(_PollableResource):
def __init__(self, pipe, receivedCallback, lostCallback):
# security attributes for pipes
self.pipe = pipe
self.receivedCallback = receivedCallback
self.lostCallback = lostCallback
def checkWork(self):
finished = 0
fullDataRead = []
while 1:
try:
buffer, bytesToRead, result = win32pipe.PeekNamedPipe(self.pipe, 1)
# finished = (result == -1)
if not bytesToRead:
break
hr, data = win32file.ReadFile(self.pipe, bytesToRead, None)
fullDataRead.append(data)
except win32api.error:
finished = 1
break
dataBuf = b"".join(fullDataRead)
if dataBuf:
self.receivedCallback(dataBuf)
if finished:
self.cleanup()
return len(dataBuf)
def cleanup(self):
self.deactivate()
self.lostCallback()
def close(self):
try:
win32api.CloseHandle(self.pipe)
except pywintypes.error:
# You can't close std handles...?
pass
def stopProducing(self):
self.close()
def pauseProducing(self):
self.deactivate()
def resumeProducing(self):
self.activate()
FULL_BUFFER_SIZE = 64 * 1024
@implementer(IConsumer)
class _PollableWritePipe(_PollableResource):
def __init__(self, writePipe, lostCallback):
self.disconnecting = False
self.producer = None
self.producerPaused = False
self.streamingProducer = 0
self.outQueue = []
self.writePipe = writePipe
self.lostCallback = lostCallback
try:
win32pipe.SetNamedPipeHandleState(
writePipe, win32pipe.PIPE_NOWAIT, None, None
)
except pywintypes.error:
# Maybe it's an invalid handle. Who knows.
pass
def close(self):
self.disconnecting = True
def bufferFull(self):
if self.producer is not None:
self.producerPaused = True
self.producer.pauseProducing()
def bufferEmpty(self):
if self.producer is not None and (
(not self.streamingProducer) or self.producerPaused
):
self.producer.producerPaused = False
self.producer.resumeProducing()
return True
return False
# almost-but-not-quite-exact copy-paste from abstract.FileDescriptor... ugh
def registerProducer(self, producer, streaming):
"""Register to receive data from a producer.
This sets this selectable to be a consumer for a producer. When this
selectable runs out of data on a write() call, it will ask the producer
to resumeProducing(). A producer should implement the IProducer
interface.
FileDescriptor provides some infrastructure for producer methods.
"""
if self.producer is not None:
raise RuntimeError(
"Cannot register producer %s, because producer %s was never "
"unregistered." % (producer, self.producer)
)
if not self.active:
producer.stopProducing()
else:
self.producer = producer
self.streamingProducer = streaming
if not streaming:
producer.resumeProducing()
def unregisterProducer(self):
"""Stop consuming data from a producer, without disconnecting."""
self.producer = None
def writeConnectionLost(self):
self.deactivate()
try:
win32api.CloseHandle(self.writePipe)
except pywintypes.error:
# OMG what
pass
self.lostCallback()
def writeSequence(self, seq):
"""
Append a C{list} or C{tuple} of bytes to the output buffer.
@param seq: C{list} or C{tuple} of C{str} instances to be appended to
the output buffer.
@raise TypeError: If C{seq} contains C{unicode}.
"""
if str in map(type, seq):
raise TypeError("Unicode not allowed in output buffer.")
self.outQueue.extend(seq)
def write(self, data):
"""
Append some bytes to the output buffer.
@param data: C{str} to be appended to the output buffer.
@type data: C{str}.
@raise TypeError: If C{data} is C{unicode} instead of C{str}.
"""
if isinstance(data, str):
raise TypeError("Unicode not allowed in output buffer.")
if self.disconnecting:
return
self.outQueue.append(data)
if sum(map(len, self.outQueue)) > FULL_BUFFER_SIZE:
self.bufferFull()
def checkWork(self):
numBytesWritten = 0
if not self.outQueue:
if self.disconnecting:
self.writeConnectionLost()
return 0
try:
win32file.WriteFile(self.writePipe, b"", None)
except pywintypes.error:
self.writeConnectionLost()
return numBytesWritten
while self.outQueue:
data = self.outQueue.pop(0)
errCode = 0
try:
errCode, nBytesWritten = win32file.WriteFile(self.writePipe, data, None)
except win32api.error:
self.writeConnectionLost()
break
else:
# assert not errCode, "wtf an error code???"
numBytesWritten += nBytesWritten
if len(data) > nBytesWritten:
self.outQueue.insert(0, data[nBytesWritten:])
break
else:
resumed = self.bufferEmpty()
if not resumed and self.disconnecting:
self.writeConnectionLost()
return numBytesWritten

View File

@@ -0,0 +1,80 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Serial Port Protocol
"""
# dependent on pyserial ( http://pyserial.sf.net/ )
# only tested w/ 1.18 (5 Dec 2002)
from serial import EIGHTBITS, PARITY_NONE, STOPBITS_ONE
from twisted.internet import abstract, fdesc
from twisted.internet.serialport import BaseSerialPort
class SerialPort(BaseSerialPort, abstract.FileDescriptor):
"""
A select()able serial device, acting as a transport.
"""
connected = 1
def __init__(
self,
protocol,
deviceNameOrPortNumber,
reactor,
baudrate=9600,
bytesize=EIGHTBITS,
parity=PARITY_NONE,
stopbits=STOPBITS_ONE,
timeout=0,
xonxoff=0,
rtscts=0,
):
abstract.FileDescriptor.__init__(self, reactor)
self._serial = self._serialFactory(
deviceNameOrPortNumber,
baudrate=baudrate,
bytesize=bytesize,
parity=parity,
stopbits=stopbits,
timeout=timeout,
xonxoff=xonxoff,
rtscts=rtscts,
)
self.reactor = reactor
self.flushInput()
self.flushOutput()
self.protocol = protocol
self.protocol.makeConnection(self)
self.startReading()
def fileno(self):
return self._serial.fd
def writeSomeData(self, data):
"""
Write some data to the serial device.
"""
return fdesc.writeToFD(self.fileno(), data)
def doRead(self):
"""
Some data's readable from serial device.
"""
return fdesc.readFromFD(self.fileno(), self.protocol.dataReceived)
def connectionLost(self, reason):
"""
Called when the serial port disconnects.
Will call C{connectionLost} on the protocol that is handling the
serial data.
"""
abstract.FileDescriptor.connectionLost(self, reason)
self._serial.close()
self.protocol.connectionLost(reason)

View File

@@ -0,0 +1,182 @@
# -*- test-case-name: twisted.test.test_stdio -*-
"""Standard input/out/err support.
Future Plans::
support for stderr, perhaps
Rewrite to use the reactor instead of an ad-hoc mechanism for connecting
protocols to transport.
Maintainer: James Y Knight
"""
from __future__ import annotations
from zope.interface import implementer
from twisted.internet import interfaces, process
from twisted.internet.interfaces import IProtocol, IReactorFDSet
from twisted.logger import Logger
from twisted.python.failure import Failure
_log = Logger()
@implementer(interfaces.IAddress)
class PipeAddress:
pass
@implementer(
interfaces.ITransport,
interfaces.IProducer,
interfaces.IConsumer,
interfaces.IHalfCloseableDescriptor,
)
class StandardIO:
_reader = None
_writer = None
disconnected = False
disconnecting = False
def __init__(
self,
proto: IProtocol,
stdin: int = 0,
stdout: int = 1,
reactor: IReactorFDSet | None = None,
):
if reactor is None:
from twisted.internet import reactor # type:ignore[assignment]
self.protocol: IProtocol = proto
self._writer = process.ProcessWriter(reactor, self, "write", stdout)
self._reader = process.ProcessReader(reactor, self, "read", stdin)
self._reader.startReading()
self.protocol.makeConnection(self)
# ITransport
# XXX Actually, see #3597.
def loseWriteConnection(self):
if self._writer is not None:
self._writer.loseConnection()
def write(self, data):
if self._writer is not None:
self._writer.write(data)
def writeSequence(self, data):
if self._writer is not None:
self._writer.writeSequence(data)
def loseConnection(self):
self.disconnecting = True
if self._writer is not None:
self._writer.loseConnection()
if self._reader is not None:
# Don't loseConnection, because we don't want to SIGPIPE it.
self._reader.stopReading()
def getPeer(self):
return PipeAddress()
def getHost(self):
return PipeAddress()
# Callbacks from process.ProcessReader/ProcessWriter
def childDataReceived(self, fd: str, data: bytes) -> None:
self.protocol.dataReceived(data)
def childConnectionLost(self, fd: str, reason: Failure) -> None:
if self.disconnected:
return
if fd == "read":
self._readConnectionLost(reason)
else:
self._writeConnectionLost(reason)
def connectionLost(self, reason):
self.disconnected = True
# Make sure to cleanup the other half
_reader = self._reader
_writer = self._writer
protocol = self.protocol
self._reader = self._writer = None
self.protocol = None # type:ignore[assignment]
if _writer is not None and not _writer.disconnected:
_writer.connectionLost(reason)
if _reader is not None and not _reader.disconnected:
_reader.connectionLost(reason)
with _log.failuresHandled("while calling stdio connectionLost:"):
protocol.connectionLost(reason)
def _writeConnectionLost(self, reason: Failure) -> None:
self._writer = None
if self.disconnecting:
self.connectionLost(reason)
return
p = interfaces.IHalfCloseableProtocol(self.protocol, None)
if p:
with _log.failuresHandled(
"while calling stdio writeConnectionLost:"
) as wcl:
p.writeConnectionLost()
if wcl.failed:
self.connectionLost(wcl.failure)
def _readConnectionLost(self, reason: Failure) -> None:
self._reader = None
p = interfaces.IHalfCloseableProtocol(self.protocol, None)
if p:
with _log.failuresHandled("while calling stdio readConnectionLost:") as rcl:
p.readConnectionLost()
if rcl.failed:
self.connectionLost(rcl.failure)
else:
self.connectionLost(reason)
# IConsumer
def registerProducer(self, producer, streaming):
if self._writer is None:
producer.stopProducing()
else:
self._writer.registerProducer(producer, streaming)
def unregisterProducer(self):
if self._writer is not None:
self._writer.unregisterProducer()
# IProducer
def stopProducing(self):
self.loseConnection()
def pauseProducing(self):
if self._reader is not None:
self._reader.pauseProducing()
def resumeProducing(self):
if self._reader is not None:
self._reader.resumeProducing()
def stopReading(self):
"""Compatibility only, don't use. Call pauseProducing."""
self.pauseProducing()
def startReading(self):
"""Compatibility only, don't use. Call resumeProducing."""
self.resumeProducing()
def readConnectionLost(self, reason):
# L{IHalfCloseableDescriptor.readConnectionLost}
raise NotImplementedError()
def writeConnectionLost(self, reason):
# L{IHalfCloseableDescriptor.writeConnectionLost}
raise NotImplementedError()

View File

@@ -0,0 +1,118 @@
# -*- test-case-name: twisted.protocols.test.test_tls,twisted.web.test.test_http2 -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Helpers for working with producers.
"""
from typing import List
from zope.interface import implementer
from twisted.internet.interfaces import IPushProducer
from twisted.internet.task import cooperate
from twisted.logger import Logger
_log = Logger()
# This module exports nothing public, it's for internal Twisted use only.
__all__: List[str] = []
@implementer(IPushProducer)
class _PullToPush:
"""
An adapter that converts a non-streaming to a streaming producer.
Because of limitations of the producer API, this adapter requires the
cooperation of the consumer. When the consumer's C{registerProducer} is
called with a non-streaming producer, it must wrap it with L{_PullToPush}
and then call C{startStreaming} on the resulting object. When the
consumer's C{unregisterProducer} is called, it must call
C{stopStreaming} on the L{_PullToPush} instance.
If the underlying producer throws an exception from C{resumeProducing},
the producer will be unregistered from the consumer.
@ivar _producer: the underling non-streaming producer.
@ivar _consumer: the consumer with which the underlying producer was
registered.
@ivar _finished: C{bool} indicating whether the producer has finished.
@ivar _coopTask: the result of calling L{cooperate}, the task driving the
streaming producer.
"""
_finished = False
def __init__(self, pullProducer, consumer):
self._producer = pullProducer
self._consumer = consumer
def _pull(self):
"""
A generator that calls C{resumeProducing} on the underlying producer
forever.
If C{resumeProducing} throws an exception, the producer is
unregistered, which should result in streaming stopping.
"""
while True:
with _log.failuresHandled(
"while calling resumeProducing on {producer}", producer=self._producer
) as op:
self._producer.resumeProducing()
if op.failed:
with _log.failuresHandled(
"while calling unregisterProducer on {consumer}",
consumer=self._consumer,
) as handlingop:
self._consumer.unregisterProducer()
if handlingop.failed:
# The consumer should now call stopStreaming() on us,
# thus stopping the streaming.
self._finished = True
return
yield None
def startStreaming(self):
"""
This should be called by the consumer when the producer is registered.
Start streaming data to the consumer.
"""
self._coopTask = cooperate(self._pull())
def stopStreaming(self):
"""
This should be called by the consumer when the producer is
unregistered.
Stop streaming data to the consumer.
"""
if self._finished:
return
self._finished = True
self._coopTask.stop()
def pauseProducing(self):
"""
@see: C{IPushProducer.pauseProducing}
"""
self._coopTask.pause()
def resumeProducing(self):
"""
@see: C{IPushProducer.resumeProducing}
"""
self._coopTask.resume()
def stopProducing(self):
"""
@see: C{IPushProducer.stopProducing}
"""
self.stopStreaming()
self._producer.stopProducing()

View File

@@ -0,0 +1,342 @@
# -*- test-case-name: twisted.internet.test.test_resolver -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
IPv6-aware hostname resolution.
@see: L{IHostnameResolver}
"""
from socket import (
AF_INET,
AF_INET6,
AF_UNSPEC,
SOCK_DGRAM,
SOCK_STREAM,
AddressFamily,
SocketKind,
gaierror,
getaddrinfo,
)
from typing import (
TYPE_CHECKING,
Callable,
List,
NoReturn,
Optional,
Sequence,
Tuple,
Type,
Union,
)
from zope.interface import implementer
from twisted.internet._idna import _idnaBytes
from twisted.internet.address import IPv4Address, IPv6Address
from twisted.internet.defer import Deferred
from twisted.internet.error import DNSLookupError
from twisted.internet.interfaces import (
IAddress,
IHostnameResolver,
IHostResolution,
IReactorThreads,
IResolutionReceiver,
IResolverSimple,
)
from twisted.internet.threads import deferToThreadPool
from twisted.logger import Logger
from twisted.python.compat import nativeString
if TYPE_CHECKING:
from twisted.python.threadpool import ThreadPool
@implementer(IHostResolution)
class HostResolution:
"""
The in-progress resolution of a given hostname.
"""
def __init__(self, name: str):
"""
Create a L{HostResolution} with the given name.
"""
self.name = name
def cancel(self) -> NoReturn:
# IHostResolution.cancel
raise NotImplementedError()
_any = frozenset([IPv4Address, IPv6Address])
_typesToAF = {
frozenset([IPv4Address]): AF_INET,
frozenset([IPv6Address]): AF_INET6,
_any: AF_UNSPEC,
}
_afToType = {
AF_INET: IPv4Address,
AF_INET6: IPv6Address,
}
_transportToSocket = {
"TCP": SOCK_STREAM,
"UDP": SOCK_DGRAM,
}
_socktypeToType = {
SOCK_STREAM: "TCP",
SOCK_DGRAM: "UDP",
}
_GETADDRINFO_RESULT = List[
Tuple[
AddressFamily,
SocketKind,
int,
str,
Union[Tuple[str, int], Tuple[str, int, int, int]],
]
]
@implementer(IHostnameResolver)
class GAIResolver:
"""
L{IHostnameResolver} implementation that resolves hostnames by calling
L{getaddrinfo} in a thread.
"""
def __init__(
self,
reactor: IReactorThreads,
getThreadPool: Optional[Callable[[], "ThreadPool"]] = None,
getaddrinfo: Callable[[str, int, int, int], _GETADDRINFO_RESULT] = getaddrinfo,
):
"""
Create a L{GAIResolver}.
@param reactor: the reactor to schedule result-delivery on
@type reactor: L{IReactorThreads}
@param getThreadPool: a function to retrieve the thread pool to use for
scheduling name resolutions. If not supplied, the use the given
C{reactor}'s thread pool.
@type getThreadPool: 0-argument callable returning a
L{twisted.python.threadpool.ThreadPool}
@param getaddrinfo: a reference to the L{getaddrinfo} to use - mainly
parameterized for testing.
@type getaddrinfo: callable with the same signature as L{getaddrinfo}
"""
self._reactor = reactor
self._getThreadPool = (
reactor.getThreadPool if getThreadPool is None else getThreadPool
)
self._getaddrinfo = getaddrinfo
def resolveHostName(
self,
resolutionReceiver: IResolutionReceiver,
hostName: str,
portNumber: int = 0,
addressTypes: Optional[Sequence[Type[IAddress]]] = None,
transportSemantics: str = "TCP",
) -> IHostResolution:
"""
See L{IHostnameResolver.resolveHostName}
@param resolutionReceiver: see interface
@param hostName: see interface
@param portNumber: see interface
@param addressTypes: see interface
@param transportSemantics: see interface
@return: see interface
"""
pool = self._getThreadPool()
addressFamily = _typesToAF[
_any if addressTypes is None else frozenset(addressTypes)
]
socketType = _transportToSocket[transportSemantics]
def get() -> _GETADDRINFO_RESULT:
try:
return self._getaddrinfo(
hostName, portNumber, addressFamily, socketType
)
except gaierror:
return []
d = deferToThreadPool(self._reactor, pool, get)
resolution = HostResolution(hostName)
resolutionReceiver.resolutionBegan(resolution)
@d.addCallback
def deliverResults(result: _GETADDRINFO_RESULT) -> None:
for family, socktype, proto, cannoname, sockaddr in result:
addrType = _afToType[family]
resolutionReceiver.addressResolved(
addrType(_socktypeToType.get(socktype, "TCP"), *sockaddr)
)
resolutionReceiver.resolutionComplete()
return resolution
@implementer(IHostnameResolver)
class SimpleResolverComplexifier:
"""
A converter from L{IResolverSimple} to L{IHostnameResolver}.
"""
_log = Logger()
def __init__(self, simpleResolver: IResolverSimple):
"""
Construct a L{SimpleResolverComplexifier} with an L{IResolverSimple}.
"""
self._simpleResolver = simpleResolver
def resolveHostName(
self,
resolutionReceiver: IResolutionReceiver,
hostName: str,
portNumber: int = 0,
addressTypes: Optional[Sequence[Type[IAddress]]] = None,
transportSemantics: str = "TCP",
) -> IHostResolution:
"""
See L{IHostnameResolver.resolveHostName}
@param resolutionReceiver: see interface
@param hostName: see interface
@param portNumber: see interface
@param addressTypes: see interface
@param transportSemantics: see interface
@return: see interface
"""
# If it's str, we need to make sure that it's just ASCII.
try:
hostName_bytes = hostName.encode("ascii")
except UnicodeEncodeError:
# If it's not just ASCII, IDNA it. We don't want to give a Unicode
# string with non-ASCII in it to Python 3, as if anyone passes that
# to a Python 3 stdlib function, it will probably use the wrong
# IDNA version and break absolutely everything
hostName_bytes = _idnaBytes(hostName)
# Make sure it's passed down as a native str, to maintain the interface
hostName = nativeString(hostName_bytes)
resolution = HostResolution(hostName)
resolutionReceiver.resolutionBegan(resolution)
(
self._simpleResolver.getHostByName(hostName)
.addCallback(
lambda address: resolutionReceiver.addressResolved(
IPv4Address("TCP", address, portNumber)
)
)
.addErrback(
lambda error: None
if error.check(DNSLookupError)
else self._log.failure(
"while looking up {name} with {resolver}",
error,
name=hostName,
resolver=self._simpleResolver,
)
)
.addCallback(lambda nothing: resolutionReceiver.resolutionComplete())
)
return resolution
@implementer(IResolutionReceiver)
class FirstOneWins:
"""
An L{IResolutionReceiver} which fires a L{Deferred} with its first result.
"""
def __init__(self, deferred: "Deferred[str]"):
"""
@param deferred: The L{Deferred} to fire when the first resolution
result arrives.
"""
self._deferred = deferred
self._resolved = False
def resolutionBegan(self, resolution: IHostResolution) -> None:
"""
See L{IResolutionReceiver.resolutionBegan}
@param resolution: See L{IResolutionReceiver.resolutionBegan}
"""
self._resolution = resolution
def addressResolved(self, address: IAddress) -> None:
"""
See L{IResolutionReceiver.addressResolved}
@param address: See L{IResolutionReceiver.addressResolved}
"""
if self._resolved:
return
self._resolved = True
# This is used by ComplexResolverSimplifier which specifies only results
# of IPv4Address.
assert isinstance(address, IPv4Address)
self._deferred.callback(address.host)
def resolutionComplete(self) -> None:
"""
See L{IResolutionReceiver.resolutionComplete}
"""
if self._resolved:
return
self._deferred.errback(DNSLookupError(self._resolution.name))
@implementer(IResolverSimple)
class ComplexResolverSimplifier:
"""
A converter from L{IHostnameResolver} to L{IResolverSimple}
"""
def __init__(self, nameResolver: IHostnameResolver):
"""
Create a L{ComplexResolverSimplifier} with an L{IHostnameResolver}.
@param nameResolver: The L{IHostnameResolver} to use.
"""
self._nameResolver = nameResolver
def getHostByName(self, name: str, timeouts: Sequence[int] = ()) -> "Deferred[str]":
"""
See L{IResolverSimple.getHostByName}
@param name: see L{IResolverSimple.getHostByName}
@param timeouts: see L{IResolverSimple.getHostByName}
@return: see L{IResolverSimple.getHostByName}
"""
result: "Deferred[str]" = Deferred()
self._nameResolver.resolveHostName(FirstOneWins(result), name, 0, [IPv4Address])
return result

View File

@@ -0,0 +1,445 @@
# -*- test-case-name: twisted.internet.test.test_sigchld -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
This module is used to integrate child process termination into a
reactor event loop. This is a challenging feature to provide because
most platforms indicate process termination via SIGCHLD and do not
provide a way to wait for that signal and arbitrary I/O events at the
same time. The naive implementation involves installing a Python
SIGCHLD handler; unfortunately this leads to other syscalls being
interrupted (whenever SIGCHLD is received) and failing with EINTR
(which almost no one is prepared to handle). This interruption can be
disabled via siginterrupt(2) (or one of the equivalent mechanisms);
however, if the SIGCHLD is delivered by the platform to a non-main
thread (not a common occurrence, but difficult to prove impossible),
the main thread (waiting on select() or another event notification
API) may not wake up leading to an arbitrary delay before the child
termination is noticed.
The basic solution to all these issues involves enabling SA_RESTART (ie,
disabling system call interruption) and registering a C signal handler which
writes a byte to a pipe. The other end of the pipe is registered with the
event loop, allowing it to wake up shortly after SIGCHLD is received. See
L{_SIGCHLDWaker} for the implementation of the event loop side of this
solution. The use of a pipe this way is known as the U{self-pipe
trick<http://cr.yp.to/docs/selfpipe.html>}.
From Python version 2.6, C{signal.siginterrupt} and C{signal.set_wakeup_fd}
provide the necessary C signal handler which writes to the pipe to be
registered with C{SA_RESTART}.
"""
from __future__ import annotations
import contextlib
import errno
import os
import signal
import socket
from types import FrameType
from typing import Callable, Optional, Sequence
from zope.interface import Attribute, Interface, implementer
from attrs import define, frozen
from typing_extensions import Protocol, TypeAlias
from twisted.internet.interfaces import IReadDescriptor
from twisted.python import failure, log, util
from twisted.python.runtime import platformType
if platformType == "posix":
from . import fdesc, process
SignalHandler: TypeAlias = Callable[[int, Optional[FrameType]], None]
def installHandler(fd: int) -> int:
"""
Install a signal handler which will write a byte to C{fd} when
I{SIGCHLD} is received.
This is implemented by installing a SIGCHLD handler that does nothing,
setting the I{SIGCHLD} handler as not allowed to interrupt system calls,
and using L{signal.set_wakeup_fd} to do the actual writing.
@param fd: The file descriptor to which to write when I{SIGCHLD} is
received.
@return: The file descriptor previously configured for this use.
"""
if fd == -1:
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
else:
def noopSignalHandler(*args):
pass
signal.signal(signal.SIGCHLD, noopSignalHandler)
signal.siginterrupt(signal.SIGCHLD, False)
return signal.set_wakeup_fd(fd)
def isDefaultHandler():
"""
Determine whether the I{SIGCHLD} handler is the default or not.
"""
return signal.getsignal(signal.SIGCHLD) == signal.SIG_DFL
class SignalHandling(Protocol):
"""
The L{SignalHandling} protocol enables customizable signal-handling
behaviors for reactors.
A value that conforms to L{SignalHandling} has install and uninstall hooks
that are called by a reactor at the correct times to have the (typically)
process-global effects necessary for dealing with signals.
"""
def install(self) -> None:
"""
Install the signal handlers.
"""
def uninstall(self) -> None:
"""
Restore signal handlers to their original state.
"""
@frozen
class _WithoutSignalHandling:
"""
A L{SignalHandling} implementation that does no signal handling.
This is the implementation of C{installSignalHandlers=False}.
"""
def install(self) -> None:
"""
Do not install any signal handlers.
"""
def uninstall(self) -> None:
"""
Do nothing because L{install} installed nothing.
"""
@frozen
class _WithSignalHandling:
"""
A reactor core helper that can manage signals: it installs signal handlers
at start time.
"""
_sigInt: SignalHandler
_sigBreak: SignalHandler
_sigTerm: SignalHandler
def install(self) -> None:
"""
Install the signal handlers for the Twisted event loop.
"""
if signal.getsignal(signal.SIGINT) == signal.default_int_handler:
# only handle if there isn't already a handler, e.g. for Pdb.
signal.signal(signal.SIGINT, self._sigInt)
signal.signal(signal.SIGTERM, self._sigTerm)
# Catch Ctrl-Break in windows
SIGBREAK = getattr(signal, "SIGBREAK", None)
if SIGBREAK is not None:
signal.signal(SIGBREAK, self._sigBreak)
def uninstall(self) -> None:
"""
At the moment, do nothing (for historical reasons).
"""
# This should really do something.
# https://github.com/twisted/twisted/issues/11761
@define
class _MultiSignalHandling:
"""
An implementation of L{SignalHandling} which propagates protocol
method calls to a number of other implementations.
This supports composition of multiple signal handling implementations into
a single object so the reactor doesn't have to be concerned with how those
implementations are factored.
@ivar _signalHandlings: The other C{SignalHandling} implementations to
which to propagate calls.
@ivar _installed: If L{install} has been called but L{uninstall} has not.
This is used to avoid double cleanup which otherwise results (at least
during test suite runs) because twisted.internet.reactormixins doesn't
keep track of whether a reactor has run or not but always invokes its
cleanup logic.
"""
_signalHandlings: Sequence[SignalHandling]
_installed: bool = False
def install(self) -> None:
for d in self._signalHandlings:
d.install()
self._installed = True
def uninstall(self) -> None:
if self._installed:
for d in self._signalHandlings:
d.uninstall()
self._installed = False
@define
class _ChildSignalHandling:
"""
Signal handling behavior which supports I{SIGCHLD} for notification about
changes to child process state.
@ivar _childWaker: L{None} or a reference to the L{_SIGCHLDWaker} which is
used to properly notice child process termination. This is L{None}
when this handling behavior is not installed and non-C{None}
otherwise. This is mostly an unfortunate implementation detail due to
L{_SIGCHLDWaker} allocating file descriptors as a side-effect of its
initializer.
"""
_addInternalReader: Callable[[IReadDescriptor], object]
_removeInternalReader: Callable[[IReadDescriptor], object]
_childWaker: Optional[_SIGCHLDWaker] = None
def install(self) -> None:
"""
Extend the basic signal handling logic to also support handling
SIGCHLD to know when to try to reap child processes.
"""
# This conditional should probably not be necessary.
# https://github.com/twisted/twisted/issues/11763
if self._childWaker is None:
self._childWaker = _SIGCHLDWaker()
self._addInternalReader(self._childWaker)
self._childWaker.install()
# Also reap all processes right now, in case we missed any
# signals before we installed the SIGCHLD waker/handler.
# This should only happen if someone used spawnProcess
# before calling reactor.run (and the process also exited
# already).
process.reapAllProcesses()
def uninstall(self) -> None:
"""
If a child waker was created and installed, uninstall it now.
Since this disables reactor functionality and is only called when the
reactor is stopping, it doesn't provide any directly useful
functionality, but the cleanup of reactor-related process-global state
that it does helps in unit tests involving multiple reactors and is
generally just a nice thing.
"""
assert self._childWaker is not None
# XXX This would probably be an alright place to put all of the
# cleanup code for all internal readers (here and in the base class,
# anyway). See #3063 for that cleanup task.
self._removeInternalReader(self._childWaker)
self._childWaker.uninstall()
self._childWaker.connectionLost(failure.Failure(Exception("uninstalled")))
# We just spoiled the current _childWaker so throw it away. We can
# make a new one later if need be.
self._childWaker = None
class _IWaker(Interface):
"""
Interface to wake up the event loop based on the self-pipe trick.
The U{I{self-pipe trick}<http://cr.yp.to/docs/selfpipe.html>}, used to wake
up the main loop from another thread or a signal handler.
This is why we have wakeUp together with doRead
This is used by threads or signals to wake up the event loop.
"""
disconnected = Attribute("")
def wakeUp() -> None:
"""
Called when the event should be wake up.
"""
def doRead() -> None:
"""
Read some data from my connection and discard it.
"""
def connectionLost(reason: failure.Failure) -> None:
"""
Called when connection was closed and the pipes.
"""
@implementer(_IWaker)
class _SocketWaker(log.Logger):
"""
The I{self-pipe trick<http://cr.yp.to/docs/selfpipe.html>}, implemented
using a pair of sockets rather than pipes (due to the lack of support in
select() on Windows for pipes), used to wake up the main loop from
another thread.
"""
disconnected = 0
def __init__(self) -> None:
"""Initialize."""
# Following select_trigger (from asyncore)'s example;
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
with contextlib.closing(
socket.socket(socket.AF_INET, socket.SOCK_STREAM)
) as server:
server.bind(("127.0.0.1", 0))
server.listen(1)
client.connect(server.getsockname())
reader, clientaddr = server.accept()
client.setblocking(False)
reader.setblocking(False)
self.r = reader
self.w = client
self.fileno = self.r.fileno
def wakeUp(self):
"""Send a byte to my connection."""
try:
util.untilConcludes(self.w.send, b"x")
except OSError as e:
if e.args[0] != errno.WSAEWOULDBLOCK:
raise
def doRead(self):
"""
Read some data from my connection.
"""
try:
self.r.recv(8192)
except OSError:
pass
def connectionLost(self, reason):
self.r.close()
self.w.close()
@implementer(IReadDescriptor)
class _FDWaker(log.Logger):
"""
The I{self-pipe trick<http://cr.yp.to/docs/selfpipe.html>}, used to wake
up the main loop from another thread or a signal handler.
L{_FDWaker} is a base class for waker implementations based on
writing to a pipe being monitored by the reactor.
@ivar o: The file descriptor for the end of the pipe which can be
written to wake up a reactor monitoring this waker.
@ivar i: The file descriptor which should be monitored in order to
be awoken by this waker.
"""
disconnected = 0
i: int
o: int
def __init__(self) -> None:
"""Initialize."""
self.i, self.o = os.pipe()
fdesc.setNonBlocking(self.i)
fdesc._setCloseOnExec(self.i)
fdesc.setNonBlocking(self.o)
fdesc._setCloseOnExec(self.o)
self.fileno = lambda: self.i
def doRead(self) -> None:
"""
Read some bytes from the pipe and discard them.
"""
fdesc.readFromFD(self.fileno(), lambda data: None)
def connectionLost(self, reason):
"""Close both ends of my pipe."""
if not hasattr(self, "o"):
return
for fd in self.i, self.o:
try:
os.close(fd)
except OSError:
pass
del self.i, self.o
@implementer(_IWaker)
class _UnixWaker(_FDWaker):
"""
This class provides a simple interface to wake up the event loop.
This is used by threads or signals to wake up the event loop.
"""
def wakeUp(self):
"""Write one byte to the pipe, and flush it."""
# We don't use fdesc.writeToFD since we need to distinguish
# between EINTR (try again) and EAGAIN (do nothing).
if self.o is not None:
try:
util.untilConcludes(os.write, self.o, b"x")
except OSError as e:
# XXX There is no unit test for raising the exception
# for other errnos. See #4285.
if e.errno != errno.EAGAIN:
raise
if platformType == "posix":
_Waker = _UnixWaker
else:
# Primarily Windows and Jython.
_Waker = _SocketWaker # type: ignore[misc,assignment]
class _SIGCHLDWaker(_FDWaker):
"""
L{_SIGCHLDWaker} can wake up a reactor whenever C{SIGCHLD} is received.
"""
def install(self) -> None:
"""
Install the handler necessary to make this waker active.
"""
installHandler(self.o)
def uninstall(self) -> None:
"""
Remove the handler which makes this waker active.
"""
installHandler(-1)
def doRead(self) -> None:
"""
Having woken up the reactor in response to receipt of
C{SIGCHLD}, reap the process which exited.
This is called whenever the reactor notices the waker pipe is
writeable, which happens soon after any call to the C{wakeUp}
method.
"""
super().doRead()
process.reapAllProcesses()

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,342 @@
# -*- test-case-name: twisted.test.test_internet -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Threaded select reactor
The threadedselectreactor is a specialized reactor for integrating with
arbitrary foreign event loop, such as those you find in GUI toolkits.
There are three things you'll need to do to use this reactor.
Install the reactor at the beginning of your program, before importing the rest
of Twisted::
| from twisted.internet import _threadedselect
| _threadedselect.install()
Interleave this reactor with your foreign event loop, at some point after your
event loop is initialized::
| from twisted.internet import reactor
| reactor.interleave(foreignEventLoopWakerFunction)
| self.addSystemEventTrigger('after', 'shutdown', foreignEventLoopStop)
Instead of shutting down the foreign event loop directly, shut down the
reactor::
| from twisted.internet import reactor
| reactor.stop()
In order for Twisted to do its work in the main thread (the thread that
interleave is called from), a waker function is necessary. The waker function
will be called from a "background" thread with one argument: func. The waker
function's purpose is to call func() from the main thread. Many GUI toolkits
ship with appropriate waker functions. One example of this is wxPython's
wx.callAfter (may be wxCallAfter in older versions of wxPython). These would
be used in place of "foreignEventLoopWakerFunction" in the above example.
The other integration point at which the foreign event loop and this reactor
must integrate is shutdown. In order to ensure clean shutdown of Twisted, you
must allow for Twisted to come to a complete stop before quitting the
application. Typically, you will do this by setting up an after shutdown
trigger to stop your foreign event loop, and call reactor.stop() where you
would normally have initiated the shutdown procedure for the foreign event
loop. Shutdown functions that could be used in place of "foreignEventloopStop"
would be the ExitMainLoop method of the wxApp instance with wxPython.
"""
from __future__ import annotations
from errno import EBADF, EINTR
from queue import Empty, Queue
from threading import Thread
from typing import Any, Callable
from zope.interface import implementer
from twisted._threads import ThreadWorker
from twisted.internet import posixbase
from twisted.internet.interfaces import IReactorFDSet, IReadDescriptor, IWriteDescriptor
from twisted.internet.selectreactor import _preenDescriptors, _select
from twisted.logger import Logger
from twisted.python.log import callWithLogger as _callWithLogger
_log = Logger()
def raiseException(e):
raise e
def _threadsafeSelect(
timeout: float | None,
readmap: dict[int, IReadDescriptor],
writemap: dict[int, IWriteDescriptor],
handleResult: Callable[
[
list[int],
list[int],
dict[int, IReadDescriptor],
dict[int, IWriteDescriptor],
bool,
],
None,
],
) -> None:
"""
Invoke C{select}. This will be called in a non-main thread, so it is very
careful to work only on integers and avoid calling any application code.
"""
preen = False
r = []
w = []
while 1:
readints = readmap.keys()
writeints = writemap.keys()
try:
result = _select(readints, writeints, [], timeout)
except ValueError:
# Possible problems with file descriptors that were passed:
# ValueError may indicate that a file descriptor has gone negative.
preen = True
break
except OSError as se:
# The select() system call encountered an error.
if se.args[0] == EINTR:
# EINTR is hard to replicate in tests using an actual select(),
# and I don't want to dedicate effort to testing this function
# when it needs to be refactored with selectreactor.
return # pragma: no cover
elif se.args[0] == EBADF:
preen = True
break
else:
# OK, I really don't know what's going on. Blow up. Never
# mind with the coverage here, since we are just trying to make
# sure we don't swallow an exception.
raise # pragma: no cover
else:
r, w, ignored = result
break
handleResult(r, w, readmap, writemap, preen)
@implementer(IReactorFDSet)
class ThreadedSelectReactor(posixbase.PosixReactorBase):
"""A threaded select() based reactor - runs on all POSIX platforms and on
Win32.
"""
def __init__(
self, waker: Callable[[Callable[[], None]], None] | None = None
) -> None:
self.reads: set[IReadDescriptor] = set()
self.writes: set[IWriteDescriptor] = set()
posixbase.PosixReactorBase.__init__(self)
self._selectorThread: ThreadWorker | None = None
self.mainWaker = waker
self._iterationQueue: Queue[Callable[[], None]] | None = None
def wakeUp(self):
# we want to wake up from any thread
self.waker.wakeUp()
def callLater(self, *args, **kw):
tple = posixbase.PosixReactorBase.callLater(self, *args, **kw)
self.wakeUp()
return tple
def _doReadOrWrite(self, selectable: object, method: str) -> None:
with _log.failuresHandled(
"while handling selectable {sel}", sel=selectable
) as op:
why = getattr(selectable, method)()
if (fail := op.failure) is not None:
why = fail.value
if why:
self._disconnectSelectable(selectable, why, method == "doRead")
def _selectOnce(self, timeout: float | None, keepGoing: bool) -> None:
reads: dict[int, Any] = {}
writes: dict[int, Any] = {}
for isRead, fdmap, d in [
(True, self.reads, reads),
(False, self.writes, writes),
]:
for each in fdmap: # type:ignore[attr-defined]
d[each.fileno()] = each
mainWaker = self.mainWaker
assert mainWaker is not None, (
"neither .interleave() nor .mainLoop() / .run() called, "
"but we are somehow running the reactor"
)
def callReadsAndWrites(
r: list[int],
w: list[int],
readmap: dict[int, IReadDescriptor],
writemap: dict[int, IWriteDescriptor],
preen: bool,
) -> None:
@mainWaker
def onMainThread() -> None:
if preen:
_preenDescriptors(
self.reads, self.writes, self._disconnectSelectable
)
return
_drdw = self._doReadOrWrite
for readable in r:
rselectable = readmap[readable]
if rselectable in self.reads:
_callWithLogger(rselectable, _drdw, rselectable, "doRead")
for writable in w:
wselectable = writemap[writable]
if wselectable in self.writes:
_callWithLogger(wselectable, _drdw, wselectable, "doWrite")
self.runUntilCurrent()
if self._started and keepGoing:
# see coverage note in .interleave()
self._selectOnce(self.timeout(), True) # pragma: no cover
else:
self._cleanUpThread()
if self._selectorThread is None:
self._selectorThread = ThreadWorker(
lambda target: Thread(target=target).start(), Queue()
)
self._selectorThread.do(
lambda: _threadsafeSelect(timeout, reads, writes, callReadsAndWrites)
)
def _cleanUpThread(self) -> None:
"""
Ensure that the selector thread is stopped.
"""
oldThread, self._selectorThread = self._selectorThread, None
if oldThread is not None:
oldThread.quit()
def interleave(
self,
waker: Callable[[Callable[[], None]], None],
installSignalHandlers: bool = True,
) -> None:
"""
interleave(waker) interleaves this reactor with the current application
by moving the blocking parts of the reactor (select() in this case) to
a separate thread. This is typically useful for integration with GUI
applications which have their own event loop already running.
See the module docstring for more information.
"""
# TODO: This method is excluded from coverage because it only happens
# in the case where we are actually running on a foreign event loop,
# and twisted's test suite isn't set up that way. It would be nice to
# add some dedicated tests for ThreadedSelectReactor that covered this
# case.
self.mainWaker = waker # pragma: no cover
self.startRunning(installSignalHandlers) # pragma: no cover
self._selectOnce(0.0, True) # pragma: no cover
def addReader(self, reader: IReadDescriptor) -> None:
"""Add a FileDescriptor for notification of data available to read."""
self.reads.add(reader)
self.wakeUp()
def addWriter(self, writer: IWriteDescriptor) -> None:
"""Add a FileDescriptor for notification of data available to write."""
self.writes.add(writer)
self.wakeUp()
def removeReader(self, reader: IReadDescriptor) -> None:
"""Remove a Selectable for notification of data available to read."""
if reader in self.reads:
self.reads.remove(reader)
def removeWriter(self, writer: IWriteDescriptor) -> None:
"""Remove a Selectable for notification of data available to write."""
if writer in self.writes:
self.writes.remove(writer)
def removeAll(self) -> list[IReadDescriptor | IWriteDescriptor]:
return self._removeAll(self.reads, self.writes) # type:ignore[no-any-return]
def getReaders(self) -> list[IReadDescriptor]:
return list(self.reads)
def getWriters(self) -> list[IWriteDescriptor]:
return list(self.writes)
def stop(self):
"""
Extend the base stop implementation to also wake up the select thread so
that C{runUntilCurrent} notices the reactor should stop.
"""
posixbase.PosixReactorBase.stop(self)
self.wakeUp()
def crash(self):
posixbase.PosixReactorBase.crash(self)
self.wakeUp()
# The following methods are mostly for test-suite support, to make
# ThreadedSelectReactor behave like another reactor you might call run()
# on.
def _testMainLoopSetup(self) -> None:
"""
Mostly for compliance with L{IReactorCore} and usability with the
tests, set up a fake blocking main-loop; make the "foreign" main loop
we are interfacing with be C{self.mainLoop()}, that is reading from a
basic Queue.
"""
self._iterationQueue = Queue()
self.mainWaker = self._iterationQueue.put
def _uninstallHandler(self) -> None:
"""
Handle uninstallation to ensure that cleanup is properly performed by
ReactorBuilder tests.
"""
super()._uninstallHandler()
self._cleanUpThread()
def iterate(self, timeout: float = 0.0) -> None:
if self._iterationQueue is None and self.mainWaker is None: # pragma: no branch
self._testMainLoopSetup()
self.wakeUp()
super().iterate(timeout)
def doIteration(self, timeout: float | None) -> None:
assert self._iterationQueue is not None
self._selectOnce(timeout, False)
try:
work = self._iterationQueue.get(timeout=timeout)
except Empty:
return
work()
def mainLoop(self) -> None:
"""
This should not normally be run.
"""
self._testMainLoopSetup()
super().mainLoop()
def install():
"""Configure the twisted mainloop to be run using the select() reactor."""
reactor = ThreadedSelectReactor()
from twisted.internet.main import installReactor
installReactor(reactor)
return reactor
__all__ = ["install"]

View File

@@ -0,0 +1,155 @@
# -*- test-case-name: twisted.internet.test.test_win32serialport -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Serial port support for Windows.
Requires PySerial and pywin32.
"""
import win32event
import win32file
# system imports
from serial import EIGHTBITS, PARITY_NONE, STOPBITS_ONE
from serial.serialutil import to_bytes
# twisted imports
from twisted.internet import abstract
# sibling imports
from twisted.internet.serialport import BaseSerialPort
class SerialPort(BaseSerialPort, abstract.FileDescriptor):
"""A serial device, acting as a transport, that uses a win32 event."""
connected = 1
def __init__(
self,
protocol,
deviceNameOrPortNumber,
reactor,
baudrate=9600,
bytesize=EIGHTBITS,
parity=PARITY_NONE,
stopbits=STOPBITS_ONE,
xonxoff=0,
rtscts=0,
):
self._serial = self._serialFactory(
deviceNameOrPortNumber,
baudrate=baudrate,
bytesize=bytesize,
parity=parity,
stopbits=stopbits,
timeout=None,
xonxoff=xonxoff,
rtscts=rtscts,
)
self.flushInput()
self.flushOutput()
self.reactor = reactor
self.protocol = protocol
self.outQueue = []
self.closed = 0
self.closedNotifies = 0
self.writeInProgress = 0
self.protocol = protocol
self._overlappedRead = win32file.OVERLAPPED()
self._overlappedRead.hEvent = win32event.CreateEvent(None, 1, 0, None)
self._overlappedWrite = win32file.OVERLAPPED()
self._overlappedWrite.hEvent = win32event.CreateEvent(None, 0, 0, None)
self.reactor.addEvent(self._overlappedRead.hEvent, self, "serialReadEvent")
self.reactor.addEvent(self._overlappedWrite.hEvent, self, "serialWriteEvent")
self.protocol.makeConnection(self)
self._finishPortSetup()
def _finishPortSetup(self):
"""
Finish setting up the serial port.
This is a separate method to facilitate testing.
"""
flags, comstat = self._clearCommError()
rc, self.read_buf = win32file.ReadFile(
self._serial._port_handle,
win32file.AllocateReadBuffer(1),
self._overlappedRead,
)
def _clearCommError(self):
return win32file.ClearCommError(self._serial._port_handle)
def serialReadEvent(self):
# get that character we set up
n = win32file.GetOverlappedResult(
self._serial._port_handle, self._overlappedRead, 0
)
first = to_bytes(self.read_buf[:n])
# now we should get everything that is already in the buffer
flags, comstat = self._clearCommError()
if comstat.cbInQue:
win32event.ResetEvent(self._overlappedRead.hEvent)
rc, buf = win32file.ReadFile(
self._serial._port_handle,
win32file.AllocateReadBuffer(comstat.cbInQue),
self._overlappedRead,
)
n = win32file.GetOverlappedResult(
self._serial._port_handle, self._overlappedRead, 1
)
# handle all the received data:
self.protocol.dataReceived(first + to_bytes(buf[:n]))
else:
# handle all the received data:
self.protocol.dataReceived(first)
# set up next one
win32event.ResetEvent(self._overlappedRead.hEvent)
rc, self.read_buf = win32file.ReadFile(
self._serial._port_handle,
win32file.AllocateReadBuffer(1),
self._overlappedRead,
)
def write(self, data):
if data:
if self.writeInProgress:
self.outQueue.append(data)
else:
self.writeInProgress = 1
win32file.WriteFile(
self._serial._port_handle, data, self._overlappedWrite
)
def serialWriteEvent(self):
try:
dataToWrite = self.outQueue.pop(0)
except IndexError:
self.writeInProgress = 0
return
else:
win32file.WriteFile(
self._serial._port_handle, dataToWrite, self._overlappedWrite
)
def connectionLost(self, reason):
"""
Called when the serial port disconnects.
Will call C{connectionLost} on the protocol that is handling the
serial data.
"""
self.reactor.removeEvent(self._overlappedRead.hEvent)
self.reactor.removeEvent(self._overlappedWrite.hEvent)
abstract.FileDescriptor.connectionLost(self, reason)
self._serial.close()
self.protocol.connectionLost(reason)

View File

@@ -0,0 +1,136 @@
# -*- test-case-name: twisted.test.test_stdio -*-
"""
Windows-specific implementation of the L{twisted.internet.stdio} interface.
"""
import msvcrt
import os
from zope.interface import implementer
import win32api
from twisted.internet import _pollingfile, main
from twisted.internet.interfaces import (
IAddress,
IConsumer,
IHalfCloseableProtocol,
IPushProducer,
ITransport,
)
from twisted.logger import Logger
from twisted.python.failure import Failure
_log = Logger()
@implementer(IAddress)
class Win32PipeAddress:
pass
@implementer(ITransport, IConsumer, IPushProducer)
class StandardIO(_pollingfile._PollingTimer):
disconnecting = False
disconnected = False
def __init__(self, proto, reactor=None):
"""
Start talking to standard IO with the given protocol.
Also, put it stdin/stdout/stderr into binary mode.
"""
if reactor is None:
from twisted.internet import reactor
for stdfd in range(0, 1, 2):
msvcrt.setmode(stdfd, os.O_BINARY)
_pollingfile._PollingTimer.__init__(self, reactor)
self.proto = proto
hstdin = win32api.GetStdHandle(win32api.STD_INPUT_HANDLE)
hstdout = win32api.GetStdHandle(win32api.STD_OUTPUT_HANDLE)
self.stdin = _pollingfile._PollableReadPipe(
hstdin, self.dataReceived, self.readConnectionLost
)
self.stdout = _pollingfile._PollableWritePipe(hstdout, self.writeConnectionLost)
self._addPollableResource(self.stdin)
self._addPollableResource(self.stdout)
self.proto.makeConnection(self)
def dataReceived(self, data):
self.proto.dataReceived(data)
def readConnectionLost(self):
with _log.failuresHandled("read connection lost") as op:
if IHalfCloseableProtocol.providedBy(self.proto):
self.proto.readConnectionLost()
self.checkConnLost()
if not op.succeeded and not self.disconnecting:
self.loseConnection()
def writeConnectionLost(self):
with _log.failuresHandled("write connection lost") as op:
if IHalfCloseableProtocol.providedBy(self.proto):
self.proto.writeConnectionLost()
self.checkConnLost()
if not op.succeeded and not self.disconnecting:
self.loseConnection()
connsLost = 0
def checkConnLost(self):
self.connsLost += 1
if self.connsLost >= 2:
self.disconnecting = True
self.disconnected = True
self.proto.connectionLost(Failure(main.CONNECTION_DONE))
# ITransport
def write(self, data):
self.stdout.write(data)
def writeSequence(self, seq):
self.stdout.write(b"".join(seq))
def loseConnection(self):
self.disconnecting = True
self.stdin.close()
self.stdout.close()
def getPeer(self):
return Win32PipeAddress()
def getHost(self):
return Win32PipeAddress()
# IConsumer
def registerProducer(self, producer, streaming):
return self.stdout.registerProducer(producer, streaming)
def unregisterProducer(self):
return self.stdout.unregisterProducer()
# def write() above
# IProducer
def stopProducing(self):
self.stdin.stopProducing()
# IPushProducer
def pauseProducing(self):
self.stdin.pauseProducing()
def resumeProducing(self):
self.stdin.resumeProducing()

View File

@@ -0,0 +1,560 @@
# -*- test-case-name: twisted.test.test_abstract -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Support for generic select()able objects.
"""
from __future__ import annotations
from socket import AF_INET, AF_INET6, inet_pton
from typing import Iterable, List, Optional, Union
from zope.interface import implementer
from twisted.internet import interfaces, main
from twisted.python import failure, reflect
# Twisted Imports
from twisted.python.compat import lazyByteSlice
def _dataMustBeBytes(obj):
if not isinstance(obj, bytes): # no, really, I mean it
raise TypeError("Data must be bytes")
# Python 3.4+ can join bytes and memoryviews; using a
# memoryview prevents the slice from copying
def _concatenate(bObj, offset, bArray):
return b"".join([memoryview(bObj)[offset:]] + bArray)
class _ConsumerMixin:
"""
L{IConsumer} implementations can mix this in to get C{registerProducer} and
C{unregisterProducer} methods which take care of keeping track of a
producer's state.
Subclasses must provide three attributes which L{_ConsumerMixin} will read
but not write:
- connected: A C{bool} which is C{True} as long as the consumer has
someplace to send bytes (for example, a TCP connection), and then
C{False} when it no longer does.
- disconnecting: A C{bool} which is C{False} until something like
L{ITransport.loseConnection} is called, indicating that the send buffer
should be flushed and the connection lost afterwards. Afterwards,
C{True}.
- disconnected: A C{bool} which is C{False} until the consumer no longer
has a place to send bytes, then C{True}.
Subclasses must also override the C{startWriting} method.
@ivar producer: L{None} if no producer is registered, otherwise the
registered producer.
@ivar producerPaused: A flag indicating whether the producer is currently
paused.
@type producerPaused: L{bool}
@ivar streamingProducer: A flag indicating whether the producer was
registered as a streaming (ie push) producer or not (ie a pull
producer). This will determine whether the consumer may ever need to
pause and resume it, or if it can merely call C{resumeProducing} on it
when buffer space is available.
@ivar streamingProducer: C{bool} or C{int}
"""
producer = None
producerPaused = False
streamingProducer = False
def startWriting(self):
"""
Override in a subclass to cause the reactor to monitor this selectable
for write events. This will be called once in C{unregisterProducer} if
C{loseConnection} has previously been called, so that the connection can
actually close.
"""
raise NotImplementedError("%r did not implement startWriting")
def registerProducer(self, producer, streaming):
"""
Register to receive data from a producer.
This sets this selectable to be a consumer for a producer. When this
selectable runs out of data on a write() call, it will ask the producer
to resumeProducing(). When the FileDescriptor's internal data buffer is
filled, it will ask the producer to pauseProducing(). If the connection
is lost, FileDescriptor calls producer's stopProducing() method.
If streaming is true, the producer should provide the IPushProducer
interface. Otherwise, it is assumed that producer provides the
IPullProducer interface. In this case, the producer won't be asked to
pauseProducing(), but it has to be careful to write() data only when its
resumeProducing() method is called.
"""
if self.producer is not None:
raise RuntimeError(
"Cannot register producer %s, because producer %s was never "
"unregistered." % (producer, self.producer)
)
if self.disconnected:
producer.stopProducing()
else:
self.producer = producer
self.streamingProducer = streaming
if not streaming:
producer.resumeProducing()
def unregisterProducer(self):
"""
Stop consuming data from a producer, without disconnecting.
"""
self.producer = None
if self.connected and self.disconnecting:
self.startWriting()
@implementer(interfaces.ILoggingContext)
class _LogOwner:
"""
Mixin to help implement L{interfaces.ILoggingContext} for transports which
have a protocol, the log prefix of which should also appear in the
transport's log prefix.
"""
def _getLogPrefix(self, applicationObject: object) -> str:
"""
Determine the log prefix to use for messages related to
C{applicationObject}, which may or may not be an
L{interfaces.ILoggingContext} provider.
@return: A C{str} giving the log prefix to use.
"""
if interfaces.ILoggingContext.providedBy(applicationObject):
return applicationObject.logPrefix()
return applicationObject.__class__.__name__
def logPrefix(self):
"""
Override this method to insert custom logging behavior. Its
return value will be inserted in front of every line. It may
be called more times than the number of output lines.
"""
return "-"
@implementer(
interfaces.IPushProducer,
interfaces.IReadWriteDescriptor,
interfaces.IConsumer,
interfaces.ITransport,
interfaces.IHalfCloseableDescriptor,
)
class FileDescriptor(_ConsumerMixin, _LogOwner):
"""
An object which can be operated on by select().
This is an abstract superclass of all objects which may be notified when
they are readable or writable; e.g. they have a file-descriptor that is
valid to be passed to select(2).
"""
# We have two buffers: a list (_tempDataBuffer), and a byte string
# (self.dataBuffer). A given write may not be able to write everything, and
# we also limit to sending at most SEND_LIMIT bytes at a time. Thus, we
# also have self.offset tracks where in self.dataBuffer we are in the
# writing process, to reduce unnecessary copying if we failed to write all
# the data.
connected = 0
disconnected = 0
disconnecting = 0
_writeDisconnecting = False
_writeDisconnected = False
dataBuffer = b""
offset = 0
SEND_LIMIT = 128 * 1024
def __init__(self, reactor: Optional[interfaces.IReactorFDSet] = None):
"""
@param reactor: An L{IReactorFDSet} provider which this descriptor will
use to get readable and writeable event notifications. If no value
is given, the global reactor will be used.
"""
if not reactor:
from twisted.internet import reactor as _reactor
reactor = _reactor # type: ignore[assignment]
self.reactor = reactor
# will be added to dataBuffer in doWrite
self._tempDataBuffer: List[bytes] = []
self._tempDataLen = 0
def connectionLost(self, reason):
"""The connection was lost.
This is called when the connection on a selectable object has been
lost. It will be called whether the connection was closed explicitly,
an exception occurred in an event handler, or the other end of the
connection closed it first.
Clean up state here, but make sure to call back up to FileDescriptor.
"""
self.disconnected = 1
self.connected = 0
if self.producer is not None:
self.producer.stopProducing()
self.producer = None
self.stopReading()
self.stopWriting()
def writeSomeData(self, data: bytes) -> Union[int, BaseException]:
"""
Write as much as possible of the given data, immediately.
This is called to invoke the lower-level writing functionality, such
as a socket's send() method, or a file's write(); this method
returns an integer or an exception. If an integer, it is the number
of bytes written (possibly zero); if an exception, it indicates the
connection was lost.
"""
raise NotImplementedError(
"%s does not implement writeSomeData" % reflect.qual(self.__class__)
)
def doRead(self):
"""
Called when data is available for reading.
Subclasses must override this method. The result will be interpreted
in the same way as a result of doWrite().
"""
raise NotImplementedError(
"%s does not implement doRead" % reflect.qual(self.__class__)
)
def doWrite(self):
"""
Called when data can be written.
@return: L{None} on success, an exception or a negative integer on
failure.
@see: L{twisted.internet.interfaces.IWriteDescriptor.doWrite}.
"""
# We only send at most SEND_LIMIT bytes at a time. If the amount of
# bytes in our send-immediately buffer is smaller than that limit,
# probably a good time to add the bytes from our secondary, list-based
# buffer (self._tempDataBuffer.)
remaining = len(self.dataBuffer) - self.offset
if remaining < self.SEND_LIMIT:
if remaining > 0:
# There is currently some data to write, extend it with the
# list data.
self.dataBuffer = _concatenate(
self.dataBuffer, self.offset, self._tempDataBuffer
)
else:
# self.dataBuffer has nothing left to write, so just convert
# the list buffer to bytes buffer in a cheaper way:
self.dataBuffer = b"".join(self._tempDataBuffer)
self.offset = 0
self._tempDataBuffer = []
self._tempDataLen = 0
# Send as much data as you can.
if self.offset:
l = self.writeSomeData(lazyByteSlice(self.dataBuffer, self.offset))
else:
# Optimization: skip lazyByteSlice() when it's unnecessary.
l = self.writeSomeData(self.dataBuffer)
# There is no writeSomeData implementation in Twisted which returns
# < 0, but the documentation for writeSomeData used to claim negative
# integers meant connection lost. Keep supporting this here,
# although it may be worth deprecating and removing at some point.
if isinstance(l, Exception) or l < 0:
return l
self.offset += l
# If there is nothing left to send,
if self.offset == len(self.dataBuffer) and not self._tempDataLen:
self.dataBuffer = b""
self.offset = 0
# stop writing.
self.stopWriting()
# If I've got a producer who is supposed to supply me with data,
if self.producer is not None and (
(not self.streamingProducer) or self.producerPaused
):
# tell them to supply some more.
self.producerPaused = False
self.producer.resumeProducing()
elif self.disconnecting:
# But if I was previously asked to let the connection die, do
# so.
return self._postLoseConnection()
elif self._writeDisconnecting:
# I was previously asked to half-close the connection. We
# set _writeDisconnected before calling handler, in case the
# handler calls loseConnection(), which will want to check for
# this attribute.
self._writeDisconnected = True
result = self._closeWriteConnection()
return result
return None
def _postLoseConnection(self):
"""Called after a loseConnection(), when all data has been written.
Whatever this returns is then returned by doWrite.
"""
# default implementation, telling reactor we're finished
return main.CONNECTION_DONE
def _closeWriteConnection(self):
# override in subclasses
pass
def writeConnectionLost(self, reason):
# in current code should never be called
self.connectionLost(reason)
def readConnectionLost(self, reason: failure.Failure) -> None:
# override in subclasses
self.connectionLost(reason)
def getHost(self):
# ITransport.getHost
raise NotImplementedError()
def getPeer(self):
# ITransport.getPeer
raise NotImplementedError()
def _isSendBufferFull(self):
"""
Determine whether the user-space send buffer for this transport is full
or not.
When the buffer contains more than C{self.bufferSize} bytes, it is
considered full. This might be improved by considering the size of the
kernel send buffer and how much of it is free.
@return: C{True} if it is full, C{False} otherwise.
"""
return len(self.dataBuffer) + self._tempDataLen > self.bufferSize
def _maybePauseProducer(self):
"""
Possibly pause a producer, if there is one and the send buffer is full.
"""
# If we are responsible for pausing our producer,
if self.producer is not None and self.streamingProducer:
# and our buffer is full,
if self._isSendBufferFull():
# pause it.
self.producerPaused = True
self.producer.pauseProducing()
def write(self, data: bytes) -> None:
"""Reliably write some data.
The data is buffered until the underlying file descriptor is ready
for writing. If there is more than C{self.bufferSize} data in the
buffer and this descriptor has a registered streaming producer, its
C{pauseProducing()} method will be called.
"""
_dataMustBeBytes(data)
if not self.connected or self._writeDisconnected:
return
if data:
self._tempDataBuffer.append(data)
self._tempDataLen += len(data)
self._maybePauseProducer()
self.startWriting()
def writeSequence(self, iovec: Iterable[bytes]) -> None:
"""
Reliably write a sequence of data.
Currently, this is a convenience method roughly equivalent to::
for chunk in iovec:
fd.write(chunk)
It may have a more efficient implementation at a later time or in a
different reactor.
As with the C{write()} method, if a buffer size limit is reached and a
streaming producer is registered, it will be paused until the buffered
data is written to the underlying file descriptor.
"""
for i in iovec:
_dataMustBeBytes(i)
if not self.connected or not iovec or self._writeDisconnected:
return
self._tempDataBuffer.extend(iovec)
for i in iovec:
self._tempDataLen += len(i)
self._maybePauseProducer()
self.startWriting()
def loseConnection(self):
"""Close the connection at the next available opportunity.
Call this to cause this FileDescriptor to lose its connection. It will
first write any data that it has buffered.
If there is data buffered yet to be written, this method will cause the
transport to lose its connection as soon as it's done flushing its
write buffer. If you have a producer registered, the connection won't
be closed until the producer is finished. Therefore, make sure you
unregister your producer when it's finished, or the connection will
never close.
"""
if self.connected and not self.disconnecting:
if self._writeDisconnected:
# doWrite won't trigger the connection close anymore
self.stopReading()
self.stopWriting()
self.connectionLost(failure.Failure(main.CONNECTION_DONE))
else:
self.stopReading()
self.startWriting()
self.disconnecting = 1
def loseWriteConnection(self):
self._writeDisconnecting = True
self.startWriting()
def stopReading(self):
"""Stop waiting for read availability.
Call this to remove this selectable from being notified when it is
ready for reading.
"""
self.reactor.removeReader(self)
def stopWriting(self):
"""Stop waiting for write availability.
Call this to remove this selectable from being notified when it is ready
for writing.
"""
self.reactor.removeWriter(self)
def startReading(self):
"""Start waiting for read availability."""
self.reactor.addReader(self)
def startWriting(self):
"""Start waiting for write availability.
Call this to have this FileDescriptor be notified whenever it is ready for
writing.
"""
self.reactor.addWriter(self)
# Producer/consumer implementation
# first, the consumer stuff. This requires no additional work, as
# any object you can write to can be a consumer, really.
producer = None
bufferSize = 2**2**2**2
def stopConsuming(self):
"""Stop consuming data.
This is called when a producer has lost its connection, to tell the
consumer to go lose its connection (and break potential circular
references).
"""
self.unregisterProducer()
self.loseConnection()
# producer interface implementation
def resumeProducing(self):
if self.connected and not self.disconnecting:
self.startReading()
def pauseProducing(self):
self.stopReading()
def stopProducing(self):
self.loseConnection()
def fileno(self):
"""File Descriptor number for select().
This method must be overridden or assigned in subclasses to
indicate a valid file descriptor for the operating system.
"""
return -1
def isIPAddress(addr: str, family: int = AF_INET) -> bool:
"""
Determine whether the given string represents an IP address of the given
family; by default, an IPv4 address.
@param addr: A string which may or may not be the decimal dotted
representation of an IPv4 address.
@param family: The address family to test for; one of the C{AF_*} constants
from the L{socket} module. (This parameter has only been available
since Twisted 17.1.0; previously L{isIPAddress} could only test for IPv4
addresses.)
@return: C{True} if C{addr} represents an IPv4 address, C{False} otherwise.
"""
if isinstance(addr, bytes): # type: ignore[unreachable]
try: # type: ignore[unreachable]
addr = addr.decode("ascii")
except UnicodeDecodeError:
return False
if family == AF_INET6:
# On some platforms, inet_ntop fails unless the scope ID is valid; this
# is a test for whether the given string *is* an IP address, so strip
# any potential scope ID before checking.
addr = addr.split("%", 1)[0]
elif family == AF_INET:
# On Windows, where 3.5+ implement inet_pton, "0" is considered a valid
# IPv4 address, but we want to ensure we have all 4 segments.
if addr.count(".") != 3:
return False
else:
raise ValueError(f"unknown address family {family!r}")
try:
# This might be a native implementation or the one from
# twisted.python.compat.
inet_pton(family, addr)
except (ValueError, OSError):
return False
return True
def isIPv6Address(addr: str) -> bool:
"""
Determine whether the given string represents an IPv6 address.
@param addr: A string which may or may not be the hex
representation of an IPv6 address.
@type addr: C{str}
@return: C{True} if C{addr} represents an IPv6 address, C{False}
otherwise.
@rtype: C{bool}
"""
return isIPAddress(addr, AF_INET6)
__all__ = ["FileDescriptor", "isIPAddress", "isIPv6Address"]

View File

@@ -0,0 +1,182 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Address objects for network connections.
"""
import os
from typing import Optional, Union
from warnings import warn
from zope.interface import implementer
import attr
from typing_extensions import Literal
from twisted.internet.interfaces import IAddress
from twisted.python.filepath import _asFilesystemBytes, _coerceToFilesystemEncoding
from twisted.python.runtime import platform
@implementer(IAddress)
@attr.s(unsafe_hash=True, auto_attribs=True)
class IPv4Address:
"""
An L{IPv4Address} represents the address of an IPv4 socket endpoint.
@ivar type: A string describing the type of transport, either 'TCP' or
'UDP'.
@ivar host: A string containing a dotted-quad IPv4 address; for example,
"127.0.0.1".
@type host: C{str}
@ivar port: An integer representing the port number.
@type port: C{int}
"""
type: Union[Literal["TCP"], Literal["UDP"]] = attr.ib(
validator=attr.validators.in_(["TCP", "UDP"])
)
host: str
port: int
@implementer(IAddress)
@attr.s(unsafe_hash=True, auto_attribs=True)
class IPv6Address:
"""
An L{IPv6Address} represents the address of an IPv6 socket endpoint.
@ivar type: A string describing the type of transport, either 'TCP' or
'UDP'.
@ivar host: A string containing a colon-separated, hexadecimal formatted
IPv6 address; for example, "::1".
@type host: C{str}
@ivar port: An integer representing the port number.
@type port: C{int}
@ivar flowInfo: the IPv6 flow label. This can be used by QoS routers to
identify flows of traffic; you may generally safely ignore it.
@type flowInfo: L{int}
@ivar scopeID: the IPv6 scope identifier - roughly analagous to what
interface traffic destined for this address must be transmitted over.
@type scopeID: L{int} or L{str}
"""
type: Union[Literal["TCP"], Literal["UDP"]] = attr.ib(
validator=attr.validators.in_(["TCP", "UDP"])
)
host: str
port: int
flowInfo: int = 0
scopeID: Union[str, int] = 0
@implementer(IAddress)
class _ProcessAddress:
"""
An L{interfaces.IAddress} provider for process transports.
"""
@attr.s(unsafe_hash=True, auto_attribs=True)
@implementer(IAddress)
class HostnameAddress:
"""
A L{HostnameAddress} represents the address of a L{HostnameEndpoint}.
@ivar hostname: A hostname byte string; for example, b"example.com".
@type hostname: L{bytes}
@ivar port: An integer representing the port number.
@type port: L{int}
"""
hostname: bytes
port: int
@attr.s(unsafe_hash=False, repr=False, eq=False, auto_attribs=True)
@implementer(IAddress)
class UNIXAddress:
"""
Object representing a UNIX socket endpoint.
@ivar name: The filename associated with this socket.
@type name: C{bytes}
"""
name: Optional[bytes] = attr.ib(
converter=attr.converters.optional(_asFilesystemBytes)
)
if getattr(os.path, "samefile", None) is not None:
def __eq__(self, other: object) -> bool:
"""
Overriding C{attrs} to ensure the os level samefile
check is done if the name attributes do not match.
"""
if not isinstance(other, self.__class__):
return NotImplemented
res = self.name == other.name
if not res and self.name and other.name:
try:
return os.path.samefile(self.name, other.name)
except OSError:
pass
except (TypeError, ValueError) as e:
# On Linux, abstract namespace UNIX sockets start with a
# \0, which os.path doesn't like.
if not platform.isLinux():
raise e
return res
else:
def __eq__(self, other: object) -> bool:
if isinstance(other, self.__class__):
return self.name == other.name
return NotImplemented
def __repr__(self) -> str:
name = self.name
show = _coerceToFilesystemEncoding("", name) if name is not None else None
return f"UNIXAddress({show!r})"
def __hash__(self):
if self.name is None:
return hash((self.__class__, None))
try:
s1 = os.stat(self.name)
return hash((s1.st_ino, s1.st_dev))
except OSError:
return hash(self.name)
# These are for buildFactory backwards compatibility due to
# stupidity-induced inconsistency.
class _ServerFactoryIPv4Address(IPv4Address):
"""Backwards compatibility hack. Just like IPv4Address in practice."""
def __eq__(self, other: object) -> bool:
if isinstance(other, tuple):
warn(
"IPv4Address.__getitem__ is deprecated. " "Use attributes instead.",
category=DeprecationWarning,
stacklevel=2,
)
return (self.host, self.port) == other
elif isinstance(other, IPv4Address):
a = (self.type, self.host, self.port)
b = (other.type, other.host, other.port)
return a == b
return NotImplemented

View File

@@ -0,0 +1,307 @@
# -*- test-case-name: twisted.test.test_internet -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
asyncio-based reactor implementation.
"""
import errno
import sys
from asyncio import AbstractEventLoop, get_event_loop
from typing import Dict, Optional, Type
from zope.interface import implementer
from twisted.internet.abstract import FileDescriptor
from twisted.internet.interfaces import IReactorFDSet
from twisted.internet.posixbase import (
_NO_FILEDESC,
PosixReactorBase,
_ContinuousPolling,
)
from twisted.logger import Logger
from twisted.python.log import callWithLogger
@implementer(IReactorFDSet)
class AsyncioSelectorReactor(PosixReactorBase):
"""
Reactor running on top of L{asyncio.SelectorEventLoop}.
On POSIX platforms, the default event loop is
L{asyncio.SelectorEventLoop}.
On Windows, the default event loop on Python 3.7 and older
is C{asyncio.WindowsSelectorEventLoop}, but on Python 3.8 and newer
the default event loop is C{asyncio.WindowsProactorEventLoop} which
is incompatible with L{AsyncioSelectorReactor}.
Applications that use L{AsyncioSelectorReactor} on Windows
with Python 3.8+ must call
C{asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())}
before instantiating and running L{AsyncioSelectorReactor}.
"""
_asyncClosed = False
_log = Logger()
def __init__(self, eventloop: Optional[AbstractEventLoop] = None):
if eventloop is None:
_eventloop: AbstractEventLoop = get_event_loop()
else:
_eventloop = eventloop
# On Python 3.8+, asyncio.get_event_loop() on
# Windows was changed to return a ProactorEventLoop
# unless the loop policy has been changed.
if sys.platform == "win32":
from asyncio import ProactorEventLoop
if isinstance(_eventloop, ProactorEventLoop):
raise TypeError(
f"ProactorEventLoop is not supported, got: {_eventloop}"
)
self._asyncioEventloop: AbstractEventLoop = _eventloop
self._writers: Dict[Type[FileDescriptor], int] = {}
self._readers: Dict[Type[FileDescriptor], int] = {}
self._continuousPolling = _ContinuousPolling(self)
self._scheduledAt = None
self._timerHandle = None
super().__init__()
def _unregisterFDInAsyncio(self, fd):
"""
Compensate for a bug in asyncio where it will not unregister a FD that
it cannot handle in the epoll loop. It touches internal asyncio code.
A description of the bug by markrwilliams:
The C{add_writer} method of asyncio event loops isn't atomic because
all the Selector classes in the selector module internally record a
file object before passing it to the platform's selector
implementation. If the platform's selector decides the file object
isn't acceptable, the resulting exception doesn't cause the Selector to
un-track the file object.
The failing/hanging stdio test goes through the following sequence of
events (roughly):
* The first C{connection.write(intToByte(value))} call hits the asyncio
reactor's C{addWriter} method.
* C{addWriter} calls the asyncio loop's C{add_writer} method, which
happens to live on C{_BaseSelectorEventLoop}.
* The asyncio loop's C{add_writer} method checks if the file object has
been registered before via the selector's C{get_key} method.
* It hasn't, so the KeyError block runs and calls the selector's
register method
* Code examples that follow use EpollSelector, but the code flow holds
true for any other selector implementation. The selector's register
method first calls through to the next register method in the MRO
* That next method is always C{_BaseSelectorImpl.register} which
creates a C{SelectorKey} instance for the file object, stores it under
the file object's file descriptor, and then returns it.
* Control returns to the concrete selector implementation, which asks
the operating system to track the file descriptor using the right API.
* The operating system refuses! An exception is raised that, in this
case, the asyncio reactor handles by creating a C{_ContinuousPolling}
object to watch the file descriptor.
* The second C{connection.write(intToByte(value))} call hits the
asyncio reactor's C{addWriter} method, which hits the C{add_writer}
method. But the loop's selector's get_key method now returns a
C{SelectorKey}! Now the asyncio reactor's C{addWriter} method thinks
the asyncio loop will watch the file descriptor, even though it won't.
"""
try:
self._asyncioEventloop._selector.unregister(fd)
except BaseException:
pass
def _readOrWrite(self, selectable, read):
method = selectable.doRead if read else selectable.doWrite
if selectable.fileno() == -1:
self._disconnectSelectable(selectable, _NO_FILEDESC, read)
return
try:
why = method()
except Exception as e:
why = e
self._log.failure(None)
if why:
self._disconnectSelectable(selectable, why, read)
def addReader(self, reader):
if reader in self._readers.keys() or reader in self._continuousPolling._readers:
return
fd = reader.fileno()
try:
self._asyncioEventloop.add_reader(
fd, callWithLogger, reader, self._readOrWrite, reader, True
)
self._readers[reader] = fd
except OSError as e:
self._unregisterFDInAsyncio(fd)
if e.errno == errno.EPERM:
# epoll(7) doesn't support certain file descriptors,
# e.g. filesystem files, so for those we just poll
# continuously:
self._continuousPolling.addReader(reader)
else:
raise
def addWriter(self, writer):
if writer in self._writers.keys() or writer in self._continuousPolling._writers:
return
fd = writer.fileno()
try:
self._asyncioEventloop.add_writer(
fd, callWithLogger, writer, self._readOrWrite, writer, False
)
self._writers[writer] = fd
except PermissionError:
self._unregisterFDInAsyncio(fd)
# epoll(7) doesn't support certain file descriptors,
# e.g. filesystem files, so for those we just poll
# continuously:
self._continuousPolling.addWriter(writer)
except BrokenPipeError:
# The kqueuereactor will raise this if there is a broken pipe
self._unregisterFDInAsyncio(fd)
except BaseException:
self._unregisterFDInAsyncio(fd)
raise
def removeReader(self, reader):
# First, see if they're trying to remove a reader that we don't have.
if not (
reader in self._readers.keys() or self._continuousPolling.isReading(reader)
):
# We don't have it, so just return OK.
return
# If it was a cont. polling reader, check there first.
if self._continuousPolling.isReading(reader):
self._continuousPolling.removeReader(reader)
return
fd = reader.fileno()
if fd == -1:
# If the FD is -1, we want to know what its original FD was, to
# remove it.
fd = self._readers.pop(reader)
else:
self._readers.pop(reader)
self._asyncioEventloop.remove_reader(fd)
def removeWriter(self, writer):
# First, see if they're trying to remove a writer that we don't have.
if not (
writer in self._writers.keys() or self._continuousPolling.isWriting(writer)
):
# We don't have it, so just return OK.
return
# If it was a cont. polling writer, check there first.
if self._continuousPolling.isWriting(writer):
self._continuousPolling.removeWriter(writer)
return
fd = writer.fileno()
if fd == -1:
# If the FD is -1, we want to know what its original FD was, to
# remove it.
fd = self._writers.pop(writer)
else:
self._writers.pop(writer)
self._asyncioEventloop.remove_writer(fd)
def removeAll(self):
return (
self._removeAll(self._readers.keys(), self._writers.keys())
+ self._continuousPolling.removeAll()
)
def getReaders(self):
return list(self._readers.keys()) + self._continuousPolling.getReaders()
def getWriters(self):
return list(self._writers.keys()) + self._continuousPolling.getWriters()
def iterate(self, timeout):
self._asyncioEventloop.call_later(timeout + 0.01, self._asyncioEventloop.stop)
self._asyncioEventloop.run_forever()
def run(self, installSignalHandlers=True):
self.startRunning(installSignalHandlers=installSignalHandlers)
self._asyncioEventloop.run_forever()
if self._justStopped:
self._justStopped = False
def stop(self):
super().stop()
# This will cause runUntilCurrent which in its turn
# will call fireSystemEvent("shutdown")
self.callLater(0, lambda: None)
def crash(self):
super().crash()
self._asyncioEventloop.stop()
def _onTimer(self):
self._scheduledAt = None
self.runUntilCurrent()
self._reschedule()
def _reschedule(self):
timeout = self.timeout()
if timeout is not None:
abs_time = self._asyncioEventloop.time() + timeout
self._scheduledAt = abs_time
if self._timerHandle is not None:
self._timerHandle.cancel()
self._timerHandle = self._asyncioEventloop.call_at(abs_time, self._onTimer)
def _moveCallLaterSooner(self, tple):
PosixReactorBase._moveCallLaterSooner(self, tple)
self._reschedule()
def callLater(self, seconds, f, *args, **kwargs):
dc = PosixReactorBase.callLater(self, seconds, f, *args, **kwargs)
abs_time = self._asyncioEventloop.time() + self.timeout()
if self._scheduledAt is None or abs_time < self._scheduledAt:
self._reschedule()
return dc
def callFromThread(self, f, *args, **kwargs):
g = lambda: self.callLater(0, f, *args, **kwargs)
self._asyncioEventloop.call_soon_threadsafe(g)
def install(eventloop=None):
"""
Install an asyncio-based reactor.
@param eventloop: The asyncio eventloop to wrap. If default, the global one
is selected.
"""
reactor = AsyncioSelectorReactor(eventloop)
from twisted.internet.main import installReactor
installReactor(reactor)

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,587 @@
# -*- test-case-name: twisted.internet.test.test_core -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
A reactor for integrating with U{CFRunLoop<http://bit.ly/cfrunloop>}, the
CoreFoundation main loop used by macOS.
This is useful for integrating Twisted with U{PyObjC<http://pyobjc.sf.net/>}
applications.
"""
from __future__ import annotations
__all__ = ["install", "CFReactor"]
import sys
from zope.interface import implementer
from CFNetwork import (
CFSocketCreateRunLoopSource,
CFSocketCreateWithNative,
CFSocketDisableCallBacks,
CFSocketEnableCallBacks,
CFSocketInvalidate,
CFSocketSetSocketFlags,
kCFSocketAutomaticallyReenableReadCallBack,
kCFSocketAutomaticallyReenableWriteCallBack,
kCFSocketConnectCallBack,
kCFSocketReadCallBack,
kCFSocketWriteCallBack,
)
from CoreFoundation import (
CFAbsoluteTimeGetCurrent,
CFRunLoopAddSource,
CFRunLoopAddTimer,
CFRunLoopGetCurrent,
CFRunLoopRemoveSource,
CFRunLoopRun,
CFRunLoopStop,
CFRunLoopTimerCreate,
CFRunLoopTimerInvalidate,
kCFAllocatorDefault,
kCFRunLoopCommonModes,
)
from twisted.internet.interfaces import IReactorFDSet
from twisted.internet.posixbase import _NO_FILEDESC, PosixReactorBase
from twisted.python import log
# We know that we're going to run on macOS so we can just pick the
# POSIX-appropriate waker. This also avoids having a dynamic base class and
# so lets more things get type checked.
from ._signals import _UnixWaker
_READ = 0
_WRITE = 1
_preserveSOError = 1 << 6
class _WakerPlus(_UnixWaker):
"""
The normal Twisted waker will simply wake up the main loop, which causes an
iteration to run, which in turn causes L{ReactorBase.runUntilCurrent}
to get invoked.
L{CFReactor} has a slightly different model of iteration, though: rather
than have each iteration process the thread queue, then timed calls, then
file descriptors, each callback is run as it is dispatched by the CFRunLoop
observer which triggered it.
So this waker needs to not only unblock the loop, but also make sure the
work gets done; so, it reschedules the invocation of C{runUntilCurrent} to
be immediate (0 seconds from now) even if there is no timed call work to
do.
"""
def __init__(self, reactor):
super().__init__()
self.reactor = reactor
def doRead(self):
"""
Wake up the loop and force C{runUntilCurrent} to run immediately in the
next timed iteration.
"""
result = super().doRead()
self.reactor._scheduleSimulate()
return result
@implementer(IReactorFDSet)
class CFReactor(PosixReactorBase):
"""
The CoreFoundation reactor.
You probably want to use this via the L{install} API.
@ivar _fdmap: a dictionary, mapping an integer (a file descriptor) to a
4-tuple of:
- source: a C{CFRunLoopSource}; the source associated with this
socket.
- socket: a C{CFSocket} wrapping the file descriptor.
- descriptor: an L{IReadDescriptor} and/or L{IWriteDescriptor}
provider.
- read-write: a 2-C{list} of booleans: respectively, whether this
descriptor is currently registered for reading or registered for
writing.
@ivar _idmap: a dictionary, mapping the id() of an L{IReadDescriptor} or
L{IWriteDescriptor} to a C{fd} in L{_fdmap}. Implemented in this
manner so that we don't have to rely (even more) on the hashability of
L{IReadDescriptor} providers, and we know that they won't be collected
since these are kept in sync with C{_fdmap}. Necessary because the
.fileno() of a file descriptor may change at will, so we need to be
able to look up what its file descriptor I{used} to be, so that we can
look it up in C{_fdmap}
@ivar _cfrunloop: the C{CFRunLoop} pyobjc object wrapped
by this reactor.
@ivar _inCFLoop: Is C{CFRunLoopRun} currently running?
@type _inCFLoop: L{bool}
@ivar _currentSimulator: if a CFTimer is currently scheduled with the CF
run loop to run Twisted callLater calls, this is a reference to it.
Otherwise, it is L{None}
"""
def __init__(self, runLoop=None, runner=None):
self._fdmap = {}
self._idmap = {}
if runner is None:
runner = CFRunLoopRun
self._runner = runner
if runLoop is None:
runLoop = CFRunLoopGetCurrent()
self._cfrunloop = runLoop
PosixReactorBase.__init__(self)
def _wakerFactory(self) -> _WakerPlus:
return _WakerPlus(self)
def _socketCallback(
self, cfSocket, callbackType, ignoredAddress, ignoredData, context
):
"""
The socket callback issued by CFRunLoop. This will issue C{doRead} or
C{doWrite} calls to the L{IReadDescriptor} and L{IWriteDescriptor}
registered with the file descriptor that we are being notified of.
@param cfSocket: The C{CFSocket} which has got some activity.
@param callbackType: The type of activity that we are being notified
of. Either C{kCFSocketReadCallBack} or C{kCFSocketWriteCallBack}.
@param ignoredAddress: Unused, because this is not used for either of
the callback types we register for.
@param ignoredData: Unused, because this is not used for either of the
callback types we register for.
@param context: The data associated with this callback by
C{CFSocketCreateWithNative} (in C{CFReactor._watchFD}). A 2-tuple
of C{(int, CFRunLoopSource)}.
"""
(fd, smugglesrc) = context
if fd not in self._fdmap:
# Spurious notifications seem to be generated sometimes if you
# CFSocketDisableCallBacks in the middle of an event. I don't know
# about this FD, any more, so let's get rid of it.
CFRunLoopRemoveSource(self._cfrunloop, smugglesrc, kCFRunLoopCommonModes)
return
src, skt, readWriteDescriptor, rw = self._fdmap[fd]
def _drdw():
why = None
isRead = False
try:
if readWriteDescriptor.fileno() == -1:
why = _NO_FILEDESC
else:
isRead = callbackType == kCFSocketReadCallBack
# CFSocket seems to deliver duplicate read/write
# notifications sometimes, especially a duplicate
# writability notification when first registering the
# socket. This bears further investigation, since I may
# have been mis-interpreting the behavior I was seeing.
# (Running the full Twisted test suite, while thorough, is
# not always entirely clear.) Until this has been more
# thoroughly investigated , we consult our own
# reading/writing state flags to determine whether we
# should actually attempt a doRead/doWrite first. -glyph
if isRead:
if rw[_READ]:
why = readWriteDescriptor.doRead()
else:
if rw[_WRITE]:
why = readWriteDescriptor.doWrite()
except BaseException:
why = sys.exc_info()[1]
log.err()
if why:
self._disconnectSelectable(readWriteDescriptor, why, isRead)
log.callWithLogger(readWriteDescriptor, _drdw)
def _watchFD(self, fd, descr, flag):
"""
Register a file descriptor with the C{CFRunLoop}, or modify its state
so that it's listening for both notifications (read and write) rather
than just one; used to implement C{addReader} and C{addWriter}.
@param fd: The file descriptor.
@type fd: L{int}
@param descr: the L{IReadDescriptor} or L{IWriteDescriptor}
@param flag: the flag to register for callbacks on, either
C{kCFSocketReadCallBack} or C{kCFSocketWriteCallBack}
"""
if fd == -1:
raise RuntimeError("Invalid file descriptor.")
if fd in self._fdmap:
src, cfs, gotdescr, rw = self._fdmap[fd]
# do I need to verify that it's the same descr?
else:
ctx = []
ctx.append(fd)
cfs = CFSocketCreateWithNative(
kCFAllocatorDefault,
fd,
kCFSocketReadCallBack
| kCFSocketWriteCallBack
| kCFSocketConnectCallBack,
self._socketCallback,
ctx,
)
CFSocketSetSocketFlags(
cfs,
kCFSocketAutomaticallyReenableReadCallBack
| kCFSocketAutomaticallyReenableWriteCallBack
|
# This extra flag is to ensure that CF doesn't (destructively,
# because destructively is the only way to do it) retrieve
# SO_ERROR and thereby break twisted.internet.tcp.BaseClient,
# which needs SO_ERROR to tell it whether or not it needs to
# call connect_ex a second time.
_preserveSOError,
)
src = CFSocketCreateRunLoopSource(kCFAllocatorDefault, cfs, 0)
ctx.append(src)
CFRunLoopAddSource(self._cfrunloop, src, kCFRunLoopCommonModes)
CFSocketDisableCallBacks(
cfs,
kCFSocketReadCallBack
| kCFSocketWriteCallBack
| kCFSocketConnectCallBack,
)
rw = [False, False]
self._idmap[id(descr)] = fd
self._fdmap[fd] = src, cfs, descr, rw
rw[self._flag2idx(flag)] = True
CFSocketEnableCallBacks(cfs, flag)
def _flag2idx(self, flag):
"""
Convert a C{kCFSocket...} constant to an index into the read/write
state list (C{_READ} or C{_WRITE}) (the 4th element of the value of
C{self._fdmap}).
@param flag: C{kCFSocketReadCallBack} or C{kCFSocketWriteCallBack}
@return: C{_READ} or C{_WRITE}
"""
return {kCFSocketReadCallBack: _READ, kCFSocketWriteCallBack: _WRITE}[flag]
def _unwatchFD(self, fd, descr, flag):
"""
Unregister a file descriptor with the C{CFRunLoop}, or modify its state
so that it's listening for only one notification (read or write) as
opposed to both; used to implement C{removeReader} and C{removeWriter}.
@param fd: a file descriptor
@type fd: C{int}
@param descr: an L{IReadDescriptor} or L{IWriteDescriptor}
@param flag: C{kCFSocketWriteCallBack} C{kCFSocketReadCallBack}
"""
if id(descr) not in self._idmap:
return
if fd == -1:
# need to deal with it in this case, I think.
realfd = self._idmap[id(descr)]
else:
realfd = fd
src, cfs, descr, rw = self._fdmap[realfd]
CFSocketDisableCallBacks(cfs, flag)
rw[self._flag2idx(flag)] = False
if not rw[_READ] and not rw[_WRITE]:
del self._idmap[id(descr)]
del self._fdmap[realfd]
CFRunLoopRemoveSource(self._cfrunloop, src, kCFRunLoopCommonModes)
CFSocketInvalidate(cfs)
def addReader(self, reader):
"""
Implement L{IReactorFDSet.addReader}.
"""
self._watchFD(reader.fileno(), reader, kCFSocketReadCallBack)
def addWriter(self, writer):
"""
Implement L{IReactorFDSet.addWriter}.
"""
self._watchFD(writer.fileno(), writer, kCFSocketWriteCallBack)
def removeReader(self, reader):
"""
Implement L{IReactorFDSet.removeReader}.
"""
self._unwatchFD(reader.fileno(), reader, kCFSocketReadCallBack)
def removeWriter(self, writer):
"""
Implement L{IReactorFDSet.removeWriter}.
"""
self._unwatchFD(writer.fileno(), writer, kCFSocketWriteCallBack)
def removeAll(self):
"""
Implement L{IReactorFDSet.removeAll}.
"""
allDesc = {descr for src, cfs, descr, rw in self._fdmap.values()}
allDesc -= set(self._internalReaders)
for desc in allDesc:
self.removeReader(desc)
self.removeWriter(desc)
return list(allDesc)
def getReaders(self):
"""
Implement L{IReactorFDSet.getReaders}.
"""
return [descr for src, cfs, descr, rw in self._fdmap.values() if rw[_READ]]
def getWriters(self):
"""
Implement L{IReactorFDSet.getWriters}.
"""
return [descr for src, cfs, descr, rw in self._fdmap.values() if rw[_WRITE]]
def _moveCallLaterSooner(self, tple):
"""
Override L{PosixReactorBase}'s implementation of L{IDelayedCall.reset}
so that it will immediately reschedule. Normally
C{_moveCallLaterSooner} depends on the fact that C{runUntilCurrent} is
always run before the mainloop goes back to sleep, so this forces it to
immediately recompute how long the loop needs to stay asleep.
"""
result = PosixReactorBase._moveCallLaterSooner(self, tple)
self._scheduleSimulate()
return result
def startRunning(self, installSignalHandlers: bool = True) -> None:
"""
Start running the reactor, then kick off the timer that advances
Twisted's clock to keep pace with CFRunLoop's.
"""
super().startRunning(installSignalHandlers)
# Before 'startRunning' is called, the reactor is not attached to the
# CFRunLoop[1]; specifically, the CFTimer that runs all of Twisted's
# timers is not active and will not have been added to the loop by any
# application code. Now that _running is probably[2] True, we need to
# ensure that timed calls will actually run on the main loop. This
# call needs to be here, rather than at the top of mainLoop, because
# it's possible to use startRunning to *attach* a reactor to an
# already-running CFRunLoop, i.e. within a plugin for an application
# that doesn't otherwise use Twisted, rather than calling it via run().
self._scheduleSimulate()
# [1]: readers & writers are still active in the loop, but arguably
# they should not be.
# [2]: application code within a 'startup' system event trigger *may*
# have already crashed the reactor and thus set _started to False,
# but that specific case is handled by mainLoop, since that case
# is inherently irrelevant in an attach-to-application case and is
# only necessary to handle mainLoop spuriously blocking.
_inCFLoop = False
def mainLoop(self) -> None:
"""
Run the runner (C{CFRunLoopRun} or something that calls it), which runs
the run loop until C{crash()} is called.
"""
if not self._started:
# If we arrive here, we were crashed by application code in a
# 'startup' system event trigger, (or crashed manually before the
# application calls 'mainLoop' directly for whatever reason; sigh,
# this method should not be public). However, application code
# doing obscure things will expect an invocation of this loop to
# have at least *one* pass over ready readers, writers, and delayed
# calls. iterate(), in particular, is emulated in exactly this way
# in this reactor implementation. In order to ensure that we enter
# the real implementation of the mainloop and do all of those
# things, we need to set _started back to True so that callLater
# actually schedules itself against the CFRunLoop, but immediately
# crash once we are in the context of the loop where we've run
# ready I/O and timers.
def docrash() -> None:
self.crash()
self._started = True
self.callLater(0, docrash)
already = False
try:
while self._started:
if already:
# Sometimes CFRunLoopRun (or its equivalents) may exit
# without CFRunLoopStop being called.
# This is really only *supposed* to happen when it runs out
# of sources & timers to process. However, in full Twisted
# test-suite runs we have observed, extremely rarely (once
# in every 3000 tests or so) CFRunLoopRun exiting in cases
# where it seems as though there *is* still some work to
# do. However, given the difficulty of reproducing the
# race conditions necessary to make this happen, it's
# possible that we have missed some nuance of when
# CFRunLoop considers the list of work "empty" and various
# callbacks and timers to be "invalidated". Therefore we
# are not fully confident that this is a platform bug, but
# it is nevertheless unexpected behavior from our reading
# of the documentation.
# To accommodate this rare and slightly ambiguous stress
# case, we make extra sure that our scheduled timer is
# re-created on the loop as a CFRunLoopTimer, which
# reliably gives the loop some work to do and 'fixes' it if
# it exited due to having no active sources or timers.
self._scheduleSimulate()
# At this point, there may be a little more code that we
# would need to put here for full correctness for a very
# peculiar type of application: if you're writing a
# command-line tool using CFReactor, adding *nothing* to
# the reactor itself, disabling even the internal Waker
# file descriptors, then there's a possibility that
# CFRunLoopRun will exit early, and if we have no timers,
# we might busy-loop here. Because we cannot seem to force
# this to happen under normal circumstances, we're leaving
# that code out.
already = True
self._inCFLoop = True
try:
self._runner()
finally:
self._inCFLoop = False
finally:
self._stopSimulating()
_currentSimulator: object | None = None
def _stopSimulating(self) -> None:
"""
If we have a CFRunLoopTimer registered with the CFRunLoop, invalidate
it and set it to None.
"""
if self._currentSimulator is None:
return
CFRunLoopTimerInvalidate(self._currentSimulator)
self._currentSimulator = None
def _scheduleSimulate(self) -> None:
"""
Schedule a call to C{self.runUntilCurrent}. This will cancel the
currently scheduled call if it is already scheduled.
"""
self._stopSimulating()
if not self._started:
# If the reactor is not running (e.g. we are scheduling callLater
# calls before starting the reactor) we should not be scheduling
# CFRunLoopTimers against the global CFRunLoop.
return
# runUntilCurrent acts on 3 things: _justStopped to process the
# side-effect of reactor.stop(), threadCallQueue to handle any calls
# from threads, and _pendingTimedCalls.
timeout = 0.0 if (self._justStopped or self.threadCallQueue) else self.timeout()
if timeout is None:
return
fireDate = CFAbsoluteTimeGetCurrent() + timeout
def simulate(cftimer, extra):
self._currentSimulator = None
self.runUntilCurrent()
self._scheduleSimulate()
c = self._currentSimulator = CFRunLoopTimerCreate(
kCFAllocatorDefault, fireDate, 0, 0, 0, simulate, None
)
CFRunLoopAddTimer(self._cfrunloop, c, kCFRunLoopCommonModes)
def callLater(self, _seconds, _f, *args, **kw):
"""
Implement L{IReactorTime.callLater}.
"""
delayedCall = PosixReactorBase.callLater(self, _seconds, _f, *args, **kw)
self._scheduleSimulate()
return delayedCall
def stop(self) -> None:
"""
Implement L{IReactorCore.stop}.
"""
PosixReactorBase.stop(self)
self._scheduleSimulate()
def crash(self):
"""
Implement L{IReactorCore.crash}
"""
PosixReactorBase.crash(self)
if not self._inCFLoop:
return
CFRunLoopStop(self._cfrunloop)
def iterate(self, delay=0):
"""
Emulate the behavior of C{iterate()} for things that want to call it,
by letting the loop run for a little while and then scheduling a timed
call to exit it.
"""
self._started = True
# Since the CoreFoundation loop doesn't have the concept of "iterate"
# we can't ask it to do this. Instead we will make arrangements to
# crash it *very* soon and then make it run. This is a rough
# approximation of "an iteration". Using crash and mainLoop here
# means that it's safe (as safe as anything using "iterate" can be) to
# do this repeatedly.
self.callLater(0, self.crash)
self.mainLoop()
def install(runLoop=None, runner=None):
"""
Configure the twisted mainloop to be run inside CFRunLoop.
@param runLoop: the run loop to use.
@param runner: the function to call in order to actually invoke the main
loop. This will default to C{CFRunLoopRun} if not specified. However,
this is not an appropriate choice for GUI applications, as you need to
run NSApplicationMain (or something like it). For example, to run the
Twisted mainloop in a PyObjC application, your C{main.py} should look
something like this::
from PyObjCTools import AppHelper
from twisted.internet.cfreactor import install
install(runner=AppHelper.runEventLoop)
# initialize your application
reactor.run()
@return: The installed reactor.
@rtype: C{CFReactor}
"""
reactor = CFReactor(runLoop=runLoop, runner=runner)
from twisted.internet.main import installReactor
installReactor(reactor)
return reactor

View File

@@ -0,0 +1,55 @@
# -*- test-case-name: twisted.internet.test.test_default -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
The most suitable default reactor for the current platform.
Depending on a specific application's needs, some other reactor may in
fact be better.
"""
__all__ = ["install"]
from twisted.python.runtime import platform
def _getInstallFunction(platform):
"""
Return a function to install the reactor most suited for the given platform.
@param platform: The platform for which to select a reactor.
@type platform: L{twisted.python.runtime.Platform}
@return: A zero-argument callable which will install the selected
reactor.
"""
# Linux: epoll(7) is the default, since it scales well.
#
# macOS: poll(2) is not exposed by Python because it doesn't support all
# file descriptors (in particular, lack of PTY support is a problem) --
# see <http://bugs.python.org/issue5154>. kqueue has the same restrictions
# as poll(2) as far PTY support goes.
#
# Windows: IOCP should eventually be default, but still has some serious
# bugs, e.g. <http://twistedmatrix.com/trac/ticket/4667>.
#
# We therefore choose epoll(7) on Linux, poll(2) on other non-macOS POSIX
# platforms, and select(2) everywhere else.
try:
if platform.isLinux():
try:
from twisted.internet.epollreactor import install
except ImportError:
from twisted.internet.pollreactor import install
elif platform.getType() == "posix" and not platform.isMacOSX():
from twisted.internet.pollreactor import install
else:
from twisted.internet.selectreactor import install
except ImportError:
from twisted.internet.selectreactor import install
return install
install = _getInstallFunction(platform)

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,259 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
An epoll() based implementation of the twisted main loop.
To install the event loop (and you should do this before any connections,
listeners or connectors are added)::
from twisted.internet import epollreactor
epollreactor.install()
"""
import errno
import select
from zope.interface import implementer
from twisted.internet import posixbase
from twisted.internet.interfaces import IReactorFDSet
from twisted.python import log
try:
# This is to keep mypy from complaining
# We don't use type: ignore[attr-defined] on import, because mypy only complains
# on on some platforms, and then the unused ignore is an issue if the undefined
# attribute isn't.
epoll = getattr(select, "epoll")
EPOLLHUP = getattr(select, "EPOLLHUP")
EPOLLERR = getattr(select, "EPOLLERR")
EPOLLIN = getattr(select, "EPOLLIN")
EPOLLOUT = getattr(select, "EPOLLOUT")
except AttributeError as e:
raise ImportError(e)
@implementer(IReactorFDSet)
class EPollReactor(posixbase.PosixReactorBase, posixbase._PollLikeMixin):
"""
A reactor that uses epoll(7).
@ivar _poller: A C{epoll} which will be used to check for I/O
readiness.
@ivar _selectables: A dictionary mapping integer file descriptors to
instances of C{FileDescriptor} which have been registered with the
reactor. All C{FileDescriptors} which are currently receiving read or
write readiness notifications will be present as values in this
dictionary.
@ivar _reads: A set containing integer file descriptors. Values in this
set will be registered with C{_poller} for read readiness notifications
which will be dispatched to the corresponding C{FileDescriptor}
instances in C{_selectables}.
@ivar _writes: A set containing integer file descriptors. Values in this
set will be registered with C{_poller} for write readiness
notifications which will be dispatched to the corresponding
C{FileDescriptor} instances in C{_selectables}.
@ivar _continuousPolling: A L{_ContinuousPolling} instance, used to handle
file descriptors (e.g. filesystem files) that are not supported by
C{epoll(7)}.
"""
# Attributes for _PollLikeMixin
_POLL_DISCONNECTED = EPOLLHUP | EPOLLERR
_POLL_IN = EPOLLIN
_POLL_OUT = EPOLLOUT
def __init__(self):
"""
Initialize epoll object, file descriptor tracking dictionaries, and the
base class.
"""
# Create the poller we're going to use. The 1024 here is just a hint
# to the kernel, it is not a hard maximum. After Linux 2.6.8, the size
# argument is completely ignored.
self._poller = epoll(1024)
self._reads = set()
self._writes = set()
self._selectables = {}
self._continuousPolling = posixbase._ContinuousPolling(self)
posixbase.PosixReactorBase.__init__(self)
def _add(self, xer, primary, other, selectables, event, antievent):
"""
Private method for adding a descriptor from the event loop.
It takes care of adding it if new or modifying it if already added
for another state (read -> read/write for example).
"""
fd = xer.fileno()
if fd not in primary:
flags = event
# epoll_ctl can raise all kinds of IOErrors, and every one
# indicates a bug either in the reactor or application-code.
# Let them all through so someone sees a traceback and fixes
# something. We'll do the same thing for every other call to
# this method in this file.
if fd in other:
flags |= antievent
self._poller.modify(fd, flags)
else:
self._poller.register(fd, flags)
# Update our own tracking state *only* after the epoll call has
# succeeded. Otherwise we may get out of sync.
primary.add(fd)
selectables[fd] = xer
def addReader(self, reader):
"""
Add a FileDescriptor for notification of data available to read.
"""
try:
self._add(
reader, self._reads, self._writes, self._selectables, EPOLLIN, EPOLLOUT
)
except OSError as e:
if e.errno == errno.EPERM:
# epoll(7) doesn't support certain file descriptors,
# e.g. filesystem files, so for those we just poll
# continuously:
self._continuousPolling.addReader(reader)
else:
raise
def addWriter(self, writer):
"""
Add a FileDescriptor for notification of data available to write.
"""
try:
self._add(
writer, self._writes, self._reads, self._selectables, EPOLLOUT, EPOLLIN
)
except OSError as e:
if e.errno == errno.EPERM:
# epoll(7) doesn't support certain file descriptors,
# e.g. filesystem files, so for those we just poll
# continuously:
self._continuousPolling.addWriter(writer)
else:
raise
def _remove(self, xer, primary, other, selectables, event, antievent):
"""
Private method for removing a descriptor from the event loop.
It does the inverse job of _add, and also add a check in case of the fd
has gone away.
"""
fd = xer.fileno()
if fd == -1:
for fd, fdes in selectables.items():
if xer is fdes:
break
else:
return
if fd in primary:
if fd in other:
flags = antievent
# See comment above modify call in _add.
self._poller.modify(fd, flags)
else:
del selectables[fd]
# See comment above _control call in _add.
self._poller.unregister(fd)
primary.remove(fd)
def removeReader(self, reader):
"""
Remove a Selectable for notification of data available to read.
"""
if self._continuousPolling.isReading(reader):
self._continuousPolling.removeReader(reader)
return
self._remove(
reader, self._reads, self._writes, self._selectables, EPOLLIN, EPOLLOUT
)
def removeWriter(self, writer):
"""
Remove a Selectable for notification of data available to write.
"""
if self._continuousPolling.isWriting(writer):
self._continuousPolling.removeWriter(writer)
return
self._remove(
writer, self._writes, self._reads, self._selectables, EPOLLOUT, EPOLLIN
)
def removeAll(self):
"""
Remove all selectables, and return a list of them.
"""
return (
self._removeAll(
[self._selectables[fd] for fd in self._reads],
[self._selectables[fd] for fd in self._writes],
)
+ self._continuousPolling.removeAll()
)
def getReaders(self):
return [
self._selectables[fd] for fd in self._reads
] + self._continuousPolling.getReaders()
def getWriters(self):
return [
self._selectables[fd] for fd in self._writes
] + self._continuousPolling.getWriters()
def doPoll(self, timeout):
"""
Poll the poller for new events.
"""
if timeout is None:
timeout = -1 # Wait indefinitely.
try:
# Limit the number of events to the number of io objects we're
# currently tracking (because that's maybe a good heuristic) and
# the amount of time we block to the value specified by our
# caller.
l = self._poller.poll(timeout, len(self._selectables))
except OSError as err:
if err.errno == errno.EINTR:
return
# See epoll_wait(2) for documentation on the other conditions
# under which this can fail. They can only be due to a serious
# programming error on our part, so let's just announce them
# loudly.
raise
_drdw = self._doReadOrWrite
for fd, event in l:
try:
selectable = self._selectables[fd]
except KeyError:
pass
else:
log.callWithLogger(selectable, _drdw, selectable, fd, event)
doIteration = doPoll
def install():
"""
Install the epoll() reactor.
"""
p = EPollReactor()
from twisted.internet.main import installReactor
installReactor(p)
__all__ = ["EPollReactor", "install"]

View File

@@ -0,0 +1,510 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Exceptions and errors for use in twisted.internet modules.
"""
import socket
from incremental import Version
from twisted.python import deprecate
class BindError(Exception):
__doc__ = MESSAGE = "An error occurred binding to an interface"
def __str__(self) -> str:
s = self.MESSAGE
if self.args:
s = "{}: {}".format(s, " ".join(self.args))
s = "%s." % s
return s
class CannotListenError(BindError):
"""
This gets raised by a call to startListening, when the object cannotstart
listening.
@ivar interface: the interface I tried to listen on
@ivar port: the port I tried to listen on
@ivar socketError: the exception I got when I tried to listen
@type socketError: L{socket.error}
"""
def __init__(self, interface, port, socketError):
BindError.__init__(self, interface, port, socketError)
self.interface = interface
self.port = port
self.socketError = socketError
def __str__(self) -> str:
iface = self.interface or "any"
return "Couldn't listen on {}:{}: {}.".format(
iface, self.port, self.socketError
)
class MulticastJoinError(Exception):
"""
An attempt to join a multicast group failed.
"""
class MessageLengthError(Exception):
__doc__ = MESSAGE = "Message is too long to send"
def __str__(self) -> str:
s = self.MESSAGE
if self.args:
s = "{}: {}".format(s, " ".join(self.args))
s = "%s." % s
return s
class DNSLookupError(IOError):
__doc__ = MESSAGE = "DNS lookup failed"
def __str__(self) -> str:
s = self.MESSAGE
if self.args:
s = "{}: {}".format(s, " ".join(self.args))
s = "%s." % s
return s
class ConnectInProgressError(Exception):
"""A connect operation was started and isn't done yet."""
# connection errors
class ConnectError(Exception):
__doc__ = MESSAGE = "An error occurred while connecting"
def __init__(self, osError=None, string=""):
self.osError = osError
Exception.__init__(self, string)
def __str__(self) -> str:
s = self.MESSAGE
if self.osError:
s = f"{s}: {self.osError}"
if self.args[0]:
s = f"{s}: {self.args[0]}"
s = "%s." % s
return s
class ConnectBindError(ConnectError):
__doc__ = MESSAGE = "Couldn't bind"
class UnknownHostError(ConnectError):
__doc__ = MESSAGE = "Hostname couldn't be looked up"
class NoRouteError(ConnectError):
__doc__ = MESSAGE = "No route to host"
class ConnectionRefusedError(ConnectError):
__doc__ = MESSAGE = "Connection was refused by other side"
class TCPTimedOutError(ConnectError):
__doc__ = MESSAGE = "TCP connection timed out"
class BadFileError(ConnectError):
__doc__ = MESSAGE = "File used for UNIX socket is no good"
class ServiceNameUnknownError(ConnectError):
__doc__ = MESSAGE = "Service name given as port is unknown"
class UserError(ConnectError):
__doc__ = MESSAGE = "User aborted connection"
class TimeoutError(UserError):
__doc__ = MESSAGE = "User timeout caused connection failure"
class SSLError(ConnectError):
__doc__ = MESSAGE = "An SSL error occurred"
class VerifyError(Exception):
__doc__ = MESSAGE = "Could not verify something that was supposed to be signed."
class PeerVerifyError(VerifyError):
__doc__ = MESSAGE = "The peer rejected our verify error."
class CertificateError(Exception):
__doc__ = MESSAGE = "We did not find a certificate where we expected to find one."
try:
import errno
errnoMapping = {
errno.ENETUNREACH: NoRouteError,
errno.ECONNREFUSED: ConnectionRefusedError,
errno.ETIMEDOUT: TCPTimedOutError,
}
if hasattr(errno, "WSAECONNREFUSED"):
errnoMapping[errno.WSAECONNREFUSED] = ConnectionRefusedError
errnoMapping[errno.WSAENETUNREACH] = NoRouteError # type: ignore[attr-defined]
except ImportError:
errnoMapping = {}
def getConnectError(e):
"""Given a socket exception, return connection error."""
if isinstance(e, Exception):
args = e.args
else:
args = e
try:
number, string = args
except ValueError:
return ConnectError(string=e)
if hasattr(socket, "gaierror") and isinstance(e, socket.gaierror):
# Only works in 2.2 in newer. Really that means always; #5978 covers
# this and other weirdnesses in this function.
klass = UnknownHostError
else:
klass = errnoMapping.get(number, ConnectError)
return klass(number, string)
class ConnectionClosed(Exception):
"""
Connection was closed, whether cleanly or non-cleanly.
"""
class ConnectionLost(ConnectionClosed):
__doc__ = MESSAGE = """
Connection to the other side was lost in a non-clean fashion
"""
def __str__(self) -> str:
s = self.MESSAGE.strip().splitlines()[:1]
if self.args:
s.append(": ")
s.append(" ".join(self.args))
s.append(".")
return "".join(s)
class ConnectionAborted(ConnectionLost):
"""
Connection was aborted locally, using
L{twisted.internet.interfaces.ITCPTransport.abortConnection}.
@since: 11.1
"""
MESSAGE = "Connection was aborted locally using " "ITCPTransport.abortConnection"
class ConnectionDone(ConnectionClosed):
__doc__ = MESSAGE = "Connection was closed cleanly"
def __str__(self) -> str:
s = self.MESSAGE
if self.args:
s = "{}: {}".format(s, " ".join(self.args))
s = "%s." % s
return s
class FileDescriptorOverrun(ConnectionLost):
"""
A mis-use of L{IUNIXTransport.sendFileDescriptor} caused the connection to
be closed.
Each file descriptor sent using C{sendFileDescriptor} must be associated
with at least one byte sent using L{ITransport.write}. If at any point
fewer bytes have been written than file descriptors have been sent, the
connection is closed with this exception.
"""
MESSAGE = (
"A mis-use of IUNIXTransport.sendFileDescriptor caused "
"the connection to be closed."
)
class ConnectionFdescWentAway(ConnectionLost):
__doc__ = MESSAGE = "Uh" # TODO
class AlreadyCalled(ValueError):
__doc__ = MESSAGE = "Tried to cancel an already-called event"
def __str__(self) -> str:
s = self.MESSAGE
if self.args:
s = "{}: {}".format(s, " ".join(self.args))
s = "%s." % s
return s
class AlreadyCancelled(ValueError):
__doc__ = MESSAGE = "Tried to cancel an already-cancelled event"
def __str__(self) -> str:
s = self.MESSAGE
if self.args:
s = "{}: {}".format(s, " ".join(self.args))
s = "%s." % s
return s
class PotentialZombieWarning(Warning):
"""
Emitted when L{IReactorProcess.spawnProcess} is called in a way which may
result in termination of the created child process not being reported.
Deprecated in Twisted 10.0.
"""
MESSAGE = (
"spawnProcess called, but the SIGCHLD handler is not "
"installed. This probably means you have not yet "
"called reactor.run, or called "
"reactor.run(installSignalHandler=0). You will probably "
"never see this process finish, and it may become a "
"zombie process."
)
deprecate.deprecatedModuleAttribute(
Version("Twisted", 10, 0, 0),
"There is no longer any potential for zombie process.",
__name__,
"PotentialZombieWarning",
)
class ProcessDone(ConnectionDone):
__doc__ = MESSAGE = "A process has ended without apparent errors"
def __init__(self, status):
Exception.__init__(self, "process finished with exit code 0")
self.exitCode = 0
self.signal = None
self.status = status
class ProcessTerminated(ConnectionLost):
__doc__ = MESSAGE = """
A process has ended with a probable error condition
@ivar exitCode: See L{__init__}
@ivar signal: See L{__init__}
@ivar status: See L{__init__}
"""
def __init__(self, exitCode=None, signal=None, status=None):
"""
@param exitCode: The exit status of the process. This is roughly like
the value you might pass to L{os._exit}. This is L{None} if the
process exited due to a signal.
@type exitCode: L{int} or L{None}
@param signal: The exit signal of the process. This is L{None} if the
process did not exit due to a signal.
@type signal: L{int} or L{None}
@param status: The exit code of the process. This is a platform
specific combination of the exit code and the exit signal. See
L{os.WIFEXITED} and related functions.
@type status: L{int}
"""
self.exitCode = exitCode
self.signal = signal
self.status = status
s = "process ended"
if exitCode is not None:
s = s + " with exit code %s" % exitCode
if signal is not None:
s = s + " by signal %s" % signal
Exception.__init__(self, s)
class ProcessExitedAlready(Exception):
"""
The process has already exited and the operation requested can no longer
be performed.
"""
class NotConnectingError(RuntimeError):
__doc__ = (
MESSAGE
) = "The Connector was not connecting when it was asked to stop connecting"
def __str__(self) -> str:
s = self.MESSAGE
if self.args:
s = "{}: {}".format(s, " ".join(self.args))
s = "%s." % s
return s
class NotListeningError(RuntimeError):
__doc__ = MESSAGE = "The Port was not listening when it was asked to stop listening"
def __str__(self) -> str:
s = self.MESSAGE
if self.args:
s = "{}: {}".format(s, " ".join(self.args))
s = "%s." % s
return s
class ReactorNotRunning(RuntimeError):
"""
Error raised when trying to stop a reactor which is not running.
"""
class ReactorNotRestartable(RuntimeError):
"""
Error raised when trying to run a reactor which was stopped.
"""
class ReactorAlreadyRunning(RuntimeError):
"""
Error raised when trying to start the reactor multiple times.
"""
class ReactorAlreadyInstalledError(AssertionError):
"""
Could not install reactor because one is already installed.
"""
class ConnectingCancelledError(Exception):
"""
An C{Exception} that will be raised when an L{IStreamClientEndpoint} is
cancelled before it connects.
@ivar address: The L{IAddress} that is the destination of the
cancelled L{IStreamClientEndpoint}.
"""
def __init__(self, address):
"""
@param address: The L{IAddress} that is the destination of the
L{IStreamClientEndpoint} that was cancelled.
"""
Exception.__init__(self, address)
self.address = address
class NoProtocol(Exception):
"""
An C{Exception} that will be raised when the factory given to a
L{IStreamClientEndpoint} returns L{None} from C{buildProtocol}.
"""
class UnsupportedAddressFamily(Exception):
"""
An attempt was made to use a socket with an address family (eg I{AF_INET},
I{AF_INET6}, etc) which is not supported by the reactor.
"""
class UnsupportedSocketType(Exception):
"""
An attempt was made to use a socket of a type (eg I{SOCK_STREAM},
I{SOCK_DGRAM}, etc) which is not supported by the reactor.
"""
class AlreadyListened(Exception):
"""
An attempt was made to listen on a file descriptor which can only be
listened on once.
"""
class InvalidAddressError(ValueError):
"""
An invalid address was specified (i.e. neither IPv4 or IPv6, or expected
one and got the other).
@ivar address: See L{__init__}
@ivar message: See L{__init__}
"""
def __init__(self, address, message):
"""
@param address: The address that was provided.
@type address: L{bytes}
@param message: A native string of additional information provided by
the calling context.
@type address: L{str}
"""
self.address = address
self.message = message
__all__ = [
"BindError",
"CannotListenError",
"MulticastJoinError",
"MessageLengthError",
"DNSLookupError",
"ConnectInProgressError",
"ConnectError",
"ConnectBindError",
"UnknownHostError",
"NoRouteError",
"ConnectionRefusedError",
"TCPTimedOutError",
"BadFileError",
"ServiceNameUnknownError",
"UserError",
"TimeoutError",
"SSLError",
"VerifyError",
"PeerVerifyError",
"CertificateError",
"getConnectError",
"ConnectionClosed",
"ConnectionLost",
"ConnectionDone",
"ConnectionFdescWentAway",
"AlreadyCalled",
"AlreadyCancelled",
"PotentialZombieWarning",
"ProcessDone",
"ProcessTerminated",
"ProcessExitedAlready",
"NotConnectingError",
"NotListeningError",
"ReactorNotRunning",
"ReactorAlreadyRunning",
"ReactorAlreadyInstalledError",
"ConnectingCancelledError",
"UnsupportedAddressFamily",
"UnsupportedSocketType",
"InvalidAddressError",
]

View File

@@ -0,0 +1,121 @@
# -*- test-case-name: twisted.test.test_fdesc -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Utility functions for dealing with POSIX file descriptors.
"""
import errno
import os
try:
import fcntl as _fcntl
except ImportError:
fcntl = None
else:
fcntl = _fcntl
# twisted imports
from twisted.internet.main import CONNECTION_DONE, CONNECTION_LOST
def setNonBlocking(fd):
"""
Set the file description of the given file descriptor to non-blocking.
"""
flags = fcntl.fcntl(fd, fcntl.F_GETFL)
flags = flags | os.O_NONBLOCK
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
def setBlocking(fd):
"""
Set the file description of the given file descriptor to blocking.
"""
flags = fcntl.fcntl(fd, fcntl.F_GETFL)
flags = flags & ~os.O_NONBLOCK
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
if fcntl is None:
# fcntl isn't available on Windows. By default, handles aren't
# inherited on Windows, so we can do nothing here.
_setCloseOnExec = _unsetCloseOnExec = lambda fd: None
else:
def _setCloseOnExec(fd):
"""
Make a file descriptor close-on-exec.
"""
flags = fcntl.fcntl(fd, fcntl.F_GETFD)
flags = flags | fcntl.FD_CLOEXEC
fcntl.fcntl(fd, fcntl.F_SETFD, flags)
def _unsetCloseOnExec(fd):
"""
Make a file descriptor close-on-exec.
"""
flags = fcntl.fcntl(fd, fcntl.F_GETFD)
flags = flags & ~fcntl.FD_CLOEXEC
fcntl.fcntl(fd, fcntl.F_SETFD, flags)
def readFromFD(fd, callback):
"""
Read from file descriptor, calling callback with resulting data.
If successful, call 'callback' with a single argument: the
resulting data.
Returns same thing FileDescriptor.doRead would: CONNECTION_LOST,
CONNECTION_DONE, or None.
@type fd: C{int}
@param fd: non-blocking file descriptor to be read from.
@param callback: a callable which accepts a single argument. If
data is read from the file descriptor it will be called with this
data. Handling exceptions from calling the callback is up to the
caller.
Note that if the descriptor is still connected but no data is read,
None will be returned but callback will not be called.
@return: CONNECTION_LOST on error, CONNECTION_DONE when fd is
closed, otherwise None.
"""
try:
output = os.read(fd, 8192)
except OSError as ioe:
if ioe.args[0] in (errno.EAGAIN, errno.EINTR):
return
else:
return CONNECTION_LOST
if not output:
return CONNECTION_DONE
callback(output)
def writeToFD(fd, data):
"""
Write data to file descriptor.
Returns same thing FileDescriptor.writeSomeData would.
@type fd: C{int}
@param fd: non-blocking file descriptor to be written to.
@type data: C{str} or C{buffer}
@param data: bytes to write to fd.
@return: number of bytes written, or CONNECTION_LOST.
"""
try:
return os.write(fd, data)
except OSError as io:
if io.errno in (errno.EAGAIN, errno.EINTR):
return 0
return CONNECTION_LOST
__all__ = ["setNonBlocking", "setBlocking", "readFromFD", "writeToFD"]

View File

@@ -0,0 +1,122 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
This module provides support for Twisted to interact with the glib
mainloop via GObject Introspection.
In order to use this support, simply do the following::
from twisted.internet import gireactor
gireactor.install()
If you wish to use a GApplication, register it with the reactor::
from twisted.internet import reactor
reactor.registerGApplication(app)
Then use twisted.internet APIs as usual.
On Python 3, pygobject v3.4 or later is required.
"""
from typing import Union
from gi.repository import GLib
from twisted.internet import _glibbase
from twisted.internet.error import ReactorAlreadyRunning
from twisted.python import runtime
if getattr(GLib, "threads_init", None) is not None:
GLib.threads_init()
class GIReactor(_glibbase.GlibReactorBase):
"""
GObject-introspection event loop reactor.
@ivar _gapplication: A C{Gio.Application} instance that was registered
with C{registerGApplication}.
"""
# By default no Application is registered:
_gapplication = None
def __init__(self, useGtk=False):
_glibbase.GlibReactorBase.__init__(self, GLib, None)
def registerGApplication(self, app):
"""
Register a C{Gio.Application} or C{Gtk.Application}, whose main loop
will be used instead of the default one.
We will C{hold} the application so it doesn't exit on its own. In
versions of C{python-gi} 3.2 and later, we exit the event loop using
the C{app.quit} method which overrides any holds. Older versions are
not supported.
"""
if self._gapplication is not None:
raise RuntimeError("Can't register more than one application instance.")
if self._started:
raise ReactorAlreadyRunning(
"Can't register application after reactor was started."
)
if not hasattr(app, "quit"):
raise RuntimeError(
"Application registration is not supported in"
" versions of PyGObject prior to 3.2."
)
self._gapplication = app
def run():
app.hold()
app.run(None)
self._run = run
self._crash = app.quit
class PortableGIReactor(_glibbase.GlibReactorBase):
"""
Portable GObject Introspection event loop reactor.
"""
def __init__(self, useGtk=False):
super().__init__(GLib, None, useGtk=useGtk)
def registerGApplication(self, app):
"""
Register a C{Gio.Application} or C{Gtk.Application}, whose main loop
will be used instead of the default one.
"""
raise NotImplementedError("GApplication is not currently supported on Windows.")
def simulate(self) -> None:
"""
For compatibility only. Do nothing.
"""
def install(useGtk: bool = False) -> Union[GIReactor, PortableGIReactor]:
"""
Configure the twisted mainloop to be run inside the glib mainloop.
@param useGtk: A hint that the Gtk GUI will or will not be used. Currently
does not modify any behavior.
"""
reactor: Union[GIReactor, PortableGIReactor]
if runtime.platform.getType() == "posix":
reactor = GIReactor(useGtk=useGtk)
else:
reactor = PortableGIReactor(useGtk=useGtk)
from twisted.internet.main import installReactor
installReactor(reactor)
return reactor
__all__ = ["install"]

View File

@@ -0,0 +1,50 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
This module provides support for Twisted to interact with the glib mainloop.
This is like gtk2, but slightly faster and does not require a working
$DISPLAY. However, you cannot run GUIs under this reactor: for that you must
use the gtk2reactor instead.
In order to use this support, simply do the following::
from twisted.internet import glib2reactor
glib2reactor.install()
Then use twisted.internet APIs as usual. The other methods here are not
intended to be called directly.
"""
from incremental import Version
from ._deprecate import deprecatedGnomeReactor
deprecatedGnomeReactor("glib2reactor", Version("Twisted", 23, 8, 0))
from twisted.internet import gtk2reactor
class Glib2Reactor(gtk2reactor.Gtk2Reactor):
"""
The reactor using the glib mainloop.
"""
def __init__(self):
"""
Override init to set the C{useGtk} flag.
"""
gtk2reactor.Gtk2Reactor.__init__(self, useGtk=False)
def install():
"""
Configure the twisted mainloop to be run inside the glib mainloop.
"""
reactor = Glib2Reactor()
from twisted.internet.main import installReactor
installReactor(reactor)
__all__ = ["install"]

View File

@@ -0,0 +1,119 @@
# -*- test-case-name: twisted.internet.test -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
This module provides support for Twisted to interact with the glib/gtk2
mainloop.
In order to use this support, simply do the following::
from twisted.internet import gtk2reactor
gtk2reactor.install()
Then use twisted.internet APIs as usual. The other methods here are not
intended to be called directly.
"""
from incremental import Version
from ._deprecate import deprecatedGnomeReactor
deprecatedGnomeReactor("gtk2reactor", Version("Twisted", 23, 8, 0))
# System Imports
import sys
# Twisted Imports
from twisted.internet import _glibbase
from twisted.python import runtime
# Certain old versions of pygtk and gi crash if imported at the same
# time. This is a problem when running Twisted's unit tests, since they will
# attempt to run both gtk2 and gtk3/gi tests. However, gireactor makes sure
# that if we are in such an old version, and gireactor was imported,
# gtk2reactor will not be importable. So we don't *need* to enforce that here
# as well; whichever is imported first will still win. Moreover, additional
# enforcement in this module is unnecessary in modern versions, and downright
# problematic in certain versions where for some reason importing gtk also
# imports some subset of gi. So we do nothing here, relying on gireactor to
# prevent the crash.
try:
if not hasattr(sys, "frozen"):
# Don't want to check this for py2exe
import pygtk
pygtk.require("2.0")
except (ImportError, AttributeError):
pass # maybe we're using pygtk before this hack existed.
import gobject
if not hasattr(gobject, "IO_HUP"):
# gi.repository's legacy compatibility helper raises an AttributeError with
# a custom error message rather than a useful ImportError, so things tend
# to fail loudly. Things that import this module expect an ImportError if,
# well, something failed to import, and treat an AttributeError as an
# arbitrary application code failure, so we satisfy that expectation here.
raise ImportError("pygobject 2.x is not installed. Use the `gi` reactor.")
if hasattr(gobject, "threads_init"):
# recent versions of python-gtk expose this. python-gtk=2.4.1
# (wrapping glib-2.4.7) does. python-gtk=2.0.0 (wrapping
# glib-2.2.3) does not.
gobject.threads_init()
class Gtk2Reactor(_glibbase.GlibReactorBase):
"""
PyGTK+ 2 event loop reactor.
"""
def __init__(self, useGtk=True):
_gtk = None
if useGtk is True:
import gtk as _gtk
_glibbase.GlibReactorBase.__init__(self, gobject, _gtk, useGtk=useGtk)
# We don't bother deprecating the PortableGtkReactor.
# The original code was removed and replaced with the
# backward compatible generic GTK reactor.
PortableGtkReactor = Gtk2Reactor
def install(useGtk=True):
"""
Configure the twisted mainloop to be run inside the gtk mainloop.
@param useGtk: should glib rather than GTK+ event loop be
used (this will be slightly faster but does not support GUI).
"""
reactor = Gtk2Reactor(useGtk)
from twisted.internet.main import installReactor
installReactor(reactor)
return reactor
def portableInstall(useGtk=True):
"""
Configure the twisted mainloop to be run inside the gtk mainloop.
"""
reactor = PortableGtkReactor()
from twisted.internet.main import installReactor
installReactor(reactor)
return reactor
if runtime.platform.getType() == "posix":
install = install
else:
install = portableInstall
__all__ = ["install"]

View File

@@ -0,0 +1,22 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
This module is a legacy compatibility alias for L{twisted.internet.gireactor}.
See that module instead.
"""
from incremental import Version
from ._deprecate import deprecatedGnomeReactor
deprecatedGnomeReactor("gtk3reactor", Version("Twisted", 23, 8, 0))
from twisted.internet import gireactor
Gtk3Reactor = gireactor.GIReactor
PortableGtk3Reactor = gireactor.PortableGIReactor
install = gireactor.install
__all__ = ["install"]

View File

@@ -0,0 +1,426 @@
# -*- test-case-name: twisted.internet.test.test_inotify -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
This module provides support for Twisted to linux inotify API.
In order to use this support, simply do the following (and start a reactor
at some point)::
from twisted.internet import inotify
from twisted.python import filepath
def notify(ignored, filepath, mask):
\"""
For historical reasons, an opaque handle is passed as first
parameter. This object should never be used.
@param filepath: FilePath on which the event happened.
@param mask: inotify event as hexadecimal masks
\"""
print("event %s on %s" % (
', '.join(inotify.humanReadableMask(mask)), filepath))
notifier = inotify.INotify()
notifier.startReading()
notifier.watch(filepath.FilePath("/some/directory"), callbacks=[notify])
notifier.watch(filepath.FilePath(b"/some/directory2"), callbacks=[notify])
Note that in the above example, a L{FilePath} which is a L{bytes} path name
or L{str} path name may be used. However, no matter what type of
L{FilePath} is passed to this module, internally the L{FilePath} is
converted to L{bytes} according to L{sys.getfilesystemencoding}.
For any L{FilePath} returned by this module, the caller is responsible for
converting from a L{bytes} path name to a L{str} path name.
@since: 10.1
"""
import os
import struct
from twisted.internet import fdesc
from twisted.internet.abstract import FileDescriptor
from twisted.python import _inotify, log
# from /usr/src/linux/include/linux/inotify.h
IN_ACCESS = 0x00000001 # File was accessed
IN_MODIFY = 0x00000002 # File was modified
IN_ATTRIB = 0x00000004 # Metadata changed
IN_CLOSE_WRITE = 0x00000008 # Writeable file was closed
IN_CLOSE_NOWRITE = 0x00000010 # Unwriteable file closed
IN_OPEN = 0x00000020 # File was opened
IN_MOVED_FROM = 0x00000040 # File was moved from X
IN_MOVED_TO = 0x00000080 # File was moved to Y
IN_CREATE = 0x00000100 # Subfile was created
IN_DELETE = 0x00000200 # Subfile was delete
IN_DELETE_SELF = 0x00000400 # Self was deleted
IN_MOVE_SELF = 0x00000800 # Self was moved
IN_UNMOUNT = 0x00002000 # Backing fs was unmounted
IN_Q_OVERFLOW = 0x00004000 # Event queued overflowed
IN_IGNORED = 0x00008000 # File was ignored
IN_ONLYDIR = 0x01000000 # only watch the path if it is a directory
IN_DONT_FOLLOW = 0x02000000 # don't follow a sym link
IN_MASK_ADD = 0x20000000 # add to the mask of an already existing watch
IN_ISDIR = 0x40000000 # event occurred against dir
IN_ONESHOT = 0x80000000 # only send event once
IN_CLOSE = IN_CLOSE_WRITE | IN_CLOSE_NOWRITE # closes
IN_MOVED = IN_MOVED_FROM | IN_MOVED_TO # moves
IN_CHANGED = IN_MODIFY | IN_ATTRIB # changes
IN_WATCH_MASK = (
IN_MODIFY
| IN_ATTRIB
| IN_CREATE
| IN_DELETE
| IN_DELETE_SELF
| IN_MOVE_SELF
| IN_UNMOUNT
| IN_MOVED_FROM
| IN_MOVED_TO
)
_FLAG_TO_HUMAN = [
(IN_ACCESS, "access"),
(IN_MODIFY, "modify"),
(IN_ATTRIB, "attrib"),
(IN_CLOSE_WRITE, "close_write"),
(IN_CLOSE_NOWRITE, "close_nowrite"),
(IN_OPEN, "open"),
(IN_MOVED_FROM, "moved_from"),
(IN_MOVED_TO, "moved_to"),
(IN_CREATE, "create"),
(IN_DELETE, "delete"),
(IN_DELETE_SELF, "delete_self"),
(IN_MOVE_SELF, "move_self"),
(IN_UNMOUNT, "unmount"),
(IN_Q_OVERFLOW, "queue_overflow"),
(IN_IGNORED, "ignored"),
(IN_ONLYDIR, "only_dir"),
(IN_DONT_FOLLOW, "dont_follow"),
(IN_MASK_ADD, "mask_add"),
(IN_ISDIR, "is_dir"),
(IN_ONESHOT, "one_shot"),
]
def humanReadableMask(mask):
"""
Auxiliary function that converts a hexadecimal mask into a series
of human readable flags.
"""
s = []
for k, v in _FLAG_TO_HUMAN:
if k & mask:
s.append(v)
return s
class _Watch:
"""
Watch object that represents a Watch point in the filesystem. The
user should let INotify to create these objects
@ivar path: The path over which this watch point is monitoring
@ivar mask: The events monitored by this watchpoint
@ivar autoAdd: Flag that determines whether this watch point
should automatically add created subdirectories
@ivar callbacks: L{list} of callback functions that will be called
when an event occurs on this watch.
"""
def __init__(self, path, mask=IN_WATCH_MASK, autoAdd=False, callbacks=None):
self.path = path.asBytesMode()
self.mask = mask
self.autoAdd = autoAdd
if callbacks is None:
callbacks = []
self.callbacks = callbacks
def _notify(self, filepath, events):
"""
Callback function used by L{INotify} to dispatch an event.
"""
filepath = filepath.asBytesMode()
for callback in self.callbacks:
callback(self, filepath, events)
class INotify(FileDescriptor):
"""
The INotify file descriptor, it basically does everything related
to INotify, from reading to notifying watch points.
@ivar _buffer: a L{bytes} containing the data read from the inotify fd.
@ivar _watchpoints: a L{dict} that maps from inotify watch ids to
watchpoints objects
@ivar _watchpaths: a L{dict} that maps from watched paths to the
inotify watch ids
"""
_inotify = _inotify
def __init__(self, reactor=None):
FileDescriptor.__init__(self, reactor=reactor)
# Smart way to allow parametrization of libc so I can override
# it and test for the system errors.
self._fd = self._inotify.init()
fdesc.setNonBlocking(self._fd)
fdesc._setCloseOnExec(self._fd)
# The next 2 lines are needed to have self.loseConnection()
# to call connectionLost() on us. Since we already created the
# fd that talks to inotify we want to be notified even if we
# haven't yet started reading.
self.connected = 1
self._writeDisconnected = True
self._buffer = b""
self._watchpoints = {}
self._watchpaths = {}
def _addWatch(self, path, mask, autoAdd, callbacks):
"""
Private helper that abstracts the use of ctypes.
Calls the internal inotify API and checks for any errors after the
call. If there's an error L{INotify._addWatch} can raise an
INotifyError. If there's no error it proceeds creating a watchpoint and
adding a watchpath for inverse lookup of the file descriptor from the
path.
"""
path = path.asBytesMode()
wd = self._inotify.add(self._fd, path, mask)
iwp = _Watch(path, mask, autoAdd, callbacks)
self._watchpoints[wd] = iwp
self._watchpaths[path] = wd
return wd
def _rmWatch(self, wd):
"""
Private helper that abstracts the use of ctypes.
Calls the internal inotify API to remove an fd from inotify then
removes the corresponding watchpoint from the internal mapping together
with the file descriptor from the watchpath.
"""
self._inotify.remove(self._fd, wd)
iwp = self._watchpoints.pop(wd)
self._watchpaths.pop(iwp.path)
def connectionLost(self, reason):
"""
Release the inotify file descriptor and do the necessary cleanup
"""
FileDescriptor.connectionLost(self, reason)
if self._fd >= 0:
try:
os.close(self._fd)
except OSError as e:
log.err(e, "Couldn't close INotify file descriptor.")
def fileno(self):
"""
Get the underlying file descriptor from this inotify observer.
Required by L{abstract.FileDescriptor} subclasses.
"""
return self._fd
def doRead(self):
"""
Read some data from the observed file descriptors
"""
fdesc.readFromFD(self._fd, self._doRead)
def _doRead(self, in_):
"""
Work on the data just read from the file descriptor.
"""
self._buffer += in_
while len(self._buffer) >= 16:
wd, mask, cookie, size = struct.unpack("=LLLL", self._buffer[0:16])
if size:
name = self._buffer[16 : 16 + size].rstrip(b"\0")
else:
name = None
self._buffer = self._buffer[16 + size :]
try:
iwp = self._watchpoints[wd]
except KeyError:
continue
path = iwp.path.asBytesMode()
if name:
path = path.child(name)
iwp._notify(path, mask)
if iwp.autoAdd and mask & IN_ISDIR and mask & IN_CREATE:
# mask & IN_ISDIR already guarantees that the path is a
# directory. There's no way you can get here without a
# directory anyway, so no point in checking for that again.
new_wd = self.watch(
path, mask=iwp.mask, autoAdd=True, callbacks=iwp.callbacks
)
# This is very very very hacky and I'd rather not do this but
# we have no other alternative that is less hacky other than
# surrender. We use callLater because we don't want to have
# too many events waiting while we process these subdirs, we
# must always answer events as fast as possible or the overflow
# might come.
self.reactor.callLater(0, self._addChildren, self._watchpoints[new_wd])
if mask & IN_DELETE_SELF:
self._rmWatch(wd)
self.loseConnection()
def _addChildren(self, iwp):
"""
This is a very private method, please don't even think about using it.
Note that this is a fricking hack... it's because we cannot be fast
enough in adding a watch to a directory and so we basically end up
getting here too late if some operations have already been going on in
the subdir, we basically need to catchup. This eventually ends up
meaning that we generate double events, your app must be resistant.
"""
try:
listdir = iwp.path.children()
except OSError:
# Somebody or something (like a test) removed this directory while
# we were in the callLater(0...) waiting. It doesn't make sense to
# process it anymore
return
# note that it's true that listdir will only see the subdirs inside
# path at the moment of the call but path is monitored already so if
# something is created we will receive an event.
for f in listdir:
# It's a directory, watch it and then add its children
if f.isdir():
wd = self.watch(f, mask=iwp.mask, autoAdd=True, callbacks=iwp.callbacks)
iwp._notify(f, IN_ISDIR | IN_CREATE)
# now f is watched, we can add its children the callLater is to
# avoid recursion
self.reactor.callLater(0, self._addChildren, self._watchpoints[wd])
# It's a file and we notify it.
if f.isfile():
iwp._notify(f, IN_CREATE | IN_CLOSE_WRITE)
def watch(
self, path, mask=IN_WATCH_MASK, autoAdd=False, callbacks=None, recursive=False
):
"""
Watch the 'mask' events in given path. Can raise C{INotifyError} when
there's a problem while adding a directory.
@param path: The path needing monitoring
@type path: L{FilePath}
@param mask: The events that should be watched
@type mask: L{int}
@param autoAdd: if True automatically add newly created
subdirectories
@type autoAdd: L{bool}
@param callbacks: A list of callbacks that should be called
when an event happens in the given path.
The callback should accept 3 arguments:
(ignored, filepath, mask)
@type callbacks: L{list} of callables
@param recursive: Also add all the subdirectories in this path
@type recursive: L{bool}
"""
if recursive:
# This behavior is needed to be compatible with the windows
# interface for filesystem changes:
# http://msdn.microsoft.com/en-us/library/aa365465(VS.85).aspx
# ReadDirectoryChangesW can do bWatchSubtree so it doesn't
# make sense to implement this at a higher abstraction
# level when other platforms support it already
for child in path.walk():
if child.isdir():
self.watch(child, mask, autoAdd, callbacks, recursive=False)
else:
wd = self._isWatched(path)
if wd:
return wd
mask = mask | IN_DELETE_SELF # need this to remove the watch
return self._addWatch(path, mask, autoAdd, callbacks)
def ignore(self, path):
"""
Remove the watch point monitoring the given path
@param path: The path that should be ignored
@type path: L{FilePath}
"""
path = path.asBytesMode()
wd = self._isWatched(path)
if wd is None:
raise KeyError(f"{path!r} is not watched")
else:
self._rmWatch(wd)
def _isWatched(self, path):
"""
Helper function that checks if the path is already monitored
and returns its watchdescriptor if so or None otherwise.
@param path: The path that should be checked
@type path: L{FilePath}
"""
path = path.asBytesMode()
return self._watchpaths.get(path, None)
INotifyError = _inotify.INotifyError
__all__ = [
"INotify",
"humanReadableMask",
"IN_WATCH_MASK",
"IN_ACCESS",
"IN_MODIFY",
"IN_ATTRIB",
"IN_CLOSE_NOWRITE",
"IN_CLOSE_WRITE",
"IN_OPEN",
"IN_MOVED_FROM",
"IN_MOVED_TO",
"IN_CREATE",
"IN_DELETE",
"IN_DELETE_SELF",
"IN_MOVE_SELF",
"IN_UNMOUNT",
"IN_Q_OVERFLOW",
"IN_IGNORED",
"IN_ONLYDIR",
"IN_DONT_FOLLOW",
"IN_MASK_ADD",
"IN_ISDIR",
"IN_ONESHOT",
"IN_CLOSE",
"IN_MOVED",
"IN_CHANGED",
]

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,10 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
I/O Completion Ports reactor
"""
from twisted.internet.iocpreactor.reactor import install
__all__ = ["install"]

Some files were not shown because too many files have changed in this diff Show More