r46790 - merge forward, resolve lots of conflicts
glyph-TA+aISz0psMTMxyoc4vAAJOcrHinNvQL0E9HWUfgJXw@public.gmane.org
| Newsgroups | gmane.comp.python.twisted.commits |
|---|---|
| Message-ID | <[email protected]> |
Author: glyph
Date: Sun Feb 14 02:54:45 2016
New Revision: 46790
Modified:
branches/getaddrinfo-4362-3/twisted/internet/base.py
branches/getaddrinfo-4362-3/twisted/internet/interfaces.py
branches/getaddrinfo-4362-3/twisted/internet/test/test_base.py
branches/getaddrinfo-4362-3/twisted/internet/test/test_core.py
branches/getaddrinfo-4362-3/twisted/internet/test/test_tcp.py
Log:
merge forward, resolve lots of conflicts
Modified: branches/getaddrinfo-4362-3/twisted/internet/base.py
==============================================================================
--- branches/getaddrinfo-4362-3/twisted/internet/base.py (original)
+++ branches/getaddrinfo-4362-3/twisted/internet/base.py Sun Feb 14 02:54:45 2016
@@ -17,9 +17,13 @@
import traceback
+from twisted.python.util import FancyEqMixin
+from twisted.python.compat import set
+from twisted.python.components import registerAdapter
from twisted.internet.interfaces import IReactorCore, IReactorTime, IReactorThreads
from twisted.internet.interfaces import IResolverSimple, IReactorPluggableResolver
from twisted.internet.interfaces import IConnector, IDelayedCall
+from twisted.internet.interfaces import INameResolver
from twisted.internet import fdesc, main, error, abstract, defer, threads
from twisted.python import log, failure, reflect
from twisted.python.compat import unicode, iteritems
@@ -209,6 +213,51 @@
return "".join(L)
+@implementer(INameResolver)
+class _ResolverComplexifier(object):
+ """
+ L{_ResolverComplexifier} adapts an L{IResolverSimple} provider to
+ L{INameResolver}.
+ """
+ def __init__(self, resolver):
+ """
+ @param resolver: A resolver which will be adapted to the new
+ L{INameResolver} interface.
+ @type resolver: A provider of L{IResolverSimple}.
+ """
+ self._resolver = resolver
+
+
+ def getAddressInformation(self, name, service, *args):
+ # In this case of a complexified IResolverSimple,
+ # getHostByName and getAddressInformation should - for
+ # consistency - both return the same single IP address. (IPv4
+ # or IPv6 depending on the behaviour of gethostbyname)
+
+ # getAddressInformation will return that IP address as part of
+ # an AddressInformation instance whose type, protocol and
+ # canonicalName are fixed.
+
+ # XXX: I don't fully understand the motivation behind
+ # _ResolverComplexifier or why lookup the address with
+ # getHostByName and then pass the address to getaddrinfo?
+ # -rwall
+ d = self._resolver.getHostByName(name)
+ def cbResolved(address):
+ family = socket.getaddrinfo(address, 0)[0][0]
+ return [
+ AddressInformation(
+ family,
+ socket.SOCK_STREAM,
+ socket.IPPROTO_TCP,
+ "",
+ (address, service))]
+ d.addCallback(cbResolved)
+ return d
+
+registerAdapter(_ResolverComplexifier, IResolverSimple, INameResolver)
+
+
@implementer(IResolverSimple)
class ThreadedResolver(object):
@@ -278,6 +327,110 @@
+@implementer(INameResolver)
+class ThreadedNameResolver(object):
+ """
+ L{ThreadedNameResolver} uses a reactor, a threadpool, and
+ L{socket.getaddrinfo} to perform name lookups without blocking the
+ reactor thread. It also supports timeouts indepedently from
+ whatever timeout logic L{socket.getaddrinfo} might have.
+
+ @ivar reactor: The reactor the threadpool of which will be used to call
+ L{socket.getaddrinfo} and the I/O thread of which the result will be
+ delivered.
+ """
+
+ def __init__(self, reactor):
+ self.reactor = reactor
+ self._runningQueries = {}
+
+
+ def getAddressInformation(self, name, service, family=None, socktype=None,
+ proto=None, flags=None, timeout=60):
+
+ query = (name, service, family, socktype, proto, flags)
+
+ userDeferred = defer.Deferred()
+
+ lookupDeferred = threads.deferToThreadPool(
+ self.reactor, self.reactor.getThreadPool(),
+ socket.getaddrinfo, *query)
+
+ cancelCall = self.reactor.callLater(
+ timeout, self._cleanup, query, lookupDeferred)
+
+ self._runningQueries[lookupDeferred] = (userDeferred, cancelCall)
+
+ lookupDeferred.addBoth(self._checkTimeout, query, lookupDeferred)
+
+ return userDeferred
+
+
+ def _fail(self, query, err):
+ return failure.Failure(
+ error.DNSLookupError(
+ "Query failure. query: %r, message: %s" % (query, err)))
+
+
+ def _cleanup(self, query, lookupDeferred):
+ userDeferred, cancelCall = self._runningQueries[lookupDeferred]
+ del self._runningQueries[lookupDeferred]
+ userDeferred.errback(self._fail(query, "timeout error"))
+
+
+ def _checkTimeout(self, result, query, lookupDeferred):
+ try:
+ userDeferred, cancelCall = self._runningQueries[lookupDeferred]
+ except KeyError:
+ pass
+ else:
+ del self._runningQueries[lookupDeferred]
+ cancelCall.cancel()
+
+ if isinstance(result, failure.Failure):
+ userDeferred.errback(self._fail(query, result.getErrorMessage()))
+ else:
+ userDeferred.callback([AddressInformation(*r) for r in result])
+
+
+
+class AddressInformation(object, FancyEqMixin):
+ """
+ A container for the results of
+ L{INameResolver.getAddressInformation}.
+ """
+ compareAttributes = ('family', 'type', 'protocol', 'canonicalName', 'address')
+
+ def __init__(self, family, type, protocol, canonicalName, address):
+ """
+ @param family: The address family.
+ @type family: One of the address family constants from the
+ socket module. For example, L{socket.AF_INET}.
+
+ @param type: The address socket type.
+ @type type: One of the socket type constants from the socket
+ module. For example, L{socket.SOCK_STREAM}.
+
+ @param protocol: The address socket protocol.
+ @type protocol: One of the protocol constants from the socket
+ module. For example, L{socket.IPPROTO_TCP}.
+
+ @param canonicalName: A string representing the canonical name
+ of the host if AI_CANONNAME is part of the flags argument;
+ else canonname will be empty.
+ @type canonicalName: C{str}
+
+ @param address: The IP address of the specified name.
+ @type address: C{str}
+ """
+ self.family = family
+ self.type = type
+ self.protocol = protocol
+ self.canonicalName = canonicalName
+ self.address = address
+
+
+
@implementer(IResolverSimple)
class BlockingResolver:
@@ -507,7 +660,7 @@
reflect.qual(self.__class__) + " did not implement installWaker")
def installResolver(self, resolver):
- assert IResolverSimple.providedBy(resolver)
+ resolver = INameResolver(resolver)
oldResolver = self.resolver
self.resolver = resolver
return oldResolver
@@ -567,7 +720,15 @@
return defer.succeed('0.0.0.0')
if abstract.isIPAddress(name):
return defer.succeed(name)
- return self.resolver.getHostByName(name, timeout)
+ d = self.resolver.getAddressInformation(name, 0)
+ def cbGotInfo(addresses):
+ for info in addresses:
+ if info.family == socket.AF_INET:
+ return info.address[0]
+ # XXX Test me
+ d.addCallback(cbGotInfo)
+ return d
+
# Installation.
Modified: branches/getaddrinfo-4362-3/twisted/internet/interfaces.py
==============================================================================
--- branches/getaddrinfo-4362-3/twisted/internet/interfaces.py (original)
+++ branches/getaddrinfo-4362-3/twisted/internet/interfaces.py Sun Feb 14 02:54:45 2016
@@ -55,6 +55,64 @@
+class INameResolver(Interface):
+ """
+ XXX Write me.
+
+ RFC 3484.
+ """
+ def getAddressInformation(name, service, family=None, socktype=None,
+ proto=None, flags=None, timeout=None):
+ """
+ Get the address information associated with the given name.
+
+ @param name: A hostname to resolve.
+ @type name: C{str}
+
+ @param service: A port number or the name of the service for which to find address
+ information. For example, C{22} or C{"ssh"}.
+ @type service: C{int} or C{str}
+
+ @param family: If specified, limit results to addresses from this family. Must be one
+ of the address family constants from the socket module. For example,
+ L{socket.AF_INET}.
+
+ @param socktype: If specified, limit results to addresses for this socket type. Must be
+ one of the socket type constants from the socket module. For example,
+ L{socket.SOCK_STREAM}.
+
+ @param proto: If specified, limit results to addresses for this socket protocol. Must
+ be one of the protocol constants from the socket module. For example,
+ L{socket.IPPROTO_TCP}.
+
+ @param flags: A bitvector specifying zero or more of the following::
+ - Yea right. Go read the `getaddrinfo(3)` man page.
+
+ @param timeout: The number of seconds after which to cancel
+ the L{Deferred} returned by this function. Can be a single
+ C{int} or an iterable of C{int} for implementations which
+ support retries, in which case the L{Deferred}
+ cancellation timeout will be the sum of all the C{int}s.
+ @type: timeout: C{int} or an iterable of C{int}
+
+ @raise ValueError: If one of the specified flags is not supported by the
+ implementation. All flags are optional.
+
+ @return: A L{Deferred} which will fire when the resolution completes. If resolution is
+ successful, the result will be a list of objects with the following attributes:
+
+ - C{family}: the family of this address
+ - C{type}: the type of this address
+ - C{protocol}: the protocol of this address
+ - C{canonicalName}: the canonical name associated with this address, or an empty
+ string.
+ - C{address}: The actual address itself, including a port number. This is suitable
+ to be passed directly to L{socket.socket.connect}.
+ """
+
+
+class IResolverSimple(Interface):
+
class IResolverSimple(Interface):
def getHostByName(name, timeout = (1, 3, 11, 45)):
"""
Modified: branches/getaddrinfo-4362-3/twisted/internet/test/test_base.py
==============================================================================
--- branches/getaddrinfo-4362-3/twisted/internet/test/test_base.py (original)
+++ branches/getaddrinfo-4362-3/twisted/internet/test/test_base.py Sun Feb 14 02:54:45 2016
@@ -12,14 +12,20 @@
from queue import Queue
from zope.interface import implementer
+from zope.interface.verify import verifyClass
from twisted.python.threadpool import ThreadPool
from twisted.internet.interfaces import IReactorTime, IReactorThreads
+from twisted.internet.interfaces import INameResolver
from twisted.internet.error import DNSLookupError
-from twisted.internet.base import ThreadedResolver, DelayedCall
+from twisted.internet.base import (
+ ThreadedNameResolver, ThreadedResolver, AddressInformation,
+ _ResolverComplexifier, DelayedCall)
from twisted.internet.task import Clock
from twisted.trial.unittest import TestCase
+from twisted.internet.test.test_tcp import FakeResolver
+
@implementer(IReactorTime, IReactorThreads)
class FakeReactor(object):
@@ -53,6 +59,206 @@
+class NameResolverAdapterTests(TestCase):
+ """
+ L{_ResolverComplexifier} adapts an L{IResolverSimple} provider to
+ L{INameResolver}.
+ """
+ def test_interface(self):
+ """
+ L{_ResolverComplexifier} implements L{INameResolver}.
+ """
+ verifyClass(INameResolver, _ResolverComplexifier)
+
+
+ def test_registeredAdapter(self):
+ """
+ L{_ResolverComplexifier} is registered as an adapter from
+ L{IResolverSimple} to L{INameResolver}.
+ """
+ simple = FakeResolver({})
+ INameResolver(simple)
+
+
+ def _successTest(self, address, family):
+ """
+ A generic test for both INET and INET6 address families.
+ """
+ simple = FakeResolver({'example.com': address})
+ resolver = _ResolverComplexifier(simple)
+ d = resolver.getAddressInformation('example.com', 1234)
+ d.addCallback(
+ self.assertEquals, [
+ AddressInformation(
+ family,
+ socket.SOCK_STREAM,
+ socket.IPPROTO_TCP,
+ "",
+ (address, 1234))])
+ return d
+
+
+ def test_ipv4Success(self):
+ """
+ L{_ResolverComplexifier} calls the wrapped object's
+ C{getHostByName} method and returns a L{Deferred} which fires
+ with a list of one element containing an AF_INET element with
+ the IPv4 address which C{getHostByName}'s L{Deferred} fired
+ with.
+ """
+ return self._successTest('192.168.1.12', socket.AF_INET)
+
+
+ def test_ipv6Success(self):
+ """
+ L{_ResolverComplexifier} calls the wrapped object's
+ C{getHostByName} method and returns a L{Deferred} which fires
+ with a list of one element containing an AF_INET6 element with
+ the IPv6 address which C{getHostByName}'s L{Deferred} fired
+ with.
+ """
+ return self._successTest('::1', socket.AF_INET6)
+
+
+ def test_failure(self):
+ """
+ The L{Deferred} L{_ResolverComplexifier.getAddressInformation}
+ returns fails if the wrapped resolver's C{getHostByName}
+ L{Deferred} fails.
+ """
+ simple = FakeResolver({})
+ resolver = _ResolverComplexifier(simple)
+ d = resolver.getAddressInformation('example.com', 1234)
+ return self.assertFailure(d, DNSLookupError)
+
+
+
+class ThreadedNameResolverTests(TestCase):
+ """
+ Tests for L{ThreadedNameResolver}.
+ """
+ def test_interface(self):
+ """
+ L{ThreadedNameResolver} implements L{INameResolver}.
+ """
+ verifyClass(INameResolver, ThreadedNameResolver)
+
+
+ def test_success(self):
+ """
+ If the underlying C{getaddrinfo} library call completes
+ successfully and returns results, the L{Deferred} returned by
+ L{ThreadedNameResolver.getAddressInformation} fires with a
+ list of L{AddressInformation} instances representing those
+ results.
+ """
+ expectedSocketResult = [
+ (2, 1, 6, '', ('192.0.43.10', 80)),
+ (2, 2, 17, '', ('192.0.43.10', 80)),
+ (2, 3, 0, '', ('192.0.43.10', 80)),
+ (10, 1, 6, '', ('2001:500:88:200::10', 80, 0, 0)),
+ (10, 2, 17, '', ('2001:500:88:200::10', 80, 0, 0)),
+ (10, 3, 0, '', ('2001:500:88:200::10', 80, 0, 0))]
+
+ query = ("example.com", 80, None, None, None, None)
+ timeout = 30
+
+ reactor = FakeReactor()
+ self.addCleanup(reactor._stop)
+
+ lookedUp = []
+ resolvedTo = []
+
+ def fakeGetAddrInfo(*args):
+ lookedUp.append(args)
+ return expectedSocketResult
+ self.patch(socket, 'getaddrinfo', fakeGetAddrInfo)
+
+ resolver = ThreadedNameResolver(reactor)
+ d = resolver.getAddressInformation(*(query + (timeout,)))
+ d.addCallback(resolvedTo.append)
+
+ reactor._runThreadCalls()
+
+ self.assertEqual(lookedUp, [query])
+ self.assertEqual(
+ resolvedTo,
+ [[AddressInformation(*x) for x in expectedSocketResult]])
+
+ # Make sure that any timeout-related stuff gets cleaned up.
+ reactor._clock.advance(timeout + 1)
+ self.assertEqual(reactor._clock.calls, [])
+
+
+ def test_failure(self):
+ """
+ L{ThreadedNameResolver.getAddressInformation} returns a
+ L{Deferred} which fires a L{Failure} if the call to
+ L{socket.getaddrinfo} raises an exception.
+ """
+ query = ("example.com", 80, None, None, None, None)
+ timeout = 30
+
+ reactor = FakeReactor()
+ self.addCleanup(reactor._stop)
+
+ def fakeGetAddrInfo(*args):
+ raise IOError("ENOBUFS (this is a funny joke)")
+
+ self.patch(socket, 'getaddrinfo', fakeGetAddrInfo)
+
+ failedWith = []
+ resolver = ThreadedNameResolver(reactor)
+ d = resolver.getAddressInformation(*query, timeout=timeout)
+# import pdb;pdb.set_trace()
+ self.assertFailure(d, DNSLookupError)
+ d.addCallback(failedWith.append)
+
+ reactor._runThreadCalls()
+
+ self.assertEqual(len(failedWith), 1)
+
+ # Make sure that any timeout-related stuff gets cleaned up.
+ reactor._clock.advance(timeout + 1)
+ self.assertEqual(reactor._clock.calls, [])
+
+
+ def test_timeout(self):
+ """
+ If L{socket.getaddrinfo} does not complete before the
+ specified timeout elapsed, the L{Deferred} returned by
+ L{ThreadedResolver.getAddressInformation} fails with
+ L{DNSLookupError}.
+ """
+ query = ("example.com", 80, None, None, None, None)
+ timeout = 10
+
+ reactor = FakeReactor()
+ self.addCleanup(reactor._stop)
+
+ result = Queue()
+ def fakeGetAddrInfo(name):
+ raise result.get()
+
+ self.patch(socket, 'getaddrinfo', fakeGetAddrInfo)
+
+ failedWith = []
+ resolver = ThreadedNameResolver(reactor)
+ d = resolver.getAddressInformation(*query, timeout=timeout)
+ self.assertFailure(d, DNSLookupError)
+ d.addCallback(failedWith.append)
+
+ reactor._clock.advance(timeout - 1)
+ self.assertEqual(failedWith, [])
+ reactor._clock.advance(1)
+ self.assertEqual(len(failedWith), 1)
+
+ # Eventually the socket.getaddrinfo does finish - in this
+ # case, with an exception. Nobody cares, though.
+ result.put(IOError("The I/O was errorful"))
+
+
+
class ThreadedResolverTests(TestCase):
"""
Tests for L{ThreadedResolver}.
Modified: branches/getaddrinfo-4362-3/twisted/internet/test/test_core.py
==============================================================================
--- branches/getaddrinfo-4362-3/twisted/internet/test/test_core.py (original)
+++ branches/getaddrinfo-4362-3/twisted/internet/test/test_core.py Sun Feb 14 02:54:45 2016
@@ -12,13 +12,19 @@
import signal
import time
import inspect
+import socket
+from zope.interface import implementer
+
+from twisted.internet.interfaces import INameResolver
from twisted.internet.abstract import FileDescriptor
from twisted.internet.error import ReactorAlreadyRunning, ReactorNotRestartable
-from twisted.internet.defer import Deferred
+from twisted.internet.defer import Deferred, succeed
+from twisted.internet.base import AddressInformation
from twisted.internet.test.reactormixins import ReactorBuilder
+
class ObjectModelIntegrationMixin(object):
"""
Helpers for tests about the object model of reactor-related objects.
@@ -41,6 +47,34 @@
+@implementer(INameResolver)
+class MemoryNameResolver(object):
+ """
+ An in-memory provider of L{INameResolver} which returns a fixed
+ list of AddressInformation for testing purposes.
+ """
+ def __init__(self, names):
+ """
+ @param names: A fixed list of results which will be returned
+ in response to L{INameResolver.getAddressInformation}
+ calls.
+ @type names: A C{list} of L{AddressInformation} instances.
+ """
+ self._names = names
+
+
+ def getAddressInformation(self, name, service, family=None, type=None,
+ protocol=None, flags=None):
+ return succeed([
+ address
+ for address
+ in self._names[name, service]
+ if family is None or family == address.family
+ and type is None or type == address.type
+ and protocol is None or protocol == address.protocol])
+
+
+
class ObjectModelIntegrationTests(ReactorBuilder, ObjectModelIntegrationMixin):
"""
Test details of object model integration against all reactors.
@@ -328,6 +362,34 @@
self.assertEqual(events, ['tested'])
+ def test_resolve(self):
+ """
+ C{reactor.resolve(name)} calls the C{getAddressInformation}
+ method of the installed resolver and returns a L{Deferred}
+ which fires with the first C{AF_INET} family element from the
+ result of C{getAddressInformation}.
+ """
+ resolver = MemoryNameResolver({
+ ('example.com', 0): [
+ AddressInformation(
+ socket.AF_INET6,
+ socket.SOCK_STREAM,
+ socket.IPPROTO_TCP,
+ "",
+ ("::1", 0)),
+ AddressInformation(
+ socket.AF_INET,
+ socket.SOCK_STREAM,
+ socket.IPPROTO_TCP,
+ "",
+ ("127.0.0.1", 22))]})
+
+ reactor = self.buildReactor()
+ reactor.installResolver(resolver)
+ d = reactor.resolve("example.com")
+ d.addCallback(self.assertEquals, "127.0.0.1")
+ return d
+
globals().update(SystemEventTestsBuilder.makeTestCaseClasses())
globals().update(ObjectModelIntegrationTests.makeTestCaseClasses())
Modified: branches/getaddrinfo-4362-3/twisted/internet/test/test_tcp.py
==============================================================================
--- branches/getaddrinfo-4362-3/twisted/internet/test/test_tcp.py (original)
+++ branches/getaddrinfo-4362-3/twisted/internet/test/test_tcp.py Sun Feb 14 02:54:45 2016
@@ -115,6 +115,28 @@
+class TCPClientTestsBuilder(ReactorBuilder):
+ """
+ Builder defining tests relating to L{IReactorTCP.connectTCP}.
+ """
+ def _freePort(self, interface='127.0.0.1'):
+ probe = socket.socket()
+ try:
+ probe.bind((interface, 0))
+ return probe.getsockname()
+ finally:
+ probe.close()
+
+ def test_clientConnectionFailedStopsReactor(self):
+ """
+ The reactor can be stopped by a client factory's
+ C{clientConnectionFailed} method.
+ """
+ host, port = self._freePort()
+ reactor = self.buildReactor()
+ reactor.connectTCP(host, port, Stop(reactor))
+
+
class FakeSocket(object):
"""
A fake for L{socket.socket} objects.