From d03dd0f2ecbaad0886bf2c22e7fbeb2dec9813b7 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 4 Mar 2013 18:01:15 -0500 Subject: [PATCH] Copy SSL certificate verification from pip. --- setup.py | 28 +++-- tuf/compatibility/__init__.py | 39 ++++++ tuf/compatibility/socket_create_connection.py | 48 ++++++++ tuf/compatibility/ssl_match_hostname.py | 70 +++++++++++ tuf/conf.py | 5 + tuf/download.py | 113 ++++++++++++++++-- tuf/interposition/README.md | 4 +- 7 files changed, 288 insertions(+), 19 deletions(-) create mode 100644 tuf/compatibility/__init__.py create mode 100644 tuf/compatibility/socket_create_connection.py create mode 100644 tuf/compatibility/ssl_match_hostname.py diff --git a/setup.py b/setup.py index 18f91b25..08709bef 100755 --- a/setup.py +++ b/setup.py @@ -2,22 +2,28 @@ from distutils.core import setup -setup(name='tuf', - version='0.0.0', +setup( + name='tuf', + version='0.1', description='A secure updater framework for Python', - author='numerous', + author='https://www.updateframework.com', author_email='info@updateframework.com', url='https://www.updateframework.com', - packages=['tuf', + packages=[ + 'evpy', + 'simplejson', + 'tuf', 'tuf.client', + 'tuf.compatibility', + 'tuf.interposition', 'tuf.pushtools', 'tuf.pushtools.transfer', - 'tuf.repo', - 'tuf.interposition', - 'evpy', - 'simplejson'], - scripts=['quickstart.py', - 'basic_client.py', + 'tuf.repo' + ], + scripts=[ + 'quickstart.py', 'tuf/pushtools/push.py', 'tuf/pushtools/receivetools/receive.py', - 'tuf/repo/signercli.py']) + 'tuf/repo/signercli.py' + ] +) diff --git a/tuf/compatibility/__init__.py b/tuf/compatibility/__init__.py new file mode 100644 index 00000000..5ae1822b --- /dev/null +++ b/tuf/compatibility/__init__.py @@ -0,0 +1,39 @@ +""" +We copy some backwards compatibility from pip. + +https://github.com/pypa/pip/tree/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/backwardcompat +""" + + +import sys + + +if sys.version_info >= (3,): + import http.client as httplib + import urllib.parse as urlparse + import urllib.request as urllib2 +else: + import httplib + import urllib2 + import urlparse + + +## py25 has no builtin ssl module +## only >=py32 has ssl.match_hostname and ssl.CertificateError +try: + import ssl + try: + from ssl import match_hostname, CertificateError + except ImportError: + from tuf.compatibility.ssl_match_hostname import match_hostname, CertificateError +except ImportError: + ssl = None + + +# patch for py25 socket to work with http://pypi.python.org/pypi/ssl/ +import socket +if not hasattr(socket, 'create_connection'): # for Python 2.5 + # monkey-patch socket module + from tuf.compatibility.socket_create_connection import create_connection + socket.create_connection = create_connection + diff --git a/tuf/compatibility/socket_create_connection.py b/tuf/compatibility/socket_create_connection.py new file mode 100644 index 00000000..1a11f4bd --- /dev/null +++ b/tuf/compatibility/socket_create_connection.py @@ -0,0 +1,48 @@ +""" +We copy some functions from the Python 2.7.3 socket module. + +http://hg.python.org/releasing/2.7.3/file/7bb96963d067/Lib/socket.py +""" + + +_GLOBAL_DEFAULT_TIMEOUT = object() + + +def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT, + source_address=None): + """Connect to *address* and return the socket object. + + Convenience function. Connect to *address* (a 2-tuple ``(host, + port)``) and return the socket object. Passing the optional + *timeout* parameter will set the timeout on the socket instance + before attempting to connect. If no *timeout* is supplied, the + global default timeout setting returned by :func:`getdefaulttimeout` + is used. If *source_address* is set it must be a tuple of (host, port) + for the socket to bind as a source address before making the connection. + An host of '' or port 0 tells the OS to use the default. + """ + + host, port = address + err = None + for res in getaddrinfo(host, port, 0, SOCK_STREAM): + af, socktype, proto, canonname, sa = res + sock = None + try: + sock = socket(af, socktype, proto) + if timeout is not _GLOBAL_DEFAULT_TIMEOUT: + sock.settimeout(timeout) + if source_address: + sock.bind(source_address) + sock.connect(sa) + return sock + + except error as _: + err = _ + if sock is not None: + sock.close() + + if err is not None: + raise err + else: + raise error("getaddrinfo returns an empty list") + diff --git a/tuf/compatibility/ssl_match_hostname.py b/tuf/compatibility/ssl_match_hostname.py new file mode 100644 index 00000000..1a2602e5 --- /dev/null +++ b/tuf/compatibility/ssl_match_hostname.py @@ -0,0 +1,70 @@ +""" +We copy some functions from the Python 3.3.0 ssl module. + +http://hg.python.org/releasing/3.3.0/file/1465cbbc8f64/Lib/ssl.py +""" + + +import re + + +class CertificateError(ValueError): + pass + + +def _dnsname_to_pat(dn): + pats = [] + for frag in dn.split(r'.'): + if frag == '*': + # When '*' is a fragment by itself, it matches a non-empty dotless + # fragment. + pats.append('[^.]+') + else: + # Otherwise, '*' matches any dotless fragment. + frag = re.escape(frag) + pats.append(frag.replace(r'\*', '[^.]*')) + return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) + + +def match_hostname(cert, hostname): + """Verify that *cert* (in decoded format as returned by + SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 rules + are mostly followed, but IP addresses are not accepted for *hostname*. + + CertificateError is raised on failure. On success, the function + returns nothing. + """ + if not cert: + raise ValueError("empty or no certificate") + dnsnames = [] + san = cert.get('subjectAltName', ()) + for key, value in san: + if key == 'DNS': + if _dnsname_to_pat(value).match(hostname): + return + dnsnames.append(value) + if not dnsnames: + # The subject is only checked when there is no dNSName entry + # in subjectAltName + for sub in cert.get('subject', ()): + for key, value in sub: + # XXX according to RFC 2818, the most specific Common Name + # must be used. + if key == 'commonName': + if _dnsname_to_pat(value).match(hostname): + return + dnsnames.append(value) + if len(dnsnames) > 1: + raise CertificateError("hostname %r " + "doesn't match either of %s" + % (hostname, ', '.join(map(repr, dnsnames)))) + elif len(dnsnames) == 1: + raise CertificateError("hostname %r " + "doesn't match %r" + % (hostname, dnsnames[0])) + else: + raise CertificateError("no appropriate commonName or " + "subjectAltName fields were found") + + + diff --git a/tuf/conf.py b/tuf/conf.py index e0c5fd7f..c66ac2ff 100755 --- a/tuf/conf.py +++ b/tuf/conf.py @@ -31,3 +31,8 @@ # which already exists and within that directory should have the file # 'metadata/current/root.txt'. This must be set! repository_directory = None + +# A directory where you may find certificate authorities +# https://en.wikipedia.org/wiki/Certificate_authority +# http://docs.python.org/2/library/ssl.html#certificates +ca_certs = None diff --git a/tuf/download.py b/tuf/download.py index 05d8b81b..61a0c953 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -22,25 +22,123 @@ """ -import urllib2 import logging +import os.path +import socket +import tuf import tuf.hash import tuf.util import tuf.formats +from tuf.compatibility import httplib, ssl, urllib2, urlparse +if ssl: + from tuf.compatibility import match_hostname +else: + raise tuf.Error( "No SSL support!" ) # TODO: degrade gracefully + + # See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger('tuf.download') +class VerifiedHTTPSConnection( httplib.HTTPSConnection ): + """ + A connection that wraps connections with ssl certificate verification. + + https://github.com/pypa/pip/blob/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/download.py#L72 + """ + def connect(self): + + self.connection_kwargs = {} + + #TODO: refactor compatibility logic into tuf.compatibility? + + # for > py2.5 + if hasattr(self, 'timeout'): + self.connection_kwargs.update(timeout = self.timeout) + + # for >= py2.7 + if hasattr(self, 'source_address'): + self.connection_kwargs.update(source_address = self.source_address) + + sock = socket.create_connection((self.host, self.port), **self.connection_kwargs) + + # for >= py2.7 + if getattr(self, '_tunnel_host', None): + self.sock = sock + self._tunnel() + + # set location of certificate authorities + assert os.path.isfile( tuf.conf.ca_certs ) + cert_path = tuf.conf.ca_certs + + self.sock = ssl.wrap_socket(sock, + self.key_file, + self.cert_file, + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=cert_path) + + match_hostname(self.sock.getpeercert(), self.host) + + +class VerifiedHTTPSHandler( urllib2.HTTPSHandler ): + """ + A HTTPSHandler that uses our own VerifiedHTTPSConnection. + + https://github.com/pypa/pip/blob/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/download.py#L109 + """ + def __init__(self, connection_class = VerifiedHTTPSConnection): + self.specialized_conn_class = connection_class + urllib2.HTTPSHandler.__init__(self) + def https_open(self, req): + return self.do_open(self.specialized_conn_class, req) + + +def _get_request(url): + """ + Wraps the URL to retrieve to protects against "creative" + interpretation of the RFC: http://bugs.python.org/issue8732 + + https://github.com/pypa/pip/blob/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/download.py#L147 + """ + + return urllib2.Request(url, headers={'Accept-encoding': 'identity'}) + + +def _get_opener( scheme = None ): + """ + Build a urllib2 opener based on whether the user now wants SSL. + + https://github.com/pypa/pip/blob/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/download.py#L178 + """ + + if scheme == "https": + assert os.path.isfile( tuf.conf.ca_certs ) + + # If we are going over https, use an opener which will provide SSL + # certificate verification. + https_handler = VerifiedHTTPSHandler() + opener = urllib2.build_opener( https_handler ) + + # strip out HTTPHandler to prevent MITM spoof + for handler in opener.handlers: + if isinstance( handler, urllib2.HTTPHandler ): + opener.handlers.remove( handler ) + else: + # Otherwise, use the default opener. + opener = urllib2.build_opener() + + return opener + + def _open_connection(url): """ Helper function that opens a connection to the url. urllib2 supports http, ftp, and file. In python (2.6+) where the ssl module is available, urllib2 also supports https. - - TODO: Do proper ssl cert/name checking. + TODO: Disallow SSLv2. TODO: Support ssl with MCrypto. TODO: Determine whether this follows http redirects and decide if we like @@ -71,11 +169,12 @@ def _open_connection(url): # servers do not recognize connections that originates from # Python-urllib/x.y. - request = urllib2.Request(url) - connection = urllib2.urlopen(request) - # urllib2.urlopen returns a file-like object: a handle to the remote data. - return connection + parsed_url = urlparse.urlparse( url ) + opener = _get_opener( scheme = parsed_url.scheme ) + request = _get_request( url ) + return opener.open( request ) except Exception, e: + raise raise tuf.DownloadError(e) diff --git a/tuf/interposition/README.md b/tuf/interposition/README.md index e1fa4cf9..3ec761a2 100644 --- a/tuf/interposition/README.md +++ b/tuf/interposition/README.md @@ -1,4 +1,4 @@ -## Methods +## Examples ```python import tuf.interposition @@ -65,3 +65,5 @@ unspecified path for the given network location. - The entire `urllib` or `urllib2` contract is not honoured. - Downloads are not thread safe. +- Uses some Python features (e.g. string formatting) not available in earlier versions (e.g. < 2.6). +- Uses some Python features (e.g. `urllib, urllib2, urlparse`) not available in later versions (e.g. >= 3).