From 2067a5498468674f0b80e3a199193ba2496c8326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20K=C3=A1rolyi?= Date: Wed, 15 Sep 2021 18:12:19 +0200 Subject: [PATCH] Closes #79, handle recovery from an `SSLError`, make use of `SSLContext` --- CHANGELOG.txt | 9 ++++++ README.rst | 18 +++++++---- setup.cfg | 2 +- setup.py | 2 +- tests/test_install.py | 7 +++- tests/test_smtp_check.py | 39 +++++++++++++++++++--- validate_email/exceptions.py | 12 ++++++- validate_email/smtp_check.py | 55 ++++++++++++++++++++++---------- validate_email/updater.py | 8 ++--- validate_email/validate_email.py | 11 +++++-- 10 files changed, 125 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 6889ada..4c378bc 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,12 @@ +1.0.2: +- Handle an SSLError during STARTTLS correctly (See https://github.com/karolyi/py3-validate-email/issues/79) +- Extend options with an `smtp_skip_tls` option. When `True`, the module won't initiate a TLS session. Defaults to `False`. +- Extend options with a `smtp_tls_context` option. When passed and `smtp_skip_tls` is `False` (or not passed), the client will use the passed `SSLContext`. +- Typo fix: 'ambigious' -> 'ambiguous' + +1.0.1: +- Fix dnspython dependency + 1.0.0: - New major release with breaking changes! They are: - Parameter names for validate_email() and validate_email_or_fail() have changed: diff --git a/README.rst b/README.rst index 87c8cfa..0e22fc5 100644 --- a/README.rst +++ b/README.rst @@ -35,6 +35,8 @@ Basic usage:: smtp_timeout=10, smtp_helo_host='my.host.name', smtp_from_address='my@from.addr.ess', + smtp_skip_tls=False, + smtp_tls_context=None, smtp_debug=False) Parameters @@ -58,6 +60,10 @@ Parameters :code:`smtp_from_address`: the email address used for the sender in the SMTP conversation; if set to :code:`None` (the default), the :code:`email_address` parameter is used as the sender as well +:code:`smtp_skip_tls`: skip the TLS negotiation with the server, even when available. defaults to :code:`False` + +:code:`smtp_tls_context`: an :code:`SSLContext` to use with the TLS negotiation when the server supports it. defaults to :code:`None` + :code:`smtp_debug`: activate :code:`smtplib`'s debug output which always goes to stderr; defaults to :code:`False` Result @@ -72,7 +78,7 @@ The function :code:`validate_email()` returns the following results: At least one of the requested checks failed for the given email address. :code:`None` - None of the requested checks failed, but at least one of them yielded an ambiguous result. Currently, the SMTP check is the only check which can actually yield an ambigous result. + None of the requested checks failed, but at least one of them yielded an ambiguous result. Currently, the SMTP check is the only check which can actually yield an ambiguous result. Getting more information ---------------------------- @@ -81,7 +87,7 @@ The function :code:`validate_email_or_fail()` works exactly like :code:`validate All these exceptions descend from :code:`EmailValidationError`. Please see below for the exact exceptions raised by the various checks. Note that all exception classes are defined in the module :code:`validate_email.exceptions`. -Please note that :code:`SMTPTemporaryError` indicates an ambigous check result rather than a check failure, so if you use :code:`validate_email_or_fail()`, you probably want to catch this exception. +Please note that :code:`SMTPTemporaryError` indicates an ambiguous check result rather than a check failure, so if you use :code:`validate_email_or_fail()`, you probably want to catch this exception. The checks ============================ @@ -138,7 +144,7 @@ Check whether the given email address exists by simulating an actual email deliv A connection to the SMTP server identified through the domain's MX record is established, and an SMTP conversation is initiated up to the point where the server confirms the existence of the email address. After that, instead of actually sending an email, the conversation is cancelled. -The module will try to negotiate a TLS connection with STARTTLS, and silently fall back to an unencrypted SMTP connection if the server doesn't support it. +Unless you set :code:`smtp_skip_tls` to :code:`True`, the module will try to negotiate a TLS connection with STARTTLS, and silently fall back to an unencrypted SMTP connection if the server doesn't support it. Additionally, depending on your client configuration, the TLS negotiation might fail which will result in an ambiguous response for the given host as the module will be unable to communicate with the host after the negotiation fails. In trying to succeed, you can pass an :code:`SSLContext` as an :code:`smtp_tls_context` parameter, but remember that the server might still deny the negotiation based on how you set the :code:`SSLContext` up, and based on its security settings as well. If the SMTP server replies to the :code:`RCPT TO` command with a code 250 (success) response, the check is considered successful. @@ -146,7 +152,7 @@ If the SMTP server replies with a code 5xx (permanent error) response at any poi If the SMTP server cannot be connected, unexpectedly closes the connection, or replies with a code 4xx (temporary error) at any stage of the conversation, the check is considered ambiguous. -If there is more than one valid MX record for the domain, they are tried in order of priority until the first time the check is either successful or failed. Only in case of an ambiguous check result, the next server is tried, and only if the check result is ambiguous for all servers, the overall check is considered ambigous as well. +If there is more than one valid MX record for the domain, they are tried in order of priority until the first time the check is either successful or failed. Only in case of an ambiguous check result, the next server is tried, and only if the check result is ambiguous for all servers, the overall check is considered ambiguous as well. On failure of this check or on ambiguous result, :code:`validate_email_or_fail()` raises one of the following exceptions, all of which descend from :code:`SMTPError`: @@ -157,9 +163,9 @@ On failure of this check or on ambiguous result, :code:`validate_email_or_fail() The SMTP server refused to even let us get to the point where we could ask it about the email address. Technically, this means that the server sent a code 5xx response either immediately after connection, or as a reply to the :code:`EHLO` (or :code:`HELO`) or :code:`MAIL FROM` commands. :code:`SMTPTemporaryError` - A temporary error occured during the check for all available MX servers. This is considered an ambigous check result. For example, greylisting is a frequent cause for this. + A temporary error occured during the check for all available MX servers. This is considered an ambiguous check result. For example, greylisting is a frequent cause for this. Make sure you check the contents of the message. -All of the above three exceptions provide further detail about the error response(s) in the exception's instance variable :code:`error_messages`. +All of the above three exceptions provide further details about the error response(s) in the exception's instance variable :code:`error_messages`. Auto-updater ============================ diff --git a/setup.cfg b/setup.cfg index 5aef279..ddb7da9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [metadata] -description-file = README.rst +description_file = README.rst diff --git a/setup.py b/setup.py index 33e2d0c..12b7745 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ class BuildPyCommand(build_py): setup( name='py3-validate-email', - version='1.0.1', + version='1.0.2', packages=find_packages(exclude=['tests']), install_requires=['dnspython~=2.1', 'idna~=3.0', 'filelock~=3.0'], author='László Károlyi', diff --git a/tests/test_install.py b/tests/test_install.py index 354ecbe..4d967b2 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -1,8 +1,9 @@ from pathlib import Path -from subprocess import check_output +from subprocess import check_output, STDOUT from tarfile import TarInfo from tarfile import open as tar_open from unittest.case import TestCase +from shutil import rmtree try: # OSX Homebrew fix: https://stackoverflow.com/a/53190037/1067833 @@ -31,8 +32,12 @@ class InstallTest(TestCase): def test_sdist_excludes_datadir(self): 'The created sdist should not contain the data dir.' + check_output( + args=[executable, 'setup.py', '-q', 'sdist'], stderr=STDOUT) latest_sdist = list(Path('dist').glob(pattern='*.tar.gz'))[-1] tar_file = tar_open(name=latest_sdist, mode='r:gz') for tarinfo in tar_file: # type: TarInfo self.assertNotIn( member='/validate_email/data/', container=tarinfo.name) + # Clean up after the test + rmtree('dist') diff --git a/tests/test_smtp_check.py b/tests/test_smtp_check.py index 26b3747..44e74bd 100644 --- a/tests/test_smtp_check.py +++ b/tests/test_smtp_check.py @@ -1,11 +1,13 @@ from smtplib import SMTPServerDisconnected from socket import timeout +from ssl import SSLError from unittest.case import TestCase from unittest.mock import patch from validate_email.email_address import EmailAddress from validate_email.exceptions import ( - AddressNotDeliverableError, SMTPCommunicationError, SMTPTemporaryError) + AddressNotDeliverableError, SMTPCommunicationError, SMTPMessage, + SMTPTemporaryError) from validate_email.smtp_check import _SMTPChecker, smtp_check @@ -25,6 +27,8 @@ class SMTPMock(_SMTPChecker): 'MAIL': (250, b'MAIL FROM successful'), 'RCPT': (250, b'RCPT TO successful'), 'QUIT': (221, b'QUIT successful'), + # This is not a real life response, only for mocking + 'STARTTLS': (220, b'Welcome'), } last_command = None @@ -33,7 +37,7 @@ class SMTPMock(_SMTPChecker): return None def send(self, s): - self.last_command = s[:4].upper() + self.last_command = s.split()[0].upper() def getreply(self): if isinstance(self.reply[self.last_command], Exception): @@ -42,6 +46,12 @@ class SMTPMock(_SMTPChecker): return self.reply[self.last_command] +def mocked_starttls(self: SMTPMock, *args, **kwargs): + 'Mocking `starttls()` to raise `SSLError`.' + (resp, reply) = self.docmd('STARTTLS') + raise SSLError('param 1', 'param 2') + + class SMTPCheckTest(TestCase): 'Collection of tests the `smtp_check` method.' @@ -83,8 +93,7 @@ class SMTPCheckTest(TestCase): with self.assertRaises(exception) as context: smtp_check( email_address=EmailAddress('alice@example.com'), - mx_records=['smtp.example.com'], - ) + mx_records=['smtp.example.com']) if isinstance(reply, tuple): error_messages = context.exception.error_messages error_info = error_messages['smtp.example.com'] @@ -98,3 +107,25 @@ class SMTPCheckTest(TestCase): for cmd, reply, exception in self.failures: with self.subTest(cmd=cmd, reply=reply): self._test_one_smtp_failure(cmd, reply, exception) + + @patch(target='validate_email.smtp_check._SMTPChecker', new=SMTPMock) + @patch(target='smtplib.SMTP.starttls', new=mocked_starttls) + def test_ssl_error(self): + 'Handles an `SSLError` during `starttls()` properly.' + with self.assertRaises(SMTPTemporaryError) as context: + smtp_check( + email_address=EmailAddress('alice@example.com'), + mx_records=['smtp.example.com']) + exc = context.exception + error_text = ( + 'During TLS negotiation, the following exception(s) happened: ' + "SSLError('param 1', 'param 2')") + smtp_message = \ + exc.error_messages['smtp.example.com'] # type: SMTPMessage + self.assertIs(type(smtp_message), SMTPMessage) + self.assertEqual(smtp_message.text, error_text) + self.assertEqual(smtp_message.command, 'STARTTLS') + self.assertEqual(smtp_message.code, -1) + error = smtp_message.exceptions[0] + self.assertIs(type(error), SSLError) + self.assertTupleEqual(error.args, ('param 1', 'param 2')) diff --git a/validate_email/exceptions.py b/validate_email/exceptions.py index bc3ef6a..eaaa8e5 100644 --- a/validate_email/exceptions.py +++ b/validate_email/exceptions.py @@ -2,7 +2,8 @@ from collections import namedtuple from typing import Dict SMTPMessage = namedtuple( - typename='SmtpErrorMessage', field_names=['command', 'code', 'text']) + typename='SmtpMessage', + field_names=['command', 'code', 'text', 'exceptions']) class Error(Exception): @@ -142,3 +143,12 @@ class SMTPTemporaryError(SMTPError): greylisting) or due to temporary server issues on the MX. """ message = 'Temporary error in email address verification:' + + +class TLSNegotiationError(EmailValidationError): + 'Raised when an error happens during the TLS negotiation.' + _str = 'During TLS negotiation, the following exception(s) happened: {exc}' + + def __str__(self): + 'Print a readable depiction of what happened.' + return self._str.format(exc=', '.join(repr(x) for x in self.args)) diff --git a/validate_email/smtp_check.py b/validate_email/smtp_check.py index 8edda3a..5a6a80c 100644 --- a/validate_email/smtp_check.py +++ b/validate_email/smtp_check.py @@ -1,12 +1,13 @@ from logging import getLogger from smtplib import ( SMTP, SMTPNotSupportedError, SMTPResponseException, SMTPServerDisconnected) +from ssl import SSLContext, SSLError from typing import List, Optional, Tuple from .email_address import EmailAddress from .exceptions import ( AddressNotDeliverableError, SMTPCommunicationError, SMTPMessage, - SMTPTemporaryError) + SMTPTemporaryError, TLSNegotiationError) LOGGER = getLogger(name=__name__) @@ -28,7 +29,8 @@ class _SMTPChecker(SMTP): def __init__( self, local_hostname: str, timeout: float, debug: bool, - sender: EmailAddress, recip: EmailAddress): + sender: EmailAddress, recip: EmailAddress, + skip_tls: bool = False, tls_context: SSLContext = None): """ Initialize the object with all the parameters which remain constant during the check of one email address on all the SMTP @@ -39,6 +41,8 @@ class _SMTPChecker(SMTP): self.__sender = sender self.__recip = recip self.__temporary_errors = {} + self.__skip_tls = skip_tls + self.__tls_context = tls_context # Avoid error on close() after unsuccessful connect self.sock = None @@ -84,6 +88,8 @@ class _SMTPChecker(SMTP): except RuntimeError: # SSL/TLS support is not available to your Python interpreter pass + except SSLError as exc: + raise TLSNegotiationError(exc) def mail(self, sender: str, options: tuple = ()): """ @@ -107,7 +113,7 @@ class _SMTPChecker(SMTP): raise AddressNotDeliverableError({ self._host: SMTPMessage( command='RCPT TO', code=code, - text=message.decode(errors='ignore'))}) + text=message.decode(errors='ignore'), exceptions=())}) elif code >= 400: raise SMTPResponseException(code=code, msg=message) return code, message @@ -125,6 +131,19 @@ class _SMTPChecker(SMTP): self.does_esmtp = False self.close() + def _handle_smtpresponseexception( + self, exc: SMTPResponseException) -> bool: + 'Handle an `SMTPResponseException`.' + smtp_message = SMTPMessage( + command=self.__command, code=exc.smtp_code, + text=exc.smtp_error.decode(errors='ignore'), exceptions=(exc,)) + if exc.smtp_code >= 500: + raise SMTPCommunicationError( + error_messages={self._host: smtp_message}) + else: + self.__temporary_errors[self._host] = smtp_message + return False + def _check_one(self, host: str) -> bool: """ Run the check for one SMTP server. @@ -138,23 +157,22 @@ class _SMTPChecker(SMTP): """ try: self.connect(host=host) - self.starttls() + if not self.__skip_tls: + self.starttls(context=self.__tls_context) self.ehlo_or_helo_if_needed() self.mail(sender=self.__sender.ace) code, message = self.rcpt(recip=self.__recip.ace) - except SMTPServerDisconnected as e: + except SMTPServerDisconnected as exc: self.__temporary_errors[self._host] = SMTPMessage( - command=self.__command, code=451, text=str(e)) + command=self.__command, code=451, text=str(exc), + exceptions=(exc,)) return False - except SMTPResponseException as e: - smtp_message = SMTPMessage( - command=self.__command, code=e.smtp_code, - text=e.smtp_error.decode(errors='ignore')) - if e.smtp_code >= 500: - raise SMTPCommunicationError( - error_messages={self._host: smtp_message}) - else: - self.__temporary_errors[self._host] = smtp_message + except SMTPResponseException as exc: + return self._handle_smtpresponseexception(exc=exc) + except TLSNegotiationError as exc: + self.__temporary_errors[self._host] = SMTPMessage( + command=self.__command, code=-1, text=str(exc), + exceptions=exc.args) return False finally: self.quit() @@ -177,7 +195,9 @@ class _SMTPChecker(SMTP): def smtp_check( email_address: EmailAddress, mx_records: List[str], timeout: float = 10, helo_host: Optional[str] = None, - from_address: Optional[EmailAddress] = None, debug: bool = False + from_address: Optional[EmailAddress] = None, + skip_tls: bool = False, tls_context: Optional[SSLContext] = None, + debug: bool = False ) -> bool: """ Returns `True` as soon as the any of the given server accepts the @@ -198,5 +218,6 @@ def smtp_check( """ smtp_checker = _SMTPChecker( local_hostname=helo_host, timeout=timeout, debug=debug, - sender=from_address or email_address, recip=email_address) + sender=from_address or email_address, recip=email_address, + skip_tls=skip_tls, tls_context=tls_context) return smtp_checker.check(hosts=mx_records) diff --git a/validate_email/updater.py b/validate_email/updater.py index 39159f3..c258bb9 100644 --- a/validate_email/updater.py +++ b/validate_email/updater.py @@ -21,8 +21,8 @@ TMP_PATH = Path(gettempdir()).joinpath( f'{gettempprefix()}-py3-validate-email-{geteuid()}') TMP_PATH.mkdir(exist_ok=True) BLACKLIST_URL = ( - 'https://github.com/disposable-email-domains/disposable-email-domains/' - 'master/disposable_email_blocklist.conf') + 'https://raw.githubusercontent.com/disposable-email-domains/' + 'disposable-email-domains/master/disposable_email_blocklist.conf') LIB_PATH_DEFAULT = Path(__file__).resolve().parent.joinpath('data') BLACKLIST_FILEPATH_INSTALLED = LIB_PATH_DEFAULT.joinpath('blacklist.txt') BLACKLIST_FILEPATH_TMP = TMP_PATH.joinpath('blacklist.txt') @@ -92,8 +92,8 @@ class BlacklistUpdater(object): """ LIB_PATH_DEFAULT.mkdir(exist_ok=True) self._download( - headers={}, blacklist_path=BLACKLIST_FILEPATH_INSTALLED, - etag_path=ETAG_FILEPATH_INSTALLED) + headers={}, blacklist_path=BLACKLIST_FILEPATH_INSTALLED, + etag_path=ETAG_FILEPATH_INSTALLED) def _process(self, force: bool = False): 'Start optionally updating the blacklist.txt file, while locked.' diff --git a/validate_email/validate_email.py b/validate_email/validate_email.py index 49671f6..127bd36 100644 --- a/validate_email/validate_email.py +++ b/validate_email/validate_email.py @@ -1,5 +1,6 @@ from logging import getLogger from typing import Optional +from ssl import SSLContext from .dns_check import dns_check from .domainlist_check import domainlist_check @@ -31,7 +32,9 @@ def validate_email_or_fail( check_blacklist: bool = True, check_dns: bool = True, dns_timeout: float = 10, check_smtp: bool = True, smtp_timeout: float = 10, smtp_helo_host: Optional[str] = None, - smtp_from_address: Optional[str] = None, smtp_debug: bool = False + smtp_from_address: Optional[str] = None, + smtp_skip_tls: bool = False, smtp_tls_context: Optional[SSLContext] = None, + smtp_debug: bool = False ) -> Optional[bool]: """ Return `True` if the email address validation is successful, `None` @@ -56,7 +59,8 @@ def validate_email_or_fail( return smtp_check( email_address=email_address, mx_records=mx_records, timeout=smtp_timeout, helo_host=smtp_helo_host, - from_address=smtp_from_address, debug=smtp_debug) + from_address=smtp_from_address, skip_tls=smtp_skip_tls, + tls_context=smtp_tls_context, debug=smtp_debug) def validate_email(email_address: str, **kwargs): @@ -69,7 +73,8 @@ def validate_email(email_address: str, **kwargs): try: return validate_email_or_fail(email_address, **kwargs) except SMTPTemporaryError as error: - LOGGER.info(msg=f'Validation for {email_address!r} is ambiguous: {error}') + LOGGER.info( + msg=f'Validation for {email_address!r} is ambiguous: {error}') return except EmailValidationError as error: LOGGER.info(msg=f'Validation for {email_address!r} failed: {error}')