diff --git a/tests/test_arbitrary_package_attack.py b/tests/test_arbitrary_package_attack.py index 639440ce..a5d840dd 100755 --- a/tests/test_arbitrary_package_attack.py +++ b/tests/test_arbitrary_package_attack.py @@ -55,6 +55,8 @@ import tuf.client.updater as updater import tuf.unittest_toolbox as unittest_toolbox +import utils + import securesystemslib import six @@ -87,9 +89,7 @@ def setUpClass(cls): logger.info('Serving on port: ' + str(cls.SERVER_PORT)) cls.url = 'http://localhost:' + str(cls.SERVER_PORT) + os.path.sep - # NOTE: Following error is raised if a delay is not applied: - # - time.sleep(1) + utils.wait_for_server('localhost', cls.SERVER_PORT) diff --git a/tests/test_download.py b/tests/test_download.py index 46d27410..c6cb595f 100755 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -48,6 +48,8 @@ import tuf.unittest_toolbox as unittest_toolbox import tuf.exceptions +import utils + import requests.exceptions import securesystemslib @@ -81,13 +83,7 @@ def setUp(self): junk, rel_target_filepath = os.path.split(target_filepath) self.url = 'http://localhost:'+str(self.PORT)+'/'+rel_target_filepath - # Provide a delay long enough to allow the HTTPS servers to start. - # Encountered an error on one test system at delay value of 0.2s, so - # increasing to 0.5s. Further increasing to 2s due to occasional failures - # in other tests in similar circumstances on AppVeyor. - # Expect to see "Connection refused" if this delay is not long enough - # (though other issues could cause that). - time.sleep(2) + utils.wait_for_server('localhost', self.PORT) # Computing hash of target file data. m = hashlib.md5() @@ -279,12 +275,8 @@ def test_https_connection(self): expd_https_server_proc = popen_python( ['simple_https_server.py', port4, expired_cert_fname]) - # Provide a delay long enough to allow the four HTTPS servers to start. - # Have encountered errors at 0.2s, 0.5s, and 2s, primarily on AppVeyor. - # Increasing to 4s for this test. - # Expect to see "Connection refused" if this delay is not long enough - # (though other issues could cause that). - time.sleep(3) + for port in range(self.PORT + 1, self.PORT + 5): + utils.wait_for_server('localhost', port) relative_target_fpath = os.path.basename(target_filepath) good_https_url = 'https://localhost:' + port1 + '/' + relative_target_fpath diff --git a/tests/test_endless_data_attack.py b/tests/test_endless_data_attack.py index d5f9713e..4e812996 100755 --- a/tests/test_endless_data_attack.py +++ b/tests/test_endless_data_attack.py @@ -57,6 +57,8 @@ import tuf.unittest_toolbox as unittest_toolbox import tuf.roledb +import utils + import securesystemslib import six @@ -89,9 +91,7 @@ def setUpClass(cls): logger.info('Serving on port: '+str(cls.SERVER_PORT)) cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep - # NOTE: Following error is raised if a delay is not applied: - # - time.sleep(.8) + utils.wait_for_server('localhost', cls.SERVER_PORT) diff --git a/tests/test_extraneous_dependencies_attack.py b/tests/test_extraneous_dependencies_attack.py index ae57e22d..fc5051a5 100755 --- a/tests/test_extraneous_dependencies_attack.py +++ b/tests/test_extraneous_dependencies_attack.py @@ -60,6 +60,8 @@ import tuf.keydb import tuf.unittest_toolbox as unittest_toolbox +import utils + import securesystemslib import six @@ -93,9 +95,7 @@ def setUpClass(cls): logger.info('Serving on port: '+str(cls.SERVER_PORT)) cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep - # NOTE: Following error is raised if a delay is not applied: - # - time.sleep(.7) + utils.wait_for_server('localhost', cls.SERVER_PORT) diff --git a/tests/test_indefinite_freeze_attack.py b/tests/test_indefinite_freeze_attack.py index f1b1fa43..1f3075ac 100755 --- a/tests/test_indefinite_freeze_attack.py +++ b/tests/test_indefinite_freeze_attack.py @@ -64,6 +64,8 @@ import tuf.keydb import tuf.exceptions +import utils + import securesystemslib import six @@ -100,13 +102,7 @@ def setUpClass(cls): logger.info('Serving on port: '+str(cls.SERVER_PORT)) cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep - # Provide a delay long enough to allow the HTTP server to start. - # Encountered an error on one test system at delay value of 0.2s, so - # increasing to 0.5s. Further increasing to 2s due to occasional failures - # in other tests in similar circumstances on AppVeyor. - # Expect to see "Connection refused" if this delay is not long enough - # (though other issues could cause that). - time.sleep(2) + utils.wait_for_server('localhost', cls.SERVER_PORT) diff --git a/tests/test_key_revocation_integration.py b/tests/test_key_revocation_integration.py index 91a794e4..b0693d10 100755 --- a/tests/test_key_revocation_integration.py +++ b/tests/test_key_revocation_integration.py @@ -56,6 +56,8 @@ import tuf.unittest_toolbox as unittest_toolbox import tuf.client.updater as updater +import utils + import securesystemslib import six @@ -89,9 +91,7 @@ def setUpClass(cls): logger.info('\tServing on port: '+str(cls.SERVER_PORT)) cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep - # NOTE: Following error is raised if a delay is not applied: - # - time.sleep(1) + utils.wait_for_server('localhost', cls.SERVER_PORT) diff --git a/tests/test_mix_and_match_attack.py b/tests/test_mix_and_match_attack.py index 3fe578a8..185bd227 100755 --- a/tests/test_mix_and_match_attack.py +++ b/tests/test_mix_and_match_attack.py @@ -57,6 +57,8 @@ import tuf.roledb import tuf.keydb +import utils + import securesystemslib import six @@ -94,9 +96,7 @@ def setUpClass(cls): logger.info('Serving on port: '+str(cls.SERVER_PORT)) cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep - # NOTE: Following error is raised if a delay is not applied: - # - time.sleep(.8) + utils.wait_for_server('localhost', cls.SERVER_PORT) diff --git a/tests/test_multiple_repositories_integration.py b/tests/test_multiple_repositories_integration.py index 35b1b094..8a29c457 100755 --- a/tests/test_multiple_repositories_integration.py +++ b/tests/test_multiple_repositories_integration.py @@ -49,6 +49,8 @@ import tuf.unittest_toolbox as unittest_toolbox import tuf.repository_tool as repo_tool +import utils + import six import securesystemslib @@ -153,13 +155,8 @@ def setUp(self): self.url = 'http://localhost:' + str(self.SERVER_PORT) + os.path.sep self.url2 = 'http://localhost:' + str(self.SERVER_PORT2) + os.path.sep - # NOTE: Following error is raised if a delay is not long enough: - # - # or, on Windows: - # Failed to establish a new connection: [Errno 111] Connection refused' - # 0.3s led to occasional failures in automated builds, primarily on - # AppVeyor, so increasing this to 2s, sadly. - time.sleep(2) + utils.wait_for_server('localhost', self.SERVER_PORT) + utils.wait_for_server('localhost', self.SERVER_PORT2) url_prefix = 'http://localhost:' + str(self.SERVER_PORT) url_prefix2 = 'http://localhost:' + str(self.SERVER_PORT2) diff --git a/tests/test_proxy_use.py b/tests/test_proxy_use.py index 76626ca7..86b8d013 100644 --- a/tests/test_proxy_use.py +++ b/tests/test_proxy_use.py @@ -50,6 +50,8 @@ import tuf.unittest_toolbox as unittest_toolbox import tuf.exceptions +import utils + import requests.exceptions import securesystemslib @@ -118,15 +120,10 @@ def setUpClass(cls): # the type of connection used with the target server. cls.https_proxy_addr = 'https://127.0.0.1:' + str(cls.https_proxy_port) - # Give the HTTP server and proxy server processes a little bit of time to - # start listening before allowing tests to begin, lest we get "Connection - # refused" errors. On the first test system. 0.1s was too short and 0.15s - # was long enough. Use 0.5s to be safe, and if issues arise, increase it. - # Observed occasional failures at 0.1s, 0.15s, 0.5s, and 2s, primarily on - # AppVeyor. Increasing to 4s. This setup runs once for the module. - time.sleep(4) - - + utils.wait_for_server('127.0.0.1', cls.http_port) + utils.wait_for_server('127.0.0.1', cls.https_port) + utils.wait_for_server('127.0.0.1', cls.http_proxy_port) + utils.wait_for_server('127.0.0.1', cls.https_proxy_port) diff --git a/tests/test_replay_attack.py b/tests/test_replay_attack.py index e6fc5d95..27d915f2 100755 --- a/tests/test_replay_attack.py +++ b/tests/test_replay_attack.py @@ -56,6 +56,8 @@ import tuf.repository_tool as repo_tool import tuf.unittest_toolbox as unittest_toolbox +import utils + import securesystemslib import six @@ -93,9 +95,7 @@ def setUpClass(cls): logger.info('Serving on port: '+str(cls.SERVER_PORT)) cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep - # NOTE: Following error is raised if a delay is not applied: - # - time.sleep(.8) + utils.wait_for_server('localhost', cls.SERVER_PORT) diff --git a/tests/test_updater.py b/tests/test_updater.py index 0b066827..79af5c8d 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -72,6 +72,8 @@ import tuf.unittest_toolbox as unittest_toolbox import tuf.client.updater as updater +import utils + import securesystemslib import six @@ -110,14 +112,7 @@ def setUpClass(cls): logger.info('\tServing on port: '+str(cls.SERVER_PORT)) cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep - # NOTE: Following error is raised if a delay is not long enough to allow - # the server process to set up and start listening: - # - # or, on Windows: - # Failed to establish a new connection: [Errno 111] Connection refused' - # While 0.3s has consistently worked on Travis and local builds, it led to - # occasional failures in AppVeyor builds, so increasing this to 2s, sadly. - time.sleep(2) + utils.wait_for_server('localhost', cls.SERVER_PORT) @@ -1100,13 +1095,7 @@ def test_6_get_one_valid_targetinfo(self): command = ['python', self.SIMPLE_SERVER_PATH, str(SERVER_PORT)] server_process = subprocess.Popen(command) - # NOTE: Following error is raised if a delay is not long enough: - # - # or, on Windows: - # Failed to establish a new connection: [Errno 111] Connection refused' - # While 0.3s has consistently worked on Travis and local builds, it led to - # occasional failures in AppVeyor builds, so increasing this to 2s, sadly. - time.sleep(2) + utils.wait_for_server('localhost', SERVER_PORT) # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'. repository_basepath = self.repository_directory[len(os.getcwd()):] @@ -1368,15 +1357,7 @@ def test_7_updated_targets(self): SERVER_PORT = random.randint(30000, 45000) command = ['python', self.SIMPLE_SERVER_PATH, str(SERVER_PORT)] server_process = subprocess.Popen(command) - - # NOTE: Following error is raised if a delay is not long enough to allow - # the server process to set up and start listening: - # - # or, on Windows: - # Failed to establish a new connection: [Errno 111] Connection refused' - # While 0.3s has consistently worked on Travis and local builds, it led to - # occasional failures in AppVeyor builds, so increasing this to 2s, sadly. - time.sleep(2) + utils.wait_for_server('localhost', SERVER_PORT) # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'. repository_basepath = self.repository_directory[len(os.getcwd()):] @@ -1501,15 +1482,7 @@ def test_8_remove_obsolete_targets(self): SERVER_PORT = random.randint(30000, 45000) command = ['python', self.SIMPLE_SERVER_PATH, str(SERVER_PORT)] server_process = subprocess.Popen(command) - - # NOTE: Following error is raised if a delay is not long enough to allow - # the server process to set up and start listening: - # - # or, on Windows: - # Failed to establish a new connection: [Errno 111] Connection refused' - # While 0.3s has consistently worked on Travis and local builds, it led to - # occasional failures in AppVeyor builds, so increasing this to 2s, sadly. - time.sleep(2) + utils.wait_for_server('localhost', SERVER_PORT) # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'. repository_basepath = self.repository_directory[len(os.getcwd()):] @@ -1908,14 +1881,8 @@ def setUp(self): self.url = 'http://localhost:' + str(self.SERVER_PORT) + os.path.sep self.url2 = 'http://localhost:' + str(self.SERVER_PORT2) + os.path.sep - # NOTE: Following error is raised if a delay is not long enough to allow - # the server process to set up and start listening: - # - # or, on Windows: - # Failed to establish a new connection: [Errno 111] Connection refused' - # While 0.3s has consistently worked on Travis and local builds, it led to - # occasional failures in AppVeyor builds, so increasing this to 2s, sadly. - time.sleep(2) + utils.wait_for_server('localhost', self.SERVER_PORT) + utils.wait_for_server('localhost', self.SERVER_PORT2) url_prefix = 'http://localhost:' + str(self.SERVER_PORT) url_prefix2 = 'http://localhost:' + str(self.SERVER_PORT2) diff --git a/tests/test_updater_root_rotation_integration.py b/tests/test_updater_root_rotation_integration.py index c73c9f70..6450cb95 100755 --- a/tests/test_updater_root_rotation_integration.py +++ b/tests/test_updater_root_rotation_integration.py @@ -63,6 +63,8 @@ import tuf.client.updater as updater import tuf.settings +import utils + import securesystemslib import six @@ -96,9 +98,8 @@ def setUpClass(cls): logger.info('\tServing on port: '+str(cls.SERVER_PORT)) cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep - # NOTE: Following error is raised if a delay is not applied: - # - time.sleep(1) + utils.wait_for_server('localhost', cls.SERVER_PORT) + diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 00000000..b9c9768f --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +# Copyright 2020, TUF contributors +# SPDX-License-Identifier: MIT OR Apache-2.0 + +""" + + utils.py + + + August 3, 2020. + + + Jussi Kukkonen + + + See LICENSE-MIT OR LICENSE for licensing information. + + + Provide common utilities for TUF tests +""" + +import errno +import logging +import socket +import time + +logger = logging.getLogger(__name__) + +try: + # is defined in Python 3 + TimeoutError +except NameError: + # Define for Python 2 + class TimeoutError(Exception): + + def __init__(self, value="Timeout"): + self.value = value + + def __str__(self): + return repr(self.value) + +# Wait until host:port accepts connections. +# Raises TimeoutError if this does not happen within timeout seconds +# There are major differences between operating systems on how this works +# but the current blocking connect() seems to work fast on Linux and seems +# to at least work on Windows (ECONNREFUSED unfortunately has a 2 second +# timeout on Windows) +def wait_for_server(host, port, timeout=5): + start = time.time() + remaining_timeout = timeout + succeeded = False + while not succeeded and remaining_timeout > 0: + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(remaining_timeout) + sock.connect((host, port)) + succeeded = True + except socket.timeout as e: + pass + except IOError as e: + # ECONNREFUSED is expected while the server is not started + if e.errno not in [errno.ECONNREFUSED]: + logger.warning("Unexpected error while waiting for server: " + str(e)) + # Avoid pegging a core just for this + time.sleep(0.01) + finally: + if sock: + sock.close() + sock = None + remaining_timeout = timeout - (time.time() - start) + + if not succeeded: + raise TimeoutError + +