tests: Avoid sleep(): make sure servers really start

* Add utility function to wait on a socket until it responds
* Use the function instead of sleeping in tests that need to wait for
  the server to start
* Increase the max timeout to 5 seconds by default (as appveyor builds
  still seem to hit the 3 second mark sometimes)

wait_for_server() functions quite differently depending on OS: Windows
can take 2 seconds to respond with ECONNREFUSED whereas Linux is almost
instant. There might be tricks to be faster on Windows (like setting
a shorter socket timeout) but this was not done here.

This makes a full Linux test run almost 40% faster and should be more
reproducible on every platform.

Signed-off-by: Jussi Kukkonen <jkukkonen@vmware.com>

tests/test_arbitrary_package_attack.py

@@ -55,6 +55,8 @@
 import tuf.client.updater as updater
 import tuf.unittest_toolbox as unittest_toolbox
 
+import utils
+
 import securesystemslib
 import six
 
@@ -87,9 +89,7 @@ def setUpClass(cls):
     logger.info('Serving on port: ' + str(cls.SERVER_PORT))
     cls.url = 'http://localhost:' + str(cls.SERVER_PORT) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not applied:
-    # <urlopen error [Errno 111] Connection refused>
-    time.sleep(1)
+    utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 

tests/test_download.py

@@ -48,6 +48,8 @@
 import tuf.unittest_toolbox as unittest_toolbox
 import tuf.exceptions
 
+import utils
+
 import requests.exceptions
 
 import securesystemslib
@@ -81,13 +83,7 @@ def setUp(self):
     junk, rel_target_filepath = os.path.split(target_filepath)
     self.url = 'http://localhost:'+str(self.PORT)+'/'+rel_target_filepath
 
-    # Provide a delay long enough to allow the HTTPS servers to start.
-    # Encountered an error on one test system at delay value of 0.2s, so
-    # increasing to 0.5s.  Further increasing to 2s due to occasional failures
-    # in other tests in similar circumstances on AppVeyor.
-    # Expect to see "Connection refused" if this delay is not long enough
-    # (though other issues could cause that).
-    time.sleep(2)
+    utils.wait_for_server('localhost', self.PORT)
 
     # Computing hash of target file data.
     m = hashlib.md5()
@@ -279,12 +275,8 @@ def test_https_connection(self):
     expd_https_server_proc = popen_python(
         ['simple_https_server.py', port4, expired_cert_fname])
 
-    # Provide a delay long enough to allow the four HTTPS servers to start.
-    # Have encountered errors at 0.2s, 0.5s, and 2s, primarily on AppVeyor.
-    # Increasing to 4s for this test.
-    # Expect to see "Connection refused" if this delay is not long enough
-    # (though other issues could cause that).
-    time.sleep(3)
+    for port in range(self.PORT + 1, self.PORT + 5):
+      utils.wait_for_server('localhost', port)
 
     relative_target_fpath = os.path.basename(target_filepath)
     good_https_url = 'https://localhost:' + port1 + '/' + relative_target_fpath

tests/test_endless_data_attack.py

@@ -57,6 +57,8 @@
 import tuf.unittest_toolbox as unittest_toolbox
 import tuf.roledb
 
+import utils
+
 import securesystemslib
 import six
 
@@ -89,9 +91,7 @@ def setUpClass(cls):
     logger.info('Serving on port: '+str(cls.SERVER_PORT))
     cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not applied:
-    # <urlopen error [Errno 111] Connection refused>
-    time.sleep(.8)
+    utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 

tests/test_extraneous_dependencies_attack.py

@@ -60,6 +60,8 @@
 import tuf.keydb
 import tuf.unittest_toolbox as unittest_toolbox
 
+import utils
+
 import securesystemslib
 import six
 
@@ -93,9 +95,7 @@ def setUpClass(cls):
     logger.info('Serving on port: '+str(cls.SERVER_PORT))
     cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not applied:
-    # <urlopen error [Errno 111] Connection refused>
-    time.sleep(.7)
+    utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 

tests/test_indefinite_freeze_attack.py

@@ -64,6 +64,8 @@
 import tuf.keydb
 import tuf.exceptions
 
+import utils
+
 import securesystemslib
 import six
 
@@ -100,13 +102,7 @@ def setUpClass(cls):
     logger.info('Serving on port: '+str(cls.SERVER_PORT))
     cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
-    # Provide a delay long enough to allow the HTTP server to start.
-    # Encountered an error on one test system at delay value of 0.2s, so
-    # increasing to 0.5s.  Further increasing to 2s due to occasional failures
-    # in other tests in similar circumstances on AppVeyor.
-    # Expect to see "Connection refused" if this delay is not long enough
-    # (though other issues could cause that).
-    time.sleep(2)
+    utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 

tests/test_key_revocation_integration.py

@@ -56,6 +56,8 @@
 import tuf.unittest_toolbox as unittest_toolbox
 import tuf.client.updater as updater
 
+import utils
+
 import securesystemslib
 import six
 
@@ -89,9 +91,7 @@ def setUpClass(cls):
     logger.info('\tServing on port: '+str(cls.SERVER_PORT))
     cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not applied:
-    # <urlopen error [Errno 111] Connection refused>
-    time.sleep(1)
+    utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 

tests/test_mix_and_match_attack.py

@@ -57,6 +57,8 @@
 import tuf.roledb
 import tuf.keydb
 
+import utils
+
 import securesystemslib
 import six
 
@@ -94,9 +96,7 @@ def setUpClass(cls):
     logger.info('Serving on port: '+str(cls.SERVER_PORT))
     cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not applied:
-    # <urlopen error [Errno 111] Connection refused>
-    time.sleep(.8)
+    utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 

tests/test_multiple_repositories_integration.py

@@ -49,6 +49,8 @@
 import tuf.unittest_toolbox as unittest_toolbox
 import tuf.repository_tool as repo_tool
 
+import utils
+
 import six
 import securesystemslib
 
@@ -153,13 +155,8 @@ def setUp(self):
     self.url = 'http://localhost:' + str(self.SERVER_PORT) + os.path.sep
     self.url2 = 'http://localhost:' + str(self.SERVER_PORT2) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not long enough:
-    # <urlopen error [Errno 111] Connection refused>
-    # or, on Windows:
-    # Failed to establish a new connection: [Errno 111] Connection refused'
-    # 0.3s led to occasional failures in automated builds, primarily on
-    # AppVeyor, so increasing this to 2s, sadly.
-    time.sleep(2)
+    utils.wait_for_server('localhost', self.SERVER_PORT)
+    utils.wait_for_server('localhost', self.SERVER_PORT2)
 
     url_prefix = 'http://localhost:' + str(self.SERVER_PORT)
     url_prefix2 = 'http://localhost:' + str(self.SERVER_PORT2)

tests/test_proxy_use.py

@@ -50,6 +50,8 @@
 import tuf.unittest_toolbox as unittest_toolbox
 import tuf.exceptions
 
+import utils
+
 import requests.exceptions
 
 import securesystemslib
@@ -118,15 +120,10 @@ def setUpClass(cls):
     # the type of connection used with the target server.
     cls.https_proxy_addr = 'https://127.0.0.1:' + str(cls.https_proxy_port)
 
-    # Give the HTTP server and proxy server processes a little bit of time to
-    # start listening before allowing tests to begin, lest we get "Connection
-    # refused" errors. On the first test system. 0.1s was too short and 0.15s
-    # was long enough. Use 0.5s to be safe, and if issues arise, increase it.
-    # Observed occasional failures at 0.1s, 0.15s, 0.5s, and 2s, primarily on
-    # AppVeyor.  Increasing to 4s.  This setup runs once for the module.
-    time.sleep(4)
-
-
+    utils.wait_for_server('127.0.0.1', cls.http_port)
+    utils.wait_for_server('127.0.0.1', cls.https_port)
+    utils.wait_for_server('127.0.0.1', cls.http_proxy_port)
+    utils.wait_for_server('127.0.0.1', cls.https_proxy_port)
 
 
 

tests/test_replay_attack.py

@@ -56,6 +56,8 @@
 import tuf.repository_tool as repo_tool
 import tuf.unittest_toolbox as unittest_toolbox
 
+import utils
+
 import securesystemslib
 import six
 
@@ -93,9 +95,7 @@ def setUpClass(cls):
     logger.info('Serving on port: '+str(cls.SERVER_PORT))
     cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not applied:
-    # <urlopen error [Errno 111] Connection refused>
-    time.sleep(.8)
+    utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 

tests/test_updater.py

@@ -72,6 +72,8 @@
 import tuf.unittest_toolbox as unittest_toolbox
 import tuf.client.updater as updater
 
+import utils
+
 import securesystemslib
 import six
 
@@ -110,14 +112,7 @@ def setUpClass(cls):
     logger.info('\tServing on port: '+str(cls.SERVER_PORT))
     cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not long enough to allow
-    # the server process to set up and start listening:
-    #     <urlopen error [Errno 111] Connection refused>
-    # or, on Windows:
-    #     Failed to establish a new connection: [Errno 111] Connection refused'
-    # While 0.3s has consistently worked on Travis and local builds, it led to
-    # occasional failures in AppVeyor builds, so increasing this to 2s, sadly.
-    time.sleep(2)
+    utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 
@@ -1100,13 +1095,7 @@ def test_6_get_one_valid_targetinfo(self):
     command = ['python', self.SIMPLE_SERVER_PATH, str(SERVER_PORT)]
     server_process = subprocess.Popen(command)
 
-    # NOTE: Following error is raised if a delay is not long enough:
-    # <urlopen error [Errno 111] Connection refused>
-    # or, on Windows:
-    # Failed to establish a new connection: [Errno 111] Connection refused'
-    # While 0.3s has consistently worked on Travis and local builds, it led to
-    # occasional failures in AppVeyor builds, so increasing this to 2s, sadly.
-    time.sleep(2)
+    utils.wait_for_server('localhost', SERVER_PORT)
 
     # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'.
     repository_basepath = self.repository_directory[len(os.getcwd()):]
@@ -1368,15 +1357,7 @@ def test_7_updated_targets(self):
     SERVER_PORT = random.randint(30000, 45000)
     command = ['python', self.SIMPLE_SERVER_PATH, str(SERVER_PORT)]
     server_process = subprocess.Popen(command)
-
-    # NOTE: Following error is raised if a delay is not long enough to allow
-    # the server process to set up and start listening:
-    #     <urlopen error [Errno 111] Connection refused>
-    # or, on Windows:
-    #     Failed to establish a new connection: [Errno 111] Connection refused'
-    # While 0.3s has consistently worked on Travis and local builds, it led to
-    # occasional failures in AppVeyor builds, so increasing this to 2s, sadly.
-    time.sleep(2)
+    utils.wait_for_server('localhost', SERVER_PORT)
 
     # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'.
     repository_basepath = self.repository_directory[len(os.getcwd()):]
@@ -1501,15 +1482,7 @@ def test_8_remove_obsolete_targets(self):
     SERVER_PORT = random.randint(30000, 45000)
     command = ['python', self.SIMPLE_SERVER_PATH, str(SERVER_PORT)]
     server_process = subprocess.Popen(command)
-
-    # NOTE: Following error is raised if a delay is not long enough to allow
-    # the server process to set up and start listening:
-    #     <urlopen error [Errno 111] Connection refused>
-    # or, on Windows:
-    #     Failed to establish a new connection: [Errno 111] Connection refused'
-    # While 0.3s has consistently worked on Travis and local builds, it led to
-    # occasional failures in AppVeyor builds, so increasing this to 2s, sadly.
-    time.sleep(2)
+    utils.wait_for_server('localhost', SERVER_PORT)
 
     # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'.
     repository_basepath = self.repository_directory[len(os.getcwd()):]
@@ -1908,14 +1881,8 @@ def setUp(self):
     self.url = 'http://localhost:' + str(self.SERVER_PORT) + os.path.sep
     self.url2 = 'http://localhost:' + str(self.SERVER_PORT2) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not long enough to allow
-    # the server process to set up and start listening:
-    #     <urlopen error [Errno 111] Connection refused>
-    # or, on Windows:
-    #     Failed to establish a new connection: [Errno 111] Connection refused'
-    # While 0.3s has consistently worked on Travis and local builds, it led to
-    # occasional failures in AppVeyor builds, so increasing this to 2s, sadly.
-    time.sleep(2)
+    utils.wait_for_server('localhost', self.SERVER_PORT)
+    utils.wait_for_server('localhost', self.SERVER_PORT2)
 
     url_prefix = 'http://localhost:' + str(self.SERVER_PORT)
     url_prefix2 = 'http://localhost:' + str(self.SERVER_PORT2)

tests/test_updater_root_rotation_integration.py

@@ -63,6 +63,8 @@
 import tuf.client.updater as updater
 import tuf.settings
 
+import utils
+
 import securesystemslib
 import six
 
@@ -96,9 +98,8 @@ def setUpClass(cls):
     logger.info('\tServing on port: '+str(cls.SERVER_PORT))
     cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not applied:
-    # <urlopen error [Errno 111] Connection refused>
-    time.sleep(1)
+    utils.wait_for_server('localhost', cls.SERVER_PORT)
+
 
 
 

tests/utils.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+# Copyright 2020, TUF contributors
+# SPDX-License-Identifier: MIT OR Apache-2.0
+
+"""
+<Program Name>
+  utils.py
+
+<Started>
+  August 3, 2020.
+
+<Author>
+  Jussi Kukkonen
+
+<Copyright>
+  See LICENSE-MIT OR LICENSE for licensing information.
+
+<Purpose>
+  Provide common utilities for TUF tests
+"""
+
+import errno
+import logging
+import socket
+import time
+
+logger = logging.getLogger(__name__)
+
+try:
+  # is defined in Python 3
+  TimeoutError
+except NameError:
+  # Define for Python 2
+  class TimeoutError(Exception):
+
+    def __init__(self, value="Timeout"):
+      self.value = value
+
+    def __str__(self):
+      return repr(self.value)
+
+# Wait until host:port accepts connections.
+# Raises TimeoutError if this does not happen within timeout seconds
+# There are major differences between operating systems on how this works
+# but the current blocking connect() seems to work fast on Linux and seems
+# to at least work on Windows (ECONNREFUSED unfortunately has a 2 second
+# timeout on Windows)
+def wait_for_server(host, port, timeout=5):
+  start = time.time()
+  remaining_timeout = timeout
+  succeeded = False
+  while not succeeded and remaining_timeout > 0:
+    try:
+      sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+      sock.settimeout(remaining_timeout)
+      sock.connect((host, port))
+      succeeded = True
+    except socket.timeout as e:
+      pass
+    except IOError as e:
+      # ECONNREFUSED is expected while the server is not started
+      if e.errno not in [errno.ECONNREFUSED]:
+        logger.warning("Unexpected error while waiting for server: " + str(e))
+      # Avoid pegging a core just for this
+      time.sleep(0.01)
+    finally:
+      if sock:
+        sock.close()
+        sock = None
+      remaining_timeout = timeout - (time.time() - start)
+
+  if not succeeded:
+    raise TimeoutError
+
+