diff --git a/.gitignore b/.gitignore index 8dbe9769..5dbf7e80 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ build/* *.egg-info .coverage .tox/* +tests/htmlcov/* diff --git a/tests/test_interpose_updater.py b/tests/test_interpose_updater.py new file mode 100755 index 00000000..89beb037 --- /dev/null +++ b/tests/test_interpose_updater.py @@ -0,0 +1,472 @@ +#!/usr/bin/env python + +""" + + test_interpose_updater.py + + + Pankhuri Goyal + + + August 2014. + + + See LICENSE for licensing information. + + + Unit test for 'tuf.interposition.updater.py'. +""" + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +import os +import sys +import tempfile +import subprocess +import random +import shutil +import logging +import time +import copy +import json + +import tuf +import tuf.util +import tuf.conf +import tuf.log +import tuf.interposition.updater as updater +import tuf.interposition.configuration as configuration +import tuf.unittest_toolbox as unittest_toolbox + +if sys.version_info >= (2, 7): + import unittest + +else: + import unittest2 as unittest + + +logger = logging.getLogger('tuf.test_interpose_updater') + + +class TestUpdaterController(unittest_toolbox.Modified_TestCase): + + @classmethod + def setUpClass(cls): + # This method is called before tests in individual class are executed. + + # Create a temporary directory to store the repository, metadata, and + # target files. 'temporary_directory' must be deleted in TearDownModule() + # so that temporary files are always removed, even when exceptions occur. + cls.temporary_directory = tempfile.mkdtemp(dir=os.getcwd()) + + # Launch a SimpleHTTPServer (serves files in the current directory). + # Test cases will request metadata and target files that have been + # pre-generated in 'tuf/tests/repository_data', which will be served + # by the SimpleHTTPServer launched here. The test cases of + # 'test_updater.py' assume the pre-generated metadata files have a specific + # structure, such as a delegated role 'targets/role1', three target + # files, five key files, etc. + cls.SERVER_PORT = random.randint(30000, 45000) + command = ['python', 'simple_server.py', str(cls.SERVER_PORT)] + cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE) + logger.info('\n\tServer process started.') + logger.info('\tServer process id: '+str(cls.server_process.pid)) + logger.info('\tServing on port: '+str(cls.SERVER_PORT)) + cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep + + time.sleep(1) + + + + @classmethod + def tearDownClass(cls): + # Remove the temporary directory after all the tests are done. + shutil.rmtree(cls.temporary_directory) + + # Kill the SimpleHTTPServer Process + if cls.server_process is None: + message = '\tServer process ' + str(cls.server_process.pid) + \ + ' terminated.' + logger.info(message) + cls.server_process.kill() + + + + def setUp(self): + # We are inheriting from custom class. + unittest_toolbox.Modified_TestCase.setUp(self) + + # Copy the original repository files provided in the test folder so that + # any modifications made to repository files are restricted to the copies. + # The 'repository_data' directory is expected to exist in 'tuf.tests/'. + original_repository_files = os.path.join(os.getcwd(), 'repository_data') + temporary_repository_root = \ + self.make_temp_directory(directory=self.temporary_directory) + + # The original repository, keystore, and client directories will be copied + # for each test case. + original_repository = os.path.join(original_repository_files, 'repository') + original_keystore = os.path.join(original_repository_files, 'keystore') + original_client = os.path.join(original_repository_files, 'client') + + # Save references to the often-needed client repository directories. + # Test cases need these references to access metadata and target files. + self.repository_directory = \ + os.path.join(temporary_repository_root, 'repository') + self.keystore_directory = \ + os.path.join(temporary_repository_root, 'keystore') + self.client_directory = os.path.join(temporary_repository_root, 'client') + self.client_metadata = os.path.join(self.client_directory, 'metadata') + self.client_metadata_current = os.path.join(self.client_metadata, 'current') + self.client_metadata_previous = \ + os.path.join(self.client_metadata, 'previous') + + # Copy the original 'repository', 'client', and 'keystore' directories + # to the temporary repository the test cases can use. + shutil.copytree(original_repository, self.repository_directory) + shutil.copytree(original_client, self.client_directory) + shutil.copytree(original_keystore, self.keystore_directory) + + # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'. + repository_basepath = self.repository_directory[len(os.getcwd()):] + + # Test Set 1 - + port = self.SERVER_PORT + url_prefix = 'http://localhost:' + str(port) + repository_basepath + + # Setting 'tuf.conf.repository_directory' with the temporary client + # directory copied from the original repository files. + tuf.conf.repository_directory = self.client_directory + + self.repository_mirrors = {'mirror': {'url_prefix': url_prefix, + 'metadata_path': 'metadata', + 'targets_path': 'targets', + 'confined_target_dirs': ['']} + } + + self.target_filepath = [{".*/targets":"/file1.txt"}] + + self.good_configuration = configuration.Configuration('localhost', 8001, + self.client_directory, + self.repository_mirrors, + self.target_filepath, None) + + self.test1_configuration = configuration.Configuration('localhost', port, + self.client_directory, + self.repository_mirrors, + 'targets', None) + + self.test2_configuration = configuration.Configuration('localhost', 8002, + self.client_directory, + self.repository_mirrors, + 'targets', None) + + test_server_port=random.randint(30000, 45000) + + self.test3_configuration = configuration.Configuration('localhost', + test_server_port, + self.client_directory, + self.repository_mirrors, + 'targets', None) + + url_prefix_test = \ + 'http://localhost:' + str(test_server_port) + repository_basepath + + + self.repository_mirrors = {'mirror': {'url_prefix': url_prefix_test, + 'metadata_path': 'metadata', + 'targets_path': 'targets', + 'confined_target_dirs': ['']} + } + + self.test4_configuration = configuration.Configuration('localhost', 8004, + self.client_directory, + self.repository_mirrors, + 'targets', None) + + + + def tearDown(self): + # We are inheriting from custom class. + unittest_toolbox.Modified_TestCase.tearDown(self) + + + + # Unit Tests + def test_add(self): + updater_controller = updater.UpdaterController() + + # Given good configuration, the UpdaterController.add() should work. + updater_controller.add(self.good_configuration) + + # Instead of configuration, if some number is given. + self.assertRaises(tuf.InvalidConfigurationError, updater_controller.add, 8) + + # Hostname already exists, should raise exception. + self.assertRaises(tuf.FormatError, updater_controller.add, + self.good_configuration) + + # Hostname already exists as a mirror, should raise an exception. + self.assertRaises(tuf.FormatError, updater_controller.add, + self.test1_configuration) + + # Repository mirror already exists as another mirror. + self.assertRaises(tuf.FormatError, updater_controller.add, + self.test2_configuration) + + # Remove the old updater. + updater_controller.remove(self.good_configuration) + + # Add a new updater for this test. + updater_controller.add(self.test3_configuration) + + # Repository mirror already exists as an updater. + self.assertRaises(tuf.FormatError, updater_controller.add, + self.test4_configuration) + + # Remove the updater once the testing is completed. + updater_controller.remove(self.test3_configuration) + + + def test_refresh(self): + updater_controller = updater.UpdaterController() + + # To check refresh() method, add a configuration for test. + updater_controller.add(self.good_configuration) + + updater_controller.refresh(self.good_configuration) + + # Check for invalid configuration error. + self.assertRaises(tuf.InvalidConfigurationError, + updater_controller.refresh, 8) + + # Check if the updater not added in the updater list is refreshed, gives an + # error or not. + self.assertRaises(tuf.NotFoundError, updater_controller.refresh, + self.test1_configuration) + + # Giving the same port number and network location as good_configuration. + self.test4_configuration.port = 8001 + self.test4_configuration.network_location = 'localhost:8001' + + # Check if the mirror not added is refreshed, gives an error or not. + self.assertRaises(tuf.NotFoundError, updater_controller.refresh, + self.test4_configuration) + + # Make an object of tuf.interposition.updater.Updater of good configuration + # for testing. + good_updater = updater.Updater(self.good_configuration) + good_updater.refresh() + + self.good_configuration.repository_mirrors['mirror']['url_prefix'] = \ + 'http://localhost:99999999' + + # To check if a bad url_prefix of a mirror raises an exception or not. + self.assertRaises(tuf.NoWorkingMirrorError, good_updater.refresh) + + + + def test_get(self): + updater_controller = updater.UpdaterController() + + updater_controller.add(self.good_configuration) + + url = 'http://localhost:8001' + updater_controller.get(url) + + wrong_url = 'http://localhost:9999' + updater_controller.get(wrong_url) + + good_updater = updater.Updater(self.good_configuration) + self.assertRaises(tuf.URLMatchesNoPatternError, + good_updater.get_target_filepath, url) + + + + def test_remove(self): + updater_controller = updater.UpdaterController() + + # To check remove() method, add a configuration for test. + updater_controller.add(self.good_configuration) + + # Check for invalid configuration error. + self.assertRaises(tuf.InvalidConfigurationError, + updater_controller.remove, 8) + + self.assertRaises(tuf.NotFoundError, updater_controller.remove, + self.test1_configuration) + + # Giving the same port number and network location as good_configuration. + self.test4_configuration.port = 8001 + self.test4_configuration.network_location = 'localhost:8001' + + self.assertRaises(tuf.NotFoundError, updater_controller.remove, + self.test4_configuration) + + +class TestUpdater(unittest_toolbox.Modified_TestCase): + + @classmethod + def setUpClass(cls): + # This method is called before tests in individual class are executed. + + # Create a temporary directory to store the repository, metadata, and + # target files. 'temporary_directory' must be deleted in TearDownModule() + # so that temporary files are always removed, even when exceptions occur. + cls.temporary_directory = tempfile.mkdtemp(dir=os.getcwd()) + + # Launch a SimpleHTTPServer (serves files in the current directory). + # Test cases will request metadata and target files that have been + # pre-generated in 'tuf/tests/repository_data', which will be served + # by the SimpleHTTPServer launched here. The test cases of + # 'test_updater.py' assume the pre-generated metadata files have a specific + # structure, such as a delegated role 'targets/role1', three target + # files, five key files, etc. + cls.SERVER_PORT = random.randint(30000, 45000) + command = ['python', 'simple_server.py', str(cls.SERVER_PORT)] + cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE) + logger.info('\n\tServer process started.') + logger.info('\tServer process id: '+str(cls.server_process.pid)) + logger.info('\tServing on port: '+str(cls.SERVER_PORT)) + cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep + + time.sleep(1) + + @classmethod + def tearDownClass(cls): + # Remove the temporary directory after all the tests are done. + shutil.rmtree(cls.temporary_directory) + + # Kill the SimpleHTTPServer Process + if cls.server_process is None: + message = '\tServer process ' + str(cls.server_process.pid) + \ + ' terminated.' + logger.info(message) + cls.server_process.kill() + + + def setUp(self): + # We are inheriting from custom class. + unittest_toolbox.Modified_TestCase.setUp(self) + + # Copy the original repository files provided in the test folder so that + # any modifications made to repository files are restricted to the copies. + # The 'repository_data' directory is expected to exist in 'tuf.tests/'. + original_repository_files = os.path.join(os.getcwd(), 'repository_data') + temporary_repository_root = \ + self.make_temp_directory(directory=self.temporary_directory) + + # The original repository, keystore, and client directories will be copied + # for each test case. + original_repository = os.path.join(original_repository_files, 'repository') + original_keystore = os.path.join(original_repository_files, 'keystore') + original_client = os.path.join(original_repository_files, 'client') + + # Save references to the often-needed client repository directories. + # Test cases need these references to access metadata and target files. + self.repository_directory = \ + os.path.join(temporary_repository_root, 'repository') + self.keystore_directory = \ + os.path.join(temporary_repository_root, 'keystore') + self.client_directory = os.path.join(temporary_repository_root, 'client') + self.client_metadata = os.path.join(self.client_directory, 'metadata') + self.client_metadata_current = os.path.join(self.client_metadata, 'current') + self.client_metadata_previous = \ + os.path.join(self.client_metadata, 'previous') + + # Copy the original 'repository', 'client', and 'keystore' directories + # to the temporary repository the test cases can use. + shutil.copytree(original_repository, self.repository_directory) + shutil.copytree(original_client, self.client_directory) + shutil.copytree(original_keystore, self.keystore_directory) + + # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'. + repository_basepath = self.repository_directory[len(os.getcwd()):] + + # Test Set 1 - + port = self.SERVER_PORT + url_prefix = 'http://localhost:' + str(port) + repository_basepath + + # Setting 'tuf.conf.repository_directory' with the temporary client + # directory copied from the original repository files. + tuf.conf.repository_directory = self.client_directory + + self.repository_mirrors = {'mirror': {'url_prefix': url_prefix, + 'metadata_path': 'metadata', + 'targets_path': 'targets', + 'confined_target_dirs': ['']} + } + + self.target_paths = [{".*/targets":"/file1.txt"}] + + self.good_configuration = configuration.Configuration('localhost', 8001, + self.client_directory, + self.repository_mirrors, + self.target_paths, None) + + + + def tearDown(self): + # We are inheriting from custom class. + unittest_toolbox.Modified_TestCase.tearDown(self) + + + # Unit Tests + def test_download_target(self): + myUpdater = updater.Updater(self.good_configuration) + + target_filepath = 'file.txt' + self.assertRaises(tuf.UnknownTargetError, myUpdater.download_target, + target_filepath) + + self.assertRaises(tuf.FormatError, myUpdater.download_target, 8) + + target_filepath = 'file1.txt' + myUpdater.download_target(target_filepath) + + + def test_get_target_filepath(self): + myUpdater = updater.Updater(self.good_configuration) + + self.assertRaises(AttributeError, myUpdater.get_target_filepath, 8) + + test_source_url = 'http://localhost:9999' + self.assertRaises(tuf.URLMatchesNoPatternError, + myUpdater.get_target_filepath, test_source_url) + + test_source_url = 'http://localhost:8001/targets/file.txt' + myUpdater.get_target_filepath(test_source_url) + + + def test_open(self): + myUpdater = updater.Updater(self.good_configuration) + + self.assertRaises(AttributeError, myUpdater.open, 8) + + url = 'http://localhost:8001/targets/file1.txt' + myUpdater.open(url, 'interposition.json') + + + def test_retrieve(self): + myUpdater = updater.Updater(self.good_configuration) + + self.assertRaises(AttributeError, myUpdater.retrieve, 8) + + test_source_url = 'http://localhost:8001/targets/file1.txt' + myUpdater.retrieve(test_source_url, 'interposition.json') + + #self.assertRaises(tuf.NoWorkingMirrorError, myUpdater.retrieve, test_source_url) + + test_source_url = 'http://6767:localhost' + self.assertRaises(tuf.URLMatchesNoPatternError, myUpdater.retrieve, + test_source_url) + + test_source_url = 'http://localhost:8001/targets/file1.txt' + myUpdater.retrieve(test_source_url) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_updater.py b/tests/test_updater.py index 5323cdd4..3c75837f 100755 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -243,18 +243,25 @@ def test_1__init__exceptions(self): shutil.move(self.client_metadata_previous, previous_backup) self.assertRaises(tuf.RepositoryError, updater.Updater, 'test_repository', self.repository_mirrors) + # Restore the client's previous directory. The required 'current' directory # is still missing. shutil.move(previous_backup, self.client_metadata_previous) - - # Test: repository with only a '{repository_directory/metadata/previous' + # Test: repository with only a '{repository_directory}/metadata/previous' # directory. self.assertRaises(tuf.RepositoryError, updater.Updater, 'test_repository', self.repository_mirrors) # Restore the client's current directory. shutil.move(current_backup, self.client_metadata_current) + # Test: repository with a '{repository_directory}/metadata/current' + # directory, but the 'previous' directory is missing. + shutil.move(self.client_metadata_previous, previous_backup) + self.assertRaises(tuf.RepositoryError, updater.Updater, 'test_repository', + self.repository_mirrors) + shutil.move(previous_backup, self.client_metadata_previous) + # Test: repository missing the required 'root.json' file. client_root_file = os.path.join(self.client_metadata_current, 'root.json') backup_root_file = client_root_file + '.backup' @@ -290,10 +297,16 @@ def test_1__load_metadata_from_file(self): # (i.e., only the 'root.json' file should have been loaded. self.assertEqual(len(self.repository_updater.metadata['current']), 5) - # Verify that the content of root metadata is valid. + # Verify that the content of root metadata is valid. self.assertEqual(self.repository_updater.metadata['current']['targets/role1'], role1_meta['signed']) + # Test invalid metadata set argument (must be either + # 'current' or 'previous'.) + self.assertRaises(tuf.Error, + self.repository_updater._load_metadata_from_file, + 'bad_metadata_set', 'targets/role1') + @@ -413,7 +426,40 @@ def test_2__import_delegations(self): for keyid in keyids: self.assertTrue(keyid in tuf.keydb._keydb_dict) + # Verify that _import_delegations() ignores invalid keytypes in the 'keys' + # field of parent role's 'delegations' + existing_keyid = keyids[0] + + self.repository_updater.metadata['current']['targets']\ + ['delegations']['keys'][existing_keyid]['keytype'] = 'bad_keytype' + self.repository_updater._import_delegations('targets') + # Restore the keytype of 'existing_keyid'. + self.repository_updater.metadata['current']['targets']\ + ['delegations']['keys'][existing_keyid]['keytype'] = 'rsa' + # Verify that _import_delegations() raises an exception if any key in + # 'delegations' is improperly formatted (i.e., bad keyid.) + tuf.keydb.clear_keydb() + self.repository_updater.metadata['current']['targets']\ + ['delegations']['keys'][existing_keyid]['keyid'] = '123' + + print(repr(self.repository_updater.metadata['current']['targets']\ + ['delegations']['keys'][existing_keyid]['keyid'])) + self.repository_updater._import_delegations('targets') + #self.assertRaises(tuf.Error, self.repository_updater._import_delegations, + # 'targets') + + # Restore the keyid of 'existing_keyids2'. + self.repository_updater.metadata['current']['targets']\ + ['delegations']['keys'][existing_keyid]['keyid'] = existing_keyid + + # Verify that _import_delegations() raises an exception if it fails to add + # one of the roles loaded from parent role's 'delegations'. + + + + + diff --git a/tests/test_util.py b/tests/test_util.py index 9c1c20ef..cd3a9b11 100755 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -394,6 +394,7 @@ def test_C2_find_delegated_role(self): self.assertTrue(tuf.formats.ROLELIST_SCHEMA.matches(role_list)) self.assertEqual(tuf.util.find_delegated_role(role_list, 'targets/tuf'), 1) self.assertEqual(tuf.util.find_delegated_role(role_list, 'targets/warehouse'), 0) + # Test for non-existent role. 'find_delegated_role()' returns 'None' # if the role is not found. self.assertEqual(tuf.util.find_delegated_role(role_list, 'targets/non-existent'), diff --git a/tuf/__init__.py b/tuf/__init__.py index 4b9d3b89..b2810181 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -293,7 +293,7 @@ class UnknownTargetError(Error): class InvalidNameError(Error): - """Indicate an error while trying to validate any type of named object""" + """Indicate an error while trying to validate any type of named object.""" pass @@ -345,3 +345,18 @@ def __str__(self): all_errors += '\n ' + repr(mirror_netloc) + ': ' + repr(mirror_error) return all_errors + + +class NotFoundError(Error): + """If a required configuration or resource is not found.""" + pass + + +class URLMatchesNoPatternError(Error): + """If a URL does not match a user-specified regular expression.""" + pass + + +class InvalidConfigurationError(Error): + """If a configuration object does not match the expected format.""" + pass diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 407bed86..dbe8d70c 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -324,7 +324,7 @@ def __init__(self, updater_name, repository_mirrors): # Ensure the current path is valid/exists before saving it. if not os.path.exists(current_path): - message = 'Missing '+repr(current_path)+'. This path must exist and, ' \ + message = 'Missing ' + repr(current_path) + '. This path must exist and, ' \ 'at a minimum, contain the root metadata file.' raise tuf.RepositoryError(message) self.metadata_directory['current'] = current_path @@ -334,7 +334,7 @@ def __init__(self, updater_name, repository_mirrors): # Ensure the previous path is valid/exists. if not os.path.exists(previous_path): - message = 'Missing '+repr(previous_path)+'. This path must exist.' + message = 'Missing ' + repr(previous_path) + '. This path must exist.' raise tuf.RepositoryError(message) self.metadata_directory['previous'] = previous_path @@ -402,7 +402,7 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): # Ensure we have a valid metadata set. if metadata_set not in ['current', 'previous']: - raise tuf.Error('Invalid metadata set: '+repr(metadata_set)) + raise tuf.Error('Invalid metadata set: ' + repr(metadata_set)) # Save and construct the full metadata path. metadata_directory = self.metadata_directory[metadata_set] @@ -515,7 +515,7 @@ def _import_delegations(self, parent_role): keys_info = current_parent_metadata['delegations'].get('keys', {}) roles_info = current_parent_metadata['delegations'].get('roles', []) - logger.debug('Adding roles delegated from '+repr(parent_role)+'.') + logger.debug('Adding roles delegated from ' + repr(parent_role) + '.') # Iterate through the keys of the delegated roles of 'parent_role' # and load them. @@ -532,12 +532,12 @@ def _import_delegations(self, parent_role): pass except (tuf.FormatError, tuf.Error) as e: - logger.exception('Failed to add keyid: '+repr(keyid)+'.') - logger.error('Aborting role delegation for parent role '+parent_role+'.') + logger.exception('Invalid key for keyid: ' + repr(keyid) + '.') + logger.error('Aborting role delegation for parent role ' + parent_role + '.') raise else: - logger.warning('Invalid key type for '+repr(keyid)+'.') + logger.warning('Invalid key type for ' + repr(keyid) + '.') continue # Add the roles to the role database. @@ -546,14 +546,14 @@ def _import_delegations(self, parent_role): # NOTE: tuf.roledb.add_role will take care of the case where rolename # is None. rolename = roleinfo.get('name') - logger.debug('Adding delegated role: '+str(rolename)+'.') + logger.debug('Adding delegated role: ' + str(rolename) + '.') tuf.roledb.add_role(rolename, roleinfo) except tuf.RoleAlreadyExistsError as e: - logger.warning('Role already exists: '+rolename) + logger.warning('Role already exists: ' + rolename) except: - logger.exception('Failed to add delegated role: '+rolename+'.') + logger.exception('Failed to add delegated role: ' + rolename + '.') raise diff --git a/tuf/interposition/README.md b/tuf/interposition/README.md index d6375169..1892f502 100644 --- a/tuf/interposition/README.md +++ b/tuf/interposition/README.md @@ -1,28 +1,56 @@ -## Examples +## Interposition -```python -import tuf.interposition -# Configurations are simply a JSON object which allows you to answer these questions: -# - Which network locations get intercepted? -# - Given a network location, which TUF mirrors should we forward requests to? -# - Given a network location, which paths should be intercepted? -# - Given a TUF mirror, how do we verify its SSL certificate? -tuf.interposition.configure() -``` +The interposition package (tuf/interposition/) can be used to integrate TUF +into a software updater. It is an integration method that requires the least +amount of effort from developers who are performing the integration. The +integration method used by interposition is considered high-level because the +integrator does not explicitly call TUF methods to refresh metadata and +download target files. For example, performing a low-level integration with +*tuf/client/updater.py* requires the integrator to instantiate an updater +object, call updater.refresh() to refresh TUF metadata, and +updater.download_target() to download target files referenced in TUF metadata. +In contrast, an integrator may utilize interposition to load some configuration +settings to indicate which URLs requested by Python urllib calls should be +interposed by TUF. This means that all the update calls for metadata and +target requests are made transparently by the low level *tuf/client/updater.py* +module. -### Option one + +### Interposition Examples + +To use interposition, integrators must: + +1. Create an interposition configuration file. +2. Import interposition, and load the configuration file with configure(). +3. Perform updater urllib calls that may be interposed. +4. Deconfigure interposition. + + +## Option 1 ```python from tuf.interposition import urllib_tuf as urllib from tuf.interposition import urllib2_tuf as urllib2 +# configure() loads the interposition configuration file that indicates which +# URLs should be interposed by TUF. Any urllib calls that occur after +# configure() are subject to interposition. + +configuration = tuf.interposition.configure() + url = 'http://example.com/path/to/document' + urllib.urlopen(url) -urllib.urlretrieve(url) +urllib.urlretrieve(url, 'mytarget') urllib2.urlopen(url) + +# deconfigure() is used to stop interposition. Any urllib calls that occur +# after deconfigure() are not interposed. +tuf.interposition.deconfigure(configuration) + ``` -### Option two +## Option 2 ```python @tuf.interposition.open_url @@ -30,6 +58,12 @@ def instancemethod(self, url, ...): ... ``` + +Note: tuf.interposition.refresh(configuration) may be called to force a +refresh of the TUF metadata. Interposition normally performs a refresh of TUF +metadata when configure() is called. + + ## Configuration A *configuration* is simply a JSON object which tells `tuf.interposition` which diff --git a/tuf/interposition/__init__.py b/tuf/interposition/__init__.py old mode 100644 new mode 100755 index 4360016e..eeac65ee --- a/tuf/interposition/__init__.py +++ b/tuf/interposition/__init__.py @@ -1,16 +1,45 @@ +""" + + __init__.py + + + Trishank Kuppusamy. + Pankhuri Goyal + + + + + See LICENSE for licensing information. + + + TODO: Add pros / cons of using interposition. Also, should we move code + here to its own module (instead of __init__.py)? +""" + +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + + import functools import imp import json import socket -import urllib -import urllib2 +import logging +import tuf.log +import tuf._vendor.six as six -# We import them directly into our namespace so that there is no name conflict. -from configuration import ConfigurationParser, InvalidConfiguration -from utility import Logger -from updater import UpdaterController +# We import the following directly into our namespace so that there is no name +# conflict. +from tuf.interposition.configuration import ConfigurationParser +from tuf.interposition.updater import UpdaterController +logger = logging.getLogger('tuf.interposition.__init__') # Export nothing when: from tuf.interposition import * __all__ = [] @@ -19,10 +48,12 @@ # TODO: # - Document design decisions. # - Interposition: Honour urllib/urllib2 contract. -# - Review security issues resulting from regular expressions (e.g. complexity attacks). +# - Review security issues resulting from regular expressions +# (e.g. complexity attacks). # - Warn user when TUF is used without any configuration. # - Override other default (e.g. HTTPS) urllib2 handlers. -# - Failsafe: If TUF fails, offer option to unsafely resort back to urllib/urllib2? +# - Failsafe: If TUF fails, offer option to unsafely resort back to +# urllib/urllib2? @@ -59,39 +90,53 @@ def __monkey_patch(): - """Build and monkey patch public copies of the urllib and urllib2 modules. + """ + Build and monkey patch public copies of the urllib and urllib2 modules. - We prefer simplicity, which leads to easier proof of security, even if it may - come at the cost of not honouring some provisions of the urllib and urllib2 - module contracts unrelated to security. + We prefer simplicity, which leads to easier proof of security, even if it may + come at the cost of not honouring some provisions of the urllib and urllib2 + module contracts unrelated to security. - References: - http://stackoverflow.com/a/11285504 - http://docs.python.org/2/library/imp.html""" + References: + http://stackoverflow.com/a/11285504 + http://docs.python.org/2/library/imp.html + """ global urllib_tuf global urllib2_tuf if urllib_tuf is None: + urllib_module_name = 'urllib' + if six.PY3: + urllib_module_name = 'urllib/request' + try: - module_file, pathname, description = imp.find_module("urllib") + module_file, pathname, description = imp.find_module(urllib_module_name) urllib_tuf = \ - imp.load_module( "urllib_tuf", module_file, pathname, description) + imp.load_module('urllib_tuf', module_file, pathname, description) module_file.close() + except: raise + else: urllib_tuf.urlopen = __urllib_urlopen urllib_tuf.urlretrieve = __urllib_urlretrieve if urllib2_tuf is None: + urllib2_module_name = 'urllib2' + if six.PY3: + urllib2_module_name = 'urllib/request' + try: - module_file, pathname, description = imp.find_module("urllib2") + module_file, pathname, description = imp.find_module(urllib2_module_name) urllib2_tuf = \ - imp.load_module( "urllib2_tuf", module_file, pathname, description) + imp.load_module('urllib2_tuf', module_file, pathname, description) module_file.close() + except: raise + else: urllib2_tuf.urlopen = __urllib2_urlopen @@ -105,7 +150,8 @@ def __urllib_urlopen(url, data=None, proxies=None): updater = __updater_controller.get(url) if updater is None: - return urllib.urlopen(url, data=data, proxies=proxies) + return six.moves.urllib.request.urlopen(url, data=data, proxies=proxies) + else: return updater.open(url, data=data) @@ -119,7 +165,8 @@ def __urllib_urlretrieve(url, filename=None, reporthook=None, data=None): updater = __updater_controller.get(url) if updater is None: - return urllib.urlretrieve(url, filename=filename, reporthook=reporthook, data=data) + return six.moves.urllib.request.urlretrieve(url, filename=filename, reporthook=reporthook, data=data) + else: return updater.retrieve(url, filename=filename, reporthook=reporthook, data=data) @@ -136,27 +183,32 @@ def __urllib2_urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): updater = None # If this is a urllib2.Request... - if isinstance(url, urllib2.Request): + if isinstance(url, six.moves.urllib.request.Request): # If this is a GET HTTP method... if url.get_method() == "GET": # ...then you should check with TUF. updater = __updater_controller.get(url.get_full_url()) + else: # ...otherwise, revert to default behaviour. - Logger.warn(NON_GET_HTTP_METHOD_MESSAGE.format(method=url.get_method(), + logger.warn(NON_GET_HTTP_METHOD_MESSAGE.format(method=url.get_method(), url=url.get_full_url())) - return urllib2.urlopen(url, data=data, timeout=timeout) + return six.moves.urllib.request.urlopen(url, data=data, timeout=timeout) + else: # ...otherwise, we assume this is a string. updater = __updater_controller.get(url) if updater is None: - return urllib2.urlopen(url, data=data, timeout=timeout) + return six.moves.urllib.request.urlopen(url, data=data, timeout=timeout) + else: response = updater.open(url, data=data) + # See urllib2.AbstractHTTPHandler.do_open # TODO: let Updater handle this response.msg = "" + return response @@ -173,12 +225,13 @@ def __read_configuration(configuration_handler, parent_ssl_certificates_directory=None): """ A generic function to read TUF interposition configurations off a file, and - then handle those configurations with a given function. configuration_handler - must be a function which accepts a tuf.interposition.Configuration - instance. + then handle those configurations with a given function. + configuration_handler must be a function which accepts a + tuf.interposition.Configuration instance. Returns the parsed configurations as a dictionary of configurations indexed - by hostnames.""" + by hostnames. + """ INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!" INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!" @@ -193,34 +246,34 @@ def __read_configuration(configuration_handler, configurations = tuf_interpositions.get("configurations", {}) if len(configurations) == 0: - raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename)) + raise tuf.InvalidConfigurationError(NO_CONFIGURATIONS.format(filename=filename)) else: - for network_location, configuration in configurations.iteritems(): + for network_location, configuration in six.iteritems(configurations): try: configuration_parser = ConfigurationParser(network_location, configuration, parent_repository_directory=parent_repository_directory, parent_ssl_certificates_directory=parent_ssl_certificates_directory) - + + # configuration_parser.parse() returns a + # 'tuf.interposition.Configuration' object, which interposition + # uses to determine which URLs should be interposed. configuration = configuration_parser.parse() configuration_handler(configuration) parsed_configurations[configuration.hostname] = configuration except: - Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location)) + logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location)) raise except: - Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename)) + logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename)) raise else: return parsed_configurations - - - # TODO: Is parent_repository_directory a security risk? For example, would it # allow the user to overwrite another TUF repository metadata on the filesystem? # On the other hand, it is beyond TUF's scope to handle filesystem permissions. @@ -230,7 +283,8 @@ def configure(filename="tuf.interposition.json", parent_repository_directory=None, parent_ssl_certificates_directory=None): - """The optional parent_repository_directory parameter is used to specify the + """ + The optional parent_repository_directory parameter is used to specify the containing parent directory of the "repository_directory" specified in a configuration for *all* network locations, because sometimes the absolute location of the "repository_directory" is only known at runtime. If you @@ -272,7 +326,8 @@ def configure(filename="tuf.interposition.json", optional; it must specify certificates bundled as PEM (RFC 1422). Returns the parsed configurations as a dictionary of configurations indexed - by hostnames.""" + by hostnames. + """ configurations = \ __read_configuration(__updater_controller.add, filename=filename, @@ -296,7 +351,7 @@ def refresh(configurations): # Although interposition was designed to remain transparent, for software # updaters that require an explicit refresh of top-level metadata, this # method is provided. - for configuration in configurations.itervalues(): + for configuration in six.itervalues(configurations): __updater_controller.refresh(configuration) @@ -306,7 +361,7 @@ def refresh(configurations): def deconfigure(configurations): """Remove TUF interposition for previously read configurations.""" - for configuration in configurations.itervalues(): + for configuration in six.itervalues(configurations): __updater_controller.remove(configuration) @@ -314,8 +369,10 @@ def deconfigure(configurations): def open_url(instancemethod): - """Decorate an instance method of the form - instancemethod(self, url, ...) with me in order to pass it to TUF.""" + """ + Decorate an instance method of the form + instancemethod(self, url, ...) with me in order to pass it to TUF. + """ @functools.wraps(instancemethod) def wrapper(self, *args, **kwargs): @@ -325,16 +382,18 @@ def wrapper(self, *args, **kwargs): data = kwargs.get("data") # If this is a urllib2.Request... - if isinstance(url_object, urllib2.Request): + if isinstance(url_object, six.moves.request.Request): # If this is a GET HTTP method... if url_object.get_method() == "GET": # ...then you should check with TUF. url = url_object.get_full_url() + else: # ...otherwise, revert to default behaviour. - Logger.warn(NON_GET_HTTP_METHOD_MESSAGE.format(method=url_object.get_method(), + logger.warn(NON_GET_HTTP_METHOD_MESSAGE.format(method=url_object.get_method(), url=url_object.get_full_url())) return instancemethod(self, *args, **kwargs) + # ...otherwise, we assume this is a string. else: url = url_object @@ -345,6 +404,7 @@ def wrapper(self, *args, **kwargs): if updater is None: # ...then revert to default behaviour. return instancemethod(self, *args, **kwargs) + else: # ...otherwise, use TUF to get this document. return updater.open(url, data=data) @@ -364,7 +424,3 @@ def wrapper(self, *args, **kwargs): # Build and monkey patch public copies of the urllib and urllib2 modules. __monkey_patch() - - - - diff --git a/tuf/interposition/configuration.py b/tuf/interposition/configuration.py old mode 100644 new mode 100755 index e165a073..a1bc7aba --- a/tuf/interposition/configuration.py +++ b/tuf/interposition/configuration.py @@ -1,38 +1,71 @@ +""" + + configuration.py + + + Trishank Kuppusamy + Pankhuri Goyal + Vladimir Diaz + + + + + See LICENSE for licensing information. + + + +""" + +# Help with Python 3 compatibility where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import os.path -import types -import urlparse - - -# We import them directly into our namespace so that there is no name conflict. -from utility import Logger, InterpositionException - - - - - -################################ GLOBAL CLASSES ################################ - - - - - -class InvalidConfiguration(InterpositionException): - """User configuration is invalid.""" - pass - +import logging +import tuf.log +import tuf._vendor.six as six +logger = logging.getLogger('tuf.interposition.configuration') class Configuration(object): - """Holds TUF interposition configuration information about a network - location which is important to an updater for that network location.""" - + """ + + Holds TUF interposition configuration information about a network + location which is important to an updater for that network location. + """ def __init__(self, hostname, port, repository_directory, repository_mirrors, target_paths, ssl_certificates): - """Constructor assumes that its parameters are valid.""" + """ + + Constructor assumes that its parameters are valid. + + + hostname: + + port: + + repository_directory: + + repository_mirrors: + + target_paths: + + ssl_certificates: + + + + + + + """ self.hostname = hostname self.port = port @@ -50,8 +83,19 @@ def __repr__(self): def get_repository_mirror_hostnames(self): - """Get a set of hostnames of every repository mirror of this - configuration.""" + """ + + Get a set of hostnames of every repository mirror of this configuration. + + + None. + + + + + + + """ # Parse TUF server repository mirrors. repository_mirrors = self.repository_mirrors @@ -59,10 +103,14 @@ def get_repository_mirror_hostnames(self): for repository_mirror in repository_mirrors: mirror_configuration = repository_mirrors[repository_mirror] + url_prefix = mirror_configuration["url_prefix"] - parsed_url = urlparse.urlparse(url_prefix) + parsed_url = six.moves.urllib.parse.urlparse(url_prefix) mirror_hostname = parsed_url.hostname - repository_mirror_hostnames.add(mirror_hostname) + mirror_port = parsed_url.port + mirror_network_location = \ + "{hostname}:{port}".format(hostname=mirror_hostname, port = mirror_port) + repository_mirror_hostnames.add(mirror_network_location) return repository_mirror_hostnames @@ -71,14 +119,36 @@ def get_repository_mirror_hostnames(self): class ConfigurationParser(object): - """Parses TUF interposition configuration information about a network - location, stored as a JSON object, and returns it as a Configuration.""" + """ + + Parses TUF interposition configuration information about a network + location, stored as a JSON object, and returns it as a Configuration. + """ def __init__(self, network_location, configuration, parent_repository_directory=None, parent_ssl_certificates_directory=None): + """ + + + network_location: + + configuration: + + parent_repository_directory: + + parent_ssl_certificates_directory: + + + + + + + None. + """ + self.network_location = network_location self.configuration = configuration self.parent_repository_directory = parent_repository_directory @@ -86,7 +156,20 @@ def __init__(self, network_location, configuration, def get_network_location(self): - """Check network location.""" + """ + + Check network location. + + + None. + + + + + + + + """ INVALID_NETWORK_LOCATION = "Invalid network location {network_location}!" @@ -97,14 +180,27 @@ def get_network_location(self): if len(network_location_tokens) > 1: port = int(network_location_tokens[1], 10) if port <= 0 or port >= 2**16: - raise InvalidConfiguration(INVALID_NETWORK_LOCATION.format( + raise tuf.InvalidConfigurationError(INVALID_NETWORK_LOCATION.format( network_location=self.network_location)) return hostname, port def get_repository_directory(self): - """Locate TUF client metadata repository.""" + """ + + Locate TUF client metadata repository. + + + None. + + + + + + + + """ INVALID_PARENT_REPOSITORY_DIRECTORY = \ "Invalid parent_repository_directory for {network_location}!" @@ -121,14 +217,27 @@ def get_repository_directory(self): # TODO: assert os.path.isdir(repository_directory) else: - raise InvalidConfiguration(INVALID_PARENT_REPOSITORY_DIRECTORY.format( + raise tuf.InvalidConfigurationError(INVALID_PARENT_REPOSITORY_DIRECTORY.format( network_location=self.network_location)) return repository_directory def get_ssl_certificates(self): - """Get any PEM certificate bundle.""" + """ + + Get any PEM certificate bundle. + + + None. + + + + + + + + """ INVALID_SSL_CERTIFICATES = \ "Invalid ssl_certificates for {network_location}!" @@ -147,11 +256,11 @@ def get_ssl_certificates(self): ssl_certificates) if not os.path.isfile(ssl_certificates): - raise InvalidConfiguration(INVALID_SSL_CERTIFICATES.format( + raise tuf.InvalidConfigurationError(INVALID_SSL_CERTIFICATES.format( network_location=self.network_location)) else: - raise InvalidConfiguration( + raise tuf.InvalidConfigurationError( INVALID_PARENT_SSL_CERTIFICATES_DIRECTORY.format( network_location=self.network_location)) @@ -159,7 +268,24 @@ def get_ssl_certificates(self): def get_repository_mirrors(self, hostname, port, ssl_certificates): - """Parse TUF server repository mirrors.""" + """ + + Parse TUF server repository mirrors. + + + hostname: + + port: + + ssl_certificates: + + + + + + + + """ INVALID_REPOSITORY_MIRROR = "Invalid repository mirror {repository_mirror}!" @@ -171,7 +297,7 @@ def get_repository_mirrors(self, hostname, port, ssl_certificates): try: url_prefix = mirror_configuration["url_prefix"] - parsed_url = urlparse.urlparse(url_prefix) + parsed_url = six.moves.urllib.parse.urlparse(url_prefix) mirror_hostname = parsed_url.hostname mirror_port = parsed_url.port or 80 mirror_scheme = parsed_url.scheme @@ -199,18 +325,29 @@ def get_repository_mirrors(self, hostname, port, ssl_certificates): except: error_message = \ INVALID_REPOSITORY_MIRROR.format(repository_mirror=repository_mirror) - Logger.exception(error_message) - raise InvalidConfiguration(error_message) + logger.exception(error_message) + raise tuf.InvalidConfigurationError(error_message) return repository_mirrors def get_target_paths(self): """ - Within a network_location, we match URLs with this list of regular - expressions, which tell us to map from a source URL to a target URL. - If there are multiple regular expressions which match a source URL, - the order of appearance will be used to resolve ambiguity. + + Within a network_location, we match URLs with this list of regular + expressions, which tell us to map from a source URL to a target URL. + If there are multiple regular expressions which match a source URL, + the order of appearance will be used to resolve ambiguity. + + + None. + + + + + + + """ INVALID_TARGET_PATH = "Invalid target path in {network_location}!" @@ -221,27 +358,40 @@ def get_target_paths(self): target_paths = self.configuration.get("target_paths", [WILD_TARGET_PATH]) # target_paths: [ target_path, ... ] - assert isinstance(target_paths, types.ListType) + assert isinstance(target_paths, list) for target_path in target_paths: try: # target_path: { "regex_with_groups", "target_with_group_captures" } # e.g. { ".*(/some/directory)/$", "{0}/index.html" } - assert isinstance(target_path, types.DictType) + assert isinstance(target_path, dict) assert len(target_path) == 1 except: error_message = \ INVALID_TARGET_PATH.format(network_location=self.network_location) - Logger.exception(error_message) - raise InvalidConfiguration(error_message) + logger.exception(error_message) + raise tuf.InvalidConfigurationError(error_message) return target_paths # TODO: more input sanity checks? def parse(self): - """Parse, check and get the required configuration parameters.""" + """ + + Parse, check, and get the required configuration parameters. + + + None. + + + + + + + + """ hostname, port = self.get_network_location() ssl_certificates = self.get_ssl_certificates() @@ -252,5 +402,5 @@ def parse(self): self.get_repository_mirrors(hostname, port, ssl_certificates) # If everything passes, we return a Configuration. - return Configuration(hostname, port, repository_directory, repository_mirrors, - target_paths, ssl_certificates) + return Configuration(hostname, port, repository_directory, + repository_mirrors, target_paths, ssl_certificates) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py old mode 100644 new mode 100755 index c2d266e2..c157f865 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -1,114 +1,456 @@ +""" + + updater.py + + + Trishank Kuppusamy + Pankhuri Goyal + + + June 2014. + Refactored and unit tested by Pankhuri. + + + See LICENSE for licensing information. + + + Assist with high-level integrations, which means that all the processes that + are taking place in the low-level 'tuf.client.updater.py' will be automated + by this module. This layer of automation will be transparent to the software + updater; urllib-type calls will be intercepted and TUF metadata automatically + fetched along with the packages requested by the software updater. + + This module provides two classes: Updater and UpdaterController: + + 'tuf.interposition.updater.Updater' contains those methods which are to be + performed on each individual updater. For example: refresh(), cleanup(), + download_target(target_filepath), get_target_filepath(source_url), open(url), + retrieve(url), switch_context(); all these methods act on particular updater. + + 'tuf.interposition.updater.UpdaterController' contains those methods which + are performed on updaters as a group. It basically keeps track of all the + updaters. For example: add(configuration), get(configuration), + refresh(configuration), remove(configuration), all these are performed on the + list of updaters. 'tuf.interposition.updater.UpdaterController' maintains a + map of updaters and a set of its mirrors. The map of updaters contains the + objects of 'tuf.interposition.updater.Updater' for each updater. The set + contains all the mirrors. The addition and removal of these updaters and + their mirrors depends on the methods of + 'tuf.interposition.updater.UpdaterController'. + + + + To integrate TUF into a software updater with interposition, integrators only + need to complete two main tasks: First, a JSON configuration file for + interposition is created. Second, the software updater is modified to import + the interposition library and configure interposition. + + 1. 'interposition.py' (code included below) is a basic example software + updater that is integrating TUF with interposition. + + # First import the interposition package, which contains all of the + # required classes and functions to use TUF and interposition. + import tuf.interposition + + # Next, explicitly import the urllib modules that interposition will be + # interposing/overwriting. 'urllib_tuf' and 'urllib2_tuf' are TUF's + # copies of urllib and urllib2 that are modified to perform updates using + # the framework and the TUF metadata. + from tuf.interposition import urllib_tuf as urllib + from tuf.interposition import urllib2_tuf as urllib2 + + # The configure() method must now be called. It takes 3 optional + # arguments, one of which is the filename of a JSON configuration file. + # This JSON file contains a set of configurations. To make this file, + # follow the second point below. Ways to call this method are as follows: + # First, configure() - By default, the configuration object is expected to + # be located in the current working directory in the file with the name + # "tuf.interposition.json". Second, configure(filename="/path/to/json") + # Configure() returns a dictionary of configurations. Internally, + # configure() calls add(configuration) function which is in the + # 'tuf.interposition.updater.UpdaterController' class. + configurations = tuf.interposition.configure() + + url = 'http://example.com/path/to/file' + + # This is the standard way of opening and retrieving URLs in Python. + # All three urllib calls below are intercepted by TUF's interposition. + urllib.urlopen(url) + urllib.urlretrieve(url) + urllib2.urlopen(url) + + # Remove TUF interposition for previously read configurations. That is + # remove the updater object. + # Deconfigure() takes only one argument (i.e. configurations). + # It calls the remove(configuration) function which is in + # 'tuf.interposition.updater.UpdaterController'. + tuf.interposition.deconfigure(configurations) + + + 2. The filename passed as a argument to configure() is a JSON file. + It is loaded as a JSON object, which tells tuf.interposition which URLs to + intercept, how to transform them (if necessary), and where to forward them + (possibly over SSL) for secure responses via TUF. By default, the name of + the file is tuf.interposition.json. An example of a configuration file + follows. + + # configurations are simply a JSON object that allows you to answer + # these questions - + # - Which network locations get intercepted? + # - Given a network location, which TUF mirrors should we forward + # requests to? + # - Given a network location, which paths should be intercepted? + # - Given a TUF mirror, how do we verify its SSL certificate? + { + # This is a required root object. + "configurations": { + # Which network location should be intercepted? + # Network locations may be specified as "hostname" or "hostname:port". + "localhost": { + + # Where do we find the client copy of the TUF server metadata? + "repository_directory": ".", + + # Where do we forward the requests to localhost? + "repository_mirrors" : { + "mirror1": { + # In this case, we forward them to http://localhost:8001 + "url_prefix": "http://localhost:8001", + + # You do not have to worry about these default parameters. + "metadata_path": "metadata", + "targets_path": "targets", + "confined_target_dirs": [""] + } + } + } + } + + # After creating 'tuf.configuration.json' and the example updater module, run + # 'interposition.py'. The urllib calls will be intercepted, and information + # about the update process is generated to a log file named 'tuf.log' in the + # same directory, which can be reviewed. +""" + +# Help with Python 3 compatibility where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import mimetypes import os.path import re import shutil import tempfile -import urllib -import urlparse - +import logging import tuf.client.updater import tuf.conf +import tuf.log +import tuf._vendor.six as six + +from tuf.interposition.configuration import Configuration -# We import them directly into our namespace so that there is no name conflict. -from configuration import Configuration, InvalidConfiguration -from utility import Logger, InterpositionException - - - - - -################################ GLOBAL CLASSES ################################ - - - - - -class URLMatchesNoPattern(InterpositionException): - """URL matches no user-specified regular expression pattern.""" - pass - - - +logger = logging.getLogger('tuf.interposition.updater') class Updater(object): - """I am an Updater model.""" + """ + + Provide a class that can download target files securely. It performs all + the actions of 'tuf/client/updater.py', but adds methods to handle HTTP + requests and multiple updater objects. + + + refresh(): + This method refreshes top-level metadata. It calls the refresh() method of + 'tuf.client.updater'. refresh() method of 'tuf.client.updater' downloads, + verifies, and loads metadata of the top-level roles in a specific order + (i.e., timestamp -> snapshot -> root -> targets). The expiration time for + downloaded metadata is also verified. + + cleanup(): + It will clean up all the temporary directories that are made following a + download request. It also logs a message when a temporary file/directory + is deleted. + + download_target(target_filepath): + It downloads the 'target_filepath' repository file. It also downloads any + required metadata to securely satisfy the 'target_filepath' request. + + get_target_filepath(source_url): + 'source_url' is the URL of the file to be updated. This method will find + the updated target for this file. + + open(url, data): + Open the 'url' URL, which can either be a string or a request object. + The file is opened in the binary read mode as a temporary file. + + retrieve(url, filename, reporthook, data): + retrieve() method first gets the target file path by calling + get_target_filepath(url), which is in 'tuf.interposition.updater.Updater' + and then calls download_target() for the above file path. + + switch_context(): + There is an updater object for each network location that is interposed. + Context switching is required because there are multiple + 'tuf.client.updater' objects and each one depends on tuf.conf settings + that are shared. + """ def __init__(self, configuration): - CREATED_TEMPDIR_MESSAGE = "Created temporary directory at {tempdir}" + """ + + Constructor for an updater object that may be used to satisfy TUF update + requests, and can be used independent of other updater objects. A + temporary directory is created when this updater object is instantiated, + which is needed by 'tuf.interposition.updater.Updater', and the top-level + roles are refreshed. The 'tuf.client.updater' module performs the + low-level calls. + + + configuration: + A dictionary holding information like the following: + + - Which network location get intercepted? + - Given a network location, which TUF mirrors should we forward requests + to? + - Given a network location, which paths should be intercepted? + - Given a TUF mirror, how do we verify its SSL certificate? + + This dictionary holds repository mirror information, conformant to + 'tuf.formats.MIRRORDICT_SCHEMA'. Information such as the directory + containing the metadata and target files, the server's URL prefix, and + the target directories the client should be confined to. + + repository_mirrors = {'mirror1': {'url_prefix': 'http://localhost:8001', + 'metadata_path': 'metadata', + 'targets_path': 'targets', + 'confined_target_dirs': ['']}} + + + tuf.FormatError: + If the arguments of 'tuf.client.updater.Updater' are improperly + formatted. + + tuf.RepositoryError: + If there is an error with the updater's repository files, such + as a missing 'root.json' file. + + tuf.NoWorkingMirrorError: + If while refreshing, the metadata for any of the top-level roles cannot + be updated. + + tuf.ExpiredMetadataError: + While refreshing, if any metadata has expired. + + + The metadata files (e.g., 'root.json', 'targets.json') for the top-level + roles are read from disk and stored in dictionaries. + + + None. + """ self.configuration = configuration + # A temporary directory used for this updater over runtime. self.tempdir = tempfile.mkdtemp() - Logger.debug(CREATED_TEMPDIR_MESSAGE.format(tempdir=self.tempdir)) + logger.debug('Created temporary directory at ' + repr(self.tempdir)) - # must switch context before instantiating updater - # because updater depends on some module (tuf.conf) variables + # Switching context before instantiating updater because updater depends + # on some module (tuf.conf) variables. self.switch_context() + + # Instantiating a 'tuf.client.updater' object causes all the configurations + # for the top-level roles to be read from disk, including the key and role + # information for the delegated targets of 'targets'. The actual metadata + # for delegated roles is not loaded in __init__. The metadata for these + # delegated roles, including nested delegated roles, are loaded, updated, + # and saved to the 'self.metadata' store by the target methods, like + # all_targets() and targets_of_role(). self.updater = tuf.client.updater.Updater(self.configuration.hostname, self.configuration.repository_mirrors) - # Update the client's top-level metadata. The download_target() method does - # not automatically refresh top-level prior to retrieving target files and - # their associated Targets metadata, so update the top-level - # metadata here. - Logger.info('Refreshing top-level metadata for interposed '+repr(configuration)) + # Update the client's top-level metadata. The download_target() method + # does not automatically refresh top-level prior to retrieving target files + # and their associated Targets metadata, so update the top-level metadata + # here. + logger.info('Refreshing top-level metadata for interposed ' + repr(configuration)) self.updater.refresh() def refresh(self): - """Refresh top-level metadata""" + """ + + This method refreshes the top-level metadata. It calls the refresh() + method of 'tuf.client.updater'. refresh() method of + 'tuf.client.updater.py' downloads, verifies, and loads metadata for the + top-level roles in a specific order (i.e., timestamp -> snapshot -> root + -> targets) The expiration time for downloaded metadata is also verified. + + This refresh() method should be called by the client before any target + requests. Therefore to automate the process, it is called here. + + + None + + + tuf.NoWorkingMirrorError: + If the metadata for any of the top-level roles cannot be updated. + + tuf.ExpiredMetadataError: + If any metadata has expired. + + + Updates the metadata files of the top-level roles with the latest + information. + + + None + """ + self.updater.refresh() def cleanup(self): - """Clean up after certain side effects, such as temporary directories.""" + """ + + Remove the updater object's temporary directory (and any sub-directories) + created when the updater object is instantiated to store downloaded + targets and metadata. + + + None - DELETED_TEMPDIR_MESSAGE = "Deleted temporary directory at {tempdir}" + + None + + + Removal of the temporary 'self.tempdir' directory. + + + None + """ + shutil.rmtree(self.tempdir) - Logger.debug(DELETED_TEMPDIR_MESSAGE.format(tempdir=self.tempdir)) + logger.debug('Deleted temporary directory at ' + repr(self.tempdir)) def download_target(self, target_filepath): - """Downloads target with TUF as a side effect.""" + """ + + Download the 'target_filepath' target file. Everything here is performed + in a temporary directory. It identifies the target information for + 'target_filepath' by calling the target() method of 'tuf.client.updater'. + This method also downloads the metadata of the updated targets. By doing + this, the client retrieves the target information for the targets they + want to update. When client retrieves all the information, the + updated_targets() method of 'tuf.client.updater' is called to determine + the list of targets which have been changed from those saved locally on + disk. tuf.client.upater.download_target() downloads all the targets in + the list in the destination directory, which is our temporary directory. + + This will only store the file in the temporary directory if the + downloaded file matches the description of the file in the trusted + metadata. - # download file into a temporary directory shared over runtime + + target_filepath: + The target's relative path on the remote repository. + + + tuf.FormatError: + If 'target_filepath', 'updated_target' in + 'tuf.client.updater.download_target', is improperly formatted. + + tuf.UnknownTargetError: + If 'target_filepath' was not found. + + tuf.NoWorkingMirrorError: + If a 'target_filepath' could not be downloaded from any of the mirrors. + + + A target file is saved to the local system. + + + It returns a (destination directory, filename) tuple where the target is + been stored and filename of the target file been stored in the directory. + """ + + tuf.formats.RELPATH_SCHEMA.check_match(target_filepath) + + # Download file into a temporary directory shared over runtime destination_directory = self.tempdir + # A new path is generated by joining the destination directory path that is + # our temporary directory path and target file path. # Note: join() discards 'destination_directory' if 'target_filepath' # contains a leading path separator (i.e., is treated as an absolute path). - filename = os.path.join(destination_directory, target_filepath.lstrip(os.sep)) + filename = \ + os.path.join(destination_directory, target_filepath.lstrip(os.sep)) - # Switch TUF context. + # Switch TUF context. Switching context before instantiating updater + # because updater depends on some module (tuf.conf) variables. self.switch_context() # Locate the fileinfo of 'target_filepath'. updater.target() searches - # Targets metadata in order of trust, according to the currently trusted + # targets metadata in order of trust, according to the currently trusted # snapshot. To prevent consecutive target file requests from referring to # different snapshots, top-level metadata is not automatically refreshed. + # It returns the target information for a specific file identified by its + # file path. This target method also downloads the metadata of updated + # targets. targets = [self.updater.target(target_filepath)] # TODO: targets are always updated if destination directory is new, right? - updated_targets = self.updater.updated_targets(targets, destination_directory) + # After the client has retrieved the target information for those targets + # they are interested in updating, updated_targets() method is called to + # determine which targets have changed from those saved locally on disk. + # All the targets that have changed are returned in a list. From this list, + # a request to download is made by calling 'download_target()'. + updated_targets = \ + self.updater.updated_targets(targets, destination_directory) + # The download_target() method in tuf.client.updater performs the actual + # download of the specified target. The file is saved to the + # 'destination_directory' argument. for updated_target in updated_targets: self.updater.download_target(updated_target, destination_directory) return destination_directory, filename - # TODO: decide prudent course of action in case of failure + # TODO: decide prudent course of action in case of failure. def get_target_filepath(self, source_url): - """Given source->target map, figure out what TUF *should* download given a - URL.""" + """ + + Given source->target map, this method will figure out what TUF should + download when a URL is given. + + + source_url: + The URL of the target we want to retrieve. - WARNING_MESSAGE = "Possibly invalid target_paths for " + \ - "{network_location}! No TUF interposition for {url}" + + tuf.URLMatchesNoPatternError: + This exception is raised when no target_path url pattern is wrong and + does match regular expression. - parsed_source_url = urlparse.urlparse(source_url) + + None + + + If the target filepath is matched, return the filepath, otherwise raise + an exception. + """ + + parsed_source_url = six.moves.urllib.parse.urlparse(source_url) target_filepath = None try: @@ -116,14 +458,17 @@ def get_target_filepath(self, source_url): # how to map the source URL to a target URL understood by TUF? for target_path in self.configuration.target_paths: + #TODO: What these two lines are doing? # target_path: { "regex_with_groups", "target_with_group_captures" } # e.g. { ".*(/some/directory)/$", "{0}/index.html" } - source_path_pattern, target_path_pattern = target_path.items()[0] - source_path_match = re.match(source_path_pattern, parsed_source_url.path) + source_path_pattern, target_path_pattern = list(target_path.items())[0] + source_path_match = \ + re.match(source_path_pattern, parsed_source_url.path) # TODO: A failure in string formatting is *critical*. if source_path_match is not None: - target_filepath = target_path_pattern.format(*source_path_match.groups()) + target_filepath = \ + target_path_pattern.format(*source_path_match.groups()) # If there is more than one regular expression which # matches source_url, we resolve ambiguity by order of @@ -133,19 +478,54 @@ def get_target_filepath(self, source_url): # If source_url does not match any regular expression... if target_filepath is None: # ...then we raise a predictable exception. - raise URLMatchesNoPattern(source_url) + raise tuf.URLMatchesNoPatternError(source_url) except: - Logger.exception(WARNING_MESSAGE.format( - network_location=self.configuration.network_location, url=source_url)) + logger.exception('Possibly invalid target_paths for ' + \ + repr(self.configuration.network_location) + \ + '! No TUF interposition for ' + repr(source_url)) raise else: return target_filepath - # TODO: distinguish between urllib and urllib2 contracts + # TODO: distinguish between urllib and urllib2 contracts. def open(self, url, data=None): + """ + + Open the URL url which can either be a string or a request object. + The file is opened in the binary read mode as a temporary file. This is + called when TUF wants to open an already existing updater's 'url'. + + + url: + The one which is to be opened. + + data: + Must be a bytes object specifying additional data to be sent to the + server or None, if no such data needed. + + + tuf.FormatError: + TODO: validate arguments. + + tuf.NoWorkingMirrorError: + If a 'target_filepath' could not be downloaded from any of the mirrors. + + tuf.URLMatchesNoPatternError: + This exception is raised when no target_path url pattern is wrong and + does match regular expression. + + + None + + + 'response' which is a file object with info() and geturl() methods added. + """ + + # TODO: validate arguments. + filename, headers = self.retrieve(url, data=data) # TUF should always open files in binary mode and remain transparent to the @@ -153,21 +533,61 @@ def open(self, url, data=None): # end-of-line characters and prevents binary files from properly loading on # Windows. # http://docs.python.org/2/tutorial/inputoutput.html#reading-and-writing-files - # TODO: like tempfile, ensure file is deleted when closed? + # TODO: like tempfile, ensure file is deleted when closed? open() in the + # line below is a predefined function in python. temporary_file = open(filename, 'rb') + #TODO: addinfourl is not in urllib package anymore. We need to check if + # other option for this is working or not. # Extend temporary_file with info(), getcode(), geturl() # http://docs.python.org/2/library/urllib.html#urllib.urlopen - response = urllib.addinfourl(temporary_file, headers, url, code=200) + # addinfourl() works as a context manager. + response = six.moves.urllib.response.addinfourl(temporary_file, headers, + url, code=200) return response # TODO: distinguish between urllib and urllib2 contracts def retrieve(self, url, filename=None, reporthook=None, data=None): - INTERPOSITION_MESSAGE = "Interposing for {url}" + """ + + Get the target file path by calling self.get_target_filepath(url) and + then self.download_target() method for the above file path. - Logger.info(INTERPOSITION_MESSAGE.format(url=url)) + + url: + The URL of the target file to retrieve. + + filename: + If given, then the given filename is used. If the filename is none, + then temporary file is used. + + + tuf.FormatError: + If 'target_filepath', 'updated_target' in + tuf.client.updater.download_target and arguments of updated_targets are + improperly formatted. + + tuf.UnknownTargetError: + If 'target_filepath' was not found. + + tuf.NoWorkingMirrorError: + If a 'target_filepath' could not be downloaded from any of the mirrors. + + tuf.URLMatchesNoPatternError: + This exception is raised when no target_path url pattern is wrong and + does match regular expression. + + + A target file is saved to the local system when the + download_target(target_filepath) is called. + + + It returns the filename and the headers of the file just retrieved. + """ + + logger.info('Interposing for '+ repr(url)) # What is the actual target to download given the URL? Sometimes we would # like to transform the given URL to the intended target; e.g. "/simple/" @@ -186,108 +606,308 @@ def retrieve(self, url, filename=None, reporthook=None, data=None): } # Download the target filepath determined by the original URL. - temporary_directory, temporary_filename = self.download_target(target_filepath) + temporary_directory, temporary_filename = \ + self.download_target(target_filepath) if filename is None: - # If no filename is given, use the temporary file. - filename = temporary_filename + # If no filename is given, use the temporary file. + filename = temporary_filename else: - # Otherwise, copy TUF-downloaded file in its own directory - # to the location user specified. - shutil.copy2(temporary_filename, filename) + # Otherwise, copy TUF-downloaded file in its own directory + # to the location user specified. + shutil.copy2(temporary_filename, filename) return filename, headers - # TODO: thread-safety, perhaps with a context manager + # TODO: thread-safety, perhaps with a context manager. def switch_context(self): - # Set the local repository directory containing the metadata files. - tuf.conf.repository_directory = self.configuration.repository_directory + """ + + There is an updater object for each network location that is interposed. + Context switching is required because there are multiple + 'tuf.client.updater' objects and each one depends on tuf.conf settings + that are shared. - # Set the local SSL certificates PEM file. - tuf.conf.ssl_certificates = self.configuration.ssl_certificates + For this, two settings are required: + 1. Setting local repository directory in 'tuf.conf'. + 2. Setting the local SSL certificate PEM file. + + None + + + None + + + The given configuration's 'repository_directory' and ssl_certificates + settings are set to 'tuf.conf.repository_directory' and + 'tuf.conf.ssl_certificates', respectively. + + + None + """ + + # Set the local repository directory containing the metadata files. + tuf.conf.repository_directory = self.configuration.repository_directory + + # Set the local SSL certificates PEM file. + tuf.conf.ssl_certificates = self.configuration.ssl_certificates class UpdaterController(object): """ - I am a controller of Updaters; given a Configuration, I will build and - store an Updater which you can get and use later. + + A controller of Updater() objects. Given a configuration, it can build and + store an Updater, which can be used later with the help of get() method. + + + + __init__(): + It creates and initializes an empty private map of updaters and an empty + private set of repository mirror network locations (hostname:port). This + is used to store the updaters added by TUF and later on TUF can get these + updater to reutilize. + + __check_configuration_on_add(configuration): + It checks if the given configuration is valid or not. + + add(configuration): + This method adds the updater by adding an object of + 'tuf.interposition.updater.Updater' in the __updater map and by adding + repository mirror's network location in the empty set initialized when + the object of 'tuf.interposition.updater.UpdaterController' is created. + + get(url): + Get the updater if it already exists. It takes the url and parses it. + Then it utilizes hostname and port of that url to check if it already + exists or not. If the updater exists, then it calls the + get_target_filepath() method which returns a target file path to be + downloaded. + + refresh(configuration): + Refreshes the top-level metadata of the given 'configuration'. It + updates the latest copies of the metadata of the top-level roles. + + remove(configuration): + Remove an Updater matching the given 'configuration' as well as its + associated mirrors. """ def __init__(self): + """ + + Initalize a private map of updaters and a private set of repository + mirror network locations (hostname:port) once the object of + 'tuf.interposition.updater.UpdaterController' is created. This empty map + and set is later used to add, get, and remove updaters and their mirrors. + + + None + + + None + + + An empty map called '__updaters' and an empty set called + '__repository_mirror_network_locations' is created. + + + None + """ + # A private map of Updaters (network_location: str -> updater: Updater) self.__updaters = {} - # A private set of repository mirror hostnames - self.__repository_mirror_hostnames = set() + # A private set of repository mirror network locations + self.__repository_mirror_network_locations = set() def __check_configuration_on_add(self, configuration): """ - If the given Configuration is invalid, I raise an exception. - Otherwise, I return some information about the Configuration, - such as repository mirror hostnames. + + If the given Configuration is invalid, an exception is raised. + Otherwise, repository mirror network locations are returned. + + + 'configuration' contains hostname, port number, repository mirrors which + are to be checked if they are valid or not. + + + tuf.InvalidConfigurationError: + If the configuration is invalid. For example - wrong hostname, invalid + port number, wrong mirror format. + + tuf.FormatError: + If the network_location is not unique or configuration.network_location + is same as repository_mirror_network_locations. + + + It logs the error message. + + + 'repository_mirror_network_locations' + In order to prove that everything worked well, a part of configuration + is returned which is the list of repository mirrors. """ - INVALID_REPOSITORY_MIRROR = "Invalid repository mirror {repository_mirror}!" - # Updater has a "global" view of configurations, so it performs - # additional checks after Configuration's own local checks. - assert isinstance(configuration, Configuration) + # additional checks after Configuration's own local checks. This will + # check if everything in tuf.interposition.configuration.ConfigurationParser + # worked or not. - # Restrict each (incoming, outgoing) hostname pair to be unique across + # According to __read_configuration() method in + # tuf.interposition.__init__, + # configuration is an instance of + # tuf.interposition.configuration.Configuration because in this method - + # configuration = configuration_parser.parse() + # configuration_parser is an instance of + # tuf.interposition.configuration.ConfigurationParser + # The configuration_parser.parse() returns + # tuf.interposition.configuration.Configuration as an object which makes + # configuration an instance of tuf.interposition.configuration.Configuration + if not isinstance(configuration, Configuration): + raise tuf.InvalidConfigurationError('Invalid configuration') + + # Restrict each (incoming, outgoing) network location pair to be unique across # configurations; this prevents interposition cycles, amongst other # things. # GOOD: A -> { A:X, A:Y, B, ... }, C -> { D }, ... # BAD: A -> { B }, B -> { C }, C -> { A }, ... - assert configuration.hostname not in self.__updaters - assert configuration.hostname not in self.__repository_mirror_hostnames + if configuration.network_location in self.__updaters: + message = 'Updater with ' + repr(configuration.network_location) + \ + ' already exists as an updater.' + raise tuf.FormatError() + + if configuration.network_location in self.__repository_mirror_network_locations: + message = 'Updater with ' + repr(configuration.network_location) + \ + ' already exists as a mirror.' + raise tuf.FormatError(message) # Check for redundancy in server repository mirrors. - repository_mirror_hostnames = configuration.get_repository_mirror_hostnames() + repository_mirror_network_locations = \ + configuration.get_repository_mirror_hostnames() - for mirror_hostname in repository_mirror_hostnames: + for mirror_network_location in repository_mirror_network_locations: try: - # Restrict each hostname in every (incoming, outgoing) pair to be + # Restrict each network location in every (incoming, outgoing) pair to be # unique across configurations; this prevents interposition cycles, # amongst other things. - assert mirror_hostname not in self.__updaters - assert mirror_hostname not in self.__repository_mirror_hostnames + if mirror_network_location in self.__updaters: + message = 'Mirror with ' + repr(mirror_network_location) + \ + ' already exists as an updater.' + raise tuf.FormatError(message) + + if mirror_network_location in self.__repository_mirror_network_locations: + message = 'Mirror with ' + repr(mirror_network_location) + \ + ' already exists as a mirror.' + raise tuf.FormatError(message) - except: - error_message = \ - INVALID_REPOSITORY_MIRROR.format(repository_mirror=mirror_hostname) - Logger.exception(error_message) - raise InvalidConfiguration(error_message) - - return repository_mirror_hostnames + except (tuf.FormatError) as e: + error_message = 'Invalid repository mirror ' + \ + repr(mirror_network_location) + logger.exception(error_message) + raise + return repository_mirror_network_locations def add(self, configuration): - """Add an Updater based on the given Configuration.""" - - repository_mirror_hostnames = self.__check_configuration_on_add(configuration) + """ + + Add an Updater based on the given 'configuration'. TUF keeps track of the + updaters so that it can be fetched for later use. - # If all is well, build and store an Updater, and remember hostnames. - Logger.info('Adding updater for interposed '+repr(configuration)) - self.__updaters[configuration.hostname] = Updater(configuration) - self.__repository_mirror_hostnames.update(repository_mirror_hostnames) - + + 'configuration' is an object and on the basis of this configuration, an + updater will be added. + + + tuf.InvalidConfigurationError: + If the configuration is invalid. For example - wrong hostname, invalid + port number, wrong mirror format. + + tuf.FormatError: + This exception is raised if the network location which tuf is trying to + add is not unique. + + + The object of 'tuf.interposition.updater.Updater' is added in the list of + updaters. Also, the mirrors of this updater are added to + 'repository_mirror_network_locations'. + + + None + """ + + repository_mirror_network_locations = \ + self.__check_configuration_on_add(configuration) + + # If all is well, build and store an Updater, and remember network + # locations. + logger.info('Adding updater for interposed ' + repr(configuration)) + + # Adding an object of the tuf.interposition.updater.Updater with the given + # configuration. + self.__updaters[configuration.network_location] = Updater(configuration) + + # Adding the new the repository mirror network locations to the list. + self.__repository_mirror_network_locations.update(repository_mirror_network_locations) def refresh(self, configuration): - """Refresh the top-level metadata of the given Configuration.""" + """ + + To refresh the top-level metadata of the given 'configuration'. + It updates the latest copies of the metadata for the top-level roles. - assert isinstance(configuration, Configuration) + + 'configuration' is the object containing the configurations of the updater + to be refreshed. - repository_mirror_hostnames = configuration.get_repository_mirror_hostnames() + + tuf.InvalidConfigurationError: + If there is anything wrong with the Format of the configuration, this + exception is raised. - assert configuration.hostname in self.__updaters - assert repository_mirror_hostnames.issubset(self.__repository_mirror_hostnames) + tuf.NotFoundError: + If the updater to be refreshed is not found in the list of updaters or + mirrors, then tuf.NotFoundError exception is raised. + + tuf.NoWorkingMirrorError: + If the metadata for any of the top-level roles cannot be updated. + + tuf.ExpiredMetadataError: + If any metadata has expired. + + + It refreshes the updater and indicate this in the log file. + + + None + """ + + # Check if the configuration is valid else raise an exception. + if not isinstance(configuration, Configuration): + raise tuf.InvalidConfigurationError('Invalid configuration') + + # Get the repository mirrors of the given configuration. + repository_mirror_network_locations = \ + configuration.get_repository_mirror_hostnames() + + # Check if the configuration.network_location is available in the updater + # or mirror list. + if configuration.network_location not in self.__updaters: + message = 'Update with ' + repr(configuration.network_location) + \ + ' not found.' + raise tuf.NotFoundError(message) + + if not repository_mirror_network_locations.issubset(self.__repository_mirror_network_locations): + message = 'Mirror with ' + repr(repository_mirror_network_locations) + \ + ' not found.' + raise tuf.NotFoundError(message) # Get the updater and refresh its top-level metadata. In the majority of # integrations, a software updater integrating TUF with interposition will @@ -297,46 +917,73 @@ def refresh(self, configuration): # Although interposition was designed to remain transparent, for software # updaters that require an explicit refresh of top-level metadata, this # method is provided. - Logger.info('Refreshing top-level metadata for '+ repr(configuration)) - updater = self.__updaters.get(configuration.hostname) + logger.info('Refreshing top-level metadata for ' + repr(configuration)) + + # If everything is good then fetch the updater from __updaters with the + # given configurations. + updater = self.__updaters.get(configuration.network_location) + + # Refresh the fetched updater. updater.refresh() - def get(self, url): - """Get an Updater, if any, for this URL. + """ + + This method is to get the updater if it already exists. It takes the url + and parse it. Then it utilizes hostname and port of that url to check if + it already exists or not. If the updater exists, then it calls the + get_target_filepath() method which returns a target file path to be + downloaded. - Assumptions: - - @url is a string.""" - - GENERIC_WARNING_MESSAGE = "No updater or interposition for url={url}" - DIFFERENT_NETLOC_MESSAGE = "We have an updater for netloc={netloc1} but not for netlocs={netloc2}" - HOSTNAME_FOUND_MESSAGE = "Found updater for interposed network location: {netloc}" - HOSTNAME_NOT_FOUND_MESSAGE = "No updater for hostname={hostname}" + + url: + URL which TUF is trying to get an updater. Assumption that url is a + string. + + + None + + + This method logs the messages in a log file if updater is not found or + not for the given url. + + + The get() method returns the updater with the given configuration. If + updater does not exists, it returns None. + """ updater = None try: - parsed_url = urlparse.urlparse(url) + # Parse the given url to access individual parts of it. + parsed_url = six.moves.urllib.parse.urlparse(url) hostname = parsed_url.hostname port = parsed_url.port or 80 netloc = parsed_url.netloc - network_location = "{hostname}:{port}".format(hostname=hostname, port=port) - # Sometimes parsed_url.netloc does not have a port (e.g. 80), - # so we do a double check. + # Combine the hostname and port number and assign it to network_location. + # The combination of hostname and port is used to identify an updater. + network_location = \ + "{hostname}:{port}".format(hostname=hostname, port=port) + + # There can be a case when parsed_url.netloc does not have a port (e.g. + # 80). To avoid errors because of this case, tuf.interposition again set + # the parameters. network_locations = set((netloc, network_location)) - updater = self.__updaters.get(hostname) + updater = self.__updaters.get(network_location) if updater is None: - Logger.warn(HOSTNAME_NOT_FOUND_MESSAGE.format(hostname=hostname)) + logger.warning('No updater for ' + repr(hostname)) else: # Ensure that the updater is meant for this (hostname, port). if updater.configuration.network_location in network_locations: - Logger.info(HOSTNAME_FOUND_MESSAGE.format(netloc=network_location)) + logger.info('Found updater for interposed network location: '+ \ + repr(network_location)) + # Raises an exception in case we do not recognize how to # transform this URL for TUF. In that case, there will be no # updater for this URL. @@ -344,45 +991,78 @@ def get(self, url): else: # Same hostname, but different (not user-specified) port. - Logger.warn(DIFFERENT_NETLOC_MESSAGE.format( - netloc1=updater.configuration.network_location, netloc2=network_locations)) + logger.warning('We have an updater for ' + \ + repr(updater.configuration.network_location) + \ + 'but not for ' + repr(network_locations)) updater = None except: - Logger.exception(GENERIC_WARNING_MESSAGE.format(url=url)) + logger.exception('No updater or interposition for ' + repr(url)) updater = None finally: if updater is None: - Logger.warn(GENERIC_WARNING_MESSAGE.format(url=url)) + logger.warning('No updater or interposition for ' + repr(url)) return updater def remove(self, configuration): - """Remove an Updater matching the given Configuration.""" + """ + + Remove an Updater matching the given 'configuration', as well as its + associated mirrors. + + + 'configuration' is the configuration object of the updater to be removed. - UPDATER_REMOVED_MESSAGE = "Updater removed for interposed {configuration}." + + tuf.InvalidConfigurationError: + If there is anything wrong with the configuration for example invalid + hostname, invalid port number etc, tuf.InvalidConfigurationError is + raised. - assert isinstance(configuration, Configuration) + tuf.NotFoundError: + If the updater with the given configuration does not exists, + tuf.NotFoundError exception is raised. - repository_mirror_hostnames = configuration.get_repository_mirror_hostnames() + + Removes the stored updater and the mirrors associated with that updater. + Then tuf logs this information in a log file. - assert configuration.hostname in self.__updaters - assert repository_mirror_hostnames.issubset(self.__repository_mirror_hostnames) + + None + """ + + # Check if the given configuration is valid or not. + if not isinstance(configuration, Configuration): + raise tuf.InvalidConfigurationError('Invalid configuration') + + # If the configuration is valid, get the repository mirrors associated with + # it. + repository_mirror_network_locations = \ + configuration.get_repository_mirror_hostnames() + + # Check if network location of the given configuration exists or not. + if configuration.network_location not in self.__updaters: + raise tuf.NotFoundError('Network location not found') + + # Check if the associated mirrors exists or not. + if not repository_mirror_network_locations.issubset(self.__repository_mirror_network_locations): + raise tuf.NotFoundError('Repository mirror does not exists') # Get the updater. - updater = self.__updaters.get(configuration.hostname) + updater = self.__updaters.get(configuration.network_location) - # If all is well, remove the stored Updater as well as its associated - # repository mirror hostnames. + # If everything works well, remove the stored Updater as well as its + # associated repository mirror network locations. updater.cleanup() - del self.__updaters[configuration.hostname] - self.__repository_mirror_hostnames.difference_update(repository_mirror_hostnames) - - Logger.info(UPDATER_REMOVED_MESSAGE.format(configuration=configuration)) - - - - + + # Delete the updater from the list of updaters. + del self.__updaters[configuration.network_location] + + # Remove the associated mirrors from the repository mirror set. + self.__repository_mirror_network_locations.difference_update(repository_mirror_network_locations) + # Log the message that the given updater is removed. + logger.info('Updater removed for interposed ' + repr(configuration)) diff --git a/tuf/interposition/utility.py b/tuf/interposition/utility.py deleted file mode 100644 index 8e1c4cd9..00000000 --- a/tuf/interposition/utility.py +++ /dev/null @@ -1,43 +0,0 @@ -import logging - - -# Import our standard logger for its side effects. -import tuf.log - - - - - -class InterpositionException(Exception): - """Base exception class.""" - pass - - - - - -class Logger(object): - """A static logging object for tuf.interposition.""" - - - __logger = logging.getLogger("tuf.interposition") - - - @staticmethod - def debug(message): - Logger.__logger.debug(message) - - - @staticmethod - def exception(message): - Logger.__logger.exception(message) - - - @staticmethod - def info(message): - Logger.__logger.info(message) - - - @staticmethod - def warn(message): - Logger.__logger.warn(message) diff --git a/tuf/keydb.py b/tuf/keydb.py index dbb34670..60ac62fb 100755 --- a/tuf/keydb.py +++ b/tuf/keydb.py @@ -164,9 +164,9 @@ def add_key(key_dict, keyid=None): # Raise 'tuf.FormatError' if the check fails. tuf.formats.KEYID_SCHEMA.check_match(keyid) - # Check if the keyid found in 'rsakey_dict' matches 'keyid'. + # Check if the keyid found in 'key_dict' matches 'keyid'. if keyid != key_dict['keyid']: - raise tuf.Error('Incorrect keyid '+key_dict['keyid']+' expected '+keyid) + raise tuf.Error('Incorrect keyid ' + key_dict['keyid'] + ' expected ' + keyid) # Check if the keyid belonging to 'rsakey_dict' is not already # available in the key database before returning. diff --git a/tuf/util.py b/tuf/util.py index ef426d77..ba401431 100755 --- a/tuf/util.py +++ b/tuf/util.py @@ -560,7 +560,7 @@ def find_delegated_role(roles, delegated_role): # This role has a different name. else: - continue + logger.debug('Skipping delegated role: ' + repr(delegated_role)) return role_index