From 03348f2dbbfb7ee90b1c802c39ffd6802155212c Mon Sep 17 00:00:00 2001 From: dachshund Date: Thu, 20 Jun 2013 19:56:34 +0800 Subject: [PATCH 01/28] Introduce name property for tuf.formats.ROLE_SCHEMA. --- tuf/formats.py | 17 ++++++++++++++--- tuf/tests/test_formats.py | 23 +++++++++++++++++++---- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/tuf/formats.py b/tuf/formats.py index 698e2463..fb0c2d43 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -267,11 +267,12 @@ targets_directory=PATH_SCHEMA, backup_directory=PATH_SCHEMA)) -# Role object in {'keyids': [keydids..], 'threshold': 1, 'paths':[filepaths..]} -# format. +# Role object in {'keyids': [keydids..], 'name': 'ABC', 'threshold': 1, +# 'paths':[filepaths..]} # format. ROLE_SCHEMA = SCHEMA.Object( object_name='role', keyids=SCHEMA.ListOf(KEYID_SCHEMA), + name=SCHEMA.Optional(NAME_SCHEMA), threshold=THRESHOLD_SCHEMA, paths=SCHEMA.Optional(SCHEMA.ListOf(RELPATH_SCHEMA))) @@ -281,6 +282,9 @@ key_schema=ROLENAME_SCHEMA, value_schema=ROLE_SCHEMA) +# Like ROLEDICT_SCHEMA, except that ROLE_SCHEMA instances are stored in order. +ROLELIST_SCHEMA = SCHEMA.ListOf(ROLENAME_SCHEMA) + # The root: indicates root keys and top-level roles. ROOT_SCHEMA = SCHEMA.Object( object_name='root', @@ -818,7 +822,7 @@ def make_fileinfo(length, hashes, custom=None): -def make_role_metadata(keyids, threshold, paths=None): +def make_role_metadata(keyids, threshold, name=None, paths=None): """ Create a dictionary conforming to 'tuf.formats.ROLE_SCHEMA', @@ -833,6 +837,9 @@ def make_role_metadata(keyids, threshold, paths=None): An integer denoting the number of required keys for the signing role. + name: + A string that is the name of this role. + paths: The 'Target' role stores the paths of target files in its metadata file. 'paths' is a list of @@ -856,6 +863,10 @@ def make_role_metadata(keyids, threshold, paths=None): role_meta = {} role_meta['keyids'] = keyids role_meta['threshold'] = threshold + + if name is not None: + role_meta['name'] = name + if paths is not None: role_meta['paths'] = paths diff --git a/tuf/tests/test_formats.py b/tuf/tests/test_formats.py index abb6152c..bf45adfa 100755 --- a/tuf/tests/test_formats.py +++ b/tuf/tests/test_formats.py @@ -491,23 +491,38 @@ def test_make_role_metadata(self): keyids = ['123abc', 'abc123'] threshold = 2 paths = ['path1/', 'path2'] + name = '123' ROLE_SCHEMA = tuf.formats.ROLE_SCHEMA make_role = tuf.formats.make_role_metadata - self.assertTrue(ROLE_SCHEMA.matches(make_role(keyids, threshold, paths))) + self.assertTrue(ROLE_SCHEMA.matches(make_role(keyids, threshold))) + self.assertTrue(ROLE_SCHEMA.matches(make_role(keyids, threshold, name=name))) + self.assertTrue(ROLE_SCHEMA.matches(make_role(keyids, threshold, paths=paths))) + self.assertTrue(ROLE_SCHEMA.matches(make_role(keyids, threshold, name=name, paths=paths))) # Test conditions for invalid arguments. bad_keyids = 'bad' bad_threshold = 'bad' bad_paths = 'bad' + bad_name = 123 - self.assertRaises(tuf.FormatError, make_role, bad_keyids, threshold, paths) - self.assertRaises(tuf.FormatError, make_role, keyids, bad_threshold, paths) - self.assertRaises(tuf.FormatError, make_role, keyids, threshold, bad_paths) self.assertRaises(tuf.FormatError, make_role, bad_keyids, threshold) self.assertRaises(tuf.FormatError, make_role, keyids, bad_threshold) + self.assertRaises(tuf.FormatError, make_role, bad_keyids, threshold, paths=paths) + self.assertRaises(tuf.FormatError, make_role, keyids, bad_threshold, paths=paths) + self.assertRaises(tuf.FormatError, make_role, keyids, threshold, paths=bad_paths) + + self.assertRaises(tuf.FormatError, make_role, bad_keyids, threshold, name=name) + self.assertRaises(tuf.FormatError, make_role, keyids, bad_threshold, name=name) + self.assertRaises(tuf.FormatError, make_role, keyids, threshold, name=bad_name) + + self.assertRaises(tuf.FormatError, make_role, bad_keyids, threshold, name=name, paths=paths) + self.assertRaises(tuf.FormatError, make_role, keyids, bad_threshold, name=name, paths=paths) + self.assertRaises(tuf.FormatError, make_role, keyids, threshold, name=bad_name, paths=paths) + self.assertRaises(tuf.FormatError, make_role, keyids, threshold, name=name, paths=bad_paths) + def test_get_role_class(self): From 4ab7d12998500aef1b0be643ff41ec977903b64b Mon Sep 17 00:00:00 2001 From: dachshund Date: Thu, 20 Jun 2013 20:43:24 +0800 Subject: [PATCH 02/28] Update tuf.formats.TARGET_SCHEMA and adjust tests.test_formats.py. --- tuf/formats.py | 6 +++--- tuf/tests/test_formats.py | 11 +++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tuf/formats.py b/tuf/formats.py index fb0c2d43..9f2fce05 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -272,7 +272,7 @@ ROLE_SCHEMA = SCHEMA.Object( object_name='role', keyids=SCHEMA.ListOf(KEYID_SCHEMA), - name=SCHEMA.Optional(NAME_SCHEMA), + name=SCHEMA.Optional(ROLENAME_SCHEMA), threshold=THRESHOLD_SCHEMA, paths=SCHEMA.Optional(SCHEMA.ListOf(RELPATH_SCHEMA))) @@ -283,7 +283,7 @@ value_schema=ROLE_SCHEMA) # Like ROLEDICT_SCHEMA, except that ROLE_SCHEMA instances are stored in order. -ROLELIST_SCHEMA = SCHEMA.ListOf(ROLENAME_SCHEMA) +ROLELIST_SCHEMA = SCHEMA.ListOf(ROLE_SCHEMA) # The root: indicates root keys and top-level roles. ROOT_SCHEMA = SCHEMA.Object( @@ -303,7 +303,7 @@ targets=FILEDICT_SCHEMA, delegations=SCHEMA.Optional(SCHEMA.Object( keys=KEYDICT_SCHEMA, - roles=ROLEDICT_SCHEMA))) + roles=ROLELIST_SCHEMA))) # A Release: indicates the latest versions of all metadata (except timestamp). RELEASE_SCHEMA = SCHEMA.Object( diff --git a/tuf/tests/test_formats.py b/tuf/tests/test_formats.py index bf45adfa..047828ef 100755 --- a/tuf/tests/test_formats.py +++ b/tuf/tests/test_formats.py @@ -199,9 +199,9 @@ def test_schemas(self): 'delegations': {'keys': {'123abc': {'keytype':'rsa', 'keyval': {'public': 'pubkey', 'private': 'privkey'}}}, - 'roles': {'root': {'keyids': ['123abc'], - 'threshold': 1, - 'paths': ['path1/', 'path2']}}}}), + 'roles': [{'name': 'root', 'keyids': ['123abc'], + 'threshold': 1, + 'paths': ['path1/', 'path2']}]}}), 'RELEASE_SCHEMA': (tuf.formats.RELEASE_SCHEMA, {'_type': 'Release', @@ -365,9 +365,8 @@ def test_TargetsFile(self): delegations = {'keys': {'123abc': {'keytype':'rsa', 'keyval': {'public': 'pubkey', 'private': 'privkey'}}}, - 'roles': {'root': {'keyids': ['123abc'], - 'threshold': 1, - 'paths': ['path1/', 'path2']}}} + 'roles': [{'name': 'root', 'keyids': ['123abc'], + 'threshold': 1, 'paths': ['path1/', 'path2']}]} make_metadata = tuf.formats.TargetsFile.make_metadata from_metadata = tuf.formats.TargetsFile.from_metadata From a8380e4768a0b0a907f7215f34a923fb3e9d8666 Mon Sep 17 00:00:00 2001 From: dachshund Date: Thu, 20 Jun 2013 21:24:53 +0800 Subject: [PATCH 03/28] Export tuf.tests; mock recursive walk in signercli.py. --- setup.py | 3 ++- tuf/tests/test_signercli.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 93a9e30b..83e9e87c 100755 --- a/setup.py +++ b/setup.py @@ -75,7 +75,8 @@ 'tuf.interposition', 'tuf.pushtools', 'tuf.pushtools.transfer', - 'tuf.repo' + 'tuf.repo', + 'tuf.tests' ], scripts=[ 'quickstart.py', diff --git a/tuf/tests/test_signercli.py b/tuf/tests/test_signercli.py index 2b8c96f3..ee36baa3 100755 --- a/tuf/tests/test_signercli.py +++ b/tuf/tests/test_signercli.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_signercli.py @@ -1350,8 +1352,10 @@ def _mock_prompt(msg, junk): return delegated_targets_dir elif msg.startswith('\nChoose and enter the parent'): return parent_role - elif msg.endswith('\nEnter the delegated role\'s name: '): + elif msg.startswith('\nEnter the delegated role\'s name: '): return delegated_role + elif msg.startswith('Recursively walk the given directory? (Y)es/(N)o: '): + return 'N' else: error_msg = ('Prompt: '+'\''+msg+'\''+ ' did not match any predefined mock prompts.') From 1edd6b611e87961dbf2063bfee70f8fcf33066e5 Mon Sep 17 00:00:00 2001 From: dachshund Date: Fri, 21 Jun 2013 00:54:29 +0800 Subject: [PATCH 04/28] Ensure uniqueness of names in making delegations. --- tuf/repo/signercli.py | 17 +++++++++--- tuf/repo/signerlib.py | 60 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/tuf/repo/signercli.py b/tuf/repo/signercli.py index aac3268c..1618d35f 100755 --- a/tuf/repo/signercli.py +++ b/tuf/repo/signercli.py @@ -1248,15 +1248,24 @@ def _update_parent_metadata(metadata_directory, delegated_role, delegated_keyids delegations['keys'] = keys # Update the 'roles' field. - roles = delegations.get('roles', {}) + roles = delegations.get('roles', []) threshold = len(delegated_keyids) delegated_role = parent_role+'/'+delegated_role relative_paths = [] for path in delegated_paths: relative_paths.append(os.path.sep.join(path.split(os.path.sep)[1:])) - roles[delegated_role] = tuf.formats.make_role_metadata(delegated_keyids, - threshold, - relative_paths) + role_metadata = tuf.formats.make_role_metadata(delegated_keyids, threshold, + name=delegated_role, + paths=relative_paths) + role_index = tuf.repo.signerlib.find_delegated_role(roles, delegated_role) + + if role_index is None: + # Append role to the end of the list of delegated roles. + roles.append(role_metadata) + else: + # Update role with the same name. + roles[role_index] = role_metadata + delegations['roles'] = roles # Update the larger metadata structure. diff --git a/tuf/repo/signerlib.py b/tuf/repo/signerlib.py index 9a1c7d0c..5b8580d9 100755 --- a/tuf/repo/signerlib.py +++ b/tuf/repo/signerlib.py @@ -1216,3 +1216,63 @@ def build_delegated_role_file(delegated_targets_directory, delegated_keyids, signable = sign_metadata(targets_metadata, delegated_keyids, targets_filepath) return write_metadata_file(signable, targets_filepath) + + + + + +def find_delegated_role(roles, delegated_role): + """ + + Find the index, if any, of a role with a given name in a list of roles. + + + roles: + The list of roles, each of which must have a name. + + delegated_role: + The name of the role to be found in the list of roles. + + + tuf.RepositoryError, if the list of roles has invalid data. + + + No known side effects. + + + None, if the role with the given name does not exist, or its unique index + in the list of roles. + + """ + + # Check argument types. + tuf.formats.ROLELIST_SCHEMA.check_match(roles) + tuf.formats.ROLENAME_SCHEMA.check_match(delegated_role) + + # The index of a role, if any, with the same name. + role_index = None + + for index in xrange(len(roles)): + role = roles[index] + name = role.get('name') + # This role has no name. + if name is None: + no_name_message = 'Role with no name!' + raise tuf.RepositoryError(no_name_message) + # Does this role have the same name? + else: + # This role has the same name, and... + if name == delegated_role: + # ...it is the only known role with the same name. + if role_index is None: + role_index = index + # ...there are at least two roles with the same name! + else: + duplicate_role_message = 'Duplicate role ('+str(delegated_role)+')!' + raise tuf.RepositoryError(duplicate_role_message) + # This role has a different name. + else: + continue + + return role_index + From e00a3549ce3aeebc35e2eddeb410400ddd397d85 Mon Sep 17 00:00:00 2001 From: dachshund Date: Fri, 21 Jun 2013 00:59:24 +0800 Subject: [PATCH 05/28] Adjust tests.test_signercli to use list of roles. --- tuf/repo/signercli.py | 2 ++ tuf/repo/signerlib.py | 8 ++++---- tuf/roledb.py | 3 ++- tuf/tests/test_signercli.py | 9 +++++++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tuf/repo/signercli.py b/tuf/repo/signercli.py index 1618d35f..5c308095 100755 --- a/tuf/repo/signercli.py +++ b/tuf/repo/signercli.py @@ -1261,9 +1261,11 @@ def _update_parent_metadata(metadata_directory, delegated_role, delegated_keyids if role_index is None: # Append role to the end of the list of delegated roles. + logger.info('Appending role '+delegated_role+' to '+parent_role) roles.append(role_metadata) else: # Update role with the same name. + logger.info('Replacing role '+delegated_role+' in '+parent_role) roles[role_index] = role_metadata delegations['roles'] = roles diff --git a/tuf/repo/signerlib.py b/tuf/repo/signerlib.py index 5b8580d9..6eb88146 100755 --- a/tuf/repo/signerlib.py +++ b/tuf/repo/signerlib.py @@ -1148,10 +1148,10 @@ def build_delegated_role_file(delegated_targets_directory, delegated_keyids, delegation_role_name): """ - Build the targets metadata file using the signing keys in 'targets_keyids'. - The generated metadata file is saved to 'metadata_directory'. The target - files located in 'targets_directory' will be tracked by the built targets - metadata. + Build the targets metadata file using the signing keys in + 'delegated_keyids'. The generated metadata file is saved to + 'metadata_directory'. The target files located in 'targets_directory' will + be tracked by the built targets metadata. delegated_targets_directory: diff --git a/tuf/roledb.py b/tuf/roledb.py index feb44265..f6446731 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -538,7 +538,8 @@ def get_delegated_rolenames(rolename): None. - A list of rolenames. + A list of rolenames. Note that the rolenames are *NOT* sorted by order of + delegation! """ diff --git a/tuf/tests/test_signercli.py b/tuf/tests/test_signercli.py index ee36baa3..97a56f25 100755 --- a/tuf/tests/test_signercli.py +++ b/tuf/tests/test_signercli.py @@ -1468,8 +1468,13 @@ def _mock_get_keyids(junk): parent_role_file = os.path.join(meta_dir, parent_role+'.txt') signable = signerlib.read_metadata_file(parent_role_file) delegated_rolename = parent_role+'/'+delegated_role - threshold = signable['signed']['delegations']['roles']\ - [delegated_rolename]['threshold'] + + roles = signable['signed']['delegations']['roles'] + role_index = signerlib.find_delegated_role(roles, delegated_rolename) + self.assertIsNotNone(role_index) + role = roles[role_index] + + threshold = role['threshold'] self.assertTrue(threshold == 2) # RESTORE From 688b0e21caee44286d35cc9d725132a7c11c6232 Mon Sep 17 00:00:00 2001 From: dachshund Date: Sat, 22 Jun 2013 16:23:52 +0800 Subject: [PATCH 06/28] WIP on adjusting tuf.client.updater to use list of roles. --- docs/tuf-spec.txt | 23 ++++++++--------------- tuf/client/updater.py | 35 +++++++++++++++++++++-------------- tuf/tests/test_formats.py | 2 ++ tuf/tests/test_updater.py | 2 ++ 4 files changed, 33 insertions(+), 29 deletions(-) diff --git a/docs/tuf-spec.txt b/docs/tuf-spec.txt index 432035cf..0c88c437 100644 --- a/docs/tuf-spec.txt +++ b/docs/tuf-spec.txt @@ -561,12 +561,12 @@ { "keys" : { KEYID : KEY, ... }, - "roles" : { - ROLE : { - "keyids" : [ KEYID, ... ] , - "threshold" : THRESHOLD, - "paths" : [ PATHPATTERN, ... ] } - , ... } + "roles" : [{ + "name": ROLE, + "keyids" : [ KEYID, ... ] , + "threshold" : THRESHOLD, + "paths" : [ PATHPATTERN, ... ] + }, ... ] } The "paths" list describes paths that the role is trusted to provide. @@ -586,15 +586,8 @@ on. The metadata of the first delegation will override that of the second delegation, the metadata of the second delegation will override that of the third delegation, and so on. In order to accommodate this scheme, the "roles" key - in the DELEGATIONS object above should point to an array, instead of a hash - table, of delegated roles, like so: - - "roles" : [{ - "role": ROLE, - "keyids" : [ KEYID, ... ] , - "threshold" : THRESHOLD, - "paths" : [ PATHPATTERN, ... ] - }, ... ] + in the DELEGATIONS object above points to an array, instead of a hash + table, of delegated roles. Another priority tag scheme would have the clients prefer the delegated role with the latest metadata for a conflicting target path. Similar ideas were diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 76a43343..a10871d0 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -106,13 +106,14 @@ import shutil import errno +import tuf.conf +import tuf.download import tuf.formats import tuf.keydb -import tuf.roledb -import tuf.mirrors -import tuf.download -import tuf.conf import tuf.log +import tuf.mirrors +import tuf.repo.signerlib +import tuf.roledb import tuf.sig import tuf.util @@ -489,7 +490,7 @@ def _import_delegations(self, parent_role): # This could be quite slow with a huge number of delegations. keys_info = current_parent_metadata['delegations'].get('keys', {}) - roles_info = current_parent_metadata['delegations'].get('roles', {}) + roles_info = current_parent_metadata['delegations'].get('roles', []) logger.debug('Adding roles delegated from '+repr(parent_role)+'.') @@ -514,14 +515,18 @@ def _import_delegations(self, parent_role): continue # Add the roles to the role database. - for rolename, roleinfo in roles_info.items(): - logger.debug('Adding delegated role: '+repr(rolename)+'.') + for roleinfo in roles_info: try: + # NOTE: tuf.roledb.add_role will take care + # of the case where rolename is None. + rolename = roleinfo.get('name') + logger.debug('Adding delegated role: '+str(rolename)+'.') tuf.roledb.add_role(rolename, roleinfo) except tuf.RoleAlreadyExistsError, e: logger.warn('Role already exists: '+rolename) - except (tuf.FormatError, tuf.InvalidNameError), e: + except: logger.exception('Failed to add delegated role: '+rolename+'.') + raise @@ -854,6 +859,7 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas # may not be trusted anymore. if metadata_role == 'targets' or metadata_role.startswith('targets/'): logger.debug('Removing delegated roles of '+repr(metadata_role)+'.') + # TODO: Should we also remove the keys of the delegated roles? tuf.roledb.remove_delegated_roles(metadata_role) self._import_delegations(metadata_role) @@ -906,14 +912,15 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): # Iterate through the targets of 'metadata_role' and confirm # these targets with the paths listed in the parent role. + roles = self.metadata['current'][parent_role]['delegations']['roles'] + role_index = tuf.repo.signerlib.find_delegated_role(roles, metadata_role) + assert role_index is not None + role = roles[role_index] for target_filepath in metadata_object['targets'].keys(): - if target_filepath not in self.metadata['current'][parent_role] \ - ['delegations']['roles'] \ - [metadata_role]['paths']: - - message = 'Role '+repr(metadata_role)+' specifies target '+ \ + if target_filepath not in role['paths']: + message = 'Role '+str(metadata_role)+' specifies target '+ \ target_filepath+' which is not an allowed path according '+ \ - 'to the delegations set by '+repr(parent_role)+'.' + 'to the delegations set by '+str(parent_role)+'.' raise tuf.RepositoryError(message) diff --git a/tuf/tests/test_formats.py b/tuf/tests/test_formats.py index 047828ef..2e41e2f6 100755 --- a/tuf/tests/test_formats.py +++ b/tuf/tests/test_formats.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_formats.py diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index 173df52f..75c7dc77 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_updater.py From 47abec8cdedc917c4b026923c7130d8df46030ba Mon Sep 17 00:00:00 2001 From: dachshund Date: Sun, 23 Jun 2013 10:36:21 +0800 Subject: [PATCH 07/28] Find target in order of trust. --- tuf/client/updater.py | 103 +++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 52 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index a10871d0..4fa97379 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -100,11 +100,11 @@ """ -import os -import time -import logging -import shutil import errno +import logging +import os +import shutil +import time import tuf.conf import tuf.download @@ -408,6 +408,7 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): if metadata_role == 'root': self._rebuild_key_and_role_db() elif metadata_object['_type'] == 'Targets': + # TODO: Should we also remove the keys of the delegated roles? tuf.roledb.remove_delegated_roles(metadata_role) self._import_delegations(metadata_role) @@ -1433,8 +1434,11 @@ def targets_of_role(self, rolename='targets'): def target(self, target_filepath): """ - Return the target file information for 'target_filepath'. - + Return the target file information for 'target_filepath'. We interrogate + the tree of target delegations in order of appearance (which implicitly + order trustworthiness), and return the matching target found in the most + trusted role. + target_filepath: The path to the target file on the repository. This @@ -1446,8 +1450,10 @@ def target(self, target_filepath): If 'target_filepath' is improperly formatted. tuf.RepositoryError: - If 'target_filepath' was not found or there were more multiple - versions (same file path but different file attributes). + If 'target_filepath' was not found. + + Exception: + In case of an unforeseen runtime error. The metadata for updated delegated roles are download and stored. @@ -1464,51 +1470,44 @@ def target(self, target_filepath): # Refresh the target metadata for all the delegated roles. self._refresh_targets_metadata(include_delegations=True) - all_rolenames = tuf.roledb.get_rolenames() - # Iterate through all the target metadata. Take precautions - # to avoid duplicate files. - target = [] - for rolename in all_rolenames: - if self.metadata['current'][rolename]['_type'] != 'Targets': - continue - # We have a target role. Extract the filepath and fileinfo - # and compare it to 'target_filepath'. Compare the fileinfo - # to avoid duplicates. - for filepath, fileinfo in self.metadata['current'][rolename] \ - ['targets'].items(): - if target_filepath == filepath: - # If 'target' is empty, we can just go ahead and add 'target_filepath' - # No need to check for duplicates in this case. - if len(target) == 0: - new_target = {} - new_target['filepath'] = filepath - new_target['fileinfo'] = fileinfo - target.append(new_target) - continue - # It appears we have a duplicate. If the fileinfo match, - # do not add the duplicate. Move on to the next target. - elif len(target) == 1: - if target[0]['fileinfo'] == fileinfo: - continue - # TODO: What if an existing file, that is listed in the targets - # metadata, gets delegated? This needs to be looked at. - # Okay, we have a matching filepath but a different fileinfo - # for the duplicate. Which one is the client expecting? - # And why would the metadata list two different versions of the - # same file? Raise an exception. - else: - message = 'Found multiple '+repr(target_filepath)+'.' - logger.error(message) - #raise tuf.RepositoryError(message) - - # Riase an exception if the target information could not be retrieved. - if len(target) == 0: - message = repr(target_filepath)+' not found.' - logger.error(message) - raise tuf.RepositoryError(message) - - return target[0] + # The target is assumed to be missing until proven otherwise. + target = None + + try: + rolenames = ['targets'] + + # Preorder depth-first traversal of the tree of target delegations. + while len(rolenames) > 0 and target is None: + # Pop the rolename from the top of the stack. + rolename = rolenames.pop(-1) + metadata = self.metadata['current'][rolename] + targets = metadata['targets'] + delegations = metadata.get('delegations', {}) + child_roles = delegations.get('roles', []) + + # Does the current rolename have our target? + logger.info('Asking role '+rolename+' about target '+target_filepath) + for filepath, fileinfo in targets.iteritems(): + if target_filepath == filepath: + logger.info('Found target '+target_filepath+' in role '+rolename) + target = {'filepath': filepath, 'fileinfo': fileinfo} + break + + # Push children in reverse order of appearance onto the stack. + for child_role in reversed(child_roles): + rolenames.append(child_role['name']) + except: + raise + finally: + # Raise an exception if the target information could not be retrieved. + if target is None: + message = target_filepath+' not found.' + logger.error(message) + raise tuf.RepositoryError(message) + # Otherwise, return the found target. + else: + return target From 42e9dffd24fec0152dd203e07b4cb593adef741e Mon Sep 17 00:00:00 2001 From: dachshund Date: Sun, 23 Jun 2013 13:08:26 +0800 Subject: [PATCH 08/28] Patch mock signercli.make_delegation prompts. --- tuf/tests/repository_setup.py | 4 +++- .../system_tests/test_extraneous_dependencies_attack.py | 4 +++- tuf/tests/system_tests/util_test_tools.py | 6 ++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tuf/tests/repository_setup.py b/tuf/tests/repository_setup.py index 3b2d20f5..969d97d0 100755 --- a/tuf/tests/repository_setup.py +++ b/tuf/tests/repository_setup.py @@ -141,8 +141,10 @@ def _mock_prompt(msg, junk): return delegated_targets_dir elif msg.startswith('\nChoose and enter the parent'): return parent_role - elif msg.endswith('\nEnter the delegated role\'s name: '): + elif msg.startswith('\nEnter the delegated role\'s name: '): return delegated_role_name + elif msg.startswith('Recursively walk the given directory? (Y)es/(N)o: '): + return 'N' else: error_msg = ('Prompt: '+'\''+msg+'\''+ ' did not match any predefined mock prompts.') diff --git a/tuf/tests/system_tests/test_extraneous_dependencies_attack.py b/tuf/tests/system_tests/test_extraneous_dependencies_attack.py index 57f1c583..597e585b 100755 --- a/tuf/tests/system_tests/test_extraneous_dependencies_attack.py +++ b/tuf/tests/system_tests/test_extraneous_dependencies_attack.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_extraneous_dependencies_attack.py @@ -179,4 +181,4 @@ def _write_rogue_metadata(): try: test_extraneous_dependencies_attack() except ExtraneousDependenciesAttackAlert, error: - print 'error' \ No newline at end of file + print 'error' diff --git a/tuf/tests/system_tests/util_test_tools.py b/tuf/tests/system_tests/util_test_tools.py index 7588e61f..6267a6ee 100755 --- a/tuf/tests/system_tests/util_test_tools.py +++ b/tuf/tests/system_tests/util_test_tools.py @@ -549,8 +549,10 @@ def _mock_prompt(msg, junk, targets_path=delegated_targets_path, return targets_path elif msg.startswith('\nChoose and enter the parent'): return parent_role - elif msg.endswith('\nEnter the delegated role\'s name: '): + elif msg.startswith('\nEnter the delegated role\'s name: '): return new_role_name + elif msg.startswith('Recursively walk the given directory? (Y)es/(N)o: '): + return 'N' else: error_msg = ('Prompt: '+'\''+msg+'\''+ ' did not match any predefined mock prompts.') @@ -585,4 +587,4 @@ def _mock_get_keyid(junk, keyid=keyid): signercli._get_keyids = original_get_keyids signercli._get_password = original_get_password signercli._prompt = original_prompt - signercli._get_metadata_directory = original_get_metadata_directory \ No newline at end of file + signercli._get_metadata_directory = original_get_metadata_directory From aa11987b11d26b333799e3b57862bd0f29a73faf Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 24 Jun 2013 08:58:24 +0800 Subject: [PATCH 09/28] First cut at adapting Konstantin's delegations test. --- tuf/tests/system_tests/test_delegations.py | 273 +++++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100755 tuf/tests/system_tests/test_delegations.py diff --git a/tuf/tests/system_tests/test_delegations.py b/tuf/tests/system_tests/test_delegations.py new file mode 100755 index 00000000..4e72a92b --- /dev/null +++ b/tuf/tests/system_tests/test_delegations.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python + +""" + + test_delegations.py + + + Konstantin Andrianov + + + February 19, 2012 + + + See LICENSE for licensing information. + + + + +""" + +import os +import sys +import tempfile +import time +import unittest +import urllib + +import tuf.formats +from tuf.interposition import urllib_tuf +import tuf.repo.keystore as keystore +import tuf.repo.signercli as signercli +import tuf.repo.signerlib as signerlib +import util_test_tools + + +# Disable logging. +#util_test_tools.disable_logging() + + + +def setup_tuf_repository(): + root_repo, url, server_proc, keyids = util_test_tools.init_repo(tuf=True) + + # Server side repository. + tuf_repo = os.path.join(root_repo, 'tuf_repo') + keystore_dir = os.path.join(tuf_repo, 'keystore') + metadata_dir = os.path.join(tuf_repo, 'metadata') + targets_dir = os.path.join(tuf_repo, 'targets') + + # Add files to the server side repository. + # target1 = 'targets_dir/target1_rand.txt' + # target2 = 'targets_dir/target2_rand.txt' + # target3 = 'targets_dir/level1_rand/target3_rand.txt' + # target4 = 'targets_dir/level1_rand/target4_rand.txt' + # target5 = 'targets_dir/level1_rand/level2_rand/target5_rand.txt' + # target6 = 'targets_dir/level1_rand/level2_rand/target6_rand.txt' + add_target = util_test_tools.add_file_to_repository + level1 = tempfile.mkdtemp(dir=targets_dir, prefix='level1_') + level2 = tempfile.mkdtemp(dir=level1, prefix='level2_') + target1_path = add_target(targets_dir, data='target1') + target2_path = add_target(targets_dir, data='target2') + target3_path = add_target(level1, data='target3') + target4_path = add_target(level1, data='target4') + target5_path = add_target(level2, data='target5') + target6_path = add_target(level2, data='target6') + + # Target paths relative to the 'targets_dir'. + # Ex: targetX = 'level1_rand/targetX_rand.txt' + target1 = os.path.relpath(target1_path, tuf_repo) + target2 = os.path.relpath(target2_path, tuf_repo) + target3 = os.path.relpath(target3_path, tuf_repo) + target4 = os.path.relpath(target4_path, tuf_repo) + target5 = os.path.relpath(target5_path, tuf_repo) + target6 = os.path.relpath(target6_path, tuf_repo) + + # Relative to repository's targets directory. + target_filepaths = [target1, target2, target3, target4, target5, target6] + + # Tracked targets. + targets_tracked_targets = [target1] + delegatee1_tracked_targets = [target1, target4] + delegatee2_tracked_targets = [target2, target4, target5] + delegatee3_tracked_targets = [target3, target4, target5, target6] + + # Assigned targets. + delegatee1_assigned_targets = [target1, target3, target4, target5, target6] + delegatee2_assigned_targets = [target2, target3, target4, target5, target6] + delegatee3_assigned_targets = [target3, target4, target5, target6] + + # Make delegation directories at the server's repository. + metadata_targets_dir = os.path.join(metadata_dir, 'targets') + metadata_delegatee1_dir = os.path.join(metadata_targets_dir, 'delegatee1') + os.makedirs(metadata_delegatee1_dir) + + # Delegations metadata paths. + delegatee1_path = os.path.join(metadata_targets_dir, 'delegatee1.txt') + delegatee2_path = os.path.join(metadata_targets_dir, 'delegatee2.txt') + delegatee3_path = os.path.join(metadata_delegatee1_dir, 'delegatee3.txt') + + # Generate delegation metadata. + generate_meta = signerlib.generate_targets_metadata + delegatee1 = generate_meta(tuf_repo, delegatee1_tracked_targets) + delegatee2 = generate_meta(tuf_repo, delegatee2_tracked_targets) + delegatee3 = generate_meta(tuf_repo, delegatee3_tracked_targets) + + # Generate a set of RSA keys that will be assigned to the delegatees. + key1 = signerlib.generate_and_save_rsa_key(keystore_dir, 'delegatee1') + key2 = signerlib.generate_and_save_rsa_key(keystore_dir, 'delegatee2') + key3 = signerlib.generate_and_save_rsa_key(keystore_dir, 'delegatee3') + + def _relative_path_to_targets(target_filepaths): + # Ex: 'targets/more_targets/somefile.txt' -> 'more_targets/somefile.txt' + # i.e. 'targets/' is removed from 'target'. + new_target_filepaths = [] + for target in target_filepaths: + relative_targetpath = os.path.sep.join(target.split(os.path.sep)[1:]) + new_target_filepaths.append(relative_targetpath) + return new_target_filepaths + + + # Create delegatee role metadata in order to later create 'delegations' + # object: + delegatee1_role_meta = \ + tuf.formats.make_role_metadata([key1['keyid']], 1, name='targets/delegatee1', + paths=_relative_path_to_targets(delegatee1_assigned_targets)) + delegatee2_role_meta = \ + tuf.formats.make_role_metadata([key2['keyid']], 1, name='targets/delegatee2', + paths=_relative_path_to_targets(delegatee2_assigned_targets)) + delegatee3_role_meta = \ + tuf.formats.make_role_metadata([key3['keyid']], 1, name='targets/delegatee1/delegatee3', + paths=_relative_path_to_targets(delegatee3_assigned_targets)) + + # Create 'delegations' object for targets metadata: + targets_delegations = {} + key1_val = tuf.rsa_key.create_in_metadata_format(key1['keyval']) + key2_val = tuf.rsa_key.create_in_metadata_format(key2['keyval']) + targets_delegations['keys'] = {key1['keyid']:key1_val, + key2['keyid']:key2_val} + targets_delegations['roles'] = [delegatee1_role_meta, delegatee2_role_meta] + + # Create 'delegations' object for delegatee2 metadata: + delegatee1_delegations = {} + key3_val = tuf.rsa_key.create_in_metadata_format(key3['keyval']) + delegatee1_delegations['keys'] = {key3['keyid']:key3_val} + delegatee1_delegations['roles'] = [delegatee3_role_meta] + delegatee1['signed']['delegations'] = delegatee1_delegations + + # Read targets.txt metadata and add the 'delegations' field. + targets_metadata_path = os.path.join(metadata_dir, 'targets.txt') + targets_signable = signerlib.read_metadata_file(targets_metadata_path) + targets_metadata = targets_signable['signed'] + targets_metadata['delegations'] = targets_delegations + + sign = signerlib.sign_metadata + write = signerlib.write_metadata_file + + # Sign and save new metadata objects. + targets_signable = sign(targets_metadata, keyids, targets_metadata_path) + delegatee1_signable = sign(delegatee1, [key1['keyid']], delegatee1_path) + delegatee2_signable = sign(delegatee2, [key2['keyid']], delegatee2_path) + delegatee3_signable = sign(delegatee3, [key3['keyid']], delegatee3_path) + write(targets_signable, targets_metadata_path) + write(delegatee1_signable, delegatee1_path) + write(delegatee2_signable, delegatee2_path) + write(delegatee3_signable, delegatee3_path) + + # Repository is set up. Refresh release and timestamp metadata to reflect + # the new changes. + signerlib.build_release_file(keyids, metadata_dir) + signerlib.build_timestamp_file(keyids, metadata_dir) + + # Unload all keys. + keystore.clear_keystore() + + # We need to provide clients with a way to reach the tuf repository. + tuf_repo_relpath = os.path.basename(tuf_repo) + tuf_url = url+tuf_repo_relpath + mirrors = {"mirror1": + {"url_prefix": tuf_url, + "metadata_path": "metadata", + "targets_path": "targets", + "confined_target_dirs": [ "" ]}} + + return (root_repo, mirrors, server_proc, keyids, + _relative_path_to_targets(target_filepaths)) + + + + + +def test(rm_repo=True): + """ + rm_repo: + Boolean signalling whether or not we should remove the created repos. + """ + + # Setup. + root_repo, mirrors, server_proc, keyids, target_filepaths = \ + setup_tuf_repository() + + # Server side repository. + tuf_repo = os.path.join(root_repo, 'tuf_repo') + keystore_dir = os.path.join(tuf_repo, 'keystore') + metadata_dir = os.path.join(tuf_repo, 'metadata') + targets_dir = os.path.join(tuf_repo, 'targets') + + # Client side repository. + tuf_client = os.path.join(root_repo, 'tuf_client') + current_dir = os.path.join(tuf_client, 'metadata', 'current') + previous_dir = os.path.join(tuf_client, 'metadata', 'previous') + downloads_dir = os.path.join(root_repo, 'downloads') + + # Adjust client's configuration file. + # Here the repository_directory is referred to client's local repository. + original_repo = tuf.conf.repository_directory + tuf.conf.repository_directory = tuf_client + + # At this point all metadata at the server's repository is in sync. + + start_time = time.time() + _client_update(mirrors, downloads_dir, target_filepaths) + end_time = time.time() + print "Client update takes", + print end_time - start_time, + print "seconds." + + # TearDown and restore value of previous repository directory. + tuf.conf.repository_directory = original_repo + if rm_repo: + server_proc.kill() + else: + util_test_tools.cleanup(root_repo, server_proc) + + + + + +def _client_update(mirrors, dest, target_filepaths=[], initial_update=False): + # We need to initialize an updater class. + updater = tuf.client.updater.Updater('my_repo', mirrors) + + # Refresh the repository's top-level roles, store the target information for + # all the targets tracked, and determine which of these targets have been + # updated. + updater.refresh() + + if initial_update: + targets = updater.all_targets() + else: + targets = [] + for target_filepath in target_filepaths: + target_info = updater.target(target_filepath) + targets.append(target_info) + + updated_targets = updater.updated_targets(targets, dest) + + # Download each of these updated targets and save them locally. + for target in updated_targets: + try: + print('Downloading target '+str(target)) + updater.download_target(target, dest) + except tuf.DownloadError, e: + pass + + + + + +if __name__ == '__main__': + start_time = time.time() + test(True) + end_time = time.time() + print end_time - start_time From cb1f552703fbf52e0a0d3164f8809a1d729ad147 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 24 Jun 2013 18:09:36 +0800 Subject: [PATCH 10/28] Neutral refactoring of schema. --- tuf/formats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tuf/formats.py b/tuf/formats.py index 9f2fce05..43cfa95a 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -274,7 +274,7 @@ keyids=SCHEMA.ListOf(KEYID_SCHEMA), name=SCHEMA.Optional(ROLENAME_SCHEMA), threshold=THRESHOLD_SCHEMA, - paths=SCHEMA.Optional(SCHEMA.ListOf(RELPATH_SCHEMA))) + paths=SCHEMA.Optional(RELPATHS_SCHEMA)) # A dict of roles where the dict keys are role names and the dict values holding # the role data/information. @@ -328,7 +328,7 @@ url_prefix=URL_SCHEMA, metadata_path=RELPATH_SCHEMA, targets_path=RELPATH_SCHEMA, - confined_target_dirs=SCHEMA.ListOf(RELPATH_SCHEMA), + confined_target_dirs=RELPATHS_SCHEMA, custom=SCHEMA.Optional(SCHEMA.Object())) # A dictionary of mirrors where the dict keys hold the mirror's name and From c138b678283db4570d501d97eecc06b4d9c16a00 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 24 Jun 2013 18:35:17 +0800 Subject: [PATCH 11/28] Ensure that we explore only delegated roles trusted with the desired target. --- tuf/client/updater.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 4fa97379..8b022f2c 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1475,28 +1475,36 @@ def target(self, target_filepath): target = None try: - rolenames = ['targets'] + current_metadata = self.metadata['current'] + role_names = ['targets'] # Preorder depth-first traversal of the tree of target delegations. - while len(rolenames) > 0 and target is None: - # Pop the rolename from the top of the stack. - rolename = rolenames.pop(-1) - metadata = self.metadata['current'][rolename] - targets = metadata['targets'] - delegations = metadata.get('delegations', {}) + while len(role_names) > 0 and target is None: + # Pop the role name from the top of the stack. + role_name = role_names.pop(-1) + role_metadata = current_metadata[role_name] + targets = role_metadata['targets'] + delegations = role_metadata.get('delegations', {}) child_roles = delegations.get('roles', []) - # Does the current rolename have our target? - logger.info('Asking role '+rolename+' about target '+target_filepath) + # Does the current role name have our target? + logger.info('Asking role '+role_name+' about target '+target_filepath) for filepath, fileinfo in targets.iteritems(): - if target_filepath == filepath: - logger.info('Found target '+target_filepath+' in role '+rolename) + if filepath == target_filepath: + logger.info('Found target '+target_filepath+' in role '+role_name) target = {'filepath': filepath, 'fileinfo': fileinfo} break # Push children in reverse order of appearance onto the stack. for child_role in reversed(child_roles): - rolenames.append(child_role['name']) + child_role_name = child_role['name'] + child_role_paths = child_role['paths'] + + # Ensure that we explore only delegated roles trusted with the target. + # Assuming conservation of delegated paths in the complete tree of + # delegations. + if target_filepath in child_role_paths: + role_names.append(child_role_name) except: raise finally: From a0179a51092113e494bc2e33c24e4951a324ba51 Mon Sep 17 00:00:00 2001 From: johnward2 Date: Tue, 25 Jun 2013 18:44:34 -0400 Subject: [PATCH 12/28] Fix several naming and scope errors. - "quickstart.py:292: No global (EEXIST) found" - "basic_client.py:194: No global (option_parser) found" - "rsa_key.py:108: No global (EnvelopeError) found" - "rsa_key.py:108: No global (KeygenError) found" - "util.py:56: No global (temp_dir) found" --- basic_client.py | 2 +- evpy/envelope.py | 3 +++ quickstart.py | 2 +- tuf/rsa_key.py | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/basic_client.py b/basic_client.py index da66ac7f..5c657fcc 100755 --- a/basic_client.py +++ b/basic_client.py @@ -191,7 +191,7 @@ def parse_options(): # Ensure the '--repo' option was set by the user. if options.REPOSITORY_MIRROR is None: message = '"--repo" must be set on the command-line.' - option_parser.error(message) + parser.error(message) # Return the repository mirror containing the metadata and target files. return options.REPOSITORY_MIRROR diff --git a/evpy/envelope.py b/evpy/envelope.py index 16ee8f68..28b2d811 100755 --- a/evpy/envelope.py +++ b/evpy/envelope.py @@ -51,6 +51,9 @@ class EnvelopeError(evp.SSLError): pass +class KeygenError(evp.SSLError): + pass + def _build_dkey_from_file(keyfile): fp = evp.fopen(keyfile, "r") if not fp: diff --git a/quickstart.py b/quickstart.py index 8761fb95..2215995f 100755 --- a/quickstart.py +++ b/quickstart.py @@ -289,7 +289,7 @@ def build_repository(project_directory): os.mkdir(keystore_directory) # 'OSError' raised if the directory cannot be created. except OSError, e: - if e.errno == EEXIST: + if e.errno == errno.EEXIST: pass else: raise diff --git a/tuf/rsa_key.py b/tuf/rsa_key.py index 075315ce..bc1767a0 100755 --- a/tuf/rsa_key.py +++ b/tuf/rsa_key.py @@ -105,7 +105,7 @@ def generate(bits=_DEFAULT_RSA_KEY_BITS): try: public_key, private_key = evpy.envelope.keygen(bits, pem=True) - except (EnvelopeError, KeygenError, MemoryError), e: + except (evpy.envelope.EnvelopeError, evpy.envelope.KeygenError, MemoryError), e: raise tuf.CryptoError(e) # Generate the keyid for the RSA key. 'key_value' corresponds to the From b40191ba1fbbcdcfd820c296eb2338128648791b Mon Sep 17 00:00:00 2001 From: dachshund Date: Sun, 30 Jun 2013 12:19:44 +0800 Subject: [PATCH 13/28] Restructing of delegations integration test. --- tuf/client/updater.py | 9 +- tuf/tests/system_tests/test_delegations.py | 399 ++++++++++----------- 2 files changed, 206 insertions(+), 202 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 8b022f2c..f5f963cb 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1501,8 +1501,13 @@ def target(self, target_filepath): child_role_paths = child_role['paths'] # Ensure that we explore only delegated roles trusted with the target. - # Assuming conservation of delegated paths in the complete tree of - # delegations. + # We assume conservation of delegated paths in the complete tree of + # delegations. Note that the call to _ensure_all_targets_allowed in + # _update_metadata should already ensure that all targets metadata is + # valid; i.e. that the targets signed by a delegatee is a proper + # subset of the targets delegated to it by the delegator. + # Nevertheless, we check it again here for performance and safety + # reasons. if target_filepath in child_role_paths: role_names.append(child_role_name) except: diff --git a/tuf/tests/system_tests/test_delegations.py b/tuf/tests/system_tests/test_delegations.py index 4e72a92b..f442c4ac 100755 --- a/tuf/tests/system_tests/test_delegations.py +++ b/tuf/tests/system_tests/test_delegations.py @@ -14,103 +14,69 @@ See LICENSE for licensing information. - - + Ensure that TUF meets expectations about target delegations. """ + + + + import os -import sys import tempfile -import time import unittest -import urllib import tuf.formats -from tuf.interposition import urllib_tuf import tuf.repo.keystore as keystore import tuf.repo.signercli as signercli import tuf.repo.signerlib as signerlib import util_test_tools -# Disable logging. -#util_test_tools.disable_logging() -def setup_tuf_repository(): - root_repo, url, server_proc, keyids = util_test_tools.init_repo(tuf=True) +class TestDelegationFunctions(unittest.TestCase): - # Server side repository. - tuf_repo = os.path.join(root_repo, 'tuf_repo') - keystore_dir = os.path.join(tuf_repo, 'keystore') - metadata_dir = os.path.join(tuf_repo, 'metadata') - targets_dir = os.path.join(tuf_repo, 'targets') - # Add files to the server side repository. - # target1 = 'targets_dir/target1_rand.txt' - # target2 = 'targets_dir/target2_rand.txt' - # target3 = 'targets_dir/level1_rand/target3_rand.txt' - # target4 = 'targets_dir/level1_rand/target4_rand.txt' - # target5 = 'targets_dir/level1_rand/level2_rand/target5_rand.txt' - # target6 = 'targets_dir/level1_rand/level2_rand/target6_rand.txt' - add_target = util_test_tools.add_file_to_repository - level1 = tempfile.mkdtemp(dir=targets_dir, prefix='level1_') - level2 = tempfile.mkdtemp(dir=level1, prefix='level2_') - target1_path = add_target(targets_dir, data='target1') - target2_path = add_target(targets_dir, data='target2') - target3_path = add_target(level1, data='target3') - target4_path = add_target(level1, data='target4') - target5_path = add_target(level2, data='target5') - target6_path = add_target(level2, data='target6') - - # Target paths relative to the 'targets_dir'. - # Ex: targetX = 'level1_rand/targetX_rand.txt' - target1 = os.path.relpath(target1_path, tuf_repo) - target2 = os.path.relpath(target2_path, tuf_repo) - target3 = os.path.relpath(target3_path, tuf_repo) - target4 = os.path.relpath(target4_path, tuf_repo) - target5 = os.path.relpath(target5_path, tuf_repo) - target6 = os.path.relpath(target6_path, tuf_repo) + def do_update(self): + # Client side repository. + tuf_client = os.path.join(self.root_repo, 'tuf_client') + downloads_dir = os.path.join(self.root_repo, 'downloads') - # Relative to repository's targets directory. - target_filepaths = [target1, target2, target3, target4, target5, target6] + # Adjust client's configuration file. + tuf.conf.repository_directory = tuf_client - # Tracked targets. - targets_tracked_targets = [target1] - delegatee1_tracked_targets = [target1, target4] - delegatee2_tracked_targets = [target2, target4, target5] - delegatee3_tracked_targets = [target3, target4, target5, target6] + updater = tuf.client.updater.Updater('my_repo', self.mirrors) - # Assigned targets. - delegatee1_assigned_targets = [target1, target3, target4, target5, target6] - delegatee2_assigned_targets = [target2, target3, target4, target5, target6] - delegatee3_assigned_targets = [target3, target4, target5, target6] + # Refresh the repository's top-level roles, store the target information for + # all the targets tracked, and determine which of these targets have been + # updated. + updater.refresh() - # Make delegation directories at the server's repository. - metadata_targets_dir = os.path.join(metadata_dir, 'targets') - metadata_delegatee1_dir = os.path.join(metadata_targets_dir, 'delegatee1') - os.makedirs(metadata_delegatee1_dir) + targets = [] + for target_filepath in self.relpath_from_targets(self.target_filepaths): + target_info = updater.target(target_filepath) + targets.append(target_info) - # Delegations metadata paths. - delegatee1_path = os.path.join(metadata_targets_dir, 'delegatee1.txt') - delegatee2_path = os.path.join(metadata_targets_dir, 'delegatee2.txt') - delegatee3_path = os.path.join(metadata_delegatee1_dir, 'delegatee3.txt') + updated_targets = updater.updated_targets(targets, downloads_dir) + # Download each of these updated targets and save them locally. + for target in updated_targets: + print('Downloading target '+str(target)) + updater.download_target(target, downloads_dir) - # Generate delegation metadata. - generate_meta = signerlib.generate_targets_metadata - delegatee1 = generate_meta(tuf_repo, delegatee1_tracked_targets) - delegatee2 = generate_meta(tuf_repo, delegatee2_tracked_targets) - delegatee3 = generate_meta(tuf_repo, delegatee3_tracked_targets) - # Generate a set of RSA keys that will be assigned to the delegatees. - key1 = signerlib.generate_and_save_rsa_key(keystore_dir, 'delegatee1') - key2 = signerlib.generate_and_save_rsa_key(keystore_dir, 'delegatee2') - key3 = signerlib.generate_and_save_rsa_key(keystore_dir, 'delegatee3') + def make_targets_metadata(self): + """Subclasses will override this method to generate metadata for all + targets roles, with the understanding that there is a fixed structure of + the targets roles.""" + + raise NotImplementedError() + + + def relpath_from_targets(self, target_filepaths): + """Ex: 'targets/more_targets/somefile.txt' -> 'more_targets/somefile.txt' + i.e. 'targets/' is removed from 'target'.""" - def _relative_path_to_targets(target_filepaths): - # Ex: 'targets/more_targets/somefile.txt' -> 'more_targets/somefile.txt' - # i.e. 'targets/' is removed from 'target'. new_target_filepaths = [] for target in target_filepaths: relative_targetpath = os.path.sep.join(target.split(os.path.sep)[1:]) @@ -118,156 +84,189 @@ def _relative_path_to_targets(target_filepaths): return new_target_filepaths - # Create delegatee role metadata in order to later create 'delegations' - # object: - delegatee1_role_meta = \ - tuf.formats.make_role_metadata([key1['keyid']], 1, name='targets/delegatee1', - paths=_relative_path_to_targets(delegatee1_assigned_targets)) - delegatee2_role_meta = \ - tuf.formats.make_role_metadata([key2['keyid']], 1, name='targets/delegatee2', - paths=_relative_path_to_targets(delegatee2_assigned_targets)) - delegatee3_role_meta = \ - tuf.formats.make_role_metadata([key3['keyid']], 1, name='targets/delegatee1/delegatee3', - paths=_relative_path_to_targets(delegatee3_assigned_targets)) + def setUp(self): + """ + The target delegations tree is fixed as such: + targets -> [T1, T2] + T1 -> [T3] + """ - # Create 'delegations' object for targets metadata: - targets_delegations = {} - key1_val = tuf.rsa_key.create_in_metadata_format(key1['keyval']) - key2_val = tuf.rsa_key.create_in_metadata_format(key2['keyval']) - targets_delegations['keys'] = {key1['keyid']:key1_val, - key2['keyid']:key2_val} - targets_delegations['roles'] = [delegatee1_role_meta, delegatee2_role_meta] + root_repo, url, server_proc, keyids = util_test_tools.init_repo(tuf=True) - # Create 'delegations' object for delegatee2 metadata: - delegatee1_delegations = {} - key3_val = tuf.rsa_key.create_in_metadata_format(key3['keyval']) - delegatee1_delegations['keys'] = {key3['keyid']:key3_val} - delegatee1_delegations['roles'] = [delegatee3_role_meta] - delegatee1['signed']['delegations'] = delegatee1_delegations + # Server side repository. + tuf_repo = os.path.join(root_repo, 'tuf_repo') + keystore_dir = os.path.join(tuf_repo, 'keystore') + metadata_dir = os.path.join(tuf_repo, 'metadata') + targets_dir = os.path.join(tuf_repo, 'targets') - # Read targets.txt metadata and add the 'delegations' field. - targets_metadata_path = os.path.join(metadata_dir, 'targets.txt') - targets_signable = signerlib.read_metadata_file(targets_metadata_path) - targets_metadata = targets_signable['signed'] - targets_metadata['delegations'] = targets_delegations + # We need to provide clients with a way to reach the tuf repository. + tuf_repo_relpath = os.path.basename(tuf_repo) + tuf_url = url+tuf_repo_relpath - sign = signerlib.sign_metadata - write = signerlib.write_metadata_file + # Add files to the server side repository. + # target1 = 'targets_dir/[random].txt' + # target2 = 'targets_dir/[random].txt' + add_target = util_test_tools.add_file_to_repository + target1_path = add_target(targets_dir, data='target1') + target2_path = add_target(targets_dir, data='target2') - # Sign and save new metadata objects. - targets_signable = sign(targets_metadata, keyids, targets_metadata_path) - delegatee1_signable = sign(delegatee1, [key1['keyid']], delegatee1_path) - delegatee2_signable = sign(delegatee2, [key2['keyid']], delegatee2_path) - delegatee3_signable = sign(delegatee3, [key3['keyid']], delegatee3_path) - write(targets_signable, targets_metadata_path) - write(delegatee1_signable, delegatee1_path) - write(delegatee2_signable, delegatee2_path) - write(delegatee3_signable, delegatee3_path) + # Target paths relative to the 'targets_dir'. + # Ex: targetX = 'targets/delegator/delegatee.txt' + target1 = os.path.relpath(target1_path, tuf_repo) + target2 = os.path.relpath(target2_path, tuf_repo) - # Repository is set up. Refresh release and timestamp metadata to reflect - # the new changes. - signerlib.build_release_file(keyids, metadata_dir) - signerlib.build_timestamp_file(keyids, metadata_dir) + # Relative to repository's targets directory. + target_filepaths = [target1, target2] - # Unload all keys. - keystore.clear_keystore() + # Store in self only the variables relevant for tests. + self.root_repo = root_repo + self.tuf_repo = tuf_repo + self.server_proc = server_proc + self.target_filepaths = target_filepaths + # Targets delegated from A to B. + self.delegated_targets = {} + # Targets actually signed by B. + self.signed_targets = {} + self.mirrors = { + "mirror1": { + "url_prefix": tuf_url, + "metadata_path": "metadata", + "targets_path": "targets", + "confined_target_dirs": [""] + } + } + # Aliases for targets roles. + self.T0 = 'targets' + self.T1 = 'targets/T1' + self.T2 = 'targets/T2' + self.T3 = 'targets/T1/T3' - # We need to provide clients with a way to reach the tuf repository. - tuf_repo_relpath = os.path.basename(tuf_repo) - tuf_url = url+tuf_repo_relpath - mirrors = {"mirror1": - {"url_prefix": tuf_url, - "metadata_path": "metadata", - "targets_path": "targets", - "confined_target_dirs": [ "" ]}} + # Get tracked and assigned targets, and generate targets metadata. + self.make_targets_metadata() + assert hasattr(self, 'T0_metadata') + assert hasattr(self, 'T1_metadata') + assert hasattr(self, 'T2_metadata') + assert hasattr(self, 'T3_metadata') - return (root_repo, mirrors, server_proc, keyids, - _relative_path_to_targets(target_filepaths)) + # Make delegation directories at the server's repository. + metadata_targets_dir = os.path.join(metadata_dir, 'targets') + metadata_T1_dir = os.path.join(metadata_targets_dir, 'T1') + os.makedirs(metadata_T1_dir) + + # Delegations metadata paths for the 3 delegated targets roles. + T0_path = os.path.join(metadata_dir, 'targets.txt') + T1_path = os.path.join(metadata_targets_dir, 'T1.txt') + T2_path = os.path.join(metadata_targets_dir, 'T2.txt') + T3_path = os.path.join(metadata_T1_dir, 'T3.txt') + + # Generate RSA keys for the 3 delegatees. + key1 = signerlib.generate_and_save_rsa_key(keystore_dir, 'T1') + key2 = signerlib.generate_and_save_rsa_key(keystore_dir, 'T2') + key3 = signerlib.generate_and_save_rsa_key(keystore_dir, 'T3') + + # ID for each of the 3 keys. + key1_id = key1['keyid'] + key2_id = key2['keyid'] + key3_id = key3['keyid'] + + # ID, in a list, for each of the 3 keys. + key1_ids = [key1_id] + key2_ids = [key2_id] + key3_ids = [key3_id] + + # Public-key JSON for each of the 3 keys. + key1_val = tuf.rsa_key.create_in_metadata_format(key1['keyval']) + key2_val = tuf.rsa_key.create_in_metadata_format(key2['keyval']) + key3_val = tuf.rsa_key.create_in_metadata_format(key3['keyval']) + + # Create delegation role metadata for each of the 3 delegated targets roles. + make_role_metadata = tuf.formats.make_role_metadata + + T1_targets = self.relpath_from_targets(self.delegated_targets[self.T1]) + T1_role = make_role_metadata(key1_ids, 1, name=self.T1, paths=T1_targets) + + T2_targets = self.relpath_from_targets(self.delegated_targets[self.T2]) + T2_role = make_role_metadata(key2_ids, 1, name=self.T2, paths=T2_targets) + + T3_targets = self.relpath_from_targets(self.delegated_targets[self.T3]) + T3_role = make_role_metadata(key3_ids, 1, name=self.T3, paths=T3_targets) + + # Assign 'delegations' object for 'targets': + self.T0_metadata['signed']['delegations'] = { + 'keys': {key1_id: key1_val, key2_id: key2_val}, + 'roles': [T1_role, T2_role] + } + + # Assign 'delegations' object for 'targets/T1': + self.T1_metadata['signed']['delegations'] = { + 'keys': {key3_id: key3_val}, + 'roles': [T3_role] + } + + sign = signerlib.sign_metadata + write = signerlib.write_metadata_file + + # Sign new metadata objects. + T0_signable = sign(self.T0_metadata, keyids, T0_path) + T1_signable = sign(self.T1_metadata, key1_ids, T1_path) + T2_signable = sign(self.T2_metadata, key2_ids, T2_path) + T3_signable = sign(self.T3_metadata, key3_ids, T3_path) + # Save new metadata objects. + write(T0_signable, T0_path) + write(T1_signable, T1_path) + write(T2_signable, T2_path) + write(T3_signable, T3_path) + + # Timestamp a new release to reflect latest targets. + signerlib.build_release_file(keyids, metadata_dir) + signerlib.build_timestamp_file(keyids, metadata_dir) + + # Unload all keys. + keystore.clear_keystore() + + + def tearDown(self): + util_test_tools.cleanup(self.root_repo, server_process=self.server_proc) -def test(rm_repo=True): - """ - rm_repo: - Boolean signalling whether or not we should remove the created repos. - """ - - # Setup. - root_repo, mirrors, server_proc, keyids, target_filepaths = \ - setup_tuf_repository() - - # Server side repository. - tuf_repo = os.path.join(root_repo, 'tuf_repo') - keystore_dir = os.path.join(tuf_repo, 'keystore') - metadata_dir = os.path.join(tuf_repo, 'metadata') - targets_dir = os.path.join(tuf_repo, 'targets') - - # Client side repository. - tuf_client = os.path.join(root_repo, 'tuf_client') - current_dir = os.path.join(tuf_client, 'metadata', 'current') - previous_dir = os.path.join(tuf_client, 'metadata', 'previous') - downloads_dir = os.path.join(root_repo, 'downloads') - - # Adjust client's configuration file. - # Here the repository_directory is referred to client's local repository. - original_repo = tuf.conf.repository_directory - tuf.conf.repository_directory = tuf_client - - # At this point all metadata at the server's repository is in sync. - - start_time = time.time() - _client_update(mirrors, downloads_dir, target_filepaths) - end_time = time.time() - print "Client update takes", - print end_time - start_time, - print "seconds." - - # TearDown and restore value of previous repository directory. - tuf.conf.repository_directory = original_repo - if rm_repo: - server_proc.kill() - else: - util_test_tools.cleanup(root_repo, server_proc) +class Test1(TestDelegationFunctions): + """We test that the basic update mechanism works.""" + def make_targets_metadata(self): + make_metadata = signerlib.generate_targets_metadata + target1, target2 = self.target_filepaths + + # Tracked targets. + self.signed_targets[self.T0] = [target1] + self.signed_targets[self.T1] = [target1] + self.signed_targets[self.T2] = [target2] + self.signed_targets[self.T3] = [target1, target2] + + # Assigned targets. + self.delegated_targets[self.T1] = [target1] + self.delegated_targets[self.T2] = [target2] + self.delegated_targets[self.T3] = [target1, target2] + + self.T0_metadata =\ + make_metadata(self.tuf_repo, self.signed_targets[self.T0]) + self.T1_metadata =\ + make_metadata(self.tuf_repo, self.signed_targets[self.T1]) + self.T2_metadata =\ + make_metadata(self.tuf_repo, self.signed_targets[self.T2]) + self.T3_metadata = \ + make_metadata(self.tuf_repo, self.signed_targets[self.T3]) - -def _client_update(mirrors, dest, target_filepaths=[], initial_update=False): - # We need to initialize an updater class. - updater = tuf.client.updater.Updater('my_repo', mirrors) - - # Refresh the repository's top-level roles, store the target information for - # all the targets tracked, and determine which of these targets have been - # updated. - updater.refresh() - - if initial_update: - targets = updater.all_targets() - else: - targets = [] - for target_filepath in target_filepaths: - target_info = updater.target(target_filepath) - targets.append(target_info) - - updated_targets = updater.updated_targets(targets, dest) - - # Download each of these updated targets and save them locally. - for target in updated_targets: - try: - print('Downloading target '+str(target)) - updater.download_target(target, dest) - except tuf.DownloadError, e: - pass + def test_update_works_with_delegations(self): + self.do_update() if __name__ == '__main__': - start_time = time.time() - test(True) - end_time = time.time() - print end_time - start_time + unittest.main() From e74689345e09f38ab5d171af37a1eb172bd40adc Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 1 Jul 2013 12:44:20 +0800 Subject: [PATCH 14/28] Test that initial update works with target delegations. --- tuf/tests/system_tests/test_delegations.py | 31 +++++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/tuf/tests/system_tests/test_delegations.py b/tuf/tests/system_tests/test_delegations.py index f442c4ac..668fb63c 100755 --- a/tuf/tests/system_tests/test_delegations.py +++ b/tuf/tests/system_tests/test_delegations.py @@ -53,17 +53,27 @@ def do_update(self): # updated. updater.refresh() + # Obtain a list of available targets. targets = [] - for target_filepath in self.relpath_from_targets(self.target_filepaths): + relative_target_filepaths = self.relpath_from_targets(self.target_filepaths) + for target_filepath in relative_target_filepaths: target_info = updater.target(target_filepath) targets.append(target_info) - updated_targets = updater.updated_targets(targets, downloads_dir) # Download each of these updated targets and save them locally. + updated_targets = updater.updated_targets(targets, downloads_dir) for target in updated_targets: - print('Downloading target '+str(target)) updater.download_target(target, downloads_dir) + # Return metadata about downloaded targets. + make_fileinfo = signerlib.get_metadata_file_info + targets_metadata = {} + for target_filepath in relative_target_filepaths: + download_filepath = os.path.join(downloads_dir, target_filepath) + target_fileinfo = signerlib.get_metadata_file_info(download_filepath) + targets_metadata[target_filepath] = target_fileinfo + return targets_metadata + def make_targets_metadata(self): """Subclasses will override this method to generate metadata for all @@ -232,8 +242,9 @@ def tearDown(self): -class Test1(TestDelegationFunctions): - """We test that the basic update mechanism works.""" +class TestInitialUpdateWithDelegations(TestDelegationFunctions): + """We show that making target delegations results in a successful initial + update of targets.""" def make_targets_metadata(self): @@ -261,8 +272,14 @@ def make_targets_metadata(self): make_metadata(self.tuf_repo, self.signed_targets[self.T3]) - def test_update_works_with_delegations(self): - self.do_update() + def test_that_initial_update_works_with_target_delegations(self): + # Get relative target paths, because that is what TUF recognizes. + relative_target_filepaths = self.relpath_from_targets(self.target_filepaths) + # Get metadata about downloaded targets. + targets_metadata = self.do_update() + # Do we have metadata about all the expected targets? + for target_filepath in relative_target_filepaths: + self.assertIn(target_filepath, targets_metadata) From e76454b4ede5438e712f0d3b4ef09f6d6867d0b0 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 1 Jul 2013 15:59:15 +0800 Subject: [PATCH 15/28] Allow deconfiguration of interposition. --- tuf/interposition/__init__.py | 69 ++++++++++++++++++++---------- tuf/interposition/configuration.py | 18 ++++++++ tuf/interposition/updater.py | 47 ++++++++++++-------- 3 files changed, 93 insertions(+), 41 deletions(-) diff --git a/tuf/interposition/__init__.py b/tuf/interposition/__init__.py index c8bac610..db2262a9 100644 --- a/tuf/interposition/__init__.py +++ b/tuf/interposition/__init__.py @@ -167,6 +167,46 @@ def __urllib2_urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): +def __read_configuration(configuration_handler, + filename="tuf.interposition.json", + parent_repository_directory=None, + parent_ssl_certificates_directory=None): + """ + A generic function to read a TUF interposition configuration off the disk, + and handle it. configuration_handler must be a function which accepts a + tuf.interposition.Configuration instance.""" + + INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!" + INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!" + NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!" + + try: + with open(filename) as tuf_interposition_json: + tuf_interpositions = json.load(tuf_interposition_json) + configurations = tuf_interpositions.get("configurations", {}) + + if len(configurations) == 0: + raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename)) + + else: + for network_location, configuration in configurations.iteritems(): + try: + configuration_parser = ConfigurationParser(network_location, + configuration, parent_repository_directory=parent_repository_directory, + parent_ssl_certificates_directory=parent_ssl_certificates_directory) + + configuration = configuration_parser.parse() + configuration_handler(configuration) + + except: + Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location)) + raise + + except: + Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename)) + raise + + # TODO: Is parent_repository_directory a security risk? For example, would it @@ -221,35 +261,18 @@ def configure(filename="tuf.interposition.json", optional; it must specify certificates bundled as PEM (RFC 1422). """ - INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!" - INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!" - NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!" + __read_configuration(__updater_controller.add, filename=filename, + parent_repository_directory=parent_repository_directory, + parent_ssl_certificates_directory=parent_ssl_certificates_directory) - try: - with open(filename) as tuf_interposition_json: - tuf_interpositions = json.load(tuf_interposition_json) - configurations = tuf_interpositions.get("configurations", {}) - if len(configurations) == 0: - raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename)) - else: - for network_location, configuration in configurations.iteritems(): - try: - configuration_parser = ConfigurationParser(network_location, - configuration, parent_repository_directory=parent_repository_directory, - parent_ssl_certificates_directory=parent_ssl_certificates_directory) - configuration = configuration_parser.parse() - __updater_controller.add(configuration) - except: - Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location)) - raise +def deconfigure(filename="tuf.interposition.json"): + """Remove TUF interposition for a previously read configuration.""" - except: - Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename)) - raise + __read_configuration(__updater_controller.remove, filename=filename) diff --git a/tuf/interposition/configuration.py b/tuf/interposition/configuration.py index de7de9e3..5a66e4af 100644 --- a/tuf/interposition/configuration.py +++ b/tuf/interposition/configuration.py @@ -51,6 +51,24 @@ def __repr__(self): return MESSAGE.format(network_location=self.network_location) + def get_repository_mirror_hostnames(self): + """Get a set of hostnames of every repository mirror of this + configuration.""" + + # Parse TUF server repository mirrors. + repository_mirrors = self.repository_mirrors + repository_mirror_hostnames = set() + + for repository_mirror in repository_mirrors: + mirror_configuration = repository_mirrors[repository_mirror] + url_prefix = mirror_configuration["url_prefix"] + parsed_url = urlparse.urlparse(url_prefix) + mirror_hostname = parsed_url.hostname + repository_mirror_hostnames.add(mirror_hostname) + + return repository_mirror_hostnames + + diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index d7dd89e6..8be219f5 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -177,7 +177,7 @@ def __init__(self): self.__repository_mirror_hostnames = set() - def __check_configuration(self, configuration): + def __check_configuration_on_add(self, configuration): """ If the given Configuration is invalid, I raise an exception. Otherwise, I return some information about the Configuration, @@ -198,42 +198,33 @@ def __check_configuration(self, configuration): assert configuration.hostname not in self.__updaters assert configuration.hostname not in self.__repository_mirror_hostnames - # Parse TUF server repository mirrors. - repository_mirrors = configuration.repository_mirrors - repository_mirror_hostnames = set() - - for repository_mirror in repository_mirrors: - mirror_configuration = repository_mirrors[repository_mirror] + # Check for redundancy in server repository mirrors. + repository_mirror_hostnames = configuration.get_repository_mirror_hostnames() + for mirror_hostname in repository_mirror_hostnames: try: - url_prefix = mirror_configuration["url_prefix"] - parsed_url = urlparse.urlparse(url_prefix) - mirror_hostname = parsed_url.hostname - - # Restrict each (incoming, outgoing) hostname pair to be unique - # across configurations; this prevents interposition cycles, + # Restrict each hostname in every (incoming, outgoing) pair to be + # unique across configurations; this prevents interposition cycles, # amongst other things. assert mirror_hostname not in self.__updaters assert mirror_hostname not in self.__repository_mirror_hostnames - # Remember this mirror's hostname for the next network_location. - repository_mirror_hostnames.add(mirror_hostname) - except: error_message = \ - INVALID_REPOSITORY_MIRROR.format(repository_mirror=repository_mirror) + INVALID_REPOSITORY_MIRROR.format(repository_mirror=mirror_hostname) Logger.exception(error_message) raise InvalidConfiguration(error_message) return repository_mirror_hostnames + def add(self, configuration): """Add an Updater based on the given Configuration.""" UPDATER_ADDED_MESSAGE = "Updater added for {configuration}." - repository_mirror_hostnames = self.__check_configuration(configuration) + repository_mirror_hostnames = self.__check_configuration_on_add(configuration) # If all is well, build and store an Updater, and remember hostnames. self.__updaters[configuration.hostname] = Updater(configuration) @@ -296,3 +287,23 @@ def get(self, url): Logger.warn(GENERIC_WARNING_MESSAGE.format(url=url)) return updater + + + def remove(self, configuration): + """Remove an Updater matching the given Configuration.""" + + UPDATER_REMOVED_MESSAGE = "Updater removed for {configuration}." + + assert isinstance(configuration, Configuration) + + repository_mirror_hostnames = configuration.get_repository_mirror_hostnames() + + assert configuration.hostname in self.__updaters + assert repository_mirror_hostnames.issubset(self.__repository_mirror_hostnames) + + # If all is well, remove the stored Updater as well as its associated + # repository mirror hostnames. + del self.__updaters[configuration.hostname] + self.__repository_mirror_hostnames.difference_update(repository_mirror_hostnames) + + Logger.info(UPDATER_REMOVED_MESSAGE.format(configuration=configuration)) From f90873572a77f5c41c4cf91b64debaad1cf835b5 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 1 Jul 2013 16:07:15 +0800 Subject: [PATCH 16/28] Deconfigure interposition on test repository cleanup. --- tuf/tests/system_tests/util_test_tools.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tuf/tests/system_tests/util_test_tools.py b/tuf/tests/system_tests/util_test_tools.py index 6267a6ee..00db5869 100755 --- a/tuf/tests/system_tests/util_test_tools.py +++ b/tuf/tests/system_tests/util_test_tools.py @@ -204,6 +204,10 @@ def cleanup(root_repo, server_process=None): # Clear the keystore. keystore.clear_keystore() + # Deconfigure interposition. + interpose_json = os.path.join(root_repo, 'tuf.interposition.json') + tuf.interposition.deconfigure(filename=interpose_json) + # Removing repository directory. try: shutil.rmtree(root_repo) @@ -381,9 +385,8 @@ def create_interposition_config(root_repo, url): "targets_path": "targets", "confined_target_dirs": [ "" ]}}}}} - # "target_paths": [ { "(.*\\.html)": "{0}" } ] - - junk, interpose_json = tempfile.mkstemp(prefix='conf_', dir=root_repo) + # We write the interposition JSON configuration at a deterministic location. + interpose_json = os.path.join(root_repo, 'tuf.interposition.json') with open(interpose_json, 'wb') as fileobj: tuf.util.json.dump(interposition_dict, fileobj) From 57bd1df14e39d99e617a908b2753b6873a1c1e96 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 1 Jul 2013 16:07:43 +0800 Subject: [PATCH 17/28] Test that delegated targets roles cannot breach delegated paths. --- tuf/tests/system_tests/test_delegations.py | 44 ++++++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/tuf/tests/system_tests/test_delegations.py b/tuf/tests/system_tests/test_delegations.py index 668fb63c..937d7b7f 100755 --- a/tuf/tests/system_tests/test_delegations.py +++ b/tuf/tests/system_tests/test_delegations.py @@ -242,7 +242,7 @@ def tearDown(self): -class TestInitialUpdateWithDelegations(TestDelegationFunctions): +class TestInitialUpdateWithTargetDelegations(TestDelegationFunctions): """We show that making target delegations results in a successful initial update of targets.""" @@ -251,13 +251,13 @@ def make_targets_metadata(self): make_metadata = signerlib.generate_targets_metadata target1, target2 = self.target_filepaths - # Tracked targets. + # Targets signed for by each of the targets roles. self.signed_targets[self.T0] = [target1] self.signed_targets[self.T1] = [target1] self.signed_targets[self.T2] = [target2] self.signed_targets[self.T3] = [target1, target2] - # Assigned targets. + # Targets delegated to each of the delegated targets roles. self.delegated_targets[self.T1] = [target1] self.delegated_targets[self.T2] = [target2] self.delegated_targets[self.T3] = [target1, target2] @@ -285,5 +285,43 @@ def test_that_initial_update_works_with_target_delegations(self): +class TestBreachOfTargetDelegation(TestDelegationFunctions): + """We show that a delegated targets role B cannot talk about targets that A + did not delegate to B.""" + + + def make_targets_metadata(self): + make_metadata = signerlib.generate_targets_metadata + target1, target2 = self.target_filepaths + + # Targets signed for by each of the targets roles. + self.signed_targets[self.T0] = [] + self.signed_targets[self.T1] = [target2] + self.signed_targets[self.T2] = [target1] + self.signed_targets[self.T3] = [] + + # Targets delegated to each of the delegated targets roles. + self.delegated_targets[self.T1] = [target1] + self.delegated_targets[self.T2] = [target2] + self.delegated_targets[self.T3] = [] + + self.T0_metadata =\ + make_metadata(self.tuf_repo, self.signed_targets[self.T0]) + self.T1_metadata =\ + make_metadata(self.tuf_repo, self.signed_targets[self.T1]) + self.T2_metadata =\ + make_metadata(self.tuf_repo, self.signed_targets[self.T2]) + self.T3_metadata = \ + make_metadata(self.tuf_repo, self.signed_targets[self.T3]) + + + def test_that_initial_update_fails_with_undelegated_signing_of_targets(self): + # Expect to see a particular exception on initial update. + self.assertRaises(tuf.MetadataNotAvailableError, self.do_update) + + + + + if __name__ == '__main__': unittest.main() From 0b8337c01fa7aaa979a948e9a13da2fd6521d81d Mon Sep 17 00:00:00 2001 From: dachshund Date: Tue, 2 Jul 2013 20:23:49 +0800 Subject: [PATCH 18/28] Conditionally deconfigure TUF interposition for system tests. --- tuf/tests/system_tests/test_endless_data_attack.py | 6 ++++-- tuf/tests/system_tests/util_test_tools.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tuf/tests/system_tests/test_endless_data_attack.py b/tuf/tests/system_tests/test_endless_data_attack.py index c8dbea90..9366accf 100755 --- a/tuf/tests/system_tests/test_endless_data_attack.py +++ b/tuf/tests/system_tests/test_endless_data_attack.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_endless_data_attack.py @@ -143,7 +145,7 @@ def test_arbitrary_package_attack(TUF=False): test_arbitrary_package_attack(TUF=False) except EndlessDataAttack, error: - print error + print('Without TUF: '+str(error)) @@ -151,4 +153,4 @@ def test_arbitrary_package_attack(TUF=False): test_arbitrary_package_attack(TUF=True) except EndlessDataAttack, error: - print error \ No newline at end of file + print('With TUF: '+str(error)) diff --git a/tuf/tests/system_tests/util_test_tools.py b/tuf/tests/system_tests/util_test_tools.py index 00db5869..7fc79c93 100755 --- a/tuf/tests/system_tests/util_test_tools.py +++ b/tuf/tests/system_tests/util_test_tools.py @@ -206,7 +206,8 @@ def cleanup(root_repo, server_process=None): # Deconfigure interposition. interpose_json = os.path.join(root_repo, 'tuf.interposition.json') - tuf.interposition.deconfigure(filename=interpose_json) + if os.path.exists(interpose_json): + tuf.interposition.deconfigure(filename=interpose_json) # Removing repository directory. try: From db8481f989a042fec64db54a45bf1ffab0f9eee8 Mon Sep 17 00:00:00 2001 From: dachshund Date: Tue, 2 Jul 2013 22:19:20 +0800 Subject: [PATCH 19/28] Test that delegation of targets works in order of appearance of roles. --- tuf/tests/system_tests/test_delegations.py | 103 +++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/tuf/tests/system_tests/test_delegations.py b/tuf/tests/system_tests/test_delegations.py index 937d7b7f..0dae80fa 100755 --- a/tuf/tests/system_tests/test_delegations.py +++ b/tuf/tests/system_tests/test_delegations.py @@ -323,5 +323,108 @@ def test_that_initial_update_fails_with_undelegated_signing_of_targets(self): +class TestOrderOfTargetDelegationWithSuccess(TestDelegationFunctions): + """We show that when multiple delegated targets roles talk about a target, + the first one in order of appearance of delegation wins. + + In this case, the first role has the correct metadata about the target.""" + + + def make_targets_metadata(self): + make_metadata = signerlib.generate_targets_metadata + target1, target2 = self.target_filepaths + + # Targets signed for by each of the targets roles. + self.signed_targets[self.T0] = [target2] + self.signed_targets[self.T1] = [] + self.signed_targets[self.T2] = [target1] + self.signed_targets[self.T3] = [target1] + + # Targets delegated to each of the delegated targets roles. + self.delegated_targets[self.T1] = [target1] + self.delegated_targets[self.T2] = [target1] + self.delegated_targets[self.T3] = [target1] + + self.T0_metadata =\ + make_metadata(self.tuf_repo, self.signed_targets[self.T0]) + self.T1_metadata =\ + make_metadata(self.tuf_repo, self.signed_targets[self.T1]) + self.T2_metadata =\ + make_metadata(self.tuf_repo, self.signed_targets[self.T2]) + self.T3_metadata = \ + make_metadata(self.tuf_repo, self.signed_targets[self.T3]) + + # Modify the hash for target1 in T2. + for target_filepath in self.relpath_from_targets([target1]): + target_metadata = self.T2_metadata['signed']['targets'][target_filepath] + sha256_hash = target_metadata['hashes']['sha256'] + last_character = sha256_hash[-1] + last_character = chr(ord(last_character)-1) + # "Subtract" the last character of the hash. + target_metadata['hashes']['sha256'] = sha256_hash[:-1] + last_character + + + def test_that_initial_update_works_with_many_roles_sharing_a_target(self): + # Get relative target paths, because that is what TUF recognizes. + relative_target_filepaths = self.relpath_from_targets(self.target_filepaths) + # Get metadata about downloaded targets. + targets_metadata = self.do_update() + # Do we have metadata about all the expected targets? + for target_filepath in relative_target_filepaths: + self.assertIn(target_filepath, targets_metadata) + + + + + +class TestOrderOfTargetDelegationWithFailure(TestDelegationFunctions): + """We show that when multiple delegated targets roles talk about a target, + the first one in order of appearance of delegation wins. + + In this case, the first role has the wrong metadata about the target.""" + + + def make_targets_metadata(self): + make_metadata = signerlib.generate_targets_metadata + target1, target2 = self.target_filepaths + + # Targets signed for by each of the targets roles. + self.signed_targets[self.T0] = [target2] + self.signed_targets[self.T1] = [] + self.signed_targets[self.T2] = [target1] + self.signed_targets[self.T3] = [target1] + + # Targets delegated to each of the delegated targets roles. + self.delegated_targets[self.T1] = [target1] + self.delegated_targets[self.T2] = [target1] + self.delegated_targets[self.T3] = [target1] + + self.T0_metadata =\ + make_metadata(self.tuf_repo, self.signed_targets[self.T0]) + self.T1_metadata =\ + make_metadata(self.tuf_repo, self.signed_targets[self.T1]) + self.T2_metadata =\ + make_metadata(self.tuf_repo, self.signed_targets[self.T2]) + self.T3_metadata = \ + make_metadata(self.tuf_repo, self.signed_targets[self.T3]) + + # Modify the hash for target1 in T3. + for target_filepath in self.relpath_from_targets([target1]): + target_metadata = self.T3_metadata['signed']['targets'][target_filepath] + sha256_hash = target_metadata['hashes']['sha256'] + last_character = sha256_hash[-1] + last_character = chr(ord(last_character)-1) + # "Subtract" the last character of the hash. + target_metadata['hashes']['sha256'] = sha256_hash[:-1] + last_character + + + def test_that_initial_update_fails_with_many_roles_sharing_a_target(self): + # Expect to see a particular exception on initial update. + self.assertRaises(tuf.DownloadError, self.do_update) + + + + + if __name__ == '__main__': unittest.main() From 4910e7a06520946236fd688a9253992b70da5759 Mon Sep 17 00:00:00 2001 From: dachshund Date: Wed, 3 Jul 2013 00:33:26 +0800 Subject: [PATCH 20/28] Test conservation of delegated targets. --- tuf/tests/system_tests/test_delegations.py | 43 ++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tuf/tests/system_tests/test_delegations.py b/tuf/tests/system_tests/test_delegations.py index 0dae80fa..e26c895a 100755 --- a/tuf/tests/system_tests/test_delegations.py +++ b/tuf/tests/system_tests/test_delegations.py @@ -426,5 +426,48 @@ def test_that_initial_update_fails_with_many_roles_sharing_a_target(self): +class TestConservationOfTargetDelegation(TestDelegationFunctions): + """We show that delegated targets roles have to neither sign for targets + delegated to them nor further delegate them.""" + + + def make_targets_metadata(self): + make_metadata = signerlib.generate_targets_metadata + target1, target2 = self.target_filepaths + + # Targets signed for by each of the targets roles. + self.signed_targets[self.T0] = [] + self.signed_targets[self.T1] = [target1] + self.signed_targets[self.T2] = [target2] + self.signed_targets[self.T3] = [] + + # Targets delegated to each of the delegated targets roles. + self.delegated_targets[self.T1] = [target1, target2] + self.delegated_targets[self.T2] = [target1, target2] + self.delegated_targets[self.T3] = [] + + self.T0_metadata =\ + make_metadata(self.tuf_repo, self.signed_targets[self.T0]) + self.T1_metadata =\ + make_metadata(self.tuf_repo, self.signed_targets[self.T1]) + self.T2_metadata =\ + make_metadata(self.tuf_repo, self.signed_targets[self.T2]) + self.T3_metadata = \ + make_metadata(self.tuf_repo, self.signed_targets[self.T3]) + + + def test_that_initial_update_works_with_unconserved_targets(self): + # Get relative target paths, because that is what TUF recognizes. + relative_target_filepaths = self.relpath_from_targets(self.target_filepaths) + # Get metadata about downloaded targets. + targets_metadata = self.do_update() + # Do we have metadata about all the expected targets? + for target_filepath in relative_target_filepaths: + self.assertIn(target_filepath, targets_metadata) + + + + + if __name__ == '__main__': unittest.main() From a3d924c9a14bb71573dd3e14aac4c64a871f9e5d Mon Sep 17 00:00:00 2001 From: dachshund Date: Sat, 6 Jul 2013 01:33:15 +0800 Subject: [PATCH 21/28] Abstraction for walking over files in a directory. --- tuf/repo/signercli.py | 60 ++++++++++++++------------------ tuf/repo/signerlib.py | 79 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 34 deletions(-) diff --git a/tuf/repo/signercli.py b/tuf/repo/signercli.py index 5c308095..4aaed950 100755 --- a/tuf/repo/signercli.py +++ b/tuf/repo/signercli.py @@ -1035,13 +1035,27 @@ def make_delegation(keystore_directory): delegated_role, delegated_keyids = _get_delegated_role(keystore_directory, metadata_directory) + # Retrieve the file paths for the delegated targets. + delegated_paths =\ + tuf.repo.signerlib.get_targets(delegated_targets_directory, + recursive_walk=recursive_walk, + followlinks=True) + + # The target paths need to be relative to the repository's targets + # directory (e.g., 'targets/role1/target_file.gif'). + # [len(repository_directory)+1:] strips the repository path, including + # its trailing path separator. + repository_directory, junk = os.path.split(metadata_directory) + + for index in xrange(len(delegated_paths)): + full_target_path = delegated_paths[index] + assert full_target_path.startswith(repository_directory) + relative_target_path = full_target_path[len(repository_directory)+1:] + delegated_paths[index] = relative_target_path + # Create, sign, and write the delegated role's metadata file. - delegated_paths = _make_delegated_metadata(metadata_directory, - delegated_targets_directory, - parent_role, delegated_role, - delegated_keyids, - recursive_walk=recursive_walk, - followlinks=True) + _make_delegated_metadata(metadata_directory, delegated_paths, parent_role, + delegated_role, delegated_keyids) # Update the parent role's metadata file. The parent role's delegation # field must be updated with the newly created delegated role. @@ -1140,39 +1154,17 @@ def _get_delegated_role(keystore_directory, metadata_directory): -def _make_delegated_metadata(metadata_directory, delegated_targets_directory, - parent_role, delegated_role, delegated_keyids, - recursive_walk=False, followlinks=False): +def _make_delegated_metadata(metadata_directory, delegated_paths, parent_role, + delegated_role, delegated_keyids): """ Create, sign, and write the metadata file for the newly added delegated - role. Determine the delegated paths for the target files found in - 'delegated_targets_directory' and the other information needed + role. Determine the delegated paths for the target files in + 'delegated_paths' and the other information needed to generate the targets metadata file for 'delegated_role'. Return the delegated paths to the caller. """ - # Retrieve the file paths for the delegated targets. - delegated_paths = [] - - # The target paths need to be relative to the repository's targets - # directory (e.g., 'targets/role1/target_file.gif'). - # [len(repository_directory)+1:] strips the repository path, including - # its trailing path separator. - repository_directory, junk = os.path.split(metadata_directory) - - for dirpath, dirnames, filenames in os.walk(delegated_targets_directory, - followlinks=followlinks): - for filename in filenames: - full_target_path = os.path.join(dirpath, filename) - relative_target_path = full_target_path[len(repository_directory)+1:] - delegated_paths.append(relative_target_path) - - # Prune the subdirectories to walk right now if we do not wish to - # recursively walk the delegated targets directory. - if recursive_walk is False: - del dirnames[:] - message = 'There are '+str(len(delegated_paths))+' target paths for '+\ str(delegated_role) logger.info(message) @@ -1191,6 +1183,7 @@ def _make_delegated_metadata(metadata_directory, delegated_targets_directory, # When creating a delegated role, if the parent directory already # exists, this means a prior delegation has been perform by the parent. parent_directory = os.path.join(metadata_directory, parent_role) + try: os.mkdir(parent_directory) except OSError, e: @@ -1198,6 +1191,7 @@ def _make_delegated_metadata(metadata_directory, delegated_targets_directory, pass else: raise + delegated_role_filename = delegated_role+'.txt' metadata_filename = os.path.join(parent_directory, delegated_role_filename) repository_directory, junk = os.path.split(metadata_directory) @@ -1206,8 +1200,6 @@ def _make_delegated_metadata(metadata_directory, delegated_targets_directory, _sign_and_write_metadata(delegated_metadata, delegated_keyids, metadata_filename) - return delegated_paths - diff --git a/tuf/repo/signerlib.py b/tuf/repo/signerlib.py index 6eb88146..2342f40d 100755 --- a/tuf/repo/signerlib.py +++ b/tuf/repo/signerlib.py @@ -1276,3 +1276,82 @@ def find_delegated_role(roles, delegated_role): return role_index + + + + +def accept_any_file(full_target_path): + """ + + Simply accept any given file. + + + full_target_path: + The absolute path to a target file. + + + None. + + + None. + + + True. + """ + + return True + + + + + +def get_targets(files_directory, recursive_walk=False, followlinks=True, + file_predicate=accept_any_file): + """ + + Walk the given files_directory to build a list of target files in it. + + + files_directory: + The path to a directory of target files. + + recursive_walk: + To recursively walk the directory, set recursive_walk=True. + + followlinks: + To follow symbolic links, set followlinks=True. + + file_predicate: + To filter a file based on a predicate, set file_predicate to a function + which accepts a full path to a file and returns a Boolean. + + + Python IO exceptions. + + + None. + + + A list of absolute paths to target files in the given files_directory. + """ + + targets = [] + + for dirpath, dirnames, filenames in os.walk(files_directory, + followlinks=followlinks): + for filename in filenames: + full_target_path = os.path.join(dirpath, filename) + if file_predicate(full_target_path): + targets.append(full_target_path) + + # Prune the subdirectories to walk right now if we do not wish to + # recursively walk files_directory. + if recursive_walk is False: + del dirnames[:] + + return targets + + + + + From a106c2ebf0940a2248178b9cd8b833b44b8f9dc7 Mon Sep 17 00:00:00 2001 From: dachshund Date: Sat, 6 Jul 2013 21:17:53 +0800 Subject: [PATCH 22/28] Comment about Unicode filenames. --- tuf/repo/signerlib.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tuf/repo/signerlib.py b/tuf/repo/signerlib.py index 2342f40d..41b97c68 100755 --- a/tuf/repo/signerlib.py +++ b/tuf/repo/signerlib.py @@ -1337,6 +1337,10 @@ def get_targets(files_directory, recursive_walk=False, followlinks=True, targets = [] + # FIXME: We need a way to tell Python 2, but not Python 3, to return + # filenames in Unicode; see # 61 and: + # http://docs.python.org/2/howto/unicode.html#unicode-filenames + for dirpath, dirnames, filenames in os.walk(files_directory, followlinks=followlinks): for filename in filenames: From f22a80808fcaf2854641f469406dc5fb01376023 Mon Sep 17 00:00:00 2001 From: dachshund Date: Sat, 6 Jul 2013 21:19:32 +0800 Subject: [PATCH 23/28] Comment about Unicode filenames. --- tuf/repo/signerlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuf/repo/signerlib.py b/tuf/repo/signerlib.py index 41b97c68..dd686430 100755 --- a/tuf/repo/signerlib.py +++ b/tuf/repo/signerlib.py @@ -1338,7 +1338,7 @@ def get_targets(files_directory, recursive_walk=False, followlinks=True, targets = [] # FIXME: We need a way to tell Python 2, but not Python 3, to return - # filenames in Unicode; see # 61 and: + # filenames in Unicode; see #61 and: # http://docs.python.org/2/howto/unicode.html#unicode-filenames for dirpath, dirnames, filenames in os.walk(files_directory, From 0074aebfc7be14a7b9a60f06eb0e16b206c7c3ed Mon Sep 17 00:00:00 2001 From: dachshund Date: Sat, 6 Jul 2013 21:19:32 +0800 Subject: [PATCH 24/28] Comment about Unicode filenames; #61. --- tuf/repo/signerlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuf/repo/signerlib.py b/tuf/repo/signerlib.py index 41b97c68..dd686430 100755 --- a/tuf/repo/signerlib.py +++ b/tuf/repo/signerlib.py @@ -1338,7 +1338,7 @@ def get_targets(files_directory, recursive_walk=False, followlinks=True, targets = [] # FIXME: We need a way to tell Python 2, but not Python 3, to return - # filenames in Unicode; see # 61 and: + # filenames in Unicode; see #61 and: # http://docs.python.org/2/howto/unicode.html#unicode-filenames for dirpath, dirnames, filenames in os.walk(files_directory, From a5597273981f976baeea2888a6ca56e0c6809fba Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 8 Jul 2013 09:11:25 +0800 Subject: [PATCH 25/28] Fast test for breach of delegation. --- tuf/client/updater.py | 54 ++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index f5f963cb..ed697e70 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -900,29 +900,47 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): None. """ - + + MISSING_ROLE_MESSAGE = '{parent_role} has not delegated to ' \ + '{metadata_role}!' + UNDELEGATED_TARGETS_MESSAGE = 'Role {metadata_role} specifies targets ' \ + '{undelegated_targets} which are not ' \ + 'allowed paths according to the ' \ + 'delegations set by {parent_role}.' + # Return if 'metadata_role' is 'targets'. 'targets' is not # a delegated role. if metadata_role == 'targets': return - - # The targets of delegated roles are stored in the parent's - # metadata file. Retrieve the parent role of 'metadata_role' - # to confirm 'metadata_role' contains valid targets. - parent_role = tuf.roledb.get_parent_rolename(metadata_role) + else: + # The targets of delegated roles are stored in the parent's + # metadata file. Retrieve the parent role of 'metadata_role' + # to confirm 'metadata_role' contains valid targets. + parent_role = tuf.roledb.get_parent_rolename(metadata_role) - # Iterate through the targets of 'metadata_role' and confirm - # these targets with the paths listed in the parent role. - roles = self.metadata['current'][parent_role]['delegations']['roles'] - role_index = tuf.repo.signerlib.find_delegated_role(roles, metadata_role) - assert role_index is not None - role = roles[role_index] - for target_filepath in metadata_object['targets'].keys(): - if target_filepath not in role['paths']: - message = 'Role '+str(metadata_role)+' specifies target '+ \ - target_filepath+' which is not an allowed path according '+ \ - 'to the delegations set by '+str(parent_role)+'.' - raise tuf.RepositoryError(message) + # Iterate through the targets of 'metadata_role' and confirm + # these targets with the paths listed in the parent role. + roles = self.metadata['current'][parent_role]['delegations']['roles'] + role_index = tuf.repo.signerlib.find_delegated_role(roles, metadata_role) + + if role_index is None: + raise tuf.RepositoryError(MISSING_ROLE_MESSAGE.format( + parent_role=parent_role, + metadata_role=metadata_role)) + else: + role = roles[role_index] + + # Test for breach of delegation with set operations; asymptotically, this + # is faster than a linear scan, but at the expense of memory. + delegated_targets = set(role['paths']) + signed_targets = set(metadata_object['targets'].keys()) + undelegated_targets = signed_targets - delegated_targets + + if len(undelegated_targets) > 0: + raise tuf.RepositoryError(UNDELEGATED_TARGETS_MESSAGE.format( + metadata_role=metadata_role, + undelegated_targets=undelegated_targets, + parent_role=parent_role)) From 20551a7989ff2bc652b40299dc6c51e2ff3b3061 Mon Sep 17 00:00:00 2001 From: zhengyuyu Date: Mon, 22 Jul 2013 00:20:12 -0400 Subject: [PATCH 26/28] Splits the real download procedure from the download_url_to_tempfileobj and put it into a new function. this makes the download_url_to_tempfileobj clearer and more modular --- tuf/download.py | 61 ++++++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/tuf/download.py b/tuf/download.py index e98c57d4..4af5905e 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -279,8 +279,6 @@ def download_url_to_tempfileobj(url, required_hashes=None, required_length=None) connection = _open_connection(url) temp_file = tuf.util.TempFile() - # Keep track of total bytes downloaded. - total_downloaded = 0 try: # info().get('Content-Length') gets the length of the url file. @@ -314,32 +312,10 @@ def download_url_to_tempfileobj(url, required_hashes=None, required_length=None) ', got '+str(file_length)+' bytes.' raise tuf.DownloadError(message) - # While-block reads data from connection 8192-bytes at a time, or less, - # until 'file_length' is reached. - while True: - data = connection.read(min(8192, file_length - total_downloaded)) - # We might have no more data to read. Let us check bytes downloaded. - if not data: - message = 'Downloaded '+str(total_downloaded)+'/' \ - +str(file_length)+' bytes.' - logger.debug(message) - # Did we download the correct amount indicated by 'Content-Length'? - if total_downloaded != file_length: - message = 'Downloaded '+str(total_downloaded)+'. Expected '+ \ - str(file_length)+' for '+url - raise tuf.DownloadError(message) - # Did we download the correct amount indicated by the user? - if required_length is not None and total_downloaded != required_length: - message = 'The user-required length of '+str(required_length)+ \ - 'did not match the '+str(len(total_downloaded))+' downloaded' - raise tuf.DownloadError(message) - break - # Data successfully read from the connection. Store it. - temp_file.write(data) - total_downloaded = total_downloaded + len(data) + # Call this function to download the target file + connection_of_download(file_length, required_length, connection, temp_file) # We appear to have downloaded the correct amount. Check the hashes. - connection.close() if required_length is not None and required_hashes is not None: _check_hashes(temp_file, required_hashes) @@ -351,3 +327,36 @@ def download_url_to_tempfileobj(url, required_hashes=None, required_length=None) raise tuf.DownloadError(e) return temp_file + + +def connection_of_download(file_length, required_length, connection, temp_file): + """ + This function is where the download really happens. While-block reads data from + connection 8192-bytes at a time, or less, until 'file_length' is reached. + """ + # Keep track of total bytes downloaded. + total_downloaded = 0 + + while True: + data = connection.read(min(8192, file_length - total_downloaded)) + # We might have no more data to read. Let us check bytes downloaded. + if not data: + message = 'Downloaded '+str(total_downloaded)+'/' \ + +str(file_length)+' bytes.' + logger.debug(message) + # Did we download the correct amount indicated by 'Content-Length' + # or user? Because file_length is always eaqual to required_length + # we just need check one of them. + if total_downloaded != file_length: + message = 'Downloaded '+str(total_downloaded)+'. Expected '+ \ + str(file_length)+' for '+url + raise tuf.DownloadError(message) + + break + # Data successfully read from the connection. Store it. + temp_file.write(data) + total_downloaded = total_downloaded + len(data) + + connection.close() + + From 6c2251c0bd1eb6187ca89c3750b50cb6efc9a11f Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 22 Jul 2013 14:23:29 -0400 Subject: [PATCH 27/28] Merge code from @zhengyuyu which makes a function more readable. His code splits tuf.download.download_url_to_tempfileobj into two major pieces. The first piece opens a connection to a URL, and computes the required and reported lengths for downloading data from that given URL. The second piece downloads data from the given URL in such a way that we can defend against endless data and slow retrieval attacks. --- tuf/download.py | 127 +++++++++++++++++++++++++++---------- tuf/tests/test_download.py | 2 + 2 files changed, 94 insertions(+), 35 deletions(-) diff --git a/tuf/download.py b/tuf/download.py index 4af5905e..162c0b7e 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -226,7 +226,91 @@ def _check_hashes(input_file, trusted_hashes): -def download_url_to_tempfileobj(url, required_hashes=None, required_length=None): +def _download_fixed_amount_of_data(connection, temp_file, file_length, + required_length): + """ + + This is a helper function, where the download really happens. While-block + reads data from connection a fixed chunk of data at a time, or less, until + 'file_length' is reached. + + + connection: + The object that the _open_connection returns for communicating with the + server about the contents of a URL. + + temp_file: + A temporary file where the contents at the URL specified by the + 'connection' object will be stored. + + file_length: + The number of bytes that the server claims is the size of the file. + + required_length: + The number of bytes that we must download for the file. This is almost + always specified by the TUF metadata for the data file in question + (except in the case of timestamp metadata, in which case we would fix a + reasonable upper bound). + + + Data from the server will be written to 'temp_file'. + + + Runtime or network exceptions will be raised without question. + + + total_downloaded: + The total number of bytes we have downloaded for the desired file and + which should be equal to 'required_length'. + + """ + + # The maximum chunk of data, in bytes, we would download in every round. + BLOCK_SIZE = 8192 + + # Keep track of total bytes downloaded. + total_downloaded = 0 + + try: + while True: + # We download a fixed chunk of data in every round. This is so that we + # can defend against slow retrieval attacks. Furthermore, we do not wish + # to download an extremely large file in one shot. + data = connection.read(min(BLOCK_SIZE, file_length-total_downloaded)) + + # We might have no more data to read. Check number of bytes downloaded. + if not data: + message = 'Downloaded '+str(total_downloaded)+'/'+ \ + str(file_length)+' bytes.' + logger.debug(message) + + # Did we download the correct amount indicated by 'Content-Length' + # or user? Because file_length is always eaqual to required_length + # we just need check one of them. + if total_downloaded != file_length: + message = 'Downloaded '+str(total_downloaded)+'. Expected '+ \ + str(file_length)+' for '+url + raise tuf.DownloadError(message) + + # Finally, we signal that the download is complete. + break + + # Data successfully read from the connection. Store it. + temp_file.write(data) + total_downloaded = total_downloaded + len(data) + except: + raise + else: + return total_downloaded + finally: + connection.close() + + + + + +def download_url_to_tempfileobj(url, required_hashes=None, + required_length=None): """ Given the url, hashes and length of the desired file, this function @@ -260,7 +344,7 @@ def download_url_to_tempfileobj(url, required_hashes=None, required_length=None) 'tuf.util.TempFile' instance. - + """ # Do all of the arguments have the appropriate format? @@ -312,8 +396,12 @@ def download_url_to_tempfileobj(url, required_hashes=None, required_length=None) ', got '+str(file_length)+' bytes.' raise tuf.DownloadError(message) - # Call this function to download the target file - connection_of_download(file_length, required_length, connection, temp_file) + # For readibility, we perform the download in a separate function, which + # returns the total number of downloaded bytes; this number should be equal + # to required_length. + total_downloaded = _download_fixed_amount_of_data(connection, temp_file, + file_length, + required_length) # We appear to have downloaded the correct amount. Check the hashes. if required_length is not None and required_hashes is not None: @@ -329,34 +417,3 @@ def download_url_to_tempfileobj(url, required_hashes=None, required_length=None) return temp_file -def connection_of_download(file_length, required_length, connection, temp_file): - """ - This function is where the download really happens. While-block reads data from - connection 8192-bytes at a time, or less, until 'file_length' is reached. - """ - # Keep track of total bytes downloaded. - total_downloaded = 0 - - while True: - data = connection.read(min(8192, file_length - total_downloaded)) - # We might have no more data to read. Let us check bytes downloaded. - if not data: - message = 'Downloaded '+str(total_downloaded)+'/' \ - +str(file_length)+' bytes.' - logger.debug(message) - # Did we download the correct amount indicated by 'Content-Length' - # or user? Because file_length is always eaqual to required_length - # we just need check one of them. - if total_downloaded != file_length: - message = 'Downloaded '+str(total_downloaded)+'. Expected '+ \ - str(file_length)+' for '+url - raise tuf.DownloadError(message) - - break - # Data successfully read from the connection. Store it. - temp_file.write(data) - total_downloaded = total_downloaded + len(data) - - connection.close() - - diff --git a/tuf/tests/test_download.py b/tuf/tests/test_download.py index 9e53be9e..cb773e78 100755 --- a/tuf/tests/test_download.py +++ b/tuf/tests/test_download.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_download.py From 29d522eb19995ea084330867d7d37bb945e9dc7a Mon Sep 17 00:00:00 2001 From: zanefisher Date: Mon, 22 Jul 2013 17:22:44 -0400 Subject: [PATCH 28/28] Merge branch 'master', remote-tracking branch 'upstream/master'