From b7dc3fccea9b110ab5b884bf1fa4f62272796946 Mon Sep 17 00:00:00 2001 From: "syrttgump@gmail.com" Date: Fri, 26 Jul 2013 15:46:43 -0400 Subject: [PATCH 1/9] Endless attack test fix --- tuf/client/updater.py | 1 + tuf/download.py | 8 +++++++- tuf/tests/system_tests/test_endless_data_attack.py | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index ed697e70..dca94cff 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1727,6 +1727,7 @@ def download_target(self, target, destination_directory): trusted_length) break except (tuf.DownloadError, tuf.FormatError), e: + raise logger.warn('Download failed from '+mirror_url+'.') target_file_object = None continue diff --git a/tuf/download.py b/tuf/download.py index 162c0b7e..b9f52261 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -394,7 +394,7 @@ def download_url_to_tempfileobj(url, required_hashes=None, if required_length is not None and file_length != required_length: message = 'Incorrect length for '+url+'. Expected '+str(required_length)+ \ ', got '+str(file_length)+' bytes.' - raise tuf.DownloadError(message) + logger.warn(message) # For readibility, we perform the download in a separate function, which # returns the total number of downloaded bytes; this number should be equal @@ -402,6 +402,12 @@ def download_url_to_tempfileobj(url, required_hashes=None, total_downloaded = _download_fixed_amount_of_data(connection, temp_file, file_length, required_length) + + # Does 'total_downloaded' match 'required_length'? + if total_downloaded != required_length: + message = 'Total downloded length '+str(total_downloaded)+ \ + ' bytes doesn\'t match required length '+str(required_length)+' bytes.' + raise tuf.DownloadError(message) # We appear to have downloaded the correct amount. Check the hashes. if required_length is not None and required_hashes is not None: diff --git a/tuf/tests/system_tests/test_endless_data_attack.py b/tuf/tests/system_tests/test_endless_data_attack.py index 9366accf..f58d9ecf 100755 --- a/tuf/tests/system_tests/test_endless_data_attack.py +++ b/tuf/tests/system_tests/test_endless_data_attack.py @@ -123,7 +123,8 @@ def test_arbitrary_package_attack(TUF=False): # If tuf.DownloadError is raised, this means that TUF has prevented # the download of an unrecognized file. Enable the logging to see, # what actually happened. - pass + #pass + raise else: # Check whether the attack succeeded by inspecting the content of the From d24567814817b93ee968757b91927c174e6f83a0 Mon Sep 17 00:00:00 2001 From: "syrttgump@gmail.com" Date: Fri, 26 Jul 2013 17:00:40 -0400 Subject: [PATCH 2/9] Endless attack test fix --- tuf/download.py | 2 +- tuf/tests/system_tests/test_endless_data_attack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tuf/download.py b/tuf/download.py index b9f52261..8741f79c 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -404,7 +404,7 @@ def download_url_to_tempfileobj(url, required_hashes=None, required_length) # Does 'total_downloaded' match 'required_length'? - if total_downloaded != required_length: + if required_length is not None and total_downloaded != required_length: message = 'Total downloded length '+str(total_downloaded)+ \ ' bytes doesn\'t match required length '+str(required_length)+' bytes.' raise tuf.DownloadError(message) diff --git a/tuf/tests/system_tests/test_endless_data_attack.py b/tuf/tests/system_tests/test_endless_data_attack.py index f58d9ecf..0e820e78 100755 --- a/tuf/tests/system_tests/test_endless_data_attack.py +++ b/tuf/tests/system_tests/test_endless_data_attack.py @@ -97,7 +97,7 @@ def test_arbitrary_package_attack(TUF=False): if TUF: # Update TUF metadata before attacker modifies anything. util_test_tools.tuf_refresh_repo(root_repo, keyids) - + print "refresh finished" # Modify the url. Remember that the interposition will intercept # urls that have 'localhost:9999' hostname, which was specified in # the json interposition configuration file. Look for 'hostname' From a2a8ba0217d87f9add064bf31c09ffc0b5049ebd Mon Sep 17 00:00:00 2001 From: ttgump Date: Tue, 30 Jul 2013 14:31:28 -0400 Subject: [PATCH 3/9] modified update.py --- tuf/client/updater.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index dca94cff..34a686fe 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1719,6 +1719,12 @@ def download_target(self, target, destination_directory): trusted_hashes = target['fileinfo']['hashes'] target_file_object = None + + # Store mirrors which doesn't work well + mirror_errors = {} + + logger.info('Trying to download: '+repr(target_filepath)) + # Iterate through the repositority mirrors until we successfully # download a target. for mirror_url in get_mirrors('target', target_filepath, self.mirrors): @@ -1727,13 +1733,13 @@ def download_target(self, target, destination_directory): trusted_length) break except (tuf.DownloadError, tuf.FormatError), e: - raise + mirror_errors[mirror_url] = e logger.warn('Download failed from '+mirror_url+'.') target_file_object = None continue # We have gone through all the mirrors. Did we get a target file object? if target_file_object == None: - raise tuf.DownloadError('No download locations known.') + raise tuf.DownloadError('No download locations known: '+repr(mirror_errors)) # We acquired a target file object from a mirror. Move the file into # place (i.e., locally to 'destination_directory'). From c5be2cd69eecfa172f251c6fae21510f6c3d44a8 Mon Sep 17 00:00:00 2001 From: ttgump Date: Wed, 31 Jul 2013 12:00:36 -0400 Subject: [PATCH 4/9] endless_attack_test_fix --- tuf/client/updater.py | 2 +- tuf/download.py | 4 ++-- tuf/tests/system_tests/test_endless_data_attack.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 34a686fe..8cf0a2b7 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1733,7 +1733,7 @@ def download_target(self, target, destination_directory): trusted_length) break except (tuf.DownloadError, tuf.FormatError), e: - mirror_errors[mirror_url] = e + mirror_errors[mirror_url] = e logger.warn('Download failed from '+mirror_url+'.') target_file_object = None continue diff --git a/tuf/download.py b/tuf/download.py index 8741f79c..561cbae0 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -404,8 +404,8 @@ def download_url_to_tempfileobj(url, required_hashes=None, required_length) # Does 'total_downloaded' match 'required_length'? - if required_length is not None and total_downloaded != required_length: - message = 'Total downloded length '+str(total_downloaded)+ \ + if total_downloaded != required_length: + message = 'Total downloaded length '+str(total_downloaded)+ \ ' bytes doesn\'t match required length '+str(required_length)+' bytes.' raise tuf.DownloadError(message) diff --git a/tuf/tests/system_tests/test_endless_data_attack.py b/tuf/tests/system_tests/test_endless_data_attack.py index 0e820e78..18d999e4 100755 --- a/tuf/tests/system_tests/test_endless_data_attack.py +++ b/tuf/tests/system_tests/test_endless_data_attack.py @@ -97,13 +97,14 @@ def test_arbitrary_package_attack(TUF=False): if TUF: # Update TUF metadata before attacker modifies anything. util_test_tools.tuf_refresh_repo(root_repo, keyids) - print "refresh finished" # Modify the url. Remember that the interposition will intercept # urls that have 'localhost:9999' hostname, which was specified in # the json interposition configuration file. Look for 'hostname' # in 'util_test_tools.py'. Further, the 'file_basename' is the target # path relative to 'targets_dir'. + print url_to_repo url_to_repo = 'http://localhost:9999/'+file_basename + print url_to_repo # Attacker modifies the file at the targets repository. target = os.path.join(tuf_targets, file_basename) @@ -119,12 +120,11 @@ def test_arbitrary_package_attack(TUF=False): # Client downloads (tries to download) the file. _download(url=url_to_repo, filename=downloaded_file, tuf=TUF) - except tuf.DownloadError: + except tuf.DownloadError,e: # If tuf.DownloadError is raised, this means that TUF has prevented # the download of an unrecognized file. Enable the logging to see, # what actually happened. - #pass - raise + logger.warn('Download failed: '+repr(e)) else: # Check whether the attack succeeded by inspecting the content of the From 1f5a1e53eaf8e155215673b574f0fd8508c48abb Mon Sep 17 00:00:00 2001 From: ttgump Date: Wed, 31 Jul 2013 12:02:37 -0400 Subject: [PATCH 5/9] endless_attack_test_fix --- tuf/tests/system_tests/test_endless_data_attack.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tuf/tests/system_tests/test_endless_data_attack.py b/tuf/tests/system_tests/test_endless_data_attack.py index 18d999e4..0530f128 100755 --- a/tuf/tests/system_tests/test_endless_data_attack.py +++ b/tuf/tests/system_tests/test_endless_data_attack.py @@ -102,9 +102,7 @@ def test_arbitrary_package_attack(TUF=False): # the json interposition configuration file. Look for 'hostname' # in 'util_test_tools.py'. Further, the 'file_basename' is the target # path relative to 'targets_dir'. - print url_to_repo url_to_repo = 'http://localhost:9999/'+file_basename - print url_to_repo # Attacker modifies the file at the targets repository. target = os.path.join(tuf_targets, file_basename) From aa6f2b6ddf432914bca2c57ece23136a1ed8863c Mon Sep 17 00:00:00 2001 From: ttgump Date: Thu, 8 Aug 2013 16:24:03 -0400 Subject: [PATCH 6/9] unicode fix in python2 --- tuf/repo/signercli.py | 4 ++++ tuf/tests/system_tests/test_endless_data_attack.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/tuf/repo/signercli.py b/tuf/repo/signercli.py index d3d7b1d9..a4c76f86 100755 --- a/tuf/repo/signercli.py +++ b/tuf/repo/signercli.py @@ -116,6 +116,10 @@ def _prompt(message, result_type=str): caller. """ + # we need to use unicode in python 2 + python_version = sys.version_info[0] + if python_version==2: + str = unicode return result_type(raw_input(message)) diff --git a/tuf/tests/system_tests/test_endless_data_attack.py b/tuf/tests/system_tests/test_endless_data_attack.py index 0530f128..99dbd6dd 100755 --- a/tuf/tests/system_tests/test_endless_data_attack.py +++ b/tuf/tests/system_tests/test_endless_data_attack.py @@ -42,11 +42,14 @@ import tuf from tuf.interposition import urllib_tuf +import logging + # Disable logging. util_test_tools.disable_logging() +logger = logging.getLogger('tuf') class EndlessDataAttack(Exception): From 1e3b9cfcad5a639f3d4ef66f271bd825705b1375 Mon Sep 17 00:00:00 2001 From: ttgump Date: Tue, 13 Aug 2013 14:57:03 -0400 Subject: [PATCH 7/9] Pull from upstream --- docs/tuf-spec.txt | 64 +++--- tuf/client/updater.py | 155 +++++++++++---- tuf/log.py | 187 +++++++++++++++--- tuf/repo/keystore.py | 12 +- .../test_extraneous_dependencies_attack.py | 1 + tuf/tests/test_keystore.py | 7 + tuf/tests/test_push.py | 0 tuf/tests/test_pushtoolslib.py | 0 8 files changed, 323 insertions(+), 103 deletions(-) mode change 100644 => 100755 tuf/tests/test_push.py mode change 100644 => 100755 tuf/tests/test_pushtoolslib.py diff --git a/docs/tuf-spec.txt b/docs/tuf-spec.txt index 0c88c437..517d17e3 100644 --- a/docs/tuf-spec.txt +++ b/docs/tuf-spec.txt @@ -32,7 +32,10 @@ in all popular Linux package managers. More information and current versions of this document can be found at https://www.updateframework.com/ - The development of TUF is supported by GENI (http://www.geni.net/). + The Global Environment for Network Innovations (GENI) and the National + Science Foundation (NSF) have provided support for the development of TUF. + (http://www.geni.net/) + (http://www.nsf.gov/) TUF's Python implementation is based heavily on Thandy, the application updater for Tor (http://www.torproject.org/). Its design and this spec are @@ -409,26 +412,24 @@ 4.2. File formats: general principles All signed files are of the format: - { "signed" : X, + { "signed" : ROLE, "signatures" : [ - { "keyid" : K, - "method" : M, - "sig" : S } + { "keyid" : KEYID, + "method" : METHOD, + "sig" : SIGNATURE } , ... ] } - where: X is a list whose first element describes the signed object. - K is the identifier of a key signing the document - M is the method to be used to make the signature - S is a signature of the canonical encoding of X using the - identified key. + where: ROLE is a dictionary whose "_type" field describes the role type. + KEYID is the identifier of the key signing the ROLE dictionary. + METHOD is the key signing method used to generate the signature. + SIGNATURE is a signature of the canonical encoding of ROLE using the + signing key belonging to KEYID. We define one signing method at present: - sha256-pkcs1 : A base64 encoded signature of the SHA256 hash of the - canonical encoding of X, using PKCS-1 padding. + "evp" : An interface to OpenSSL's EVP functions. - All times are given as strings of the format "YYYY-MM-DD HH:MM:SS", - in UTC. + All times are given as strings of the format "YYYY-MM-DD HH:MM:SS UTC". All keys are of the format: { "keytype" : KEYTYPE, @@ -443,13 +444,12 @@ We define one keytype at present: 'rsa'. Its format is: { "keytype" : "rsa", - "keyval" : { "e" : E, - "n" : N } + "keyval" : { "public" : PUBLIC, + "private" : PRIVATE } } - where E and N are the binary representations of the exponent and - modulus, encoded as big-endian numbers in base64. All RSA keys must - be at least 2048 bits long. + where PUBLIC and PRIVATE are in PEM format and are strings. All RSA keys + must be at least 2048 bits long. 4.3. File formats: root.txt @@ -462,7 +462,7 @@ The format of root.txt is as follows: { "_type" : "Root", - "ts" : TIME, + "version" : VERSION, "expires" : EXPIRES, "keys" : { KEYID : KEY @@ -474,12 +474,11 @@ , ... } } - The "ts" line describes when this file was updated. Clients - MUST NOT replace a file with an older one, and SHOULD NOT accept a - file too far in the future. + VERSION is an integer that is greater than 0. Clients MUST NOT replace a + metadata file with a version number less than the one currently trusted. - The "expires" line states when the metadata should be considered expired - and no longer trusted by clients. Clients MUST NOT trust an expired file. + EXPIRES determines when metadata should be considered expired and no longer + trusted by clients. Clients MUST NOT trust an expired file. A ROLE is one of "root", "release", "targets", "timestamp", or "mirrors". A role for each of "root", "release", "timestamp", and "targets" MUST be @@ -505,7 +504,7 @@ The format of release.txt is as follows: { "_type" : "Release", - "ts" : TIME, + "version" : VERSION, "expires" : EXPIRES, "meta" : METAFILES } @@ -527,7 +526,7 @@ The format of targets.txt is as follows: { "_type" : "Targets", - "ts" : TIME, + "version" : VERSION, "expires" : EXPIRES, "targets" : TARGETS, ("delegations" : DELEGATIONS) @@ -572,10 +571,9 @@ The "paths" list describes paths that the role is trusted to provide. Clients MUST check that a target is in one of the trusted paths of all roles in a delegation chain, not just in a trusted path of the role that describes - the target file. The format of a PATHPATTERN may be either a path to a - single file or a path to a directory and end with "/**" to indicate all - files under that directory. The value of "/**" by itself therefore means - all files. + the target file. The format of a PATHPATTERN may be either a path to a single + file, or a path to a directory to indicate all files and/or subdirectories + under that directory. We are currently investigating a few "priority tag" schemes to resolve conflicts between delegated roles that share responsibility for overlapping @@ -610,7 +608,7 @@ The format of the timestamp file is as follows: { "_type" : "Timestamp", - "ts" : TIME, + "version" : VERSION, "expires" : EXPIRES, "meta" : METAFILES } @@ -628,7 +626,7 @@ The format of mirrors.txt is as follows: { "_type" : "Mirrorlist", - "ts" : TIME, + "version" : VERSION, "expires" : EXPIRES, "mirrors" : [ { "urlbase" : URLBASE, diff --git a/tuf/client/updater.py b/tuf/client/updater.py index b976ecf5..c0710e73 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -912,7 +912,9 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): under 'paths'. A parent role may delegate trust to all files under a particular directory, including files in subdirectories, by simply listing the directory (e.g., 'packages/source/Django/', the equivalent - of 'packages/source/Django/*'). + of 'packages/source/Django/*'). Targets listed in hashed bins are + also validated (i.e., its calculated path hash prefix must be delegated + by the parent role. metadata_role: @@ -928,7 +930,8 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): tuf.RepositoryError: If the targets of 'metadata_role' are not allowed according to - the parent's metadata file. + the parent's metadata file. The 'paths' and 'path_hash_prefix' fields + are verified. None. @@ -938,6 +941,13 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): """ + # The algorithm used by the repository to generate the hashes of the + # target filepaths. The repository may optionally organize + # targets into hashed bins to ease target delegations and role metadata + # management. The use of consistent hashing allows for a uniform + # distribution of targets into bins. + HASH_PATH_ALGORITHM = 'sha256' + # Return if 'metadata_role' is 'targets'. 'targets' is not # a delegated role. if metadata_role == 'targets': @@ -955,30 +965,60 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): role_index = tuf.repo.signerlib.find_delegated_role(roles, metadata_role) # Ensure the delegated role exists prior to extracting trusted paths - # from the parent's 'paths'. + # from the parent's 'paths', or trusted path hash prefixes from the parent's + # 'path_hash_prefix'. if role_index is not None: role = roles[role_index] - allowed_child_paths = role['paths'] + allowed_child_paths = role.get('paths') + allowed_child_path_hash_prefix = role.get('path_hash_prefix') actual_child_targets = metadata_object['targets'].keys() - - # Check that each delegated target is either explicitly listed or a parent - # directory is found under role['paths'], otherwise raise an exception. - # If the parent role explicitly lists target file paths in 'paths', - # this loop will run in O(n^2), the worst-case. The repository - # maintainer will likely delegate entire directories, and opt for - # explicit file paths if the targets in a directory are delegated to - # different roles/developers. - for child_target in actual_child_targets: - for allowed_child_path in allowed_child_paths: - prefix = os.path.commonprefix([child_target, allowed_child_path]) - if prefix == allowed_child_path: - break - else: - message = 'Role '+repr(metadata_role)+' specifies target '+\ - repr(child_target)+' which is not an allowed path according '+\ - 'to the delegations set by '+repr(parent_role)+'.' - raise tuf.RepositoryError(message) + if allowed_child_path_hash_prefix is not None: + for child_target in actual_child_targets: + # Calculate the hash of 'child_target' to determine if it has been + # placed in the correct bin. The client currently assumes the + # repository uses 'HASH_PATH_ALGORITHM' to generate hashes. + # TODO: Should the TUF spec restrict the repository to one particular + # algorithm? Should we allow the repository to specify in the role + # dictionary the algorithm used for these generated hashed paths? + digest_object = tuf.hash.digest(HASH_PATH_ALGORITHM) + digest_object.update(child_target) + child_target_path_hash = digest_object.hexdigest() + + if not child_target_path_hash.startswith(allowed_child_path_hash_prefix): + message = 'Role '+repr(metadata_role)+' specifies target '+\ + repr(child_target)+ ' which does not have a path hash prefix '+\ + 'matching the prefix listed by the parent role '+\ + repr(parent_role)+'.' + raise tuf.RepositoryError(message) + elif allowed_child_paths is not None: + + # Check that each delegated target is either explicitly listed or a parent + # directory is found under role['paths'], otherwise raise an exception. + # If the parent role explicitly lists target file paths in 'paths', + # this loop will run in O(n^2), the worst-case. The repository + # maintainer will likely delegate entire directories, and opt for + # explicit file paths if the targets in a directory are delegated to + # different roles/developers. + for child_target in actual_child_targets: + for allowed_child_path in allowed_child_paths: + prefix = os.path.commonprefix([child_target, allowed_child_path]) + if prefix == allowed_child_path: + break + else: + message = 'Role '+repr(metadata_role)+' specifies target '+\ + repr(child_target)+' which is not an allowed path according '+\ + 'to the delegations set by '+repr(parent_role)+'.' + raise tuf.RepositoryError(message) + else: + + # 'role' should have been validated when it was downloaded. + # The 'paths' or 'path_hash_prefix' fields should not be missing, + # so log a warning if this else clause is reached. + message = repr(role)+' unexpectedly did not contain one of '+\ + 'the required fields ("paths" or "path_hash_prefix").' + logger.warn(message) + # Raise an exception if the parent has not delegated to the specified # 'metadata_role' child role. else: @@ -1014,7 +1054,7 @@ def _fileinfo_has_changed(self, metadata_filename, new_fileinfo): dict conforms to 'tuf.formats.FILEINFO_SCHEMA' and has the form: {'length': 23423 - 'hashes': {'sha256': adfbc32343..}} + 'hashes': {'sha256': /dfbc32343..}} None. @@ -1534,6 +1574,13 @@ def target(self, target_filepath): # Raise 'tuf.FormatError' if there is a mismatch. tuf.formats.RELPATH_SCHEMA.check_match(target_filepath) + # The algorithm used by the repository to generate the hashes of the + # target filepaths. The repository may optionally organize + # targets into hashed bins to ease target delegations and role metadata + # management. The use of consistent hashing allows for a uniform + # distribution of targets into bins. + HASH_PATH_ALGORITHM = 'sha256' + # Ensure the client has the most up-to-date version of 'targets.txt'. # Raise 'tuf.MetadataNotAvailableError' if the changed metadata # cannot be successfully downloaded and 'tuf.RepositoryError' if the @@ -1545,12 +1592,23 @@ def target(self, target_filepath): # The target is assumed to be missing until proven otherwise. target = None + # Calculate the hash of the filepath to determine which bin to find the + # target. The client currently assumes the repository uses + # 'HASH_PATH_ALGORITHM' to generate hashes. + # TODO: Should the TUF spec restrict the repository to one particular + # algorithm? Should we allow the repository to specify in the role + # dictionary the algorithm used for these generated hashed paths? + digest_object = tuf.hash.digest(HASH_PATH_ALGORITHM) + digest_object.update(target_filepath) + target_file_path_hash = digest_object.hexdigest() + try: current_metadata = self.metadata['current'] role_names = ['targets'] # Preorder depth-first traversal of the tree of target delegations. while len(role_names) > 0 and target is None: + # Pop the role name from the top of the stack. role_name = role_names.pop(-1) @@ -1575,20 +1633,49 @@ def target(self, target_filepath): break # Push children in reverse order of appearance onto the stack. + # NOTE: This may be a slow operation if there are many delegated roles + # or bins. for child_role in reversed(child_roles): child_role_name = child_role['name'] - child_role_paths = child_role['paths'] + child_role_paths = child_role.get('paths') + child_role_path_hash_prefix = child_role.get('path_hash_prefix') - # Ensure that we explore only delegated roles trusted with the target. - # We assume conservation of delegated paths in the complete tree of - # delegations. Note that the call to _ensure_all_targets_allowed in - # _update_metadata should already ensure that all targets metadata is - # valid; i.e. that the targets signed by a delegatee is a proper - # subset of the targets delegated to it by the delegator. - # Nevertheless, we check it again here for performance and safety - # reasons. - if target_filepath in child_role_paths: - role_names.append(child_role_name) + if child_role_path_hash_prefix is not None: + if target_file_path_hash.startswith(child_role_path_hash_prefix): + + # Found a matching path hash prefix. The metadata for + # 'child_role_name' will be retrieved on the next iteration + # of the while-loop. + role_names.append(child_role_name) + elif child_role_paths is not None: + + # Ensure that we explore only delegated roles trusted with the target. + # We assume conservation of delegated paths in the complete tree of + # delegations. Note that the call to _ensure_all_targets_allowed in + # _update_metadata should already ensure that all targets metadata is + # valid; i.e. that the targets signed by a delegatee is a proper + # subset of the targets delegated to it by the delegator. + # Nevertheless, we check it again here for performance and safety + # reasons. + for child_role_path in child_role_paths: + + # A child role path may be a filepath or directory. The child + # role 'child_role_name' is added if 'target_filepath' is located + # under 'child_role_path'. Explicit filepaths are also added. + prefix = os.path.commonprefix([target_filepath, child_role_path]) + if prefix == child_role_path: + + # The metadata for 'child_role_name' will be retrieved on the next + # iteration of the while-loop. + role_names.append(child_role_name) + else: + + # 'role_name' should have been validated when it was downloaded. + # The 'paths' or 'path_hash_prefix' fields should not be missing, + # so log a warning if this else clause is reached. + message = repr(child_role)+' unexpectedly did not contain one of '+\ + 'the required fields ("paths" or "path_hash_prefix").' + logger.warn(message) except: raise finally: diff --git a/tuf/log.py b/tuf/log.py index cd61ecf7..f3c018bc 100755 --- a/tuf/log.py +++ b/tuf/log.py @@ -12,10 +12,9 @@ See LICENSE for licensing information. - A central location for all logging-related configuration. - This module should be imported once by the main program. - If other modules wish to incorporate 'tuf' logging, they - should do the following: + A central location for all logging-related configuration. This module should + be imported once by the main program. If other modules wish to incorporate + 'tuf' logging, they should do the following: import logging logger = logging.getLogger('tuf') @@ -26,18 +25,28 @@ instance. In this 'log.py' module, we perform the initial setup for the name 'tuf'. The 'log.py' module should only be imported once by the main program. When any other module does a logging.getLogger('tuf'), it is referring to the - same 'tuf' instance and its associated settings we set up here in 'log.py'. - See http://docs.python.org/library/logging.html#logger-objects - for more information. + same 'tuf' instance, and its associated settings, set here in 'log.py'. + See http://docs.python.org/library/logging.html#logger-objects for more + information. We use multiple handlers to process log messages in various ways and to configure each one independently. Instead of using one single manner of processing log messages, we can use two built-in handlers that have already been configured for us. For example, the built-in FileHandler will catch - log message and dump them to a file. If we wanted, we could set this file - handler to only catch CRITICAL (and greater) messages and save them to a - file. The other stream handler would still handle DEBUG-level (and greater) - messages. + log messages and dump them to a file. If we wanted, we could set this file + handler to only catch CRITICAL (and greater) messages and save them to a + file. Other handlers (e.g., StreamHandler) could handle INFO-level + (and greater) messages. + + Logging Levels: + + --Level-- --Value-- + logging.CRITICAL 50 + logging.ERROR 40 + logging.WARNING 30 + logging.INFO 20 + logging.DEBUG 10 + logging.NOTSET 0 """ @@ -45,35 +54,46 @@ import logging import time +import tuf -_DEFAULT_LOG_LEVEL = logging.INFO +# Setting a handler's log level filters only logging messages of that level +# (and above). For example, setting the built-in StreamHandler's log level to +# 'logging.WARNING' will cause the stream handler to only process messages +# of levels: WARNING, ERROR, and CRITICAL. _DEFAULT_LOG_FILENAME = 'tuf.log' +_DEFAULT_LOG_LEVEL = logging.DEBUG +_DEFAULT_CONSOLE_LOG_LEVEL = logging.INFO +_DEFAULT_FILE_LOG_LEVEL = logging.DEBUG # Set the format for logging messages. +# Example format for '_FORMAT_STRING': +# [2013-08-13 15:21:18,068 UTC] [tuf] [INFO][_update_metadata:851@updater.py] _FORMAT_STRING = '[%(asctime)s UTC] [%(name)s] [%(levelname)s]'+\ '[%(funcName)s:%(lineno)s@%(filename)s] %(message)s' + + logging.Formatter.converter = time.gmtime formatter = logging.Formatter(_FORMAT_STRING) -# Set the handlers for the logger. -# The built-in stream handler will log -# messages to 'sys.stderr' and capture -# '_DEFAULT_LOG_LEVEL' messages. -stream_handler = logging.StreamHandler() -stream_handler.setLevel(_DEFAULT_LOG_LEVEL) -stream_handler.setFormatter(formatter) +# Set the handlers for the logger. The console handler is unset by default. A +# module importing 'log.py' should explicitly set the console handler if +# outputting log messages to the screen is needed. Adding a console handler +# can be done with tuf.log.add_console_handler(). Logging messages to a file +# *is* set by default. +console_handler = None -# Set the built-in file handler. Messages -# will be logged to '_DEFAULT_LOG_FILENAME' -# and use the logger's default log level. -# The file will be opened in append mode. +# Set the built-in file handler. Messages will be logged to +# '_DEFAULT_LOG_FILENAME', and only those messages with a log level of +# '_DEFAULT_LOG_LEVEL'. The log level of messages handled by 'file_handler' +# may be modified with 'set_filehandler_log_level()'. '_DEFAULT_LOG_FILENAME' +# will be opened in append mode. file_handler = logging.FileHandler(_DEFAULT_LOG_FILENAME) +file_handler.setLevel(_DEFAULT_LOG_LEVEL) file_handler.setFormatter(formatter) # Set the logger and its settings. logger = logging.getLogger('tuf') logger.setLevel(_DEFAULT_LOG_LEVEL) -logger.addHandler(stream_handler) logger.addHandler(file_handler) # Silently ignore logger exceptions. @@ -83,27 +103,132 @@ -def set_log_level(log_level): +def set_log_level(log_level=_DEFAULT_LOG_LEVEL): """ Allow the default log level to be overridden. log_level: - The log level to set for the logger and handler(s). - E.g., logging.INFO; logging.CRITICAL. + The log level to set for the 'log.py' file handler. + 'log_level' examples: logging.INFO; logging.CRITICAL. None. - Overrides the logging level for the internal - 'logger' and 'handler'. + Overrides the logging level for the 'log.py' file handler. None. """ - + + # Does 'log_level' have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.LENGTH_SCHEMA.check_match(log_level) + logger.setLevel(log_level) - stream_handler.setLevel(log_level) + + + + + +def set_filehandler_log_level(log_level=_DEFAULT_FILE_LOG_LEVEL): + """ + + Allow the default file handler log level to be overridden. + + + log_level: + The log level to set for the 'log.py' file handler. + 'log_level' examples: logging.INFO; logging.CRITICAL. + + + None. + + + Overrides the logging level for the 'log.py' file handler. + + + None. + + """ + + # Does 'log_level' have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.LENGTH_SCHEMA.check_match(log_level) + + file_handler.setLevel(log_level) + + + + + +def set_console_log_level(log_level=_DEFAULT_CONSOLE_LOG_LEVEL): + """ + + Allow the default log level for console messages to be overridden. + + + log_level: + The log level to set for the console handler. + 'log_level' examples: logging.INFO; logging.CRITICAL. + + + tuf.Error, if the 'log.py' console handler has not been set yet with + add_console_handler(). + + + Overrides the logging level for the console handler. + + + None. + + """ + + # Does 'log_level' have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.LENGTH_SCHEMA.check_match(log_level) + + if console_handler is not None: + console_handler.setLevel(log_level) + else: + message = 'The console handler has not been set with add_console_handler().' + raise tuf.Error(message) + + + + +def add_console_handler(log_level=_DEFAULT_CONSOLE_LOG_LEVEL): + """ + + Add a console handler and set its log level to 'log_level'. + + + log_level: + The log level to set for the console handler. + 'log_level' examples: logging.INFO; logging.CRITICAL. + + + None. + + + Adds a console handler to the 'log.py' logger and sets its logging level to + 'log_level'. + + + None. + + """ + + # Does 'log_level' have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.LENGTH_SCHEMA.check_match(log_level) + + # Set the console handler for the logger. The built-in console handler will + # log messages to 'sys.stderr' and capture 'log_level' messages. + console_handler = logging.StreamHandler() + console_handler.setLevel(log_level) + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) diff --git a/tuf/repo/keystore.py b/tuf/repo/keystore.py index c2ca98c0..c8fe5afd 100755 --- a/tuf/repo/keystore.py +++ b/tuf/repo/keystore.py @@ -147,7 +147,7 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): directory_name: The name of the directory containing the key files ('.key'), - conformant to tuf.formats.RELPATH_SCHEMA. + conformant to 'tuf.formats.RELPATH_SCHEMA'. keyids: A list containing the keyids of the signing keys to load. @@ -188,10 +188,8 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): logger.info('Loading private key(s) from '+repr(directory_name)) - # Make sure the directory exists. - if not os.path.exists(directory_name): - logger.warn('...no such directory. Keystore cannot be loaded.') - else: + # Load the private key(s) if 'directory_name' exists, otherwise log a warning. + if os.path.exists(directory_name): # Decrypt the keys we can from those stored in 'keyids'. for keyid in keyids: try: @@ -243,7 +241,11 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): logger.warn(repr(full_filepath)+' contains an invalid key type.') continue + else: + logger.warn('...no such directory. Keystore cannot be loaded.') + logger.info('Done.') + return loaded_keys diff --git a/tuf/tests/system_tests/test_extraneous_dependencies_attack.py b/tuf/tests/system_tests/test_extraneous_dependencies_attack.py index 72937d65..25a258f0 100755 --- a/tuf/tests/system_tests/test_extraneous_dependencies_attack.py +++ b/tuf/tests/system_tests/test_extraneous_dependencies_attack.py @@ -187,4 +187,5 @@ def _write_rogue_metadata(): try: test_extraneous_dependencies_attack() except ExtraneousDependenciesAttackAlert, error: + raise print 'error' diff --git a/tuf/tests/test_keystore.py b/tuf/tests/test_keystore.py index 196bb7d4..816f400a 100755 --- a/tuf/tests/test_keystore.py +++ b/tuf/tests/test_keystore.py @@ -19,12 +19,19 @@ import unittest import shutil import os +import logging import tuf.repo.keystore import tuf.rsa_key import tuf.formats import tuf.util +logger = logging.getLogger('tuf') + +# Disable all logging calls of level CRITICAL and below. +# Comment the line below to enable logging. +logging.disable(logging.CRITICAL) + # We'll need json module for testing '_encrypt()' and '_decrypt()' # internal function. json = tuf.util.import_json() diff --git a/tuf/tests/test_push.py b/tuf/tests/test_push.py old mode 100644 new mode 100755 diff --git a/tuf/tests/test_pushtoolslib.py b/tuf/tests/test_pushtoolslib.py old mode 100644 new mode 100755 From f8d699e62b751be5c5aa49aa7d93981bcd59cc8f Mon Sep 17 00:00:00 2001 From: zanefisher Date: Tue, 13 Aug 2013 18:41:38 -0400 Subject: [PATCH 8/9] Randomized order of aggragate tests. Added teardown code to some unit tests. --- tuf/tests/aggregate_tests.py | 3 +++ tuf/tests/test_signercli.py | 4 ++++ tuf/tests/test_signerlib.py | 4 ++++ tuf/tests/test_updater.py | 4 ++++ 4 files changed, 15 insertions(+) diff --git a/tuf/tests/aggregate_tests.py b/tuf/tests/aggregate_tests.py index 3ac9ecac..b5dd21b7 100755 --- a/tuf/tests/aggregate_tests.py +++ b/tuf/tests/aggregate_tests.py @@ -27,12 +27,15 @@ import tuf.keydb as keydb import tuf.repo.keystore as keystore import tuf.roledb as roledb +import random tests_list = glob.glob('test_*.py') # Remove '.py' from each filename. tests_list = [test[:-3] for test in tests_list] +random.shuffle(tests_list) + suite = unittest.TestLoader().loadTestsFromNames(tests_list) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tuf/tests/test_signercli.py b/tuf/tests/test_signercli.py index aa91d862..ea9fccc5 100755 --- a/tuf/tests/test_signercli.py +++ b/tuf/tests/test_signercli.py @@ -1559,5 +1559,9 @@ def _mock_get_keyids(junk): signercli._get_metadata_directory = original_get_metadata_directory +def tearDownModule(): + unittest_toolbox.Modified_TestCase.clear_toolbox() + + if __name__ == '__main__': unittest.main() diff --git a/tuf/tests/test_signerlib.py b/tuf/tests/test_signerlib.py index dcb47993..8331ea74 100755 --- a/tuf/tests/test_signerlib.py +++ b/tuf/tests/test_signerlib.py @@ -975,6 +975,10 @@ def _get_signed_role_info(self, role, directory=None): filename) return signed_meta, role_info +def tearDownModule(): + unit_tbox.clear_toolbox() + tuf.repo.keystore.clear_keystore() + if __name__ == '__main__': unittest.main() diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index d32664e7..8386338e 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -1150,5 +1150,9 @@ def test_8_remove_obsolete_targets(self): tuf.download.download_url_to_tempfileobj = original_download +def tearDownModule(): + setup.remove_all_repositories(TestUpdater.repositories['main_repository']) + unittest_toolbox.Modified_TestCase.clear_toolbox() + if __name__ == '__main__': unittest.main() From 2048a3eccbf59095d0acb58c0863f49c86f7f9e0 Mon Sep 17 00:00:00 2001 From: ttgump Date: Tue, 27 Aug 2013 17:02:42 -0400 Subject: [PATCH 9/9] Update fix for endless-data-attack test. --- tuf/download.py | 6 +++--- tuf/tests/system_tests/test_endless_data_attack.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tuf/download.py b/tuf/download.py index 561cbae0..861e2966 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -403,10 +403,10 @@ def download_url_to_tempfileobj(url, required_hashes=None, file_length, required_length) - # Does 'total_downloaded' match 'required_length'? - if total_downloaded != required_length: + # Does 'total_downloaded' matches 'required_length'? + if required_length is not None and total_downloaded != required_length: message = 'Total downloaded length '+str(total_downloaded)+ \ - ' bytes doesn\'t match required length '+str(required_length)+' bytes.' + ' bytes doesn\'t match required length '+str(required_length)+' bytes.' raise tuf.DownloadError(message) # We appear to have downloaded the correct amount. Check the hashes. diff --git a/tuf/tests/system_tests/test_endless_data_attack.py b/tuf/tests/system_tests/test_endless_data_attack.py index 196e4af9..476a134b 100755 --- a/tuf/tests/system_tests/test_endless_data_attack.py +++ b/tuf/tests/system_tests/test_endless_data_attack.py @@ -44,6 +44,7 @@ import logging +logger = logging.getLogger('tuf.test_endless_data_attack') class EndlessDataAttack(Exception): pass