diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 7b9e91c2..bb91f939 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -55,7 +55,7 @@ # client accesses this module solely to set the repository directory. # This directory will hold the files downloaded from a remote repository. settings.repository_directory = 'local-repository' - + # Next, the client creates a dictionary object containing the repository # mirrors. The client may download content from any one of these mirrors. # In the example below, a single mirror named 'mirror1' is defined. The @@ -67,7 +67,7 @@ # interpreted as no confinement. In other words, the client can download # targets from any directory or subdirectories. If the client had chosen # 'targets1/', they would have been confined to the '/targets/targets1/' - # directory on the 'http://localhost:8001' mirror. + # directory on the 'http://localhost:8001' mirror. repository_mirrors = {'mirror1': {'url_prefix': 'http://localhost:8001', 'metadata_path': 'metadata', 'targets_path': 'targets', @@ -85,7 +85,7 @@ # The target file information for all the repository targets is determined. targets = updater.all_targets() - + # Among these targets, determine the ones that have changed since the client's # last refresh(). A target is considered updated if it does not exist in # 'destination_directory' (current directory) or the target located there has @@ -145,65 +145,65 @@ class Updater(object): Provide a class that can download target files securely. The updater keeps track of currently and previously trusted metadata, target files - available to the client, target file attributes such as file size and + available to the client, target file attributes such as file size and hashes, key and role information, metadata signatures, and the ability to determine when the download of a file should be permitted. self.metadata: Dictionary holding the currently and previously trusted metadata. - + Example: {'current': {'root': ROOT_SCHEMA, 'targets':TARGETS_SCHEMA, ...}, 'previous': {'root': ROOT_SCHEMA, 'targets':TARGETS_SCHEMA, ...}} - + self.metadata_directory: The directory where trusted metadata is stored. - + self.versioninfo: A cache of version numbers for the roles available on the repository. - + Example: {'targets.json': {'version': 128}, ...} self.mirrors: The repository mirrors from which metadata and targets are available. Conformant to 'tuf.ssl_crypto.formats.MIRRORDICT_SCHEMA'. - + self.updater_name: The name of the updater instance. - + refresh(): This method downloads, verifies, and loads metadata for the top-level roles in a specific order (i.e., timestamp -> snapshot -> root -> targets) The expiration time for downloaded metadata is also verified. - + The metadata for delegated roles are not refreshed by this method, but by the target methods (e.g., all_targets(), targets_of_role(), get_one_valid_targetinfo()). The refresh() method should be called by the client before any target requests. - + all_targets(): Returns the target information for the 'targets' and delegated roles. Prior to extracting the target information, this method attempts a file download of all the target metadata that have changed. - + targets_of_role('targets'): Returns the target information for the targets of a specified role. Like all_targets(), delegated metadata is updated if it has changed. - + get_one_valid_targetinfo(file_path): Returns the target information for a specific file identified by its file path. This target method also downloads the metadata of updated targets. - + updated_targets(targets, destination_directory): After the client has retrieved the target information for those targets they are interested in updating, they would call this method to determine which targets have changed from those saved locally on disk. All the targets that have changed are returns in a list. From this list, they can request a download by calling 'download_target()'. - + download_target(target, destination_directory): This method performs the actual download of the specified target. The file is saved to the 'destination_directory' argument. @@ -231,40 +231,40 @@ def __init__(self, updater_name, repository_mirrors): for these delegated roles, including nested delegated roles, are loaded, updated, and saved to the 'self.metadata' store by the target methods, like all_targets() and targets_of_role(). - + The initial set of metadata files are provided by the software update system utilizing TUF. - + In order to use an updater, the following directories must already exist locally: - + {settings.repository_directory}/metadata/current {settings.repository_directory}/metadata/previous - + and, at a minimum, the root metadata file must exist: {settings.repository_directory}/metadata/current/root.json - + updater_name: The name of the updater. - + repository_mirrors: A dictionary holding repository mirror information, conformant to 'tuf.ssl_crypto.formats.MIRRORDICT_SCHEMA'. This dictionary holds information such as the directory containing the metadata and target files, the server's URL prefix, and the target content directories the client should be confined to. - + repository_mirrors = {'mirror1': {'url_prefix': 'http://localhost:8001', 'metadata_path': 'metadata', 'targets_path': 'targets', 'confined_target_dirs': ['']}} - + tuf.ssl_commons.exceptions.FormatError: - If the arguments are improperly formatted. - + If the arguments are improperly formatted. + tuf.ssl_commons.exceptions.RepositoryError: If there is an error with the updater's repository files, such as a missing 'root.json' file. @@ -277,7 +277,7 @@ def __init__(self, updater_name, repository_mirrors): None. """ - + # Do the arguments have the correct format? # These checks ensure the arguments have the appropriate # number of objects and object types and that all dict @@ -285,17 +285,17 @@ def __init__(self, updater_name, repository_mirrors): # Raise 'tuf.ssl_commons.exceptions.FormatError' if there is a mistmatch. tuf.ssl_crypto.formats.NAME_SCHEMA.check_match(updater_name) tuf.ssl_crypto.formats.MIRRORDICT_SCHEMA.check_match(repository_mirrors) - + # Save the validated arguments. self.updater_name = updater_name self.mirrors = repository_mirrors # Store the trusted metadata read from disk. self.metadata = {} - + # Store the currently trusted/verified metadata. - self.metadata['current'] = {} - + self.metadata['current'] = {} + # Store the previously trusted/verified metadata. self.metadata['previous'] = {} @@ -310,7 +310,7 @@ def __init__(self, updater_name, repository_mirrors): # determine whether a metadata file has changed and so needs to be # re-downloaded. self.fileinfo = {} - + # Store the location of the client's metadata directory. self.metadata_directory = {} @@ -318,17 +318,17 @@ def __init__(self, updater_name, repository_mirrors): # determines if metadata and target files downloaded from remote # repositories include the digest. self.consistent_snapshot = False - + # Ensure the repository metadata directory has been set. if settings.repository_directory is None: raise tuf.ssl_commons.exceptions.RepositoryError('The TUF update client' ' module must specify the directory containing the local repository' ' files. "settings.repository_directory" MUST be set.') - # Set the path for the current set of metadata files. + # Set the path for the current set of metadata files. repository_directory = settings.repository_directory current_path = os.path.join(repository_directory, 'metadata', 'current') - + # Ensure the current path is valid/exists before saving it. if not os.path.exists(current_path): raise tuf.ssl_commons.exceptions.RepositoryError('Missing' @@ -336,22 +336,22 @@ def __init__(self, updater_name, repository_mirrors): ' contain the Root metadata file.') self.metadata_directory['current'] = current_path - - # Set the path for the previous set of metadata files. - previous_path = os.path.join(repository_directory, 'metadata', 'previous') - + + # Set the path for the previous set of metadata files. + previous_path = os.path.join(repository_directory, 'metadata', 'previous') + # Ensure the previous path is valid/exists. if not os.path.exists(previous_path): raise tuf.ssl_commons.exceptions.RepositoryError('Missing ' + repr(previous_path) + '.' ' This path MUST exist.') self.metadata_directory['previous'] = previous_path - + # Load current and previous metadata. for metadata_set in ['current', 'previous']: for metadata_role in ['root', 'targets', 'snapshot', 'timestamp']: self._load_metadata_from_file(metadata_set, metadata_role) - + # Raise an exception if the repository is missing the required 'root' # metadata. if 'root' not in self.metadata['current']: @@ -366,7 +366,7 @@ def __str__(self): """ The string representation of an Updater object. """ - + return self.updater_name @@ -381,12 +381,12 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): 'root.json') cannot be loaded, raise an exception. The extracted metadata object loaded from file is saved to the metadata store (i.e., self.metadata). - - + + metadata_set: The string 'current' or 'previous', depending on whether one wants to load the currently or previously trusted metadata file. - + metadata_role: The name of the metadata. This is a role name and should not end in '.json'. Examples: 'root', 'targets', 'unclaimed'. @@ -398,7 +398,7 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): tuf.ssl_commons.exceptions.Error: If there was an error importing a delegated role of 'metadata_role' or the 'metadata_set' is not one currently supported. - + If the metadata is loaded successfully, it is saved to the metadata store. If 'metadata_role' is 'root', the role and key databases @@ -417,29 +417,29 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): metadata_directory = self.metadata_directory[metadata_set] metadata_filename = metadata_role + '.json' metadata_filepath = os.path.join(metadata_directory, metadata_filename) - - # Ensure the metadata path is valid/exists, else ignore the call. + + # Ensure the metadata path is valid/exists, else ignore the call. if os.path.exists(metadata_filepath): # Load the file. The loaded object should conform to # 'tuf.ssl_crypto.formats.SIGNABLE_SCHEMA'. try: metadata_signable = tuf.ssl_crypto.util.load_json_file(metadata_filepath) - + # Although the metadata file may exist locally, it may not # be a valid json file. On the next refresh cycle, it will be # updated as required. If Root if cannot be loaded from disk # successfully, an exception should be raised by the caller. except tuf.ssl_commons.exceptions.Error: return - + tuf.formats.check_signable_object_format(metadata_signable) # Extract the 'signed' role object from 'metadata_signable'. metadata_object = metadata_signable['signed'] - + # Save the metadata object to the metadata store. self.metadata[metadata_set][metadata_role] = metadata_object - + # If 'metadata_role' is 'root' or targets metadata, the key and role # databases must be rebuilt. If 'root', ensure self.consistent_snaptshots # is updated. @@ -447,7 +447,7 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): if metadata_role == 'root': self._rebuild_key_and_role_db() self.consistent_snapshot = metadata_object['consistent_snapshot'] - + elif metadata_object['_type'] == 'Targets': # TODO: Should we also remove the keys of the delegated roles? self._import_delegations(metadata_role) @@ -482,7 +482,7 @@ def _rebuild_key_and_role_db(self): None. """ - + # Clobbering this means all delegated metadata files are rendered outdated # and will need to be reloaded. However, reloading the delegated metadata # files is avoided here because fetching target information with methods @@ -503,11 +503,11 @@ def _import_delegations(self, parent_role): """ Non-public method that imports all the roles delegated by 'parent_role'. - + parent_role: The role whose delegations will be imported. - + tuf.ssl_commons.exceptions.FormatError: If a key attribute of a delegated role's signing key is @@ -523,9 +523,9 @@ def _import_delegations(self, parent_role): None. """ - + current_parent_metadata = self.metadata['current'][parent_role] - + if 'delegations' not in current_parent_metadata: return @@ -534,12 +534,12 @@ def _import_delegations(self, parent_role): roles_info = current_parent_metadata['delegations'].get('roles', []) logger.debug('Adding roles delegated from ' + repr(parent_role) + '.') - + # Iterate the keys of the delegated roles of 'parent_role' and load them. for keyid, keyinfo in six.iteritems(keys_info): if keyinfo['keytype'] in ['rsa', 'ed25519']: key, keyids = tuf.ssl_crypto.keys.format_metadata_to_key(keyinfo) - + # We specify the keyid to ensure that it's the correct keyid # for the key. try: @@ -550,12 +550,12 @@ def _import_delegations(self, parent_role): except tuf.ssl_commons.exceptions.KeyAlreadyExistsError: pass - + except (tuf.ssl_commons.exceptions.FormatError, tuf.ssl_commons.exceptions.Error): logger.exception('Invalid key for keyid: ' + repr(keyid) + '.') logger.error('Aborting role delegation for parent role ' + parent_role + '.') raise - + else: logger.warning('Invalid key type for ' + repr(keyid) + '.') continue @@ -568,10 +568,10 @@ def _import_delegations(self, parent_role): rolename = roleinfo.get('name') logger.debug('Adding delegated role: ' + str(rolename) + '.') tuf.roledb.add_role(rolename, roleinfo, self.updater_name) - + except tuf.ssl_commons.exceptions.RoleAlreadyExistsError: logger.warning('Role already exists: ' + rolename) - + except: logger.exception('Failed to add delegated role: ' + rolename + '.') raise @@ -587,7 +587,7 @@ def refresh(self, unsafely_update_root_if_necessary=True): update request process follows a specific order to ensure the metadata files are securely updated: timestamp -> snapshot -> root (if necessary) -> targets. - + Delegated metadata is not refreshed by this method. After this method is called, the use of target methods (e.g., all_targets(), targets_of_role(), or get_one_valid_targetinfo()) will update delegated @@ -617,8 +617,8 @@ def refresh(self, unsafely_update_root_if_necessary=True): tuf.ssl_commons.exceptions.ExpiredMetadataError: If any of the top-level metadata is expired (whether a new version was downloaded expired or no new version was found and the existing - version is now expired). - + version is now expired). + Updates the metadata files of the top-level roles with the latest information. @@ -626,9 +626,9 @@ def refresh(self, unsafely_update_root_if_necessary=True): None. """ - - # Do the arguments have the correct format? - # This check ensures the arguments have the appropriate + + # Do the arguments have the correct format? + # This check ensures the arguments have the appropriate # number of objects and object types, and that all dict # keys are properly named. # Raise 'tuf.ssl_commons.exceptions.FormatError' if the check fail. @@ -651,10 +651,10 @@ def refresh(self, unsafely_update_root_if_necessary=True): # is insufficient trusted signatures for the specified metadata. # Raise 'tuf.ssl_commons.exceptions.NoWorkingMirrorError' if an update fails. root_metadata = self.metadata['current']['root'] - - try: + + try: self._ensure_not_expired(root_metadata, 'root') - + except tuf.ssl_commons.exceptions.ExpiredMetadataError: # Raise 'tuf.ssl_commons.exceptions.NoWorkingMirrorError' if a valid (not # expired, properly signed, and valid metadata) 'root.json' cannot be @@ -662,7 +662,7 @@ def refresh(self, unsafely_update_root_if_necessary=True): if unsafely_update_root_if_necessary: logger.info('Expired Root metadata was loaded from disk.' ' Try to update it now.' ) - + # The caller explicitly requested not to unsafely fetch an expired Root. else: logger.info('An expired Root metadata was loaded and must be updated.') @@ -671,9 +671,9 @@ def refresh(self, unsafely_update_root_if_necessary=True): # TODO: How should the latest root metadata be verified? According to the # currently trusted root keys? What if all of the currently trusted # root keys have since been revoked by the latest metadata? Alternatively, - # do we blindly trust the downloaded root metadata here? + # do we blindly trust the downloaded root metadata here? self._update_root_metadata(root_metadata) - + # Use default but sane information for timestamp metadata, and do not # require strict checks on its required length. self._update_metadata('timestamp', DEFAULT_TIMESTAMP_UPPERLENGTH) @@ -683,7 +683,7 @@ def refresh(self, unsafely_update_root_if_necessary=True): self._update_metadata_if_changed('snapshot', referenced_metadata='timestamp') self._update_metadata_if_changed('targets') - + def _update_root_metadata(self, current_root_metadata, compression_algorithm=None): @@ -693,22 +693,22 @@ def _update_root_metadata(self, current_root_metadata, compression_algorithm=Non well as the previous root threshold and keys. The update process for root files means that each intermediate root file must be downloaded, to build a chain of trusted root keys from keys already trusted by the client: - + 1.root -> 2.root -> 3.root - + 3.root must be signed by the threshold and keys of 2.root, and 2.root must be signed by the threshold and keys of 1.root. - + current_root_metadata: The currently held version of root. - + compresison_algorithm: The compression algorithm used to compress remote metadata. - + Updates the root metadata files with the latest information. - + None. """ @@ -716,15 +716,15 @@ def _update_root_metadata(self, current_root_metadata, compression_algorithm=Non # Retrieve the latest, remote root.json. latest_root_metadata_file = \ self._get_metadata_file('root', 'root.json', - settings.DEFAULT_ROOT_REQUIRED_LENGTH, None, + settings.DEFAULT_ROOT_REQUIRED_LENGTH, None, compression_algorithm=compression_algorithm) latest_root_metadata = \ tuf.ssl_crypto.util.load_json_string(latest_root_metadata_file.read().decode('utf-8')) - - + + next_version = current_root_metadata['version'] + 1 latest_version = latest_root_metadata['signed']['version'] - + # update from the next version of root up to (and including) the latest # version. For example: # current = version 1 @@ -735,7 +735,7 @@ def _update_root_metadata(self, current_root_metadata, compression_algorithm=Non # in the latest root.json after running through the intermediates with # _update_metadata(). self.consistent_snapshot = True - self._update_metadata('root', settings.DEFAULT_ROOT_REQUIRED_LENGTH, version=version, + self._update_metadata('root', settings.DEFAULT_ROOT_REQUIRED_LENGTH, version=version, compression_algorithm=compression_algorithm) @@ -775,7 +775,7 @@ def _check_hashes(self, file_object, trusted_hashes): digest_object = tuf.ssl_crypto.hash.digest(algorithm) digest_object.update(file_object.read()) computed_hash = digest_object.hexdigest() - + # Raise an exception if any of the hashes are incorrect. if trusted_hash != computed_hash: raise tuf.ssl_commons.exceptions.BadHashError(trusted_hash, computed_hash) @@ -817,7 +817,7 @@ def _hard_check_file_length(self, file_object, trusted_file_length): # Read the entire contents of 'file_object', a 'tuf.ssl_crypto.util.TempFile' file-like # object that ensures the entire file is read. observed_length = len(file_object.read()) - + # Return and log a message if the length 'file_object' is equal to # 'trusted_file_length', otherwise raise an exception. A hard check # ensures that a downloaded file strictly matches a known, or trusted, @@ -866,8 +866,8 @@ def _soft_check_file_length(self, file_object, trusted_file_length): # Read the entire contents of 'file_object', a # 'tuf.ssl_crypto.util.TempFile' file-like object that ensures the entire # file is read. - observed_length = len(file_object.read()) - + observed_length = len(file_object.read()) + # Return and log a message if 'file_object' is less than or equal to # 'trusted_file_length', otherwise raise an exception. A soft check # ensures that an upper bound restricts how large a file is downloaded. @@ -919,7 +919,7 @@ def _get_target_file(self, target_filepath, file_length, file_hashes): # and called. The 'verify_target_file' function ensures the file length # and hashes of 'target_filepath' are strictly equal to the trusted values. def verify_target_file(target_file_object): - + # Every target file must have its length and hashes inspected. self._hard_check_file_length(target_file_object, file_length) self._check_hashes(target_file_object, file_hashes) @@ -983,13 +983,13 @@ def _verify_uncompressed_metadata_file(self, metadata_file_object, """ metadata = metadata_file_object.read().decode('utf-8') - + try: metadata_signable = tuf.ssl_crypto.util.load_json_string(metadata) - + except Exception as exception: raise tuf.ssl_commons.exceptions.InvalidMetadataJSONError(exception) - + else: # Ensure the loaded 'metadata_signable' is properly formatted. Raise # 'tuf.ssl_commons.exceptions.FormatError' if not. @@ -997,15 +997,15 @@ def _verify_uncompressed_metadata_file(self, metadata_file_object, # Is 'metadata_signable' expired? self._ensure_not_expired(metadata_signable['signed'], metadata_role) - + # We previously verified version numbers in this function, but have since # moved version number verification to the functions that retrieve # metadata. # Verify the signature on the downloaded metadata object. - + valid = tuf.sig.verify(metadata_signable, metadata_role, self.updater_name) - + if not valid: raise tuf.ssl_commons.exceptions.BadSignatureError(metadata_role) @@ -1040,7 +1040,7 @@ def _get_metadata_file(self, metadata_role, remote_filename, compression_algorithm: The name of the compression algorithm (e.g., 'gzip'). The algorithm is - needed if the remote metadata file is compressed. + needed if the remote metadata file is compressed. tuf.ssl_commons.exceptions.NoWorkingMirrorError: @@ -1070,21 +1070,21 @@ def _get_metadata_file(self, metadata_role, remote_filename, if compression_algorithm is not None: logger.info('Decompressing ' + str(file_mirror)) file_object.decompress_temp_file_object(compression_algorithm) - + else: logger.info('Not decompressing ' + str(file_mirror)) - + # Verify 'file_object' according to the callable function. # 'file_object' is also verified if decompressed above (i.e., the # uncompressed version). metadata_signable = \ tuf.ssl_crypto.util.load_json_string(file_object.read().decode('utf-8')) - + # If the version number is unspecified, ensure that the version number # downloaded is greater than the currently trusted version number for # 'metadata_role'. - version_downloaded = metadata_signable['signed']['version'] - + version_downloaded = metadata_signable['signed']['version'] + if expected_version is not None: # Verify that the downloaded version matches the version expected by # the caller. @@ -1092,7 +1092,7 @@ def _get_metadata_file(self, metadata_role, remote_filename, raise tuf.ssl_commons.exceptions.BadVersionNumberError('Downloaded' ' version number: ' + repr(version_downloaded) + '. Version' ' number MUST be: ' + repr(expected_version)) - + # The caller does not know which version to download. Verify that the # downloaded version is at least greater than the one locally available. else: @@ -1100,49 +1100,49 @@ def _get_metadata_file(self, metadata_role, remote_filename, # 'timestamp.json', if available, is less than what was downloaded. # Otherwise, accept the new timestamp with version number # 'version_downloaded'. - + try: current_version = \ self.metadata['current'][metadata_role]['version'] - + if version_downloaded < current_version: raise tuf.ssl_commons.exceptions.ReplayedMetadataError(metadata_role, version_downloaded, current_version) - + except KeyError: logger.info(metadata_role + ' not available locally.') self._verify_uncompressed_metadata_file(file_object, metadata_role) - + except Exception as exception: # Remember the error from this mirror, and "reset" the target file. logger.exception('Update failed from ' + file_mirror + '.') file_mirror_errors[file_mirror] = exception file_object = None - + else: break if file_object: return file_object - + else: logger.error('Failed to update ' + repr(remote_filename) + ' from all' ' mirrors: ' + repr(file_mirror_errors)) raise tuf.ssl_commons.exceptions.NoWorkingMirrorError(file_mirror_errors) - - + + def _verify_root_chain_link(self, role, current, next): if role != 'root': return True - + current_role = current['roles'][role] - + # Verify next metadata with current keys/threshold valid = tuf.sig.verify(next, role, self.updater_name, current_role['threshold'], current_role['keyids']) - + if not valid: raise tuf.ssl_commons.exceptions.BadSignatureError('Root is not signed by previous threshold' ' of keys.') @@ -1182,12 +1182,12 @@ def _get_file(self, filepath, verify_file_function, file_type, compression: The name of the compression algorithm (e.g., 'gzip'), if the metadata - file is compressed. - + file is compressed. + verify_compressed_file_function: If compression is specified, in the case of metadata files, this callable function may be set to perform verification of the compressed - version of the metadata file. Decompressed metadata is also verified. + version of the metadata file. Decompressed metadata is also verified. download_safely: A boolean switch to toggle safe or unsafe download of the file. @@ -1206,7 +1206,7 @@ def _get_file(self, filepath, verify_file_function, file_type, A 'tuf.ssl_crypto.util.TempFile' file-like object containing the metadata or target. """ - + file_mirrors = tuf.mirrors.get_list_of_mirrors(file_type, filepath, self.mirrors) # file_mirror (URL): error (Exception) @@ -1226,14 +1226,14 @@ def _get_file(self, filepath, verify_file_function, file_type, file_length) if compression is not None: - if verify_compressed_file_function is not None: - verify_compressed_file_function(file_object) + if verify_compressed_file_function is not None: + verify_compressed_file_function(file_object) logger.info('Decompressing ' + str(file_mirror)) file_object.decompress_temp_file_object(compression) - + else: logger.info('Not decompressing ' + str(file_mirror)) - + # Verify 'file_object' according to the callable function. # 'file_object' is also verified if decompressed above (i.e., the # uncompressed version). @@ -1244,13 +1244,13 @@ def _get_file(self, filepath, verify_file_function, file_type, logger.exception('Update failed from ' + file_mirror + '.') file_mirror_errors[file_mirror] = exception file_object = None - + else: break if file_object: return file_object - + else: logger.error('Failed to update {0} from all mirrors: {1}'.format( filepath, file_mirror_errors)) @@ -1269,12 +1269,12 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None, has been updated by the repository and thus needs to be re-downloaded. The current and previous metadata stores are updated if the newly downloaded metadata is successfully downloaded and verified. - + metadata_role: The name of the metadata. This is a role name and should not end in '.json'. Examples: 'root', 'targets', 'targets/linux/x86'. - + upperbound_filelength: The expected length, or upper bound, of the metadata file to be downloaded. @@ -1282,7 +1282,7 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None, version: The expected and required version number of the 'metadata_role' file downloaded. 'expected_version' is an integer. - + compression_algorithm: A string designating the compression type of 'metadata_role'. The 'snapshot' metadata file may be optionally downloaded and stored in @@ -1297,7 +1297,7 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None, The metadata file belonging to 'metadata_role' is downloaded from a - repository mirror. If the metadata is valid, it is stored in the + repository mirror. If the metadata is valid, it is stored in the metadata store. @@ -1307,9 +1307,9 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None, # Construct the metadata filename as expected by the download/mirror modules. metadata_filename = metadata_role + '.json' uncompressed_metadata_filename = metadata_filename - + # The 'snapshot' or Targets metadata may be compressed. Add the appropriate - # extension to 'metadata_filename'. + # extension to 'metadata_filename'. if compression_algorithm == 'gzip': metadata_filename = metadata_filename + '.gz' @@ -1332,7 +1332,7 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None, # Note also that we presently support decompression of only "safe" # metadata, but this is easily extend to "unsafe" metadata as well as # "safe" targets. - + remote_filename = metadata_filename filename_version = '' @@ -1340,7 +1340,7 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None, filename_version = version dirname, basename = os.path.split(remote_filename) remote_filename = os.path.join(dirname, str(filename_version) + '.' + basename) - + metadata_file_object = \ self._get_metadata_file(metadata_role, remote_filename, upperbound_filelength, version, @@ -1353,11 +1353,11 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None, metadata_filename) current_filepath = os.path.abspath(current_filepath) tuf.ssl_crypto.util.ensure_parent_dir(current_filepath) - + previous_filepath = os.path.join(self.metadata_directory['previous'], metadata_filename) previous_filepath = os.path.abspath(previous_filepath) - + if os.path.exists(current_filepath): # Previous metadata might not exist, say when delegations are added. tuf.ssl_crypto.util.ensure_parent_dir(previous_filepath) @@ -1368,7 +1368,7 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None, # 'metadata_file_object' is an instance of tuf.ssl_crypto.util.TempFile. metadata_signable = \ tuf.ssl_crypto.util.load_json_string(metadata_file_object.read().decode('utf-8')) - + if compression_algorithm == 'gzip': current_uncompressed_filepath = \ os.path.join(self.metadata_directory['current'], @@ -1376,7 +1376,7 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None, current_uncompressed_filepath = \ os.path.abspath(current_uncompressed_filepath) metadata_file_object.move(current_uncompressed_filepath) - + else: metadata_file_object.move(current_filepath) @@ -1385,7 +1385,7 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None, # stored for 'metadata_role'. updated_metadata_object = metadata_signable['signed'] current_metadata_object = self.metadata['current'].get(metadata_role) - + self._verify_root_chain_link(metadata_role, current_metadata_object, metadata_signable) @@ -1419,7 +1419,7 @@ def _update_metadata_if_changed(self, metadata_role, is updated in refresh() by calling _update_metadata('timestamp'). This method is also called for delegated role metadata, which are referenced by 'snapshot'. - + If the metadata needs to be updated but an update cannot be obtained, this method will delete the file (with the exception of the root metadata, which never gets removed without a replacement). @@ -1446,12 +1446,12 @@ def _update_metadata_if_changed(self, metadata_role, other words, it is updated by calling _update_metadata('timestamp') and not by this method. The referenced metadata for 'snapshot' is 'timestamp'. See refresh(). - + tuf.ssl_commons.exceptions.NoWorkingMirrorError: If 'metadata_role' could not be downloaded after determining that it had changed. - + tuf.ssl_commons.exceptions.RepositoryError: If the referenced metadata is missing. @@ -1466,25 +1466,25 @@ def _update_metadata_if_changed(self, metadata_role, None. """ - + uncompressed_metadata_filename = metadata_role + '.json' expected_versioninfo = None expected_fileinfo = None # Ensure the referenced metadata has been loaded. The 'root' role may be - # updated without having 'snapshot' available. + # updated without having 'snapshot' available. if referenced_metadata not in self.metadata['current']: raise tuf.ssl_commons.exceptions.RepositoryError('Cannot update' ' ' + repr(metadata_role) + ' because ' + referenced_metadata + ' is' ' missing.') - + # The referenced metadata has been loaded. Extract the new versioninfo for - # 'metadata_role' from it. + # 'metadata_role' from it. else: logger.debug(repr(metadata_role) + ' referenced in ' + repr(referenced_metadata)+ '. ' + repr(metadata_role) + ' may be updated.') - + # Simply return if the metadata for 'metadata_role' has not been updated, # according to the uncompressed metadata provided by the referenced # metadata. The metadata is considered updated if its version number is @@ -1492,23 +1492,23 @@ def _update_metadata_if_changed(self, metadata_role, expected_versioninfo = self.metadata['current'][referenced_metadata] \ ['meta'] \ [uncompressed_metadata_filename] - + if not self._versioninfo_has_been_updated(uncompressed_metadata_filename, expected_versioninfo): logger.info(repr(uncompressed_metadata_filename) + ' up-to-date.') - + # Since we have not downloaded a new version of this metadata, we # should check to see if our local version is stale and notify the user # if so. This raises tuf.ssl_commons.exceptions.ExpiredMetadataError if the metadata we # have is expired. Resolves issue #322. self._ensure_not_expired(self.metadata['current'][metadata_role], metadata_role) - # TODO: If 'metadata_role' is root or snapshot, we should verify that + # TODO: If 'metadata_role' is root or snapshot, we should verify that # root's hash matches what's in snapshot, and that snapshot hash matches # what's listed in timestamp.json. - + return - + logger.debug('Metadata ' + repr(uncompressed_metadata_filename) + ' has changed.') # There might be a compressed version of 'snapshot.json' or Targets @@ -1538,7 +1538,7 @@ def _update_metadata_if_changed(self, metadata_role, logger.debug('Compressed version of ' + repr(uncompressed_metadata_filename) + ' is available at ' + repr(gzip_metadata_filename) + '.') - + else: logger.debug('Compressed version of ' + repr(uncompressed_metadata_filename) + ' not available.') @@ -1547,16 +1547,16 @@ def _update_metadata_if_changed(self, metadata_role, # known. Set an upper limit for the length of the downloaded file for each # expected role. Note: The Timestamp role is not updated via this # function. - if metadata_role == 'snapshot': + if metadata_role == 'snapshot': upperbound_filelength = settings.DEFAULT_SNAPSHOT_REQUIRED_LENGTH - + elif metadata_role == 'root': upperbound_filelength = settings.DEFAULT_ROOT_REQUIRED_LENGTH - + # The metadata is considered Targets (or delegated Targets metadata). else: upperbound_filelength = settings.DEFAULT_TARGETS_REQUIRED_LENGTH - + try: self._update_metadata(metadata_role, upperbound_filelength, expected_versioninfo['version'], compression) @@ -1566,14 +1566,14 @@ def _update_metadata_if_changed(self, metadata_role, # metadata. We shouldn't use the old metadata anymore. This will get rid # of in-memory knowledge of the role and delegated roles, but will leave # delegated metadata files as current files on disk. - # + # # TODO: Should we get rid of the delegated metadata files? We shouldn't # need to, but we need to check the trust implications of the current # implementation. self._delete_metadata(metadata_role) logger.error('Metadata for ' + repr(metadata_role) + ' cannot be updated.') raise - + else: # We need to import the delegated roles of 'metadata_role', since its # list of delegations might have changed from what was previously @@ -1594,7 +1594,7 @@ def _versioninfo_has_been_updated(self, metadata_filename, new_versioninfo): extracted from the latest copy of the metadata that references 'metadata_filename'. Example: 'root.json' would be referenced by 'snapshot.json'. - + 'new_versioninfo' should only be 'None' if this is for updating 'root.json' without having 'snapshot.json' available. @@ -1609,9 +1609,9 @@ def _versioninfo_has_been_updated(self, metadata_filename, new_versioninfo): updating 'root' without having 'snapshot' available. This dict conforms to 'tuf.ssl_crypto.formats.VERSIONINFO_SCHEMA' and has the form: - + {'version': 288} - + None. @@ -1622,7 +1622,7 @@ def _versioninfo_has_been_updated(self, metadata_filename, new_versioninfo): Boolean. True if the versioninfo has changed, False otherwise. """ - + # If there is no versioninfo currently stored for 'metadata_filename', # try to load the file, calculate the versioninfo, and store it. if metadata_filename not in self.versioninfo: @@ -1638,7 +1638,7 @@ def _versioninfo_has_been_updated(self, metadata_filename, new_versioninfo): if new_versioninfo['version'] > current_versioninfo['version']: return True - + else: return False @@ -1670,11 +1670,11 @@ def _update_versioninfo(self, metadata_filename): None. """ - + # In case we delayed loading the metadata and didn't do it in - # __init__ (such as with delegated metadata), then get the version + # __init__ (such as with delegated metadata), then get the version # info now. - + # Save the path to the current metadata file for 'metadata_filename'. current_filepath = os.path.join(self.metadata_directory['current'], metadata_filename) @@ -1682,7 +1682,7 @@ def _update_versioninfo(self, metadata_filename): if not os.path.exists(current_filepath): self.versioninfo[metadata_filename] = None return - + # Extract the version information from the trusted snapshot role and save # it to the 'self.versioninfo' store. if metadata_filename == 'timestamp.json': @@ -1695,7 +1695,7 @@ def _update_versioninfo(self, metadata_filename): # root.json initially, and perform a refresh of top-level metadata to # obtain the remaining roles. elif metadata_filename == 'snapshot.json': - + # Verify the version number of the currently trusted snapshot.json in # snapshot.json itself. Checking the version number specified in # timestamp.json may be greater than the version specified in the @@ -1703,13 +1703,13 @@ def _update_versioninfo(self, metadata_filename): try: timestamp_version_number = self.metadata['current']['snapshot']['version'] trusted_versioninfo = tuf.formats.make_versioninfo(timestamp_version_number) - + except KeyError: trusted_versioninfo = \ self.metadata['current']['timestamp']['meta']['snapshot.json'] - + else: - + try: # The metadata file names in 'self.metadata' exclude the role # extension. Strip the '.json' extension when checking if @@ -1718,7 +1718,7 @@ def _update_versioninfo(self, metadata_filename): self.metadata['current'][metadata_filename[:-len('.json')]]['version'] trusted_versioninfo = \ tuf.formats.make_versioninfo(targets_version_number) - + except KeyError: trusted_versioninfo = \ self.metadata['current']['snapshot']['meta'][metadata_filename] @@ -1737,36 +1737,36 @@ def _fileinfo_has_changed(self, metadata_filename, new_fileinfo): argument should be extracted from the latest copy of the metadata that references 'metadata_filename'. Example: 'root.json' would be referenced by 'snapshot.json'. - + 'new_fileinfo' should only be 'None' if this is for updating 'root.json' without having 'snapshot.json' available. - + metadadata_filename: The metadata filename for the role. For the 'root' role, 'metadata_filename' would be 'root.json'. - + new_fileinfo: A dict object representing the new file information for 'metadata_filename'. 'new_fileinfo' may be 'None' when updating 'root' without having 'snapshot' available. This dict conforms to 'tuf.ssl_crypto.formats.FILEINFO_SCHEMA' and has the form: - + {'length': 23423 'hashes': {'sha256': adfbc32343..}} - + None. - + If there is no fileinfo currently loaded for 'metada_filename', try to load it. - + Boolean. True if the fileinfo has changed, false otherwise. """ - + # If there is no fileinfo currently stored for 'metadata_filename', # try to load the file, calculate the fileinfo, and store it. if metadata_filename not in self.fileinfo: @@ -1792,7 +1792,7 @@ def _fileinfo_has_changed(self, metadata_filename, new_fileinfo): for algorithm, hash_value in six.iteritems(new_fileinfo['hashes']): # We're only looking for a single match. This isn't a security # check, we just want to prevent unnecessary downloads. - if algorithm in current_fileinfo['hashes']: + if algorithm in current_fileinfo['hashes']: if hash_value == current_fileinfo['hashes'][algorithm]: return False @@ -1810,27 +1810,27 @@ def _update_fileinfo(self, metadata_filename): 'metadata_filename' cannot be loaded, set its fileinfo' to 'None' to signal that it is not in the 'self.fileinfo' AND it also doesn't exist locally. - + metadata_filename: The metadata filename for the role. For the 'root' role, 'metadata_filename' would be 'root.json'. - + None. - + The file details of 'metadata_filename' is calculated and stored in 'self.fileinfo'. - + None. """ - + # In case we delayed loading the metadata and didn't do it in # __init__ (such as with delegated metadata), then get the file # info now. - + # Save the path to the current metadata file for 'metadata_filename'. current_filepath = os.path.join(self.metadata_directory['current'], metadata_filename) @@ -1838,7 +1838,7 @@ def _update_fileinfo(self, metadata_filename): if not os.path.exists(current_filepath): self.fileinfo[metadata_filename] = None return - + # Extract the file information from the actual file and save it # to the fileinfo store. file_length, hashes = tuf.ssl_crypto.util.get_file_details(current_filepath) @@ -1861,7 +1861,7 @@ def _move_current_to_previous(self, metadata_role): metadata_role: The name of the metadata. This is a role name and should not end in '.json'. Examples: 'root', 'targets', 'targets/linux/x86'. - + None. @@ -1884,7 +1884,7 @@ def _move_current_to_previous(self, metadata_role): if os.path.exists(previous_filepath): os.remove(previous_filepath) - # Move the current path to the previous path. + # Move the current path to the previous path. if os.path.exists(current_filepath): tuf.ssl_crypto.util.ensure_parent_dir(previous_filepath) os.rename(current_filepath, previous_filepath) @@ -1912,18 +1912,18 @@ def _delete_metadata(self, metadata_role): The role database is modified and the metadata for 'metadata_role' removed from the 'self.metadata' store. - + None. """ - + # The root metadata role is never deleted without a replacement. if metadata_role == 'root': return - + # Get rid of the current metadata file. self._move_current_to_previous(metadata_role) - + # Remove knowledge of the role. if metadata_role in self.metadata['current']: del self.metadata['current'][metadata_role] @@ -1938,7 +1938,7 @@ def _ensure_not_expired(self, metadata_object, metadata_rolename): Non-public method that raises an exception if the current specified metadata has expired. - + metadata_object: The metadata that should be expired, a 'tuf.ssl_crypto.formats.ANYROLE_SCHEMA' @@ -1947,7 +1947,7 @@ def _ensure_not_expired(self, metadata_object, metadata_rolename): metadata_rolename: The name of the metadata. This is a role name and should not end in '.json'. Examples: 'root', 'targets', 'targets/linux/x86'. - + tuf.ssl_commons.exceptions.ExpiredMetadataError: If 'metadata_rolename' has expired. @@ -1961,7 +1961,7 @@ def _ensure_not_expired(self, metadata_object, metadata_rolename): # Extract the expiration time. expires = metadata_object['expires'] - + # If the current time has surpassed the expiration date, raise an # exception. 'expires' is in # 'tuf.ssl_crypto.formats.ISO8601_DATETIME_SCHEMA' format (e.g., @@ -1974,7 +1974,7 @@ def _ensure_not_expired(self, metadata_object, metadata_rolename): # current time (i.e., a local time.) expires_datetime = iso8601.parse_date(expires) expires_timestamp = tuf.formats.datetime_to_unix_timestamp(expires_datetime) - + if expires_timestamp < current_time: message = 'Metadata '+repr(metadata_rolename)+' expired on ' + \ expires_datetime.ctime() + ' (UTC).' @@ -1988,13 +1988,13 @@ def _ensure_not_expired(self, metadata_object, metadata_rolename): def all_targets(self): """ - + Get a list of the target information for all the trusted targets on the repository. This list also includes all the targets of delegated roles. Targets of the list returned are ordered according the trusted order of the delegated roles, where parent roles come before children. The list conforms to 'tuf.ssl_crypto.formats.TARGETINFOS_SCHEMA' and has the form: - + [{'filepath': 'a/b/c.txt', 'fileinfo': {'length': 13323, 'hashes': {'sha256': dbfac345..}} @@ -2018,11 +2018,11 @@ def all_targets(self): A list of targets, conformant to 'tuf.ssl_crypto.formats.TARGETINFOS_SCHEMA'. """ - + # Load the most up-to-date targets of the 'targets' role and all # delegated roles. self._refresh_targets_metadata(refresh_all_delegated_roles=True) - + # Fetch the targets for the 'targets' role. all_targets = self._targets_of_role('targets', skip_refresh=True) @@ -2032,12 +2032,12 @@ def all_targets(self): for role in tuf.roledb.get_rolenames(self.updater_name): if role in ['root', 'snapshot', 'targets', 'timestamp']: continue - - else: + + else: delegated_targets.extend(self._targets_of_role(role, skip_refresh=True)) - + all_targets.extend(delegated_targets) - + return all_targets @@ -2061,10 +2061,10 @@ def _refresh_targets_metadata(self, rolename='targets', rolename: This is a delegated role name and should not end in '.json'. Example: 'unclaimed'. - + refresh_all_delegated_roles: Boolean indicating if all the delegated roles available in the - repository (via snapshot.json) should be refreshed. + repository (via snapshot.json) should be refreshed. tuf.ssl_commons.exceptions.RepositoryError: @@ -2081,12 +2081,12 @@ def _refresh_targets_metadata(self, rolename='targets', """ roles_to_update = [] - + if rolename + '.json' in self.metadata['current']['snapshot']['meta']: roles_to_update.append(rolename) - + if refresh_all_delegated_roles: - + for role in six.iterkeys(self.metadata['current']['snapshot']['meta']): # snapshot.json keeps track of root.json, targets.json, and delegated # roles (e.g., django.json, unclaimed.json). Remove the 'targets' role @@ -2096,10 +2096,10 @@ def _refresh_targets_metadata(self, rolename='targets', role = role[:-len('.json')] if role not in ['root', 'targets', rolename]: roles_to_update.append(role) - + else: continue - + # If there is nothing to refresh, we are done. if not roles_to_update: return @@ -2124,7 +2124,7 @@ def _targets_of_role(self, rolename, targets=None, skip_refresh=False): Non-public method that returns the target information of all the targets of 'rolename'. The returned information is a list conformant to 'tuf.ssl_crypto.formats.TARGETINFOS_SCHEMA', and has the form: - + [{'filepath': 'a/b/c.txt', 'fileinfo': {'length': 13323, 'hashes': {'sha256': dbfac345..}} @@ -2134,7 +2134,7 @@ def _targets_of_role(self, rolename, targets=None, skip_refresh=False): rolename: This is a role name and should not end in '.json'. Examples: 'targets', 'unclaimed'. - + targets: A list of targets containing target information, conformant to 'tuf.ssl_crypto.formats.TARGETINFOS_SCHEMA'. @@ -2158,7 +2158,7 @@ def _targets_of_role(self, rolename, targets=None, skip_refresh=False): if targets is None: targets = [] - + targets_of_role = list(targets) logger.debug('Getting targets of role: ' + repr(rolename) + '.') @@ -2169,19 +2169,19 @@ def _targets_of_role(self, rolename, targets=None, skip_refresh=False): # this is enforced before any new metadata is accepted. if not skip_refresh: self._refresh_targets_metadata(rolename) - + # Do we have metadata for 'rolename'? if rolename not in self.metadata['current']: logger.debug('No metadata for ' + repr(rolename) + '.' ' Unable to determine targets.') return [] - + # Get the targets specified by the role itself. for filepath, fileinfo in six.iteritems(self.metadata['current'][rolename].get('targets', [])): - new_target = {} - new_target['filepath'] = filepath + new_target = {} + new_target['filepath'] = filepath new_target['fileinfo'] = fileinfo - + targets_of_role.append(new_target) return targets_of_role @@ -2192,11 +2192,11 @@ def _targets_of_role(self, rolename, targets=None, skip_refresh=False): def targets_of_role(self, rolename='targets'): """ - + Return a list of trusted targets directly specified by 'rolename'. The returned information is a list conformant to 'tuf.ssl_crypto.formats.TARGETINFOS_SCHEMA', and has the form: - + [{'filepath': 'a/b/c.txt', 'fileinfo': {'length': 13323, 'hashes': {'sha256': dbfac345..}} @@ -2210,11 +2210,11 @@ def targets_of_role(self, rolename='targets'): rolename: The name of the role whose list of targets are wanted. The name of the role should start with 'targets'. - + tuf.ssl_commons.exceptions.FormatError: If 'rolename' is improperly formatted. - + tuf.ssl_commons.exceptions.RepositoryError: If the metadata of 'rolename' cannot be updated. @@ -2223,19 +2223,19 @@ def targets_of_role(self, rolename='targets'): The metadata of updated delegated roles are downloaded and stored. - + A list of targets, conformant to - 'tuf.ssl_crypto.formats.TARGETINFOS_SCHEMA'. + 'tuf.ssl_crypto.formats.TARGETINFOS_SCHEMA'. """ - + # Does 'rolename' have the correct format? # Raise 'tuf.ssl_commons.exceptions.FormatError' if there is a mismatch. tuf.ssl_crypto.formats.RELPATH_SCHEMA.check_match(rolename) if not tuf.roledb.role_exists(rolename, self.updater_name): raise tuf.ssl_commons.exceptions.UnknownRoleError(rolename) - + self._refresh_targets_metadata(rolename) return self._targets_of_role(rolename, skip_refresh=True) @@ -2250,7 +2250,7 @@ def get_one_valid_targetinfo(self, target_filepath): Return the target information of 'target_filepath', and update its corresponding metadata, if necessary. - + target_filepath: The path to the target file on the repository. This will be relative to the 'targets' (or equivalent) directory on a given mirror. @@ -2263,10 +2263,10 @@ def get_one_valid_targetinfo(self, target_filepath): If 'target_filepath' was not found. Any other unforeseen runtime exception. - + The metadata for updated delegated roles are downloaded and stored. - + The target information for 'target_filepath', conformant to 'tuf.ssl_crypto.formats.TARGETINFO_SCHEMA'. @@ -2275,7 +2275,7 @@ def get_one_valid_targetinfo(self, target_filepath): # Does 'target_filepath' have the correct format? # Raise 'tuf.ssl_commons.exceptions.FormatError' if there is a mismatch. tuf.ssl_crypto.formats.RELPATH_SCHEMA.check_match(target_filepath) - + # 'target_filepath' might contain URL encoding escapes. # http://docs.python.org/2/library/urllib.html#urllib.unquote target_filepath = six.moves.urllib.parse.unquote(target_filepath) @@ -2290,7 +2290,7 @@ def get_one_valid_targetinfo(self, target_filepath): if target is None: logger.error(target_filepath + ' not found.') raise tuf.ssl_commons.exceptions.UnknownTargetError(target_filepath + ' not found.') - + # Otherwise, return the found target. else: return target @@ -2306,7 +2306,7 @@ def _preorder_depth_first_walk(self, target_filepath): order of appearance (which implicitly order trustworthiness), and returns the matching target found in the most trusted role. - + target_filepath: The path to the target file on the repository. This will be relative to the 'targets' (or equivalent) directory on a given mirror. @@ -2317,10 +2317,10 @@ def _preorder_depth_first_walk(self, target_filepath): tuf.ssl_commons.exceptions.RepositoryError: If 'target_filepath' is not found. - + The metadata for updated delegated roles are downloaded and stored. - + The target information for 'target_filepath', conformant to 'tuf.ssl_crypto.formats.TARGETINFO_SCHEMA'. @@ -2344,7 +2344,7 @@ def _preorder_depth_first_walk(self, target_filepath): # Pop the role name from the top of the stack. role_name = role_names.pop(-1) - + # Skip any visited current role to prevent cycles. if role_name in visited_role_names: logger.debug('Skipping visited current role ' + repr(role_name)) @@ -2366,7 +2366,7 @@ def _preorder_depth_first_walk(self, target_filepath): target_filepath) # After preorder check, add current role to set of visited roles. visited_role_names.add(role_name) - + # And also decrement number of visited roles. number_of_delegations -= 1 @@ -2383,10 +2383,10 @@ def _preorder_depth_first_walk(self, target_filepath): role_names = [] child_roles_to_visit.append(child_role_name) break - + elif child_role_name is None: logger.debug('Skipping child role ' + repr(child_role_name)) - + else: logger.debug('Adding child role ' + repr(child_role_name)) child_roles_to_visit.append(child_role_name) @@ -2423,17 +2423,17 @@ def _get_target_from_targets_role(self, role_name, targets, target_filepath): targets: The targets of the Targets role with the name 'role_name'. - + target_filepath: The path to the target file on the repository. This will be relative to the 'targets' (or equivalent) directory on a given mirror. None. - + None. - + The target information for 'target_filepath', conformant to 'tuf.ssl_crypto.formats.TARGETINFO_SCHEMA'. @@ -2444,13 +2444,13 @@ def _get_target_from_targets_role(self, role_name, targets, target_filepath): # Does the current role name have our target? logger.debug('Asking role ' + repr(role_name) + ' about target ' +\ repr(target_filepath)) - + for filepath, fileinfo in six.iteritems(targets): if filepath == target_filepath: logger.debug('Found target ' + target_filepath + ' in role ' + role_name) target = {'filepath': filepath, 'fileinfo': fileinfo} break - + else: logger.debug('No target ' + target_filepath + ' in role ' + role_name) @@ -2494,10 +2494,10 @@ def _visit_child_role(self, child_role, target_filepath, parent_delegations): None. - + None. - + If 'child_role' has been delegated the target with the name 'target_filepath', then we return the role name of 'child_role'. @@ -2518,7 +2518,7 @@ def _visit_child_role(self, child_role, target_filepath, parent_delegations): for child_role_path_hash_prefix in child_role_path_hash_prefixes: if target_filepath_hash.startswith(child_role_path_hash_prefix): child_role_is_relevant = True - + else: continue @@ -2545,21 +2545,21 @@ def _visit_child_role(self, child_role, target_filepath, parent_delegations): if child_role_is_relevant: # Is the child role allowed by its parent role to specify this path # in its metadata? - try: + try: tuf.ssl_crypto.util.ensure_all_targets_allowed(child_role_name, [target_filepath], parent_delegations) - + except tuf.ssl_commons.exceptions.ForbiddenTargetError: logger.debug('Child role ' + repr(child_role_name) + ' has target ' + \ repr(target_filepath) + ', but is not allowed to sign for' ' it according to its delegating role.') return None - + else: logger.debug('Child role ' + repr(child_role_name) + ' has target ' + \ repr(target_filepath)) return child_role_name - + else: logger.debug('Child role ' + repr(child_role_name) + \ ' does not have target ' + repr(target_filepath)) @@ -2587,25 +2587,25 @@ def _get_target_hash(self, target_filepath, hash_function='sha256'): target filepaths. The repository may optionally organize targets into hashed bins to ease target delegations and role metadata management. The use of consistent hashing allows for a uniform distribution of - targets into bins. + targets into bins. None. - + None. - + The hash of 'target_filepath'. """ - # Calculate the hash of the filepath to determine which bin to find the + # Calculate the hash of the filepath to determine which bin to find the # target. The client currently assumes the repository (i.e., repository # tool) uses 'hash_function' to generate hashes and UTF-8. digest_object = tuf.ssl_crypto.hash.digest(hash_function) encoded_target_filepath = target_filepath.encode('utf-8') digest_object.update(encoded_target_filepath) - target_filepath_hash = digest_object.hexdigest() + target_filepath_hash = digest_object.hexdigest() return target_filepath_hash @@ -2619,7 +2619,7 @@ def remove_obsolete_targets(self, destination_directory): Remove any files that are in 'previous' but not 'current'. This makes it so if you remove a file from a repository, it actually goes away. The targets for the 'targets' role and all delegated roles are checked. - + destination_directory: The directory containing the target files tracked by TUF. @@ -2627,7 +2627,7 @@ def remove_obsolete_targets(self, destination_directory): tuf.ssl_commons.exceptions.FormatError: If 'destination_directory' is improperly formatted. - + tuf.ssl_commons.exceptions.RepositoryError: If an error occurred removing any files. @@ -2637,7 +2637,7 @@ def remove_obsolete_targets(self, destination_directory): None. """ - + # Does 'destination_directory' have the correct format? # Raise 'tuf.ssl_commons.exceptions.FormatError' if there is a mismatch. tuf.ssl_crypto.formats.PATH_SCHEMA.check_match(destination_directory) @@ -2651,22 +2651,22 @@ def remove_obsolete_targets(self, destination_directory): if target not in self.metadata['current'][role]['targets']: # 'target' is only in 'previous', so remove it. logger.warning('Removing obsolete file: ' + repr(target) + '.') - + # Remove the file if it hasn't been removed already. destination = \ os.path.join(destination_directory, target.lstrip(os.sep)) try: os.remove(destination) - + except OSError as e: # If 'filename' already removed, just log it. if e.errno == errno.ENOENT: logger.info('File ' + repr(destination) + ' was already' ' removed.') - + else: logger.error(str(e)) - + else: logger.debug('Skipping: ' + repr(target) + '. It is still' ' a current target.') @@ -2687,7 +2687,7 @@ def updated_targets(self, targets, destination_directory): The returned information is a list conformant to 'tuf.ssl_crypto.formats.TARGETINFOS_SCHEMA' and has the form: - + [{'filepath': 'a/b/c.txt', 'fileinfo': {'length': 13323, 'hashes': {'sha256': dbfac345..}} @@ -2706,7 +2706,7 @@ def updated_targets(self, targets, destination_directory): If the arguments are improperly formatted. - The files in 'targets' are read and their hashes computed. + The files in 'targets' are read and their hashes computed. A list of targets, conformant to @@ -2734,10 +2734,10 @@ def updated_targets(self, targets, destination_directory): if filepath[0] == '/': filepath = filepath[1:] target_filepath = os.path.join(destination_directory, filepath) - + if target_filepath in updated_targetpaths: continue - + # Try one of the algorithm/digest combos for a mismatch. We break # as soon as we find a mismatch. for algorithm, digest in six.iteritems(target['fileinfo']['hashes']): @@ -2745,19 +2745,19 @@ def updated_targets(self, targets, destination_directory): try: digest_object = tuf.ssl_crypto.hash.digest_filename(target_filepath, algorithm=algorithm) - - # This exception would occur if the target does not exist locally. + + # This exception would occur if the target does not exist locally. except IOError: updated_targets.append(target) updated_targetpaths.append(target_filepath) break - - # The file does exist locally, check if its hash differs. + + # The file does exist locally, check if its hash differs. if digest_object.hexdigest() != digest: updated_targets.append(target) updated_targetpaths.append(target_filepath) break - + return updated_targets @@ -2768,11 +2768,11 @@ def download_target(self, target, destination_directory): """ Download 'target' and verify it is trusted. - + This will only store the file at 'destination_directory' if the downloaded file matches the description of the file in the trusted metadata. - + target: The target to be downloaded. Conformant to @@ -2790,7 +2790,7 @@ def download_target(self, target, destination_directory): Although expected to be rare, there might be OSError exceptions (except errno.EEXIST) raised when creating the destination directory (if it - doesn't exist). + doesn't exist). A target file is saved to the local system. @@ -2799,8 +2799,8 @@ def download_target(self, target, destination_directory): None. """ - # Do the arguments have the correct format? - # This check ensures the arguments have the appropriate + # Do the arguments have the correct format? + # This check ensures the arguments have the appropriate # number of objects and object types, and that all dict # keys are properly named. # Raise 'tuf.ssl_commons.exceptions.FormatError' if the check fail. @@ -2816,7 +2816,7 @@ def download_target(self, target, destination_directory): # that passes verification. target_file_object = self._get_target_file(target_filepath, trusted_length, trusted_hashes) - + # We acquired a target file object from a mirror. Move the file into place # (i.e., locally to 'destination_directory'). Note: join() discards # 'destination_directory' if 'target_path' contains a leading path @@ -2825,7 +2825,7 @@ def download_target(self, target, destination_directory): target_filepath.lstrip(os.sep)) destination = os.path.abspath(destination) target_dirpath = os.path.dirname(destination) - + # When attempting to create the leaf directory of 'target_dirpath', ignore # any exceptions raised if the root directory already exists. All other # exceptions potentially thrown by os.makedirs() are re-raised. @@ -2833,11 +2833,11 @@ def download_target(self, target, destination_directory): # or cannot be created. try: os.makedirs(target_dirpath) - + except OSError as e: if e.errno == errno.EEXIST: pass - + else: raise diff --git a/tuf/download.py b/tuf/download.py index 481a5ee5..825166d6 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -1,17 +1,17 @@ """ download.py - + February 21, 2012. Based on previous version by Geremy Condra. Konstantin Andrianov Vladimir Diaz - + See LICENSE for licensing information. - + Download metadata and target files and check their validity. The hash and length of a downloaded file has to match the hash and length supplied by the @@ -65,17 +65,17 @@ def safe_download(url, required_length): the length of the downloaded file matches 'required_length' exactly. tuf.download.unsafe_download() may be called if an upper download limit is preferred. - + 'tuf.ssl_crypto.util.TempFile', the file-like object returned, is used instead of regular tempfile object because of additional functionality provided, such as handling compressed metadata and automatically closing files after moving to final destination. - + url: A URL string that represents the location of the file. The URI scheme component must be one of 'settings.SUPPORTED_URI_SCHEMES'. - + required_length: An integer value representing the length of the file. This is an exact limit. @@ -83,21 +83,21 @@ def safe_download(url, required_length): A 'tuf.ssl_crypto.util.TempFile' object is created on disk to store the contents of 'url'. - + tuf.ssl_commons.exceptions.DownloadLengthMismatchError, if there was a mismatch of observed vs expected lengths while downloading the file. - + tuf.ssl_commons.exceptions.FormatError, if any of the arguments are improperly formatted. Any other unforeseen runtime exception. - + A 'tuf.ssl_crypto.util.TempFile' file-like object that points to the contents of 'url'. """ - + # Do all of the arguments have the appropriate format? # Raise 'tuf.ssl_commons.exceptions.FormatError' if there is a mismatch. tuf.ssl_crypto.formats.URL_SCHEMA.check_match(url) @@ -108,7 +108,7 @@ def safe_download(url, required_length): # supported. If the URI scheme of 'url' is empty or "file", files on the # local system can be accessed. Unexpected files may be accessed by # compromised metadata (unlikely to happen if targets.json metadata is signed - # with offline keys). + # with offline keys). parsed_url = six.moves.urllib.parse.urlparse(url) if parsed_url.scheme not in settings.SUPPORTED_URI_SCHEMES: @@ -116,7 +116,7 @@ def safe_download(url, required_length): repr(url) + ' specifies an unsupported URI scheme. Supported ' + \ ' URI Schemes: ' + repr(settings.SUPPORTED_URI_SCHEMES) raise tuf.ssl_commons.exceptions.FormatError(message) - + return _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True) @@ -131,17 +131,17 @@ def unsafe_download(url, required_length): the length of the downloaded file is up to 'required_length', and no larger. tuf.download.safe_download() may be called if an exact download limit is preferred. - + 'tuf.ssl_crypto.util.TempFile', the file-like object returned, is used instead of regular tempfile object because of additional functionality provided, such as handling compressed metadata and automatically closing files after moving to final destination. - + url: A URL string that represents the location of the file. The URI scheme component must be one of 'settings.SUPPORTED_URI_SCHEMES'. - + required_length: An integer value representing the length of the file. This is an upper limit. @@ -149,40 +149,40 @@ def unsafe_download(url, required_length): A 'tuf.ssl_crypto.util.TempFile' object is created on disk to store the contents of 'url'. - + tuf.ssl_commons.exceptions.DownloadLengthMismatchError, if there was a mismatch of observed vs expected lengths while downloading the file. - + tuf.ssl_commons.exceptions.FormatError, if any of the arguments are improperly formatted. Any other unforeseen runtime exception. - + A 'tuf.ssl_crypto.util.TempFile' file-like object that points to the contents of 'url'. """ - + # Do all of the arguments have the appropriate format? # Raise 'tuf.ssl_commons.exceptions.FormatError' if there is a mismatch. tuf.ssl_crypto.formats.URL_SCHEMA.check_match(url) tuf.ssl_crypto.formats.LENGTH_SCHEMA.check_match(required_length) - + # Ensure 'url' specifies one of the URI schemes in # 'settings.SUPPORTED_URI_SCHEMES'. Be default, ['http', 'https'] is # supported. If the URI scheme of 'url' is empty or "file", files on the # local system can be accessed. Unexpected files may be accessed by # compromised metadata (unlikely to happen if targets.json metadata is signed - # with offline keys). + # with offline keys). parsed_url = six.moves.urllib.parse.urlparse(url) if parsed_url.scheme not in settings.SUPPORTED_URI_SCHEMES: message = \ repr(url) + ' specifies an unsupported URI scheme. Supported ' + \ - ' URI Schemes: ' + repr(settings.SUPPORTED_URI_SCHEMES) + ' URI Schemes: ' + repr(settings.SUPPORTED_URI_SCHEMES) raise tuf.ssl_commons.exceptions.FormatError(message) - + return _download_file(url, required_length, STRICT_REQUIRED_LENGTH=False) @@ -192,18 +192,20 @@ def unsafe_download(url, required_length): def _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True): """ - Given the url, hashes and length of the desired file, this function - opens a connection to 'url' and downloads the file while ensuring its - length and hashes match 'required_hashes' and 'required_length'. - + Given the url and length of the desired file, this function opens a + connection to 'url' and downloads the file while ensuring its length + matches 'required_length' if 'STRICT_REQUIRED_LENGH' is True (If False, + the file's length is not checked and a slow retrieval exception is raised + if the downloaded rate falls below the acceptable rate). + tuf.ssl_crypto.util.TempFile is used instead of regular tempfile object because of additional functionality provided by 'tuf.ssl_crypto.util.TempFile'. - + url: - A URL string that represents the location of the file. - + A URL string that represents the location of the file. + required_length: An integer value representing the length of the file. @@ -216,16 +218,16 @@ def _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True): A 'tuf.ssl_crypto.util.TempFile' object is created on disk to store the contents of 'url'. - + tuf.ssl_commons.exceptions.DownloadLengthMismatchError, if there was a mismatch of observed vs expected lengths while downloading the file. - + tuf.ssl_commons.exceptions.FormatError, if any of the arguments are improperly formatted. Any other unforeseen runtime exception. - + A 'tuf.ssl_crypto.util.TempFile' file-like object that points to the contents of 'url'. @@ -238,7 +240,7 @@ def _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True): # 'url.replace()' is for compatibility with Windows-based systems because # they might put back-slashes in place of forward-slashes. This converts it - # to the common format. + # to the common format. url = url.replace('\\', '/') logger.info('Downloading: ' + repr(url)) @@ -261,7 +263,7 @@ def _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True): # temporary file, and get the total number of downloaded bytes. total_downloaded, average_download_speed = \ _download_fixed_amount_of_data(connection, temp_file, required_length) - + # Does the total number of downloaded bytes match the required length? _check_downloaded_length(total_downloaded, required_length, STRICT_REQUIRED_LENGTH=STRICT_REQUIRED_LENGTH, @@ -286,7 +288,7 @@ def _download_fixed_amount_of_data(connection, temp_file, required_length): This is a helper function, where the download really happens. While-block reads data from connection a fixed chunk of data at a time, or less, until 'required_length' is reached. - + connection: The object that the _open_connection returns for communicating with the @@ -301,20 +303,20 @@ def _download_fixed_amount_of_data(connection, temp_file, required_length): always specified by the TUF metadata for the data file in question (except in the case of timestamp metadata, in which case we would fix a reasonable upper bound). - + Data from the server will be written to 'temp_file'. - + Runtime or network exceptions will be raised without question. - + A (total_downloaded, average_download_speed) tuple, where 'total_downloaded' is the total number of bytes downloaded for the desired file and the 'average_download_speed' calculated for the download attempt. """ - + # Tolerate servers with a slow start by ignoring their delivery speed for # 'settings.SLOW_START_GRACE_PERIOD' seconds. Set 'seconds_spent_receiving' # to negative SLOW_START_GRACE_PERIOD seconds, and begin checking the average @@ -324,7 +326,7 @@ def _download_fixed_amount_of_data(connection, temp_file, required_length): # Keep track of total bytes downloaded. number_of_bytes_received = 0 average_download_speed = 0 - + start_time = timeit.default_timer() try: @@ -335,60 +337,60 @@ def _download_fixed_amount_of_data(connection, temp_file, required_length): # round, sleep for a short amount of time so that the CPU is not hogged # in the while loop. time.sleep(0.05) - data = b'' + data = b'' read_amount = min(settings.CHUNK_SIZE, required_length - number_of_bytes_received) - - try: + + try: data = connection.read(read_amount) - - # Python 3.2 returns 'IOError' if the remote file object has timed out. + + # Python 3.2 returns 'IOError' if the remote file object has timed out. except (socket.error, IOError): pass - + number_of_bytes_received = number_of_bytes_received + len(data) - - # Data successfully read from the connection. Store it. + + # Data successfully read from the connection. Store it. temp_file.write(data) if number_of_bytes_received == required_length: - break + break stop_time = timeit.default_timer() seconds_spent_receiving = stop_time - start_time - + if (seconds_spent_receiving + grace_period) < 0: - continue - + continue + # Measure the average download speed. average_download_speed = number_of_bytes_received / seconds_spent_receiving - + if average_download_speed < settings.MIN_AVERAGE_DOWNLOAD_SPEED: logger.debug('The average download speed dropped below the minimum' - ' average download speed set in settings.py.') + ' average download speed set in settings.py.') break - + else: logger.debug('The average download speed has not dipped below the' ' mimimum average download speed set in settings.py.') - # We might have no more data to read. Check number of bytes downloaded. + # We might have no more data to read. Check number of bytes downloaded. if not data: logger.debug('Downloaded ' + repr(number_of_bytes_received) + '/' + repr(required_length) + ' bytes.') # Finally, we signal that the download is complete. break - + except: raise - + else: # This else block returns and skips closing the connection in the finally # block, so close the connection here. connection.close() return number_of_bytes_received, average_download_speed - + finally: # Whatever happens, make sure that we always close the connection. connection.close() @@ -430,7 +432,7 @@ def _get_opener(scheme=None): for handler in opener.handlers: if isinstance(handler, six.moves.urllib.request.HTTPHandler): opener.handlers.remove(handler) - + else: # Otherwise, use the default opener. opener = six.moves.urllib.request.build_opener() @@ -444,40 +446,40 @@ def _get_opener(scheme=None): def _open_connection(url): """ - Helper function that opens a connection to the url. urllib2 supports http, - ftp, and file. In python (2.6+) where the ssl module is available, urllib2 + Helper function that opens a connection to the url. urllib2 supports http, + ftp, and file. In python (2.6+) where the ssl module is available, urllib2 also supports https. TODO: Determine whether this follows http redirects and decide if we like that. For example, would we not want to allow redirection from ssl to non-ssl urls? - + url: - URL string (e.g., 'http://...' or 'ftp://...' or 'file://...') - + URL string (e.g., 'http://...' or 'ftp://...' or 'file://...') + None. - + Opens a connection to a remote server. - + File-like object. """ - # urllib2.Request produces a Request object that allows for a finer control + # urllib2.Request produces a Request object that allows for a finer control # of the requesting process. Request object allows to add headers or data to # the HTTP request. For instance, request method add_header(key, val) can be - # used to change/spoof 'User-Agent' from default Python-urllib/x.y to + # used to change/spoof 'User-Agent' from default Python-urllib/x.y to # 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)' this can be useful if - # servers do not recognize connections that originates from + # servers do not recognize connections that originates from # Python-urllib/x.y. parsed_url = six.moves.urllib.parse.urlparse(url) opener = _get_opener(scheme=parsed_url.scheme) request = _get_request(url) - + return opener.open(request, timeout = settings.SOCKET_TIMEOUT) @@ -488,18 +490,18 @@ def _get_content_length(connection): """ A helper function that gets the purported file length from server. - + connection: The object that the _open_connection function returns for communicating with the server about the contents of a URL. - + No known side effects. - + Runtime exceptions will be suppressed but logged. - + reported_length: The total number of bytes reported by server. If the process fails, we @@ -509,19 +511,19 @@ def _get_content_length(connection): try: # What is the length of this document according to the HTTP spec? reported_length = connection.info().get('Content-Length') - + # Try casting it as a decimal number. reported_length = int(reported_length, 10) - + # Make sure that it is a nonnegative integer. assert reported_length > -1 - + except: message = \ 'Could not get content length about ' + str(connection) + ' from server.' logger.exception(message) reported_length = None - + finally: return reported_length @@ -534,7 +536,7 @@ def _check_content_length(reported_length, required_length, strict_length=True): A helper function that checks whether the length reported by server is equal to the length we expected. - + reported_length: The total number of bytes reported by the server. @@ -548,30 +550,30 @@ def _check_content_length(reported_length, required_length, strict_length=True): No known side effects. - + No known exceptions. - + None. """ logger.debug('The server reported a length of '+repr(reported_length)+' bytes.') comparison_result = None - + if reported_length < required_length: - comparison_result = 'less than' - + comparison_result = 'less than' + elif reported_length > required_length: - comparison_result = 'greater than' - + comparison_result = 'greater than' + else: - comparison_result = 'equal to' + comparison_result = 'equal to' if strict_length: logger.debug('The reported length is ' + comparison_result + ' the' ' required length of '+repr(required_length)+' bytes.') - + else: logger.debug('The reported length is ' + comparison_result + ' the upper' ' limit of ' + repr(required_length) + ' bytes.') @@ -586,8 +588,8 @@ def _check_downloaded_length(total_downloaded, required_length, """ A helper function which checks whether the total number of downloaded bytes - matches our expectation. - + matches our expectation. + total_downloaded: The total number of bytes supposedly downloaded for the file in question. @@ -606,11 +608,11 @@ def _check_downloaded_length(total_downloaded, required_length, timestamp metadata, which has no signed required_length. average_download_speed: - The average download speed for the downloaded file. - + The average download speed for the downloaded file. + None. - + tuf.ssl_commons.exceptions.DownloadLengthMismatchError, if STRICT_REQUIRED_LENGTH is True and total_downloaded is not equal @@ -627,7 +629,7 @@ def _check_downloaded_length(total_downloaded, required_length, if total_downloaded == required_length: logger.info('Downloaded ' + str(total_downloaded) + ' bytes out of the' ' expected ' + str(required_length) + ' bytes.') - + else: difference_in_bytes = abs(total_downloaded - required_length) @@ -637,21 +639,21 @@ def _check_downloaded_length(total_downloaded, required_length, logger.error('Downloaded ' + str(total_downloaded) + ' bytes, but' ' expected ' + str(required_length) + ' bytes. There is a difference' ' of ' + str(difference_in_bytes) + ' bytes.') - + # If the average download speed is below a certain threshold, we flag # this as a possible slow-retrieval attack. logger.debug('Average download speed: ' + repr(average_download_speed)) logger.debug('Minimum average download speed: ' + repr(settings.MIN_AVERAGE_DOWNLOAD_SPEED)) - + if average_download_speed < settings.MIN_AVERAGE_DOWNLOAD_SPEED: raise tuf.ssl_commons.exceptions.SlowRetrievalError(average_download_speed) else: logger.debug('Good average download speed: ' + repr(average_download_speed) + ' bytes per second') - + raise tuf.ssl_commons.exceptions.DownloadLengthMismatchError(required_length, total_downloaded) - + else: # We specifically disabled strict checking of required length, but we # will log a warning anyway. This is useful when we wish to download the @@ -663,7 +665,7 @@ def _check_downloaded_length(total_downloaded, required_length, else: logger.debug('Good average download speed: ' + repr(average_download_speed) + ' bytes per second') - + logger.info('Downloaded ' + str(total_downloaded) + ' bytes out of an' ' upper limit of ' + str(required_length) + ' bytes.')