From 5970ad8c253f79a14b86b0321e03f40b2a8c209f Mon Sep 17 00:00:00 2001 From: zanefisher Date: Wed, 4 Sep 2013 20:12:51 -0400 Subject: [PATCH] Refactor client.updater, removing verification from download.py. --- tuf/client/updater.py | 262 ++++++++++++++++++++++++++---------------- tuf/download.py | 75 ++---------- 2 files changed, 174 insertions(+), 163 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 1ce51548..3b583e23 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -582,8 +582,7 @@ def refresh(self): # Use default but sane information for timestamp metadata, and do not # require strict checks on its required length. - self._update_metadata('timestamp', DEFAULT_TIMESTAMP_FILEINFO, - STRICT_REQUIRED_LENGTH=False) + self._update_metadata('timestamp', DEFAULT_TIMESTAMP_FILEINFO) self._update_metadata_if_changed('release', referenced_metadata='timestamp') @@ -601,8 +600,158 @@ def refresh(self): - def _update_metadata(self, metadata_role, fileinfo, compression=None, - STRICT_REQUIRED_LENGTH=True): +def _check_hashes(input_file, trusted_hashes=None): + """ + + A helper function that verifies multiple secure hashes of the downloaded + file. If any of these fail it raises an exception. This is to conform + with the TUF specs, which support clients with different hashing + algorithms. The 'hash.py' module is used to compute the hashes of the + 'input_file'. + + + input_file: + A file-like object. + + trusted_hashes: + A dictionary with hash-algorithm names as keys and hashes as dict values. + The hashes should be in the hexdigest format. + + + tuf.BadHashError, if the hashes don't match. + + + Hash digest object is created using the 'tuf.hash' module. + + + None. + + """ + + if trusted_hashes: + # Verify each trusted hash of 'trusted_hashes'. Raise exception if + # any of the hashes are incorrect and return if all are correct. + for algorithm, trusted_hash in trusted_hashes.items(): + digest_object = tuf.hash.digest(algorithm) + digest_object.update(input_file.read()) + computed_hash = digest_object.hexdigest() + if trusted_hash != computed_hash: + raise tuf.BadHashError('Hashes do not match! Expected '+ + trusted_hash+' got '+computed_hash) + else: + logger.info('The file\'s '+algorithm+' hash is correct: '+trusted_hash) + else: + logger.warn('No trusted hashes supplied to verify file at: '+ + str(input_file)) + + + + + + def get_target_file(self, target_filepath, file_length, file_hashes): + + def verify_target_file(self, target_file_object): + self._check_hashes(target_file_object, file_hashes) + + return self.__get_file(target_filepath, verify_target_file, 'target', + file_length, download_safely=True) + + + + + + def __verify_metadata_file(self, metadata_file_object, metadata_role, file_hashes): + # FIXME: Decompression should be handled elsewhere. + #if compression: + # metadata_file_object.decompress_temp_file_object(compression) + + self._check_hashes(metadata_file_object, file_hashes) + + # Read and load the downloaded file. + try: + metadata_signable = \ + tuf.util.load_json_string(metadata_file_object.read()) + except: + logger.exception('Invalid metadata from '+mirror_url+'.') + raise + else: + # Verify the signature on the downloaded metadata object. + try: + valid = tuf.sig.verify(metadata_signable, metadata_role) + except: + message = 'Unable to verify '+metadata_filename + logger.exception(message) + raise + else: + if valid: + logger.debug('Good signature on '+mirror_url+'.') + else: + raise tuf.BadSignatureError('Bad signature on '+mirror_url+'.') + + + + + + def unsafely_get_metadata_file(self, metadata_role, metadata_filepath, file_length): + + def unsafely_verify_metadata_file(metadata_file_object): + self.__verify_metadata_file(metadata_file_object, metadata_role, None) + + return self.__get_file(metadata_filepath, unsafely_verify_metadata_file, + 'meta', file_length, download_safely=False) + + + def safely_get_metadata_file(self, metadata_role, metadata_filepath, file_length, file_hashes): + + def safely_verify_metadata_file(metadata_file_object): + self.__verify_metadata_file(metadata_file_object, metadata_role, file_hashes) + + return self.__get_file(metadata_filepath, _verify_metadata_file, + 'meta', file_length, download_safely=True) + + + + + + def __get_file(self, filepath, verify_file, reference_metadata, trusted_length, download_safely): + file_mirrors = tuf.mirrors.get_list_of_mirrors(reference_metadata, filepath, + self.mirrors) + # file_mirror (URL): error (Exception) + file_mirror_errors = {} + target_file_object = None + + for file_mirror in file_mirrors: + try: + if (download_safely): + target_file_object = tuf.download.safe_download(file_mirror, trusted_length) + else: + target_file_object = tuf.download.unsafe_download(file_mirror, trusted_length) + + except Exception, e: + # Remember the error from this mirror, and "reset" the target file. + logger.exception('Download failed from '+file_mirror+'.') + file_mirror_errors[file_mirror] = e + target_file_object = None + else: + try: + verify_file(file_object) + except Exception, e: + file_mirror_errors[file_mirror] = e + target_file_object = None + else: + break + + if target_file_object: + return target_file_object + else: + # TODO: wrap file_mirror_errors in an Exception + raise tuf.UpdateError(file_mirror_errors) + + + + + + def _update_metadata(self, metadata_role, fileinfo, compression=None): """ Download, verify, and 'install' the metadata belonging to 'metadata_role'. @@ -659,15 +808,15 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None, metadata_filename = metadata_filename + '.gz' # Reference to the 'get_list_of_mirrors' function. - get_mirrors = tuf.mirrors.get_list_of_mirrors + # get_mirrors = tuf.mirrors.get_list_of_mirrors # Reference to the 'download_url_to_tempfileobj' function. download_file = tuf.download.download_url_to_tempfileobj # Extract file length and file hashes. They will be passed as arguments # to 'download_file' function. - file_length=fileinfo['length'] - file_hashes=fileinfo['hashes'] + file_length = fileinfo['length'] + file_hashes = fileinfo['hashes'] # A dictionary to keep the error from every mirror that we try. mirror_errors = {} @@ -681,63 +830,15 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None, # 'tuf.formats.SIGNABLE_SCHEMA'. metadata_file_object = None metadata_signable = None - - for mirror_url in get_mirrors('meta', - metadata_filename.encode("utf-8"), - self.mirrors): - try: + if metadata_role == 'timestamp': metadata_file_object = \ - download_file(mirror_url, file_length, file_hashes, - STRICT_REQUIRED_LENGTH=STRICT_REQUIRED_LENGTH) - except: - logger.exception('Download failed from '+mirror_url+'.') - mirror_errors[mirror_url] = traceback.format_exc(1) - else: - # FIXME: mirror_errors for a mirror_url must not be overwritten! + self.unsafely_get_metadata_file(metadata_role, metadata_filename, file_length) + else: + metadata_file_object = \ + self.safely_get_metadata_file(metadata_role, metadata_filename, file_length, file_hashes) - # FIXME: Another point of failure which we should handle. - if compression: - metadata_file_object.decompress_temp_file_object(compression) - - # Read and load the downloaded file. - try: - metadata_signable = \ - tuf.util.load_json_string(metadata_file_object.read()) - except: - logger.exception('Invalid metadata from '+mirror_url+'.') - mirror_errors[mirror_url] = traceback.format_exc(1) - metadata_signable = None - else: - # Verify the signature on the downloaded metadata object. - try: - valid = tuf.sig.verify(metadata_signable, metadata_role) - except (tuf.UnknownRoleError, tuf.FormatError, tuf.Error), e: - # FIXME: Exception.message is deprecated in 2.6, and gone in 3.0, - # but this is a workaround for Unicode messages. We need a - # long-term solution with #61. - # http://bugs.python.org/issue2517 - message = 'Unable to verify '+metadata_filename+':'+\ - e.message.encode("utf-8") - logger.exception(message) - mirror_errors[mirror_url] = message - metadata_signable = None - else: - if valid: - logger.debug('Good signature on '+mirror_url+'.') - break - else: - message = 'Bad signature on '+mirror_url+'.' - logger.warn(message) - mirror_errors[mirror_url] = message - metadata_signable = None - - # Raise an exception if a valid metadata signable could not be downloaded - # from any of the mirrors. - if metadata_signable is None: - message = 'Unable to update '+str(metadata_filename)+\ - ' from all known mirrors: '+str(mirror_errors) - logger.error(message) - raise tuf.RepositoryError(message) + # Read and load the downloaded file. + metadata_signable = tuf.util.load_json_string(metadata_file_object.read()) # Ensure the loaded 'metadata_signable' is properly formatted. try: @@ -1885,45 +1986,14 @@ def download_target(self, target, destination_directory): tuf.formats.TARGETFILE_SCHEMA.check_match(target) tuf.formats.PATH_SCHEMA.check_match(destination_directory) - # Reference to the 'get_list_of_mirrors' function. - get_mirrors = tuf.mirrors.get_list_of_mirrors - - # Reference to the 'download_url_to_tempfileobj' function. - download_file = tuf.download.download_url_to_tempfileobj - # Extract the target file information. target_filepath = target['filepath'] trusted_length = target['fileinfo']['length'] trusted_hashes = target['fileinfo']['hashes'] - # A dictionary to keep the error from every mirror that we try. - mirror_errors = {} - - # Wherein the downloaded target file will be stored. - target_file_object = None - - logger.info('Trying to download: '+str(target_filepath)) - - # Iterate through the repositority mirrors until we successfully - # download a target. - for mirror_url in get_mirrors('target', target_filepath, self.mirrors): - try: - target_file_object = download_file(mirror_url, trusted_length, - trusted_hashes) - break - except: - # Remember the error from this mirror, and "reset" the target file. - logger.exception('Download failed from '+mirror_url+'.') - mirror_errors[mirror_url] = traceback.format_exc(1) - target_file_object = None - continue - - # We have gone through all the mirrors. Did we get a target file object? - if target_file_object == None: - raise tuf.DownloadError('Failed to download target '+\ - str(target_filepath)+\ - ' from all known mirrors: '+\ - str(mirror_errors)) + # get_target_file checks every mirror and returns the first target + # that passes verification. + target_file_object = get_target_file(target_filepath, trusted_length, trusted_hashes) # We acquired a target file object from a mirror. Move the file into # place (i.e., locally to 'destination_directory'). diff --git a/tuf/download.py b/tuf/download.py index 0321a72d..5fe0bc3e 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -363,54 +363,6 @@ def _open_connection(url): -def _check_hashes(input_file, trusted_hashes=None): - """ - - A helper function that verifies multiple secure hashes of the downloaded - file. If any of these fail it raises an exception. This is to conform - with the TUF specs, which support clients with different hashing - algorithms. The 'hash.py' module is used to compute the hashes of the - 'input_file'. - - - input_file: - A file-like object. - - trusted_hashes: - A dictionary with hash-algorithm names as keys and hashes as dict values. - The hashes should be in the hexdigest format. - - - tuf.BadHashError, if the hashes don't match. - - - Hash digest object is created using the 'tuf.hash' module. - - - None. - - """ - - if trusted_hashes: - # Verify each trusted hash of 'trusted_hashes'. Raise exception if - # any of the hashes are incorrect and return if all are correct. - for algorithm, trusted_hash in trusted_hashes.items(): - digest_object = tuf.hash.digest(algorithm) - digest_object.update(input_file.read()) - computed_hash = digest_object.hexdigest() - if trusted_hash != computed_hash: - raise tuf.BadHashError('Hashes do not match! Expected '+ - trusted_hash+' got '+computed_hash) - else: - logger.info('The file\'s '+algorithm+' hash is correct: '+trusted_hash) - else: - logger.warn('No trusted hashes supplied to verify file at: '+ - str(input_file)) - - - - - def _download_fixed_amount_of_data(connection, temp_file, required_length): """ @@ -624,8 +576,7 @@ def _check_downloaded_length(total_downloaded, required_length, -def download_url_to_tempfileobj(url, required_length, required_hashes=None, - STRICT_REQUIRED_LENGTH=True): +def _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True): """ Given the url, hashes and length of the desired file, this function @@ -641,13 +592,6 @@ def download_url_to_tempfileobj(url, required_length, required_hashes=None, required_length: An integer value representing the length of the file. - - required_hashes: - A dictionary, where the keys represent the hashing algorithm used to - hash the file and the dict values the hexdigest. - - For instance, a hash pair might look something like this: - {'md5': '37544f383be1fc1a32f42801c9c4b4d6'} STRICT_REQUIRED_LENGTH: A Boolean indicator used to signal whether we should perform strict @@ -678,13 +622,6 @@ def download_url_to_tempfileobj(url, required_length, required_hashes=None, tuf.formats.URL_SCHEMA.check_match(url) tuf.formats.LENGTH_SCHEMA.check_match(required_length) - # FIXME: This function should only download files up to an expected length, - # and not check hashes; that is the job of the updater. - if required_hashes: - tuf.formats.HASHDICT_SCHEMA.check_match(required_hashes) - else: - logger.warn('Missing hashes for: '+str(url)) - # 'url.replace()' is for compatibility with Windows-based systems because # they might put back-slashes in place of forward-slashes. This converts it # to the common format. @@ -725,9 +662,6 @@ def download_url_to_tempfileobj(url, required_length, required_hashes=None, _check_downloaded_length(total_downloaded, required_length, STRICT_REQUIRED_LENGTH=STRICT_REQUIRED_LENGTH) - # Finally, check the hashes expected of the file. - _check_hashes(temp_file, trusted_hashes=required_hashes) - except: # Close 'temp_file'; any written data is lost. temp_file.close_temp_file() @@ -744,6 +678,13 @@ def download_url_to_tempfileobj(url, required_length, required_hashes=None, socket.setdefaulttimeout(previous_socket_timeout) +def safe_download(url, required_length): + return _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True) + + +def unsafe_download(url, required_length): + return _download_file(url, required_length, STRICT_REQUIRED_LENGTH=False) +