From 1e360846bf73bb454fc63aacb4cc82ddb9f4937d Mon Sep 17 00:00:00 2001 From: vladdd Date: Mon, 28 Apr 2014 22:31:42 -0400 Subject: [PATCH 1/2] Reject expired metadata without loading. Reject downloaded metadata as early as possible. The top-level roles were all downloaded as a group and then their expiration inspected. All metadata provided by a mirror that has already expired is discarded immediately and the next mirror tried. The update process stops if a requested role cannot be successfully validated, or one of its parents. [2014-04-29 02:00:32,308 UTC] [tuf.download] [INFO] [_download_file:745@download.py] Downloading: http://localhost:8001/metadata/timestamp.json [2014-04-29 02:00:32,324 UTC] [tuf.download] [INFO] [_check_downloaded_length:676@download.py] Downloaded 544 bytes out of an upper limit of 16384 bytes. [2014-04-29 02:00:32,324 UTC] [tuf.client.updater] [INFO] [_get_file:1189@updater.py] Not decompressing http://localhost:8001/metadata/timestamp.json [2014-04-29 02:00:32,331 UTC] [tuf.download] [INFO] [_download_file:745@download.py] Downloading: http://localhost:8001/metadata/snapshot.json [2014-04-29 02:00:32,333 UTC] [tuf.download] [INFO] [_check_downloaded_length:654@download.py] Downloaded 1003 bytes out of the expected 1003 bytes. [2014-04-29 02:00:32,334 UTC] [tuf.client.updater] [INFO] [_get_file:1189@updater.py] Not decompressing http://localhost:8001/metadata/snapshot.json [2014-04-29 02:00:32,334 UTC] [tuf.client.updater] [INFO] [_check_hashes:696@updater.py] The file's sha256 hash is correct: 5b3aec7cf295a25e4b39d875c7474511da9645bc6d27f9e86fb7e439c82e0ec7 [2014-04-29 02:00:32,335 UTC] [tuf.client.updater] [ERROR] [_ensure_not_expired:1789@updater.py] Metadata 'snapshot' expired on Tue Apr 29 01:59:01 2014 (UTC). Do not request, download, and install top-level roles if the root of trust has already expired after the inital load. If requested, update an expired root role: [2014-04-29 01:18:02,457 UTC] [tuf.client.updater] [ERROR] [_ensure_not_expired:1789@updater.py] Metadata 'root' expired on Mon Apr 28 23:23:57 2014 (UTC). [2014-04-29 01:18:02,458 UTC] [tuf.client.updater] [INFO] [refresh:628@updater.py] Expired Root metadata was loaded from disk. Try to update it now. [2014-04-29 01:18:02,458 UTC] [tuf.download] [INFO] [_download_file:745@download.py] Downloading: http://localhost:8001/metadata/root.json [2014-04-29 01:18:02,461 UTC] [tuf.download] [INFO] [_check_downloaded_length:676@download.py] Downloaded 1198 bytes out of an upper limit of 512000 bytes. [2014-04-29 01:18:02,461 UTC] [tuf.client.updater] [INFO] [_get_file:1189@updater.py] Not decompressing http://localhost:8001/metadata/root.json [2014-04-29 01:18:02,462 UTC] [tuf.client.updater] [ERROR] [_ensure_not_expired:1789@updater.py] Metadata 'root' expired on Mon Apr 28 23:23:57 2014 (UTC). Note: An expired 'root' was provided by the server. The requested root must also be signed by keys trusted by the client. --- tests/test_indefinite_freeze_attack.py | 8 ++- tests/test_updater.py | 17 ++++-- tuf/__init__.py | 4 ++ tuf/client/updater.py | 80 +++++++++++++------------- tuf/download.py | 3 + 5 files changed, 65 insertions(+), 47 deletions(-) diff --git a/tests/test_indefinite_freeze_attack.py b/tests/test_indefinite_freeze_attack.py index f6cc1c72..4aae5b5c 100755 --- a/tests/test_indefinite_freeze_attack.py +++ b/tests/test_indefinite_freeze_attack.py @@ -236,8 +236,12 @@ def test_with_tuf(self): # continue the update process. Sleep for at least 2 seconds to ensure # 'repository.timestamp.expiration' is reached. time.sleep(2) - self.assertRaises(tuf.ExpiredMetadataError, - self.repository_updater.refresh) + try: + self.repository_updater.refresh() + + except tuf.NoWorkingMirrorError as e: + for mirror_url, mirror_error in e.mirror_errors.iteritems(): + self.assertTrue(isinstance(mirror_error, tuf.ExpiredMetadataError)) if __name__ == '__main__': diff --git a/tests/test_updater.py b/tests/test_updater.py index 9397177f..ec2414b6 100755 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -460,20 +460,21 @@ def test_2__delete_metadata(self): def test_2__ensure_not_expired(self): # This test condition will verify that nothing is raised when a metadata # file has a future expiration date. - self.repository_updater._ensure_not_expired('root') + root_metadata = self.repository_updater.metadata['current']['root'] + self.repository_updater._ensure_not_expired(root_metadata, 'root') # 'tuf.ExpiredMetadataError' should be raised in this next test condition, # because the expiration_date has expired by 10 seconds. expires = tuf.formats.unix_timestamp_to_datetime(int(time.time() - 10)) expires = expires.isoformat() + 'Z' - self.repository_updater.metadata['current']['root']['expires'] = expires + root_metadata['expires'] = expires # Ensure the 'expires' value of the root file is valid by checking the # the formats of the 'root.json' object. - root_object = self.repository_updater.metadata['current']['root'] - self.assertTrue(tuf.formats.ROOT_SCHEMA.matches(root_object)) + self.assertTrue(tuf.formats.ROOT_SCHEMA.matches(root_metadata)) self.assertRaises(tuf.ExpiredMetadataError, - self.repository_updater._ensure_not_expired, 'root') + self.repository_updater._ensure_not_expired, + root_metadata, 'root') @@ -657,6 +658,12 @@ def test_4_refresh(self): # This unit test is based on adding an extra target file to the # server and rebuilding all server-side metadata. All top-level metadata # should be updated when the client calls refresh(). + + # First verify that an expired root metadata is updated. + expired_date = '1960-01-01T12:00:00Z' + self.repository_updater.metadata['current']['root']['expires'] = expired_date + self.repository_updater.refresh() + repository = repo_tool.load_repository(self.repository_directory) target3 = os.path.join(self.repository_directory, 'targets', 'file3.txt') diff --git a/tuf/__init__.py b/tuf/__init__.py index 61a83dc4..cd6de9e7 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -21,7 +21,11 @@ """ import urlparse +import logging +import tuf.log + +logging = logging.getLogger('tuf.__init__') # Import 'tuf.formats' if a module tries to import the # entire tuf package (i.e., from tuf import *). diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 658dab24..c0c82e8b 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -608,6 +608,26 @@ def refresh(self, unsafely_update_root_if_necessary=True): # is insufficient trusted signatures for the specified metadata. # Raise 'tuf.NoWorkingMirrorError' if an update fails. + # Is the Root role expired? When the top-level roles are initially loaded + # from disk, their expiration is not checked to allow their updating when + # requested (and give the updater the chance to continue, rather than always + # failing with an expired metadata error.) If + # 'unsafely_update_root_if_necessary' is True, update an expired Root role + # now. Updating the other top-level roles, regardless of their validity, + # should only occur if the root of trust is up-to-date. + if unsafely_update_root_if_necessary: + root_metadata = self.metadata['current']['root'] + try: + self._ensure_not_expired(root_metadata, 'root') + + except tuf.ExpiredMetadataError as e: + # Raise 'tuf.NoWorkingMirrorError' if a valid (not expired, properly + # signed, and valid metadata) 'root' cannot be installed. + message = \ + 'Expired Root metadata was loaded from disk. Try to update it now.' + logger.info(message) + self._update_metadata('root', DEFAULT_ROOT_FILEINFO) + # Use default but sane information for timestamp metadata, and do not # require strict checks on its required length. try: @@ -629,13 +649,6 @@ def refresh(self, unsafely_update_root_if_necessary=True): else: raise - else: - # Updated the top-level metadata (which all had valid signatures), - # however, have they expired? Raise 'tuf.ExpiredMetadataError' if any of - # the metadata has expired. - for metadata_role in ['timestamp', 'root', 'snapshot', 'targets']: - self._ensure_not_expired(metadata_role) - @@ -883,13 +896,18 @@ def _verify_uncompressed_metadata_file(self, metadata_file_object, try: metadata_signable = tuf.util.load_json_string(metadata) + except Exception, exception: raise tuf.InvalidMetadataJSONError(exception) + else: # Ensure the loaded 'metadata_signable' is properly formatted. Raise # 'tuf.FormatError' if not. tuf.formats.check_signable_object_format(metadata_signable) + # Is 'metadata_signable' expired? + self._ensure_not_expired(metadata_signable['signed'], metadata_role) + # Is 'metadata_signable' newer than the currently installed # version? current_metadata_role = self.metadata['current'].get(metadata_role) @@ -990,10 +1008,10 @@ def unsafely_verify_uncompressed_metadata_file(metadata_file_object): self._check_hashes(metadata_file_object, uncompressed_file_hashes) self._verify_uncompressed_metadata_file(metadata_file_object, metadata_role) - - def unsafely_verify_compressed_metadata_file(metadata_file_object): - self._hard_check_file_length(metadata_file_object, compressed_file_length) - self._check_hashes(metadata_file_object, compressed_file_hashes) + + def unsafely_verify_compressed_metadata_file(metadata_signable): + self._hard_check_file_length(metadata_signable, compressed_file_length) + self._check_hashes(metadata_signable, compressed_file_hashes) if compression is None: unsafely_verify_compressed_metadata_file = None @@ -1069,7 +1087,7 @@ def safely_verify_uncompressed_metadata_file(metadata_file_object): uncompressed_file_length) self._check_hashes(metadata_file_object, uncompressed_file_hashes) self._verify_uncompressed_metadata_file(metadata_file_object, - metadata_role) + metadata_role) def safely_verify_compressed_metadata_file(metadata_file_object): self._hard_check_file_length(metadata_file_object, compressed_file_length) @@ -1724,20 +1742,24 @@ def _delete_metadata(self, metadata_role): - def _ensure_not_expired(self, metadata_role): + def _ensure_not_expired(self, metadata_object, metadata_rolename): """ Non-public method that raises an exception if the current specified metadata has expired. - metadata_role: + metadata_object: + The metadata that should be expired, a 'tuf.formats.ANYROLE_SCHEMA' + object. + + metadata_rolename: The name of the metadata. This is a role name and should not end in '.json'. Examples: 'root', 'targets', 'targets/linux/x86'. tuf.ExpiredMetadataError: - If 'metadata_role' has expired. + If 'metadata_rolename' has expired. None. @@ -1745,17 +1767,9 @@ def _ensure_not_expired(self, metadata_role): None. """ - - # Construct the full metadata filename and the location of its - # current path. The current path of 'metadata_role' is needed - # to log the exact filename of the expired metadata. - metadata_filename = metadata_role + '.json' - rolepath = os.path.join(self.metadata_directory['current'], - metadata_filename) - rolepath = os.path.abspath(rolepath) - + # Extract the expiration time. - expires = self.metadata['current'][metadata_role]['expires'] + expires = metadata_object['expires'] # If the current time has surpassed the expiration date, raise # an exception. 'expires' is in 'tuf.formats.ISO8601_DATETIME_SCHEMA' @@ -1770,7 +1784,7 @@ def _ensure_not_expired(self, metadata_role): expires_timestamp = tuf.formats.datetime_to_unix_timestamp(expires_datetime) if expires_timestamp < current_time: - message = 'Metadata '+repr(rolepath)+' expired on ' + \ + message = 'Metadata '+repr(metadata_rolename)+' expired on ' + \ expires_datetime.ctime() + ' (UTC).' logger.error(message) @@ -1908,13 +1922,6 @@ def _refresh_targets_metadata(self, rolename='targets', include_delegations=Fals self._update_metadata_if_changed(rolename) - # Remove the role if it has expired. - try: - self._ensure_not_expired(rolename) - - except tuf.ExpiredMetadataError: - tuf.roledb.remove_role(rolename) - @@ -2033,13 +2040,6 @@ def refresh_targets_metadata_chain(self, rolename): self._update_metadata_if_changed(rolename) - # Remove the role if it has expired. - try: - self._ensure_not_expired(rolename) - refreshed_chain.append(rolename) - except tuf.ExpiredMetadataError: - tuf.roledb.remove_role(rolename) - return refreshed_chain diff --git a/tuf/download.py b/tuf/download.py index aba22fdb..d1fe6539 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -502,10 +502,13 @@ def _download_fixed_amount_of_data(connection, temp_file, required_length): # Data successfully read from the connection. Store it. temp_file.write(data) total_downloaded = total_downloaded + len(data) + except: raise + else: return total_downloaded + finally: # Whatever happens, make sure that we always close the connection. connection.close() From 91480c86285d682c59f76e1381dd32ae95192ced Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Tue, 29 Apr 2014 08:15:18 -0400 Subject: [PATCH 2/2] Edit updater.refresh() and install of compressed metadata. 1. Raise an exception for clients that do not wish to automatically fetch a Root file when an expired version is loaded from disk. 2. Properly 'install' compressed metadata downloaded. The compressed version of the rolename was added to the fileinfo store, which prevented detection of changed metadata and would unintentionally cause compressed metadata to always refresh (only the fileinfo of uncompressed metadata is stored and compared.) 3. Rename unsafely_verify_compressed_metadata_file() variable names, so that they match the other verify functions. --- tuf/client/updater.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index c0c82e8b..726ce482 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -615,18 +615,23 @@ def refresh(self, unsafely_update_root_if_necessary=True): # 'unsafely_update_root_if_necessary' is True, update an expired Root role # now. Updating the other top-level roles, regardless of their validity, # should only occur if the root of trust is up-to-date. - if unsafely_update_root_if_necessary: - root_metadata = self.metadata['current']['root'] - try: - self._ensure_not_expired(root_metadata, 'root') - - except tuf.ExpiredMetadataError as e: - # Raise 'tuf.NoWorkingMirrorError' if a valid (not expired, properly - # signed, and valid metadata) 'root' cannot be installed. + root_metadata = self.metadata['current']['root'] + try: + self._ensure_not_expired(root_metadata, 'root') + + except tuf.ExpiredMetadataError as e: + # Raise 'tuf.NoWorkingMirrorError' if a valid (not expired, properly + # signed, and valid metadata) 'root' cannot be installed. + if unsafely_update_root_if_necessary: message = \ 'Expired Root metadata was loaded from disk. Try to update it now.' logger.info(message) self._update_metadata('root', DEFAULT_ROOT_FILEINFO) + + # The caller explicitly requested not to unsafely fetch an expired Root. + else: + logger.info('An expired Root metadata was loaded and must be updated.') + raise # Use default but sane information for timestamp metadata, and do not # require strict checks on its required length. @@ -1009,9 +1014,9 @@ def unsafely_verify_uncompressed_metadata_file(metadata_file_object): self._verify_uncompressed_metadata_file(metadata_file_object, metadata_role) - def unsafely_verify_compressed_metadata_file(metadata_signable): - self._hard_check_file_length(metadata_signable, compressed_file_length) - self._check_hashes(metadata_signable, compressed_file_hashes) + def unsafely_verify_compressed_metadata_file(metadata_file_object): + self._hard_check_file_length(metadata_file_object, compressed_file_length) + self._check_hashes(metadata_file_object, compressed_file_hashes) if compression is None: unsafely_verify_compressed_metadata_file = None @@ -1372,7 +1377,7 @@ def _update_metadata(self, metadata_role, uncompressed_fileinfo, logger.debug('Updated '+repr(current_filepath)+'.') self.metadata['previous'][metadata_role] = current_metadata_object self.metadata['current'][metadata_role] = updated_metadata_object - self._update_fileinfo(metadata_filename) + self._update_fileinfo(uncompressed_metadata_filename) # Ensure the role and key information of the top-level roles is also updated # according to the newly-installed Root metadata.