From b8ce4bbfcdee1edd0a3b9e97150363817b2dfd97 Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Thu, 14 Apr 2016 08:13:50 -0400 Subject: [PATCH] Implement changes to write and load the new format of snapshot.json The repository tools should properly write and load consistent versions of root.json and snapshot.json. Version numbers were previously prepended to these two roles. --- tests/test_formats.py | 23 +++++----- tests/test_repository_tool.py | 2 +- tuf/client/updater.py | 2 +- tuf/formats.py | 28 ++++++------- tuf/repository_lib.py | 79 +++++++++++++++++++++-------------- tuf/repository_tool.py | 4 +- 6 files changed, 79 insertions(+), 59 deletions(-) diff --git a/tests/test_formats.py b/tests/test_formats.py index 4bcff487..b23491bf 100755 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -224,13 +224,14 @@ def test_schemas(self): {'_type': 'Snapshot', 'version': 8, 'expires': '1985-10-21T13:20:00Z', - 'meta': {'metadata/snapshot.json': {'version': 1024}}}), + 'meta': {'snapshot.json': {'version': 1024}}}), 'TIMESTAMP_SCHEMA': (tuf.formats.TIMESTAMP_SCHEMA, {'_type': 'Timestamp', 'version': 8, 'expires': '1985-10-21T13:20:00Z', - 'meta': {'metadata/timestamp.json': {'version': 1024}}}), + 'meta': {'metadattimestamp.json': {'length': 1024, + 'hashes': {'sha256': 'AB1245'}}}}), 'MIRROR_SCHEMA': (tuf.formats.MIRROR_SCHEMA, {'url_prefix': 'http://localhost:8001', @@ -298,29 +299,31 @@ def __init__(self, version, expires): def test_TimestampFile(self): # Test conditions for valid instances of 'tuf.formats.TimestampFile'. - version = 8 + version = 8 + length = 88 + hashes = {'sha256': '3c7fe3eeded4a34'} expires = '1985-10-21T13:20:00Z' - versiondict = {'targets.json': {'version': version}} + filedict = {'snapshot.json': {'length': length, 'hashes': hashes}} make_metadata = tuf.formats.TimestampFile.make_metadata from_metadata = tuf.formats.TimestampFile.from_metadata TIMESTAMP_SCHEMA = tuf.formats.TIMESTAMP_SCHEMA self.assertTrue(TIMESTAMP_SCHEMA.matches(make_metadata(version, expires, - versiondict))) - metadata = make_metadata(version, expires, versiondict) + filedict))) + metadata = make_metadata(version, expires, filedict) self.assertTrue(isinstance(from_metadata(metadata), tuf.formats.TimestampFile)) # Test conditions for invalid arguments. bad_version = 'eight' bad_expires = '2000' - bad_versiondict = 123 + bad_filedict = 123 self.assertRaises(tuf.FormatError, make_metadata, bad_version, - expires, versiondict) + expires, filedict) self.assertRaises(tuf.FormatError, make_metadata, version, - bad_expires, versiondict) + bad_expires, filedict) self.assertRaises(tuf.FormatError, make_metadata, version, - expires, bad_versiondict) + expires, bad_filedict) self.assertRaises(tuf.FormatError, from_metadata, 123) diff --git a/tests/test_repository_tool.py b/tests/test_repository_tool.py index f5921d23..662c818f 100755 --- a/tests/test_repository_tool.py +++ b/tests/test_repository_tool.py @@ -325,7 +325,7 @@ def test_write_and_write_partial(self): repository.root.load_signing_key(root_privkey) repository.snapshot.load_signing_key(snapshot_privkey) - # Verify that consistent snapshot can be written and loaded. + # Verify that a consistent snapshot can be written and loaded. repository.write(consistent_snapshot=True) repo_tool.load_repository(repository_directory) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 5fb45bb3..12a2b550 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -991,7 +991,7 @@ def _unsafely_get_metadata_file(self, metadata_role, metadata_filepath, metadata_role: The role name of the metadata (e.g., 'root', 'targets', - 'targets/linux/x86'). + 'claimed'). metadata_filepath: The metadata filepath (i.e., relative to the repository metadata diff --git a/tuf/formats.py b/tuf/formats.py index d9d4dc84..dd5873e1 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -281,12 +281,12 @@ object_name = 'VERSIONINFO_SCHEMA', version = METADATAVERSION_SCHEMA) -# A dict holding the version information for a particular metadata role. The -# dict keys hold the relative file paths, and the dict values the corresponding -# version numbers. -VERSIONDICT_SCHEMA = SCHEMA.DictOf( +# A dict holding the version or file information for a particular metadata +# role. The dict keys hold the relative file paths, and the dict values the +# corresponding version numbers or file information. +FILEINFODICT_SCHEMA = SCHEMA.DictOf( key_schema = RELPATH_SCHEMA, - value_schema = VERSIONINFO_SCHEMA) + value_schema = SCHEMA.OneOf([VERSIONINFO_SCHEMA, FILEINFO_SCHEMA])) # A dict holding the information for a particular target / file. The dict keys # hold the relative file paths, and the dict values the corresponding file @@ -477,7 +477,7 @@ _type = SCHEMA.String('Snapshot'), version = METADATAVERSION_SCHEMA, expires = ISO8601_DATETIME_SCHEMA, - meta = VERSIONDICT_SCHEMA) + meta = FILEINFODICT_SCHEMA) # Timestamp role: indicates the latest version of the snapshot file. TIMESTAMP_SCHEMA = SCHEMA.Object( @@ -485,7 +485,7 @@ _type = SCHEMA.String('Timestamp'), version = METADATAVERSION_SCHEMA, expires = ISO8601_DATETIME_SCHEMA, - meta = VERSIONDICT_SCHEMA) + meta = FILEDICT_SCHEMA) # project.cfg file: stores information about the project in a json dictionary PROJECT_CFG_SCHEMA = SCHEMA.Object( @@ -573,11 +573,11 @@ def __getattr__(self, name): class TimestampFile(MetaFile): - def __init__(self, version, expires, versiondict): + def __init__(self, version, expires, filedict): self.info = {} self.info['version'] = version self.info['expires'] = expires - self.info['meta'] = versiondict + self.info['meta'] = filedict @staticmethod @@ -588,17 +588,17 @@ def from_metadata(object): version = object['version'] expires = object['expires'] - versiondict = object['meta'] + filedict = object['meta'] - return TimestampFile(version, expires, versiondict) + return TimestampFile(version, expires, filedict) @staticmethod - def make_metadata(version, expiration_date, versiondict): + def make_metadata(version, expiration_date, filedict): result = {'_type' : 'Timestamp'} result['version'] = version result['expires'] = expiration_date - result['meta'] = versiondict + result['meta'] = filedict # Is 'result' a Timestamp metadata file? # Raise 'tuf.FormatError' if not. @@ -1048,7 +1048,7 @@ def make_versioninfo(version_number): information of a metadata role. """ - versioninfo = {'version' : version_number} + versioninfo = {'version': version_number} # Raise 'tuf.FormatError' if 'versioninfo' is improperly formatted. try: diff --git a/tuf/repository_lib.py b/tuf/repository_lib.py index 9aab12c3..6c81d9c8 100755 --- a/tuf/repository_lib.py +++ b/tuf/repository_lib.py @@ -612,17 +612,21 @@ def _load_top_level_metadata(repository, top_level_filenames): else: pass - # Load 'snapshot.json'. A consistent snapshot of Snapshot must be calculated - # if 'consistent_snapshot' is True. + # Load 'snapshot.json'. A consistent snapshot.json must be calculated if + # 'consistent_snapshot' is True. + # The Snapshot and Root roles are both accessed by their hashes. if consistent_snapshot: - snapshot_version = timestamp_metadata['meta'][SNAPSHOT_FILENAME]['version'] + snapshot_hashes = timestamp_metadata['meta'][SNAPSHOT_FILENAME]['hashes'] + snapshot_hash = random.choice(list(snapshot_hashes.values())) + dirname, basename = os.path.split(snapshot_filename) - snapshot_filename = os.path.join(dirname, str(snapshot_version) + '.' + basename) - + snapshot_filename = os.path.join(dirname, str(snapshot_hash) + '.' + basename) + if os.path.exists(snapshot_filename): signable = tuf.util.load_json_file(snapshot_filename) tuf.formats.check_signable_object_format(signable) snapshot_metadata = signable['signed'] + for signature in signable['signatures']: repository.snapshot.add_signature(signature, mark_role_as_dirty=False) @@ -1652,11 +1656,14 @@ def generate_snapshot_metadata(metadata_directory, version, expiration_date, metadata_directory = _check_directory(metadata_directory) - # Retrieve the versioninfo of 'root.json' and 'targets.json'. 'versiondict' - # shall contain the version number of all roles available on the repository. - versiondict = {} - versiondict[ROOT_FILENAME] = get_metadata_versioninfo(root_filename) - versiondict[TARGETS_FILENAME] = get_metadata_versioninfo(targets_filename) + # Set the fileinfo of 'root.json', and the versioninfo of + # 'targets.json'. 'fileinfodict' shall contain the version number of all + # available delegated roles on the repository. + fileinfodict = {} + root_path = os.path.join(metadata_directory, root_filename + '.json') + length, hashes = tuf.util.get_file_details(root_path) + fileinfodict[ROOT_FILENAME] = tuf.formats.make_fileinfo(length, hashes) + fileinfodict[TARGETS_FILENAME] = get_metadata_versioninfo(targets_filename) # We previously also stored the compressed versions of roles in # snapshot.json, however, this is no longer needed as their hashes and @@ -1678,18 +1685,18 @@ def generate_snapshot_metadata(metadata_directory, version, expiration_date, if metadata_filename.endswith(metadata_extension): rolename = metadata_filename[:-len(metadata_extension)] - # Obsolete role files may still be found. Ensure only roles loaded # in the roledb are included in the Snapshot metadata. Since the # snapshot and timestamp roles are not listed in snapshot.json, do not # list these roles found in the metadata directory. - if tuf.roledb.role_exists(rolename) and rolename not in ['snapshot', 'timestamp']: - versiondict[metadata_name] = get_metadata_versioninfo(rolename) + if tuf.roledb.role_exists(rolename) and \ + rolename not in ['root', 'snapshot', 'timestamp', 'targets']: + fileinfodict[metadata_name] = get_metadata_versioninfo(rolename) # Generate the Snapshot metadata object. snapshot_metadata = tuf.formats.SnapshotFile.make_metadata(version, expiration_date, - versiondict) + fileinfodict) return snapshot_metadata @@ -1737,8 +1744,9 @@ def generate_timestamp_metadata(snapshot_filename, version, expiration_date): tuf.formats.ISO8601_DATETIME_SCHEMA.check_match(expiration_date) # Retrieve the versioninfo of the Snapshot metadata file. - versioninfo = {} - versioninfo[SNAPSHOT_FILENAME] = get_metadata_versioninfo('snapshot') + snapshot_fileinfo = {} + length, hashes = tuf.util.get_file_details(snapshot_filename) + snapshot_fileinfo[SNAPSHOT_FILENAME] = tuf.formats.make_fileinfo(length, hashes) # We previously saved the versioninfo of the compressed versions of # 'snapshot.json' in 'versioninfo'. Since version numbers are now stored, @@ -1748,7 +1756,7 @@ def generate_timestamp_metadata(snapshot_filename, version, expiration_date): # Generate the timestamp metadata object. timestamp_metadata = tuf.formats.TimestampFile.make_metadata(version, expiration_date, - versioninfo) + snapshot_fileinfo) return timestamp_metadata @@ -1907,7 +1915,6 @@ def write_metadata_file(metadata, filename, version_number, # path so that temporary files are moved to their expected destinations. filename = os.path.abspath(filename) written_filename = filename - written_consistent_filename = None _check_directory(os.path.dirname(filename)) # Generate the actual metadata file content of 'metadata'. Metadata is @@ -1916,11 +1923,6 @@ def write_metadata_file(metadata, filename, version_number, # if re-saving is required. file_content = _get_written_metadata(metadata) - if consistent_snapshot: - dirname, basename = os.path.split(filename) - version_and_filename = str(version_number) + '.' + basename - written_consistent_filename = os.path.join(dirname, version_and_filename) - # Verify whether new metadata needs to be written (i.e., has not been # previously written or has changed. write_new_metadata = False @@ -1930,8 +1932,7 @@ def write_metadata_file(metadata, filename, version_number, # Compressed metadata should only be written if it does not exist or the # uncompressed version has changed). new_digests = {} - hash_algorithms = tuf.conf.REPOSITORY_HASH_ALGORITHMS - for hash_algorithm in hash_algorithms: + for hash_algorithm in tuf.conf.REPOSITORY_HASH_ALGORITHMS: digest_object = tuf.hash.digest(hash_algorithm) digest_object.update(file_content) new_digests.update({hash_algorithm: digest_object.hexdigest()}) @@ -1942,14 +1943,14 @@ def write_metadata_file(metadata, filename, version_number, write_new_metadata = True # 'tuf.Error' raised if 'filename' does not exist. - except tuf.Error as e: + except tuf.Error: write_new_metadata = True if write_new_metadata: # The 'metadata' object is written to 'file_object', including compressed # versions. To avoid partial metadata from being written, 'metadata' is - # first written to a temporary location (i.e., 'file_object') and then moved - # to 'filename'. + # first written to a temporary location (i.e., 'file_object') and then + # moved to 'filename'. file_object = tuf.util.TempFile() # Serialize 'metadata' to the file-like object and then write @@ -1960,7 +1961,22 @@ def write_metadata_file(metadata, filename, version_number, logger.debug('Saving ' + repr(written_filename)) file_object.move(written_filename) - if consistent_snapshot: + if consistent_snapshot: + dirname, basename = os.path.split(written_filename) + + if basename in ['root.json', 'snapshot.json']: + hash_algorithms = tuf.conf.REPOSITORY_HASH_ALGORITHMS + file_length_junk, digests = \ + tuf.util.get_file_details(written_filename, hash_algorithms) + + for digest in digests.values(): + digest_and_filename = str(digest) + '.' + basename + written_consistent_filename = os.path.join(dirname, digest_and_filename) + + else: + version_and_filename = str(version_number) + '.' + basename + written_consistent_filename = os.path.join(dirname, version_and_filename) + logger.info('Linking ' + repr(written_consistent_filename)) os.link(written_filename, written_consistent_filename) @@ -1994,8 +2010,9 @@ def write_metadata_file(metadata, filename, version_number, raise tuf.FormatError('Unknown compression algorithm: ' + repr(compressio_algorithm)) # Save the compressed version, ensuring an unchanged file is not re-saved. - # Re-saving the same compressed version may cause its digest to unexpectedly - # change (gzip includes a timestamp) even though content has not changed. + # Re-saving the same compressed version may cause its digest to + # unexpectedly change (gzip includes a timestamp) even though content has + # not changed. _write_compressed_metadata(file_object, compressed_filename, write_new_metadata, consistent_snapshot) return written_filename diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index bbe3c46f..5c512e62 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -2785,8 +2785,8 @@ def load_repository(repository_directory): if metadata_name in ['root', 'snapshot', 'targets', 'timestamp']: continue - # Keep a store metadata previously loaded metadata to prevent - # re-loading duplicate versions. Duplicate versions may occur with + # Keep a store of metadata previously loaded metadata to prevent re-loading + # duplicate versions. Duplicate versions may occur with # 'consistent_snapshot', where the same metadata may be available in # multiples files (the different hash is included in each filename. if metadata_name in loaded_metadata: