diff --git a/tests/test_formats.py b/tests/test_formats.py index 4bcff487..b23491bf 100755 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -224,13 +224,14 @@ def test_schemas(self): {'_type': 'Snapshot', 'version': 8, 'expires': '1985-10-21T13:20:00Z', - 'meta': {'metadata/snapshot.json': {'version': 1024}}}), + 'meta': {'snapshot.json': {'version': 1024}}}), 'TIMESTAMP_SCHEMA': (tuf.formats.TIMESTAMP_SCHEMA, {'_type': 'Timestamp', 'version': 8, 'expires': '1985-10-21T13:20:00Z', - 'meta': {'metadata/timestamp.json': {'version': 1024}}}), + 'meta': {'metadattimestamp.json': {'length': 1024, + 'hashes': {'sha256': 'AB1245'}}}}), 'MIRROR_SCHEMA': (tuf.formats.MIRROR_SCHEMA, {'url_prefix': 'http://localhost:8001', @@ -298,29 +299,31 @@ def __init__(self, version, expires): def test_TimestampFile(self): # Test conditions for valid instances of 'tuf.formats.TimestampFile'. - version = 8 + version = 8 + length = 88 + hashes = {'sha256': '3c7fe3eeded4a34'} expires = '1985-10-21T13:20:00Z' - versiondict = {'targets.json': {'version': version}} + filedict = {'snapshot.json': {'length': length, 'hashes': hashes}} make_metadata = tuf.formats.TimestampFile.make_metadata from_metadata = tuf.formats.TimestampFile.from_metadata TIMESTAMP_SCHEMA = tuf.formats.TIMESTAMP_SCHEMA self.assertTrue(TIMESTAMP_SCHEMA.matches(make_metadata(version, expires, - versiondict))) - metadata = make_metadata(version, expires, versiondict) + filedict))) + metadata = make_metadata(version, expires, filedict) self.assertTrue(isinstance(from_metadata(metadata), tuf.formats.TimestampFile)) # Test conditions for invalid arguments. bad_version = 'eight' bad_expires = '2000' - bad_versiondict = 123 + bad_filedict = 123 self.assertRaises(tuf.FormatError, make_metadata, bad_version, - expires, versiondict) + expires, filedict) self.assertRaises(tuf.FormatError, make_metadata, version, - bad_expires, versiondict) + bad_expires, filedict) self.assertRaises(tuf.FormatError, make_metadata, version, - expires, bad_versiondict) + expires, bad_filedict) self.assertRaises(tuf.FormatError, from_metadata, 123) diff --git a/tests/test_repository_tool.py b/tests/test_repository_tool.py index f5921d23..662c818f 100755 --- a/tests/test_repository_tool.py +++ b/tests/test_repository_tool.py @@ -325,7 +325,7 @@ def test_write_and_write_partial(self): repository.root.load_signing_key(root_privkey) repository.snapshot.load_signing_key(snapshot_privkey) - # Verify that consistent snapshot can be written and loaded. + # Verify that a consistent snapshot can be written and loaded. repository.write(consistent_snapshot=True) repo_tool.load_repository(repository_directory) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 5fb45bb3..12a2b550 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -991,7 +991,7 @@ def _unsafely_get_metadata_file(self, metadata_role, metadata_filepath, metadata_role: The role name of the metadata (e.g., 'root', 'targets', - 'targets/linux/x86'). + 'claimed'). metadata_filepath: The metadata filepath (i.e., relative to the repository metadata diff --git a/tuf/formats.py b/tuf/formats.py index d9d4dc84..dd5873e1 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -281,12 +281,12 @@ object_name = 'VERSIONINFO_SCHEMA', version = METADATAVERSION_SCHEMA) -# A dict holding the version information for a particular metadata role. The -# dict keys hold the relative file paths, and the dict values the corresponding -# version numbers. -VERSIONDICT_SCHEMA = SCHEMA.DictOf( +# A dict holding the version or file information for a particular metadata +# role. The dict keys hold the relative file paths, and the dict values the +# corresponding version numbers or file information. +FILEINFODICT_SCHEMA = SCHEMA.DictOf( key_schema = RELPATH_SCHEMA, - value_schema = VERSIONINFO_SCHEMA) + value_schema = SCHEMA.OneOf([VERSIONINFO_SCHEMA, FILEINFO_SCHEMA])) # A dict holding the information for a particular target / file. The dict keys # hold the relative file paths, and the dict values the corresponding file @@ -477,7 +477,7 @@ _type = SCHEMA.String('Snapshot'), version = METADATAVERSION_SCHEMA, expires = ISO8601_DATETIME_SCHEMA, - meta = VERSIONDICT_SCHEMA) + meta = FILEINFODICT_SCHEMA) # Timestamp role: indicates the latest version of the snapshot file. TIMESTAMP_SCHEMA = SCHEMA.Object( @@ -485,7 +485,7 @@ _type = SCHEMA.String('Timestamp'), version = METADATAVERSION_SCHEMA, expires = ISO8601_DATETIME_SCHEMA, - meta = VERSIONDICT_SCHEMA) + meta = FILEDICT_SCHEMA) # project.cfg file: stores information about the project in a json dictionary PROJECT_CFG_SCHEMA = SCHEMA.Object( @@ -573,11 +573,11 @@ def __getattr__(self, name): class TimestampFile(MetaFile): - def __init__(self, version, expires, versiondict): + def __init__(self, version, expires, filedict): self.info = {} self.info['version'] = version self.info['expires'] = expires - self.info['meta'] = versiondict + self.info['meta'] = filedict @staticmethod @@ -588,17 +588,17 @@ def from_metadata(object): version = object['version'] expires = object['expires'] - versiondict = object['meta'] + filedict = object['meta'] - return TimestampFile(version, expires, versiondict) + return TimestampFile(version, expires, filedict) @staticmethod - def make_metadata(version, expiration_date, versiondict): + def make_metadata(version, expiration_date, filedict): result = {'_type' : 'Timestamp'} result['version'] = version result['expires'] = expiration_date - result['meta'] = versiondict + result['meta'] = filedict # Is 'result' a Timestamp metadata file? # Raise 'tuf.FormatError' if not. @@ -1048,7 +1048,7 @@ def make_versioninfo(version_number): information of a metadata role. """ - versioninfo = {'version' : version_number} + versioninfo = {'version': version_number} # Raise 'tuf.FormatError' if 'versioninfo' is improperly formatted. try: diff --git a/tuf/repository_lib.py b/tuf/repository_lib.py index 9aab12c3..6c81d9c8 100755 --- a/tuf/repository_lib.py +++ b/tuf/repository_lib.py @@ -612,17 +612,21 @@ def _load_top_level_metadata(repository, top_level_filenames): else: pass - # Load 'snapshot.json'. A consistent snapshot of Snapshot must be calculated - # if 'consistent_snapshot' is True. + # Load 'snapshot.json'. A consistent snapshot.json must be calculated if + # 'consistent_snapshot' is True. + # The Snapshot and Root roles are both accessed by their hashes. if consistent_snapshot: - snapshot_version = timestamp_metadata['meta'][SNAPSHOT_FILENAME]['version'] + snapshot_hashes = timestamp_metadata['meta'][SNAPSHOT_FILENAME]['hashes'] + snapshot_hash = random.choice(list(snapshot_hashes.values())) + dirname, basename = os.path.split(snapshot_filename) - snapshot_filename = os.path.join(dirname, str(snapshot_version) + '.' + basename) - + snapshot_filename = os.path.join(dirname, str(snapshot_hash) + '.' + basename) + if os.path.exists(snapshot_filename): signable = tuf.util.load_json_file(snapshot_filename) tuf.formats.check_signable_object_format(signable) snapshot_metadata = signable['signed'] + for signature in signable['signatures']: repository.snapshot.add_signature(signature, mark_role_as_dirty=False) @@ -1652,11 +1656,14 @@ def generate_snapshot_metadata(metadata_directory, version, expiration_date, metadata_directory = _check_directory(metadata_directory) - # Retrieve the versioninfo of 'root.json' and 'targets.json'. 'versiondict' - # shall contain the version number of all roles available on the repository. - versiondict = {} - versiondict[ROOT_FILENAME] = get_metadata_versioninfo(root_filename) - versiondict[TARGETS_FILENAME] = get_metadata_versioninfo(targets_filename) + # Set the fileinfo of 'root.json', and the versioninfo of + # 'targets.json'. 'fileinfodict' shall contain the version number of all + # available delegated roles on the repository. + fileinfodict = {} + root_path = os.path.join(metadata_directory, root_filename + '.json') + length, hashes = tuf.util.get_file_details(root_path) + fileinfodict[ROOT_FILENAME] = tuf.formats.make_fileinfo(length, hashes) + fileinfodict[TARGETS_FILENAME] = get_metadata_versioninfo(targets_filename) # We previously also stored the compressed versions of roles in # snapshot.json, however, this is no longer needed as their hashes and @@ -1678,18 +1685,18 @@ def generate_snapshot_metadata(metadata_directory, version, expiration_date, if metadata_filename.endswith(metadata_extension): rolename = metadata_filename[:-len(metadata_extension)] - # Obsolete role files may still be found. Ensure only roles loaded # in the roledb are included in the Snapshot metadata. Since the # snapshot and timestamp roles are not listed in snapshot.json, do not # list these roles found in the metadata directory. - if tuf.roledb.role_exists(rolename) and rolename not in ['snapshot', 'timestamp']: - versiondict[metadata_name] = get_metadata_versioninfo(rolename) + if tuf.roledb.role_exists(rolename) and \ + rolename not in ['root', 'snapshot', 'timestamp', 'targets']: + fileinfodict[metadata_name] = get_metadata_versioninfo(rolename) # Generate the Snapshot metadata object. snapshot_metadata = tuf.formats.SnapshotFile.make_metadata(version, expiration_date, - versiondict) + fileinfodict) return snapshot_metadata @@ -1737,8 +1744,9 @@ def generate_timestamp_metadata(snapshot_filename, version, expiration_date): tuf.formats.ISO8601_DATETIME_SCHEMA.check_match(expiration_date) # Retrieve the versioninfo of the Snapshot metadata file. - versioninfo = {} - versioninfo[SNAPSHOT_FILENAME] = get_metadata_versioninfo('snapshot') + snapshot_fileinfo = {} + length, hashes = tuf.util.get_file_details(snapshot_filename) + snapshot_fileinfo[SNAPSHOT_FILENAME] = tuf.formats.make_fileinfo(length, hashes) # We previously saved the versioninfo of the compressed versions of # 'snapshot.json' in 'versioninfo'. Since version numbers are now stored, @@ -1748,7 +1756,7 @@ def generate_timestamp_metadata(snapshot_filename, version, expiration_date): # Generate the timestamp metadata object. timestamp_metadata = tuf.formats.TimestampFile.make_metadata(version, expiration_date, - versioninfo) + snapshot_fileinfo) return timestamp_metadata @@ -1907,7 +1915,6 @@ def write_metadata_file(metadata, filename, version_number, # path so that temporary files are moved to their expected destinations. filename = os.path.abspath(filename) written_filename = filename - written_consistent_filename = None _check_directory(os.path.dirname(filename)) # Generate the actual metadata file content of 'metadata'. Metadata is @@ -1916,11 +1923,6 @@ def write_metadata_file(metadata, filename, version_number, # if re-saving is required. file_content = _get_written_metadata(metadata) - if consistent_snapshot: - dirname, basename = os.path.split(filename) - version_and_filename = str(version_number) + '.' + basename - written_consistent_filename = os.path.join(dirname, version_and_filename) - # Verify whether new metadata needs to be written (i.e., has not been # previously written or has changed. write_new_metadata = False @@ -1930,8 +1932,7 @@ def write_metadata_file(metadata, filename, version_number, # Compressed metadata should only be written if it does not exist or the # uncompressed version has changed). new_digests = {} - hash_algorithms = tuf.conf.REPOSITORY_HASH_ALGORITHMS - for hash_algorithm in hash_algorithms: + for hash_algorithm in tuf.conf.REPOSITORY_HASH_ALGORITHMS: digest_object = tuf.hash.digest(hash_algorithm) digest_object.update(file_content) new_digests.update({hash_algorithm: digest_object.hexdigest()}) @@ -1942,14 +1943,14 @@ def write_metadata_file(metadata, filename, version_number, write_new_metadata = True # 'tuf.Error' raised if 'filename' does not exist. - except tuf.Error as e: + except tuf.Error: write_new_metadata = True if write_new_metadata: # The 'metadata' object is written to 'file_object', including compressed # versions. To avoid partial metadata from being written, 'metadata' is - # first written to a temporary location (i.e., 'file_object') and then moved - # to 'filename'. + # first written to a temporary location (i.e., 'file_object') and then + # moved to 'filename'. file_object = tuf.util.TempFile() # Serialize 'metadata' to the file-like object and then write @@ -1960,7 +1961,22 @@ def write_metadata_file(metadata, filename, version_number, logger.debug('Saving ' + repr(written_filename)) file_object.move(written_filename) - if consistent_snapshot: + if consistent_snapshot: + dirname, basename = os.path.split(written_filename) + + if basename in ['root.json', 'snapshot.json']: + hash_algorithms = tuf.conf.REPOSITORY_HASH_ALGORITHMS + file_length_junk, digests = \ + tuf.util.get_file_details(written_filename, hash_algorithms) + + for digest in digests.values(): + digest_and_filename = str(digest) + '.' + basename + written_consistent_filename = os.path.join(dirname, digest_and_filename) + + else: + version_and_filename = str(version_number) + '.' + basename + written_consistent_filename = os.path.join(dirname, version_and_filename) + logger.info('Linking ' + repr(written_consistent_filename)) os.link(written_filename, written_consistent_filename) @@ -1994,8 +2010,9 @@ def write_metadata_file(metadata, filename, version_number, raise tuf.FormatError('Unknown compression algorithm: ' + repr(compressio_algorithm)) # Save the compressed version, ensuring an unchanged file is not re-saved. - # Re-saving the same compressed version may cause its digest to unexpectedly - # change (gzip includes a timestamp) even though content has not changed. + # Re-saving the same compressed version may cause its digest to + # unexpectedly change (gzip includes a timestamp) even though content has + # not changed. _write_compressed_metadata(file_object, compressed_filename, write_new_metadata, consistent_snapshot) return written_filename diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index bbe3c46f..5c512e62 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -2785,8 +2785,8 @@ def load_repository(repository_directory): if metadata_name in ['root', 'snapshot', 'targets', 'timestamp']: continue - # Keep a store metadata previously loaded metadata to prevent - # re-loading duplicate versions. Duplicate versions may occur with + # Keep a store of metadata previously loaded metadata to prevent re-loading + # duplicate versions. Duplicate versions may occur with # 'consistent_snapshot', where the same metadata may be available in # multiples files (the different hash is included in each filename. if metadata_name in loaded_metadata: