Elgato_dark/python-tuf
1
0
Fork 0
mirror of https://github.com/theupdateframework/python-tuf synced 2026-05-24 10:08:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implement changes to write and load the new format of snapshot.json

Browse source 
The repository tools should properly write and load consistent versions of root.json and snapshot.json.  Version numbers were previously prepended to these two roles.
This commit is contained in:
Vladimir Diaz 2016-04-14 08:13:50 -04:00
parent c0958f11f2
commit b8ce4bbfcd
6 changed files with 79 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
tests/test_formats.py
View file

@ -224,13 +224,14 @@ def test_schemas(self):
{'_type': 'Snapshot',
'version': 8,
'expires': '1985-10-21T13:20:00Z',
'meta': {'metadata/snapshot.json': {'version': 1024}}}),
'meta': {'snapshot.json': {'version': 1024}}}),
'TIMESTAMP_SCHEMA': (tuf.formats.TIMESTAMP_SCHEMA,
{'_type': 'Timestamp',
'version': 8,
'expires': '1985-10-21T13:20:00Z',
'meta': {'metadata/timestamp.json': {'version': 1024}}}),
'meta': {'metadattimestamp.json': {'length': 1024,
'hashes': {'sha256': 'AB1245'}}}}),
'MIRROR_SCHEMA': (tuf.formats.MIRROR_SCHEMA,
{'url_prefix': 'http://localhost:8001',
@ -298,29 +299,31 @@ def __init__(self, version, expires):
def test_TimestampFile(self):
# Test conditions for valid instances of 'tuf.formats.TimestampFile'.
version = 8
version = 8
length = 88
hashes = {'sha256': '3c7fe3eeded4a34'}
expires = '1985-10-21T13:20:00Z'
versiondict = {'targets.json': {'version': version}}
filedict = {'snapshot.json': {'length': length, 'hashes': hashes}}
make_metadata = tuf.formats.TimestampFile.make_metadata
from_metadata = tuf.formats.TimestampFile.from_metadata
TIMESTAMP_SCHEMA = tuf.formats.TIMESTAMP_SCHEMA
self.assertTrue(TIMESTAMP_SCHEMA.matches(make_metadata(version, expires,
versiondict)))
metadata = make_metadata(version, expires, versiondict)
filedict)))
metadata = make_metadata(version, expires, filedict)
self.assertTrue(isinstance(from_metadata(metadata), tuf.formats.TimestampFile))
# Test conditions for invalid arguments.
bad_version = 'eight'
bad_expires = '2000'
bad_versiondict = 123
bad_filedict = 123
self.assertRaises(tuf.FormatError, make_metadata, bad_version,
expires, versiondict)
expires, filedict)
self.assertRaises(tuf.FormatError, make_metadata, version,
bad_expires, versiondict)
bad_expires, filedict)
self.assertRaises(tuf.FormatError, make_metadata, version,
expires, bad_versiondict)
expires, bad_filedict)
self.assertRaises(tuf.FormatError, from_metadata, 123)

2
tests/test_repository_tool.py
View file

@ -325,7 +325,7 @@ def test_write_and_write_partial(self):
repository.root.load_signing_key(root_privkey)
repository.snapshot.load_signing_key(snapshot_privkey)
# Verify that consistent snapshot can be written and loaded.
# Verify that a consistent snapshot can be written and loaded.
repository.write(consistent_snapshot=True)
repo_tool.load_repository(repository_directory)

2
tuf/client/updater.py
View file

@ -991,7 +991,7 @@ def _unsafely_get_metadata_file(self, metadata_role, metadata_filepath,
<Arguments>
metadata_role:
The role name of the metadata (e.g., 'root', 'targets',
'targets/linux/x86').
'claimed').
metadata_filepath:
The metadata filepath (i.e., relative to the repository metadata

28
tuf/formats.py
View file

@ -281,12 +281,12 @@
object_name = 'VERSIONINFO_SCHEMA',
version = METADATAVERSION_SCHEMA)
# A dict holding the version information for a particular metadata role. The
# dict keys hold the relative file paths, and the dict values the corresponding
# version numbers.
VERSIONDICT_SCHEMA = SCHEMA.DictOf(
# A dict holding the version or file information for a particular metadata
# role. The dict keys hold the relative file paths, and the dict values the
# corresponding version numbers or file information.
FILEINFODICT_SCHEMA = SCHEMA.DictOf(
key_schema = RELPATH_SCHEMA,
value_schema = VERSIONINFO_SCHEMA)
value_schema = SCHEMA.OneOf([VERSIONINFO_SCHEMA, FILEINFO_SCHEMA]))
# A dict holding the information for a particular target / file. The dict keys
# hold the relative file paths, and the dict values the corresponding file
@ -477,7 +477,7 @@
_type = SCHEMA.String('Snapshot'),
version = METADATAVERSION_SCHEMA,
expires = ISO8601_DATETIME_SCHEMA,
meta = VERSIONDICT_SCHEMA)
meta = FILEINFODICT_SCHEMA)
# Timestamp role: indicates the latest version of the snapshot file.
TIMESTAMP_SCHEMA = SCHEMA.Object(
@ -485,7 +485,7 @@
_type = SCHEMA.String('Timestamp'),
version = METADATAVERSION_SCHEMA,
expires = ISO8601_DATETIME_SCHEMA,
meta = VERSIONDICT_SCHEMA)
meta = FILEDICT_SCHEMA)
# project.cfg file: stores information about the project in a json dictionary
PROJECT_CFG_SCHEMA = SCHEMA.Object(
@ -573,11 +573,11 @@ def __getattr__(self, name):
class TimestampFile(MetaFile):
def __init__(self, version, expires, versiondict):
def __init__(self, version, expires, filedict):
self.info = {}
self.info['version'] = version
self.info['expires'] = expires
self.info['meta'] = versiondict
self.info['meta'] = filedict
@staticmethod
@ -588,17 +588,17 @@ def from_metadata(object):
version = object['version']
expires = object['expires']
versiondict = object['meta']
filedict = object['meta']
return TimestampFile(version, expires, versiondict)
return TimestampFile(version, expires, filedict)
@staticmethod
def make_metadata(version, expiration_date, versiondict):
def make_metadata(version, expiration_date, filedict):
result = {'_type' : 'Timestamp'}
result['version'] = version
result['expires'] = expiration_date
result['meta'] = versiondict
result['meta'] = filedict
# Is 'result' a Timestamp metadata file?
# Raise 'tuf.FormatError' if not.
@ -1048,7 +1048,7 @@ def make_versioninfo(version_number):
information of a metadata role.
"""
versioninfo = {'version' : version_number}
versioninfo = {'version': version_number}
# Raise 'tuf.FormatError' if 'versioninfo' is improperly formatted.
try:

79
tuf/repository_lib.py
View file

@ -612,17 +612,21 @@ def _load_top_level_metadata(repository, top_level_filenames):
else:
pass
# Load 'snapshot.json'. A consistent snapshot of Snapshot must be calculated
# if 'consistent_snapshot' is True.
# Load 'snapshot.json'. A consistent snapshot.json must be calculated if
# 'consistent_snapshot' is True.
# The Snapshot and Root roles are both accessed by their hashes.
if consistent_snapshot:
snapshot_version = timestamp_metadata['meta'][SNAPSHOT_FILENAME]['version']
snapshot_hashes = timestamp_metadata['meta'][SNAPSHOT_FILENAME]['hashes']
snapshot_hash = random.choice(list(snapshot_hashes.values()))
dirname, basename = os.path.split(snapshot_filename)
snapshot_filename = os.path.join(dirname, str(snapshot_version) + '.' + basename)
snapshot_filename = os.path.join(dirname, str(snapshot_hash) + '.' + basename)
if os.path.exists(snapshot_filename):
signable = tuf.util.load_json_file(snapshot_filename)
tuf.formats.check_signable_object_format(signable)
snapshot_metadata = signable['signed']
for signature in signable['signatures']:
repository.snapshot.add_signature(signature, mark_role_as_dirty=False)
@ -1652,11 +1656,14 @@ def generate_snapshot_metadata(metadata_directory, version, expiration_date,
metadata_directory = _check_directory(metadata_directory)
# Retrieve the versioninfo of 'root.json' and 'targets.json'. 'versiondict'
# shall contain the version number of all roles available on the repository.
versiondict = {}
versiondict[ROOT_FILENAME] = get_metadata_versioninfo(root_filename)
versiondict[TARGETS_FILENAME] = get_metadata_versioninfo(targets_filename)
# Set the fileinfo of 'root.json', and the versioninfo of
# 'targets.json'. 'fileinfodict' shall contain the version number of all
# available delegated roles on the repository.
fileinfodict = {}
root_path = os.path.join(metadata_directory, root_filename + '.json')
length, hashes = tuf.util.get_file_details(root_path)
fileinfodict[ROOT_FILENAME] = tuf.formats.make_fileinfo(length, hashes)
fileinfodict[TARGETS_FILENAME] = get_metadata_versioninfo(targets_filename)
# We previously also stored the compressed versions of roles in
# snapshot.json, however, this is no longer needed as their hashes and
@ -1678,18 +1685,18 @@ def generate_snapshot_metadata(metadata_directory, version, expiration_date,
if metadata_filename.endswith(metadata_extension):
rolename = metadata_filename[:-len(metadata_extension)]
# Obsolete role files may still be found. Ensure only roles loaded
# in the roledb are included in the Snapshot metadata. Since the
# snapshot and timestamp roles are not listed in snapshot.json, do not
# list these roles found in the metadata directory.
if tuf.roledb.role_exists(rolename) and rolename not in ['snapshot', 'timestamp']:
versiondict[metadata_name] = get_metadata_versioninfo(rolename)
if tuf.roledb.role_exists(rolename) and \
rolename not in ['root', 'snapshot', 'timestamp', 'targets']:
fileinfodict[metadata_name] = get_metadata_versioninfo(rolename)
# Generate the Snapshot metadata object.
snapshot_metadata = tuf.formats.SnapshotFile.make_metadata(version,
expiration_date,
versiondict)
fileinfodict)
return snapshot_metadata
@ -1737,8 +1744,9 @@ def generate_timestamp_metadata(snapshot_filename, version, expiration_date):
tuf.formats.ISO8601_DATETIME_SCHEMA.check_match(expiration_date)
# Retrieve the versioninfo of the Snapshot metadata file.
versioninfo = {}
versioninfo[SNAPSHOT_FILENAME] = get_metadata_versioninfo('snapshot')
snapshot_fileinfo = {}
length, hashes = tuf.util.get_file_details(snapshot_filename)
snapshot_fileinfo[SNAPSHOT_FILENAME] = tuf.formats.make_fileinfo(length, hashes)
# We previously saved the versioninfo of the compressed versions of
# 'snapshot.json' in 'versioninfo'. Since version numbers are now stored,
@ -1748,7 +1756,7 @@ def generate_timestamp_metadata(snapshot_filename, version, expiration_date):
# Generate the timestamp metadata object.
timestamp_metadata = tuf.formats.TimestampFile.make_metadata(version,
expiration_date,
versioninfo)
snapshot_fileinfo)
return timestamp_metadata
@ -1907,7 +1915,6 @@ def write_metadata_file(metadata, filename, version_number,
# path so that temporary files are moved to their expected destinations.
filename = os.path.abspath(filename)
written_filename = filename
written_consistent_filename = None
_check_directory(os.path.dirname(filename))
# Generate the actual metadata file content of 'metadata'. Metadata is
@ -1916,11 +1923,6 @@ def write_metadata_file(metadata, filename, version_number,
# if re-saving is required.
file_content = _get_written_metadata(metadata)
if consistent_snapshot:
dirname, basename = os.path.split(filename)
version_and_filename = str(version_number) + '.' + basename
written_consistent_filename = os.path.join(dirname, version_and_filename)
# Verify whether new metadata needs to be written (i.e., has not been
# previously written or has changed.
write_new_metadata = False
@ -1930,8 +1932,7 @@ def write_metadata_file(metadata, filename, version_number,
# Compressed metadata should only be written if it does not exist or the
# uncompressed version has changed).
new_digests = {}
hash_algorithms = tuf.conf.REPOSITORY_HASH_ALGORITHMS
for hash_algorithm in hash_algorithms:
for hash_algorithm in tuf.conf.REPOSITORY_HASH_ALGORITHMS:
digest_object = tuf.hash.digest(hash_algorithm)
digest_object.update(file_content)
new_digests.update({hash_algorithm: digest_object.hexdigest()})
@ -1942,14 +1943,14 @@ def write_metadata_file(metadata, filename, version_number,
write_new_metadata = True
# 'tuf.Error' raised if 'filename' does not exist.
except tuf.Error as e:
except tuf.Error:
write_new_metadata = True
if write_new_metadata:
# The 'metadata' object is written to 'file_object', including compressed
# versions. To avoid partial metadata from being written, 'metadata' is
# first written to a temporary location (i.e., 'file_object') and then moved
# to 'filename'.
# first written to a temporary location (i.e., 'file_object') and then
# moved to 'filename'.
file_object = tuf.util.TempFile()
# Serialize 'metadata' to the file-like object and then write
@ -1960,7 +1961,22 @@ def write_metadata_file(metadata, filename, version_number,
logger.debug('Saving ' + repr(written_filename))
file_object.move(written_filename)
if consistent_snapshot:
if consistent_snapshot:
dirname, basename = os.path.split(written_filename)
if basename in ['root.json', 'snapshot.json']:
hash_algorithms = tuf.conf.REPOSITORY_HASH_ALGORITHMS
file_length_junk, digests = \
tuf.util.get_file_details(written_filename, hash_algorithms)
for digest in digests.values():
digest_and_filename = str(digest) + '.' + basename
written_consistent_filename = os.path.join(dirname, digest_and_filename)
else:
version_and_filename = str(version_number) + '.' + basename
written_consistent_filename = os.path.join(dirname, version_and_filename)
logger.info('Linking ' + repr(written_consistent_filename))
os.link(written_filename, written_consistent_filename)
@ -1994,8 +2010,9 @@ def write_metadata_file(metadata, filename, version_number,
raise tuf.FormatError('Unknown compression algorithm: ' + repr(compressio_algorithm))
# Save the compressed version, ensuring an unchanged file is not re-saved.
# Re-saving the same compressed version may cause its digest to unexpectedly
# change (gzip includes a timestamp) even though content has not changed.
# Re-saving the same compressed version may cause its digest to
# unexpectedly change (gzip includes a timestamp) even though content has
# not changed.
_write_compressed_metadata(file_object, compressed_filename,
write_new_metadata, consistent_snapshot)
return written_filename

4
tuf/repository_tool.py
View file

@ -2785,8 +2785,8 @@ def load_repository(repository_directory):
if metadata_name in ['root', 'snapshot', 'targets', 'timestamp']:
continue
# Keep a store metadata previously loaded metadata to prevent
# re-loading duplicate versions. Duplicate versions may occur with
# Keep a store of metadata previously loaded metadata to prevent re-loading
# duplicate versions. Duplicate versions may occur with
# 'consistent_snapshot', where the same metadata may be available in
# multiples files (the different hash is included in each filename.
if metadata_name in loaded_metadata: