From 81f235de8af7a8b7d07ae18cf8fc9d8dd59283bd Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Wed, 22 Jan 2014 21:03:23 -0500 Subject: [PATCH 01/51] Added the base skeleton for the tuf developer tools file. This version is not a working one, and breaks on the status() call. Some dependencies are not met yet --- tuf/devtools.py | 811 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 811 insertions(+) create mode 100644 tuf/devtools.py diff --git a/tuf/devtools.py b/tuf/devtools.py new file mode 100644 index 00000000..e62fcf1c --- /dev/null +++ b/tuf/devtools.py @@ -0,0 +1,811 @@ +""" + + tuf-devtoools.py + + + Santiago Torres + Zane Fisher + + Based on the work done by Vladimir Diaz + + + January 22, 2014 + + + See LICENSE for licensing information. + + + See 'tuf/README' for a complete guide on using 'tuf.devtools.py'. +""" + +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +#this import is the interface to the whole tuf module, since the imports are +# made there. +import os +import errno +import sys +import logging +import shutil +import tempfile + +import tuf +import tuf.formats +import tuf.util +import tuf.keydb +import tuf.roledb +import tuf.keys +import tuf.sig +import tuf.log +import tuf.conf +import tuf.libtuf + +from tuf.libtuf import Targets +from tuf.libtuf import get_metadata_file_info +from tuf.libtuf import get_metadata_filenames +from tuf.libtuf import generate_and_write_rsa_keypair +from tuf.libtuf import import_rsa_publickey_from_file +from tuf.libtuf import import_rsa_privatekey_from_file +from tuf.libtuf import generate_and_write_ed25519_keypair +from tuf.libtuf import import_ed25519_publickey_from_file +from tuf.libtuf import import_ed25519_privatekey_from_file +#from tuf.libtuf import _generate_and_write_metadata +from tuf.libtuf import generate_targets_metadata +from tuf.libtuf import sign_metadata + +# See 'log.py' to learn how logging is handled in TUF. +logger = logging.getLogger('tuf.devtools') + +# Recommended RSA key sizes: +# http://www.emc.com/emc-plus/rsa-labs/historical/twirl-and-rsa-key-size.htm#table1 +# According to the document above, revised May 6, 2003, RSA keys of +# size 3072 provide security through 2031 and beyond. 2048-bit keys +# are the recommended minimum and are good from the present through 2030. +DEFAULT_RSA_KEY_BITS = 3072 + +# The algorithm used by the repository to generate the hashes of the +# target filepaths. The repository may optionally organize targets into +HASH_FUNCTION = 'sha256' + +# The extension of TUF metadata. +METADATA_EXTENSION = '.txt' + +# The metadata filename for the targets metadata information. +TARGETS_FILENAME = 'targets' + METADATA_EXTENSION + +# The targets and metadata directory names. Metadata files are written +# to the staged metadata directory instead of the "live" one. +METADATA_STAGED_DIRECTORY_NAME = 'metadata.staged' +METADATA_DIRECTORY_NAME = 'metadata' +TARGETS_DIRECTORY_NAME = 'targets' + +# The full list of supported TUF metadata extensions. +METADATA_EXTENSIONS = ['.txt', '.txt.gz'] + +# The recognized compression extensions. +SUPPORTED_COMPRESSION_EXTENSIONS = ['.gz'] + +# Supported key types. +SUPPORTED_KEY_TYPES = ['rsa', 'ed25519'] + +# Expiration date delta, in seconds, of the top-level roles. A metadata +# expiration date is set by taking the current time and adding the expiration +# seconds listed below. + +# Initial 'targets.txt' expiration time of 3 months. +TARGETS_EXPIRATION = 7889230 + + +class Project(object): + """ + + This class works as the abstraction of the developer's files. this module + was created with the objective of simplifying the publishing process using + TUF by taking care of all of the bookkeeping, signature handling and + metadata integrity verification. + + This class is the direct representation of a metadata file* with the + intention to provide the ability to modify this data in an OOP manner + without messing with syntax and sanity-checking. + + + + project_directory: + The root folder of the project that contains the metadata and targets + sub-directories. + + metadata_directory: + The metadata sub-directory contains the files of the top-level + roles, including all roles delegated from 'targets.txt'. + + targets_directory: + The targets sub-directory contains all the target files that are + downloaded by clients and are referenced in TUF Metadata. The hashes and + file lengths are listed in Metadata files so that they are securely + downloaded. Metadata files are similarly referenced in the top-level + metadata. + + + tuf.FormatError, if the arguments are improperly formatted. + + + Creates top-level role objects and assigns them as attributes. + + + A project object that contains default Metadata objects for the top-level + roles. + """ + + def __init__(self, + repository_directory, + metadata_directory, + targets_directory, + file_prefix, + ): + + # Do the arguments have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if any are improperly formatted. + tuf.formats.PATH_SCHEMA.check_match(repository_directory) + tuf.formats.PATH_SCHEMA.check_match(metadata_directory) + tuf.formats.PATH_SCHEMA.check_match(targets_directory) + + self._repository_directory = repository_directory + self._metadata_directory = metadata_directory + self._targets_directory = targets_directory + + # Set the top-level role objects. + self.targets = Targets(self._targets_directory, 'targets') + + + + #TODO: continue where we left off. + def write(self, write_partial=False, consistent_snapshots=False): + """ + + Write all the JSON Metadata objects to their corresponding files. + write() raises an exception if any of the role metadata to be written to + disk is invalid, such as an insufficient threshold of signatures, missing + private keys, etc. + + + write_partial: + A boolean indicating whether partial metadata should be written to + disk. Partial metadata may be written to allow multiple maintainters + to independently sign and update role metadata. write() raises an + exception if a metadata role cannot be written due to not having enough + signatures. + + consistent_snapshots: + A boolean indicating whether written metadata and target files should + include a digest in the filename (i.e., root..txt, + targets..txt.gz, README..txt, where is the + file's SHA256 digest. Example: + 'root.1f4e35a60c8f96d439e27e858ce2869c770c1cdd54e1ef76657ceaaf01da18a3.txt' + + + tuf.Error, if any of the top-level roles do not have a minimum + threshold of signatures. + + + Creates metadata files in the repository's metadata directory. + + + None. + """ + + # Does 'write_partial' have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if any are improperly formatted. + tuf.formats.BOOLEAN_SCHEMA.check_match(write_partial) + tuf.formats.BOOLEAN_SCHEMA.check_match(consistent_snapshots) + + # At this point the tuf.keydb and tuf.roledb stores must be fully + # populated, otherwise write() throwns a 'tuf.Repository' exception if + # any of the top-level roles are missing signatures, keys, etc. + + # Write the metadata files of all the delegated roles. + delegated_rolenames = tuf.roledb.get_delegated_rolenames('targets') + for delegated_rolename in delegated_rolenames: + roleinfo = tuf.roledb.get_roleinfo(delegated_rolename) + delegated_filename = os.path.join(self._metadata_directory, + delegated_rolename + METADATA_EXTENSION) + + # Ensure the parent directories of 'metadata_filepath' exist, otherwise an + # IO exception is raised if 'metadata_filepath' is written to a + # sub-directory. + tuf.util.ensure_parent_dir(delegated_filename) + + _generate_and_write_metadata(delegated_rolename, delegated_filename, + write_partial, self._targets_directory, + self._metadata_directory, + consistent_snapshots) + + + # Generate the 'targets.txt' metadata file. + targets_filename = 'targets' + METADATA_EXTENSION + targets_filename = os.path.join(self._metadata_directory, targets_filename) + signable_junk, targets_filename = \ + _generate_and_write_metadata('targets', targets_filename, write_partial, + self._targets_directory, + self._metadata_directory, + consistent_snapshots) + + + # Delete the metadata of roles no longer in 'tuf.roledb'. Obsolete roles + # may have been revoked. + _delete_obsolete_metadata(self._metadata_directory, + release_signable['signed'], consistent_snapshots) + + + + def write_partial(self): + """ + + Write all the JSON Metadata objects to their corresponding files, but + allow metadata files to contain an invalid threshold of signatures. + + + None. + + + None. + + + Creates metadata files in the repository's metadata directory. + + + None. + """ + + self.write(write_partial=True) + + + + def status(self): + """ + + Determine the status of the top-level roles, including those delegated. + status() checks if each role provides sufficient public keys, signatures, + and that a valid metadata file is generated if write() were to be called. + Metadata files are temporary written to check that proper metadata files + are written, where file hashes and lengths are calculated and referenced + by the top-level roles. status() does not do a simple check for number + of threshold keys and signatures. + + + None. + + + None. + + + Generates and writes temporary metadata files. + + + None. + """ + + temp_project_directory = None + + try: + temp_project_directory = tempfile.mkdtemp() + metadata_directory = os.path.join(temp_project_directory, + METADATA_STAGED_DIRECTORY_NAME) + os.mkdir(metadata_directory) + + filenames = get_metadata_filenames(metadata_directory) + + # Delegated roles. + delegated_roles = tuf.roledb.get_delegated_rolenames('targets') + insufficient_keys = [] + insufficient_signatures = [] + + for delegated_role in delegated_roles: + try: + _check_role_keys(delegated_role) + except tuf.InsufficientKeysError, e: + insufficient_keys.append(delegated_role) + continue + + roleinfo = tuf.roledb.get_roleinfo(delegated_role) + try: + write_delegated_metadata_file(temp_project_directory, + self._targets_directory, + delegated_role, roleinfo, + write_partial=False) + except tuf.Error, e: + insufficient_signatures.append(delegated_role) + + if len(insufficient_keys): + message = 'Delegated roles with insufficient keys: '+ \ + repr(insufficient_keys) + print(message) + return + + if len(insufficient_signatures): + message = 'Delegated roles with insufficient signatures: '+ \ + repr(insufficient_signatures) + print(message) + return + + # Targets role. + try: + _check_role_keys(self.targets.rolename) + except tuf.InsufficientKeysError, e: + print(str(e)) + return + + try: + signable = _generate_and_write_metadata(self.targets.rolename, + filenames, False, + self._targets_directory, + self._metadata_directory, + False) + #_print_status(self.targets.rolename, signable) + except tuf.Error, e: + print(str(e)) + signable = e[1] + #_print_status(self.targets.rolename, signable) + return + + finally: + shutil.rmtree(temp_project_directory, ignore_errors=True) + + + + def get_filepaths_in_directory(self, files_directory, recursive_walk=False, + followlinks=True): + """ + + Walk the given 'files_directory' and build a list of target files found. + + + files_directory: + The path to a directory of target files. + + recursive_walk: + To recursively walk the directory, set recursive_walk=True. + + followlinks: + To follow symbolic links, set followlinks=True. + + + tuf.FormatError, if the arguments are improperly formatted. + + tuf.Error, if 'file_directory' is not a valid directory. + + Python IO exceptions. + + + None. + + + A list of absolute paths to target files in the given 'files_directory'. + """ + + # Do the arguments have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if any are improperly formatted. + tuf.formats.PATH_SCHEMA.check_match(files_directory) + tuf.formats.BOOLEAN_SCHEMA.check_match(recursive_walk) + tuf.formats.BOOLEAN_SCHEMA.check_match(followlinks) + + # Ensure a valid directory is given. + if not os.path.isdir(files_directory): + message = repr(files_directory)+' is not a directory.' + raise tuf.Error(message) + + # A list of the target filepaths found in 'file_directory'. + targets = [] + + # FIXME: We need a way to tell Python 2, but not Python 3, to return + # filenames in Unicode; see #61 and: + # http://docs.python.org/2/howto/unicode.html#unicode-filenames + for dirpath, dirnames, filenames in os.walk(files_directory, + followlinks=followlinks): + for filename in filenames: + full_target_path = os.path.join(dirpath, filename) + targets.append(full_target_path) + + # Prune the subdirectories to walk right now if we do not wish to + # recursively walk files_directory. + if recursive_walk is False: + del dirnames[:] + + return targets + +def _delete_obsolete_metadata(metadata_directory, release_metadata, + consistent_snapshots): + print("missing implementation") + + +def _check_role_keys(rolename): + print("missing implementation") + + +def _print_status(rolename, signable): + """ + Non-public function prints the number of (good/threshold) signatures of + 'rolename'. + """ + + status = tuf.sig.get_signature_status(signable, rolename) + + message = repr(rolename)+' role contains '+ \ + repr(len(status['good_sigs']))+' / '+ \ + repr(status['threshold'])+' signatures.' + print(message) + +def _generate_and_write_metadata(rolename, metadata_filename, write_partial, + targets_directory, metadata_directory, + consistent_snapshots, filenames=None): + """ + Non-public function that can generate and write the metadata of the specified + top-level 'rolename'. It also increments version numbers if: + + 1. write_partial==True and the metadata is the first to be written. + + 2. write_partial=False (i.e., write()), the metadata was not loaded as + partially written, and a write_partial is not needed. + """ + + metadata = None + + # Retrieve the roleinfo of 'rolename' to extract the needed metadata + # attributes, such as version number, expiration, etc. + roleinfo = tuf.roledb.get_roleinfo(rolename) + #release_compressions = tuf.roledb.get_roleinfo('release')['compressions'] + + metadata = generate_targets_metadata(targets_directory, + roleinfo['paths'], + roleinfo['version'], + roleinfo['expires'], + roleinfo['delegations'], + consistent_snapshots) + + signable = sign_metadata(metadata, roleinfo['signing_keyids'], + metadata_filename) + + # Check if the version number of 'rolename' may be automatically incremented, + # depending on whether if partial metadata is loaded or if the metadata is + # written with write() / write_partial(). + # Increment the version number if this is the first partial write. + if write_partial: + temp_signable = sign_metadata(metadata, [], metadata_filename) + temp_signable['signatures'].extend(roleinfo['signatures']) + status = tuf.sig.get_signature_status(temp_signable, rolename) + if len(status['good_sigs']) == 0: + metadata['version'] = metadata['version'] + 1 + signable = sign_metadata(metadata, roleinfo['signing_keyids'], + metadata_filename) + + # non-partial write() + else: + if tuf.sig.verify(signable, rolename) and not roleinfo['partial_loaded']: + metadata['version'] = metadata['version'] + 1 + signable = sign_metadata(metadata, roleinfo['signing_keyids'], + metadata_filename) + + # Write the metadata to file if contains a threshold of signatures. + signable['signatures'].extend(roleinfo['signatures']) + + if tuf.sig.verify(signable, rolename) or write_partial: + _remove_invalid_and_duplicate_signatures(signable) + compressions = roleinfo['compressions'] + filename = write_metadata_file(signable, metadata_filename, compressions, + consistent_snapshots) + + # The root and timestamp files should also be written without a digest if + # 'consistent_snaptshots' is True. Client may request a timestamp and root + # file without knowing its digest and file size. + if rolename == 'root' or rolename == 'timestamp': + write_metadata_file(signable, metadata_filename, compressions, + consistent_snapshots=False) + + return signable, filename + + # 'signable' contains an invalid threshold of signatures. + else: + message = 'Not enough signatures for '+repr(metadata_filename) + raise tuf.Error(message, signable) + + + + +def _prompt(message, result_type=str): + """ + Non-public function that prompts the user for input by printing 'message', + converting the input to 'result_type', and returning the value to the + caller. + """ + + return result_type(raw_input(message)) + + + + + +def _get_password(prompt='Password: ', confirm=False): + """ + Non-public function that returns the password entered by the user. If + 'confirm' is True, the user is asked to enter the previously entered + password once again. If they match, the password is returned to the caller. + """ + + while True: + # getpass() prompts the user for a password without echoing + # the user input. + password = getpass.getpass(prompt, sys.stderr) + if not confirm: + return password + password2 = getpass.getpass('Confirm: ', sys.stderr) + if password == password2: + return password + else: + print('Mismatch; try again.') + + +def create_new_project(project_directory,prefix): + """ + + Create a new project object, instantiate barebones metadata for the + targets, and return a blank project object. On disk, create_new_project() + only creates the directories needed to hold the metadata and targets files. + The project object returned can be directly modified to meet the designer's + criteria and then written using the method project.write(). + + + project_directory: + The directory that will eventually hold the metadata and target files of + the project. + + prefix: + a string determining the "upstream" filepath to sign the metadata + appropiately + + + tuf.FormatError, if the arguments are improperly formatted. + + + The 'projet_directory' directory is created if it does not exist, + including its metadata and targets sub-directories. + + + A 'tuf.devtools.Repository' object. + """ + + # Does 'project_directory' have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(project_directory) + + # Do the same for the prefix + tuf.formats.PATH_SCHEMA.check_match(prefix) + + # Set the repository, metadata, and targets directories. These directories + # are created if they do not exist. + project_directory = os.path.abspath(project_directory) + metadata_directory = None + targets_directory = None + + # Try to create 'repository_directory' if it does not exist. + try: + message = 'Creating '+repr(project_directory) + logger.info(message) + os.makedirs(project_directory) + + # 'OSError' raised if the leaf directory already exists or cannot be created. + # Check for case where 'repository_directory' has already been created. + except OSError, e: + if e.errno == errno.EEXIST: + pass + else: + raise + + # Set the metadata and targets directories. The metadata directory is a + # staged one so that the "live" repository is not affected. The + # staged metadata changes may be moved over to "live" after all updated + # have been completed. + metadata_directory = \ + os.path.join(project_directory, METADATA_STAGED_DIRECTORY_NAME) + targets_directory = \ + os.path.join(project_directory, TARGETS_DIRECTORY_NAME) + + # Try to create the metadata directory that will hold all of the metadata + # files, such as 'root.txt' and 'release.txt'. + try: + message = 'Creating '+repr(metadata_directory) + logger.info(message) + os.mkdir(metadata_directory) + + # 'OSError' raised if the leaf directory already exists or cannot be created. + except OSError, e: + if e.errno == errno.EEXIST: + pass + else: + raise + + # Try to create the targets directory that will hold all of the target files. + try: + message = 'Creating '+repr(targets_directory) + logger.info(message) + os.mkdir(targets_directory) + except OSError, e: + if e.errno == errno.EEXIST: + pass + else: + raise + + # Create the bare bones repository object, where only the top-level roles + # have been set and contain default values (e.g., Root roles has a threshold + # of 1, expires 1 year into the future, etc.) + project = Project(project_directory, + metadata_directory, + targets_directory, + prefix + ) + + return project + + + +def load_repository(repository_directory): + """ + + Return a repository object containing the contents of metadata files loaded + from the repository. + + + repository_directory: + + + tuf.FormatError, if 'repository_directory' or any of the metadata files + are improperly formatted. Also raised if, at a minimum, the Root role + cannot be found. + + + All the metadata files found in the repository are loaded and their contents + stored in a libtuf.Repository object. + + + libtuf.Repository object. + + BEGIN ORIGINAL + # Does 'repository_directory' have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(repository_directory) + + # Load top-level metadata. + repository_directory = os.path.abspath(repository_directory) + metadata_directory = os.path.join(repository_directory, + METADATA_STAGED_DIRECTORY_NAME) + targets_directory = os.path.join(repository_directory, + TARGETS_DIRECTORY_NAME) + + # The Repository() object loaded (i.e., containing all the metadata roles + # found) and returned. + repository = Repository(repository_directory, metadata_directory, + targets_directory) + + filenames = get_metadata_filenames(metadata_directory) + + # The Root file is always available without a consistent snapshots digest + # attached to the filename. Store the 'consistent_snapshots' value read the + # loaded Root file so that other metadata files may be located. + # 'consistent_snapshots' value. + consistent_snapshots = False + + # Load the metadata of the top-level roles (i.e., Root, Timestamp, Targets, + # and Release). + repository, consistent_snapshots = _load_top_level_metadata(repository, + filenames) + + # Load delegated targets metadata. + # Walk the 'targets/' directory and generate the fileinfo of all the files + # listed. This information is stored in the 'meta' field of the release + # metadata object. + targets_objects = {} + loaded_metadata = [] + targets_objects['targets'] = repository.targets + targets_metadata_directory = os.path.join(metadata_directory, + TARGETS_DIRECTORY_NAME) + if os.path.exists(targets_metadata_directory) and \ + os.path.isdir(targets_metadata_directory): + for root, directories, files in os.walk(targets_metadata_directory): + + # 'files' here is a list of target file names. + for basename in files: + metadata_path = os.path.join(root, basename) + metadata_name = \ + metadata_path[len(metadata_directory):].lstrip(os.path.sep) + + # Strip the digest if 'consistent_snapshots' is True. + # Example: 'targets/unclaimed/13df98ab0.django.txt' --> + # 'targets/unclaimed/django.txt' + metadata_name, digest_junk = \ + _strip_consistent_snapshots_digest(metadata_name, consistent_snapshots) + + if metadata_name.endswith(METADATA_EXTENSION): + extension_length = len(METADATA_EXTENSION) + metadata_name = metadata_name[:-extension_length] + else: + continue + + # Keep a store metadata previously loaded metadata to prevent + # re-loading duplicate versions. Duplicate versions may occur with + # consistent_snapshots, where the same metadata may be available in + # multiples files (the different hash is included in each filename. + if metadata_name in loaded_metadata: + continue + + signable = None + try: + signable = tuf.util.load_json_file(metadata_path) + except (ValueError, IOError), e: + continue + + metadata_object = signable['signed'] + + roleinfo = tuf.roledb.get_roleinfo(metadata_name) + roleinfo['signatures'].extend(signable['signatures']) + roleinfo['version'] = metadata_object['version'] + roleinfo['expires'] = metadata_object['expires'] + roleinfo['paths'] = metadata_object['targets'].keys() + roleinfo['delegations'] = metadata_object['delegations'] + + if os.path.exists(metadata_path+'.gz'): + roleinfo['compressions'].append('gz') + + _check_if_partial_loaded(metadata_name, signable, roleinfo) + tuf.roledb.update_roleinfo(metadata_name, roleinfo) + loaded_metadata.append(metadata_name) + + new_targets_object = Targets(targets_directory, metadata_name, roleinfo) + targets_object = \ + targets_objects[tuf.roledb.get_parent_rolename(metadata_name)] + targets_objects[metadata_name] = new_targets_object + + targets_object._delegated_roles[(os.path.basename(metadata_name))] = \ + new_targets_object + + # Add the keys specified in the delegations field of the Targets role. + for key_metadata in metadata_object['delegations']['keys'].values(): + key_object = tuf.keys.format_metadata_to_key(key_metadata) + try: + tuf.keydb.add_key(key_object) + except tuf.KeyAlreadyExistsError, e: + pass + + for role in metadata_object['delegations']['roles']: + rolename = role['name'] + roleinfo = {'name': role['name'], 'keyids': role['keyids'], + 'threshold': role['threshold'], + 'compressions': [''], 'signing_keyids': [], + 'signatures': [], + 'partial_loaded': False, + 'delegations': {'keys': {}, + 'roles': []}} + tuf.roledb.add_role(rolename, roleinfo) + + return repository + """ + raise Exception("To be implemented") + + +if __name__ == '__main__': + # The interactive sessions of the documentation strings can + # be tested by running libtuf.py as a standalone module: + # $ python libtuf.py. + # import doctest + # doctest.testmod() + print("main") From 44d4acdd77cd75d12eca97d58221bcc5d644fea0 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Fri, 24 Jan 2014 17:24:03 -0500 Subject: [PATCH 02/51] updated devtools file to include repository tools instead of libtuf --- tuf/devtools.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tuf/devtools.py b/tuf/devtools.py index e62fcf1c..9f7c2cd5 100644 --- a/tuf/devtools.py +++ b/tuf/devtools.py @@ -43,20 +43,21 @@ import tuf.sig import tuf.log import tuf.conf -import tuf.libtuf +import tuf.repository_tool -from tuf.libtuf import Targets -from tuf.libtuf import get_metadata_file_info -from tuf.libtuf import get_metadata_filenames -from tuf.libtuf import generate_and_write_rsa_keypair -from tuf.libtuf import import_rsa_publickey_from_file -from tuf.libtuf import import_rsa_privatekey_from_file -from tuf.libtuf import generate_and_write_ed25519_keypair -from tuf.libtuf import import_ed25519_publickey_from_file -from tuf.libtuf import import_ed25519_privatekey_from_file -#from tuf.libtuf import _generate_and_write_metadata -from tuf.libtuf import generate_targets_metadata -from tuf.libtuf import sign_metadata +from tuf.repository_tool import Targets +from tuf.repository_tool import get_metadata_file_info +from tuf.repository_tool import get_metadata_filenames +from tuf.repository_tool import generate_and_write_rsa_keypair +from tuf.repository_tool import import_rsa_publickey_from_file +from tuf.repository_tool import import_rsa_privatekey_from_file +from tuf.repository_tool import generate_and_write_ed25519_keypair +from tuf.repository_tool import import_ed25519_publickey_from_file +from tuf.repository_tool import import_ed25519_privatekey_from_file +#from tuf.import _generate_and_write_metadata +from tuf.repository_tool import generate_targets_metadata +from tuf.repository_tool import sign_metadata +#from tuf.repository-tool import write_delegated_metadata_file # See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger('tuf.devtools') @@ -471,7 +472,7 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, roleinfo['expires'], roleinfo['delegations'], consistent_snapshots) - + print(metadata) signable = sign_metadata(metadata, roleinfo['signing_keyids'], metadata_filename) From e4de851d28263aec0032471315b421bee2d9a0ec Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Sat, 25 Jan 2014 12:04:56 -0500 Subject: [PATCH 03/51] Fixed the status and write methods on devtools.py I need to review the whole code and refactor some sections to meet the specifications, optimize performance and minimize filesize. Thorough checking of this module is needed. Expect unused variables and functions. --- tuf/devtools.py | 44 +++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/tuf/devtools.py b/tuf/devtools.py index 9f7c2cd5..600ddce7 100644 --- a/tuf/devtools.py +++ b/tuf/devtools.py @@ -55,9 +55,12 @@ from tuf.repository_tool import import_ed25519_publickey_from_file from tuf.repository_tool import import_ed25519_privatekey_from_file #from tuf.import _generate_and_write_metadata +from tuf.repository_tool import _remove_invalid_and_duplicate_signatures +from tuf.repository_tool import _check_role_keys +from tuf.repository_tool import _delete_obsolete_metadata from tuf.repository_tool import generate_targets_metadata from tuf.repository_tool import sign_metadata -#from tuf.repository-tool import write_delegated_metadata_file +from tuf.repository_tool import write_metadata_file # See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger('tuf.devtools') @@ -233,7 +236,7 @@ def write(self, write_partial=False, consistent_snapshots=False): # Generate the 'targets.txt' metadata file. targets_filename = 'targets' + METADATA_EXTENSION targets_filename = os.path.join(self._metadata_directory, targets_filename) - signable_junk, targets_filename = \ + release_signable, targets_filename = \ _generate_and_write_metadata('targets', targets_filename, write_partial, self._targets_directory, self._metadata_directory, @@ -293,7 +296,6 @@ def status(self): None. """ - temp_project_directory = None try: @@ -302,8 +304,11 @@ def status(self): METADATA_STAGED_DIRECTORY_NAME) os.mkdir(metadata_directory) - filenames = get_metadata_filenames(metadata_directory) - + #filenames = get_metadata_filenames(metadata_directory)A + # we should do the schema check + filenames = {} + filenames['targets'] = os.path.join(metadata_directory,TARGETS_FILENAME) + # Delegated roles. delegated_roles = tuf.roledb.get_delegated_rolenames('targets') insufficient_keys = [] @@ -345,14 +350,13 @@ def status(self): return try: - signable = _generate_and_write_metadata(self.targets.rolename, - filenames, False, + signable = _generate_and_write_metadata(self.targets.rolename, + filenames['targets'], False, self._targets_directory, self._metadata_directory, False) - #_print_status(self.targets.rolename, signable) + #_print_status(self.targets.rolename, signable) except tuf.Error, e: - print(str(e)) signable = e[1] #_print_status(self.targets.rolename, signable) return @@ -424,13 +428,8 @@ def get_filepaths_in_directory(self, files_directory, recursive_walk=False, return targets -def _delete_obsolete_metadata(metadata_directory, release_metadata, - consistent_snapshots): - print("missing implementation") -def _check_role_keys(rolename): - print("missing implementation") def _print_status(rolename, signable): @@ -460,7 +459,7 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, """ metadata = None - + print("we do get here") # Retrieve the roleinfo of 'rolename' to extract the needed metadata # attributes, such as version number, expiration, etc. roleinfo = tuf.roledb.get_roleinfo(rolename) @@ -472,10 +471,8 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, roleinfo['expires'], roleinfo['delegations'], consistent_snapshots) - print(metadata) signable = sign_metadata(metadata, roleinfo['signing_keyids'], metadata_filename) - # Check if the version number of 'rolename' may be automatically incremented, # depending on whether if partial metadata is loaded or if the metadata is # written with write() / write_partial(). @@ -499,20 +496,13 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, # Write the metadata to file if contains a threshold of signatures. signable['signatures'].extend(roleinfo['signatures']) + #import pdb; pdb.set_trace() if tuf.sig.verify(signable, rolename) or write_partial: _remove_invalid_and_duplicate_signatures(signable) compressions = roleinfo['compressions'] filename = write_metadata_file(signable, metadata_filename, compressions, consistent_snapshots) - # The root and timestamp files should also be written without a digest if - # 'consistent_snaptshots' is True. Client may request a timestamp and root - # file without knowing its digest and file size. - if rolename == 'root' or rolename == 'timestamp': - write_metadata_file(signable, metadata_filename, compressions, - consistent_snapshots=False) - - return signable, filename # 'signable' contains an invalid threshold of signatures. else: @@ -520,6 +510,10 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, raise tuf.Error(message, signable) + # The root and timestamp files should also be written without a digest if + # 'consistent_snaptshots' is True. Client may request a timestamp and root + # file without knowing its digest and file size. + return signable, filename def _prompt(message, result_type=str): From 940142b2dc3eba8912d6a8397fe57ffa45904fa8 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 28 Jan 2014 22:46:10 -0500 Subject: [PATCH 04/51] Prefix property handling Updated the module so it prepends the prefix property by prepending it to the filenames before signing the metadata on the write method. --- tuf/devtools.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tuf/devtools.py b/tuf/devtools.py index 600ddce7..0ba9da33 100644 --- a/tuf/devtools.py +++ b/tuf/devtools.py @@ -159,7 +159,8 @@ def __init__(self, tuf.formats.PATH_SCHEMA.check_match(repository_directory) tuf.formats.PATH_SCHEMA.check_match(metadata_directory) tuf.formats.PATH_SCHEMA.check_match(targets_directory) - + tuf.formats.PATH_SCHEMA.check_match(file_prefix) + self._repository_directory = repository_directory self._metadata_directory = metadata_directory self._targets_directory = targets_directory @@ -167,7 +168,7 @@ def __init__(self, # Set the top-level role objects. self.targets = Targets(self._targets_directory, 'targets') - + self.prefix = file_prefix #TODO: continue where we left off. def write(self, write_partial=False, consistent_snapshots=False): @@ -230,7 +231,7 @@ def write(self, write_partial=False, consistent_snapshots=False): _generate_and_write_metadata(delegated_rolename, delegated_filename, write_partial, self._targets_directory, self._metadata_directory, - consistent_snapshots) + consistent_snapshots,prefix=self.prefix) # Generate the 'targets.txt' metadata file. @@ -240,7 +241,7 @@ def write(self, write_partial=False, consistent_snapshots=False): _generate_and_write_metadata('targets', targets_filename, write_partial, self._targets_directory, self._metadata_directory, - consistent_snapshots) + consistent_snapshots,prefix=self.prefix) # Delete the metadata of roles no longer in 'tuf.roledb'. Obsolete roles @@ -447,7 +448,8 @@ def _print_status(rolename, signable): def _generate_and_write_metadata(rolename, metadata_filename, write_partial, targets_directory, metadata_directory, - consistent_snapshots, filenames=None): + consistent_snapshots, filenames=None, + prefix=''): """ Non-public function that can generate and write the metadata of the specified top-level 'rolename'. It also increments version numbers if: @@ -459,20 +461,27 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, """ metadata = None - print("we do get here") # Retrieve the roleinfo of 'rolename' to extract the needed metadata # attributes, such as version number, expiration, etc. roleinfo = tuf.roledb.get_roleinfo(rolename) #release_compressions = tuf.roledb.get_roleinfo('release')['compressions'] - metadata = generate_targets_metadata(targets_directory, roleinfo['paths'], roleinfo['version'], roleinfo['expires'], roleinfo['delegations'], - consistent_snapshots) + consistent_snapshots) + + # preprend the prefix to the project's filepath to avoid signature errors + # in upstream + for element in metadata['targets'].keys(): + metadata['targets'][prefix+element] = metadata['targets'][element] + if prefix != '': + del(metadata['targets'][element]) + signable = sign_metadata(metadata, roleinfo['signing_keyids'], metadata_filename) + # Check if the version number of 'rolename' may be automatically incremented, # depending on whether if partial metadata is loaded or if the metadata is # written with write() / write_partial(). From 2ab9be71a9588721959dbe20610a5a9b3ac451b9 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Fri, 31 Jan 2014 22:02:28 -0500 Subject: [PATCH 05/51] Added the load project function Load_project(path) now loads a projects metadata and builds the corresponding object. Methods to save supporting metadata in the project.cfg file were also refined --- tuf/devtools.py | 227 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 174 insertions(+), 53 deletions(-) diff --git a/tuf/devtools.py b/tuf/devtools.py index 0ba9da33..4ad2ea3b 100644 --- a/tuf/devtools.py +++ b/tuf/devtools.py @@ -33,6 +33,7 @@ import logging import shutil import tempfile +import json import tuf import tuf.formats @@ -77,11 +78,14 @@ HASH_FUNCTION = 'sha256' # The extension of TUF metadata. -METADATA_EXTENSION = '.txt' +METADATA_EXTENSION = '.json' # The metadata filename for the targets metadata information. TARGETS_FILENAME = 'targets' + METADATA_EXTENSION +# Project configuration filename +PROJECT_FILENAME = 'project.cfg' + # The targets and metadata directory names. Metadata files are written # to the staged metadata directory instead of the "live" one. METADATA_STAGED_DIRECTORY_NAME = 'metadata.staged' @@ -171,7 +175,7 @@ def __init__(self, self.prefix = file_prefix #TODO: continue where we left off. - def write(self, write_partial=False, consistent_snapshots=False): + def write(self, write_partial=False): """ Write all the JSON Metadata objects to their corresponding files. @@ -210,7 +214,6 @@ def write(self, write_partial=False, consistent_snapshots=False): # types, and that all dict keys are properly named. # Raise 'tuf.FormatError' if any are improperly formatted. tuf.formats.BOOLEAN_SCHEMA.check_match(write_partial) - tuf.formats.BOOLEAN_SCHEMA.check_match(consistent_snapshots) # At this point the tuf.keydb and tuf.roledb stores must be fully # populated, otherwise write() throwns a 'tuf.Repository' exception if @@ -231,7 +234,7 @@ def write(self, write_partial=False, consistent_snapshots=False): _generate_and_write_metadata(delegated_rolename, delegated_filename, write_partial, self._targets_directory, self._metadata_directory, - consistent_snapshots,prefix=self.prefix) + prefix=self.prefix) # Generate the 'targets.txt' metadata file. @@ -241,13 +244,14 @@ def write(self, write_partial=False, consistent_snapshots=False): _generate_and_write_metadata('targets', targets_filename, write_partial, self._targets_directory, self._metadata_directory, - consistent_snapshots,prefix=self.prefix) - + prefix=self.prefix) + save_project_configuration(self._metadata_directory, self.targets.keys, + self.prefix, self.targets.threshold) # Delete the metadata of roles no longer in 'tuf.roledb'. Obsolete roles # may have been revoked. _delete_obsolete_metadata(self._metadata_directory, - release_signable['signed'], consistent_snapshots) + release_signable['signed'], False) @@ -448,7 +452,7 @@ def _print_status(rolename, signable): def _generate_and_write_metadata(rolename, metadata_filename, write_partial, targets_directory, metadata_directory, - consistent_snapshots, filenames=None, + filenames=None, prefix=''): """ Non-public function that can generate and write the metadata of the specified @@ -470,12 +474,14 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, roleinfo['version'], roleinfo['expires'], roleinfo['delegations'], - consistent_snapshots) + False) # preprend the prefix to the project's filepath to avoid signature errors # in upstream for element in metadata['targets'].keys(): - metadata['targets'][prefix+element] = metadata['targets'][element] + junk_path, relative_target = os.path.split(element) + prefixed_path = os.path.join(prefix,relative_target) + metadata['targets'][prefixed_path] = metadata['targets'][element] if prefix != '': del(metadata['targets'][element]) @@ -510,7 +516,7 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, _remove_invalid_and_duplicate_signatures(signable) compressions = roleinfo['compressions'] filename = write_metadata_file(signable, metadata_filename, compressions, - consistent_snapshots) + False) # 'signable' contains an invalid threshold of signatures. @@ -661,66 +667,183 @@ def create_new_project(project_directory,prefix): return project - - -def load_repository(repository_directory): +def save_project_configuration(metadata_directory, public_keys, prefix, + threshold): """ - Return a repository object containing the contents of metadata files loaded - from the repository. + Persist the project's information in a file to provide the information + for the load routine - repository_directory: + metadata_directory: where the project's metadata is located + public_keys: a list containing the public keys for the toplevel targets + role + prefix: the project's prefix (if any) + threshold: the threshold value for the toplevel targets role - tuf.FormatError, if 'repository_directory' or any of the metadata files - are improperly formatted. Also raised if, at a minimum, the Root role - cannot be found. + Exceptions may rise if the metadata_directory/project.cfg file exists and + is non-writeable + + Exceptions are also expected if either the prefix or the metadata directory + are malformed - All the metadata files found in the repository are loaded and their contents + A project.cfg file is created or overwritten + + + nothing + """ + # schema check for metadata_directory and prefix + tuf.formats.PATH_SCHEMA.check_match(metadata_directory) + tuf.formats.PATH_SCHEMA.check_match(prefix) + + # get the absolute filepath to our metadata_directory for consistency + metadata_directory = os.path.abspath(metadata_directory) + + # is the file open-able? open for overwriting + project_filename = os.path.join(metadata_directory,PROJECT_FILENAME) + try: + fp = open(project_filename,"wt") + except OSError, e: + raise + + # build the data structure + project_config = {} + project_config['prefix'] = prefix + project_config['public_keys'] = {} + + project_config['threshold'] = threshold + # build a dictionary containing the actual keys + for key in public_keys: + key_info = tuf.keydb.get_key(key) + project_config['public_keys'][key] = {} + project_config['public_keys'][key]['keytype'] = key_info['keytype'] + project_config['public_keys'][key]['public'] = key_info['keyval']['public'] + + # save the actual data + json.dump(project_config,fp) + + # clean our mess + fp.close() + + +def load_project(project_directory, prefix=''): + """ + + Return a project object initialized with the contents of the metadata + files loaded from the project_directory path + + + project_directory: The path to the project folder + prefix: the prefix for the metadata + + + tuf.FormatError, if 'project_directory' or any of the metadata files + are improperly formatted. + + + All the metadata files found in the project are loaded and their contents stored in a libtuf.Repository object. libtuf.Repository object. BEGIN ORIGINAL + return repository + """ # Does 'repository_directory' have the correct format? # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.PATH_SCHEMA.check_match(repository_directory) + tuf.formats.PATH_SCHEMA.check_match(project_directory) + # do the same for the prefix + tuf.formats.PATH_SCHEMA.check_match(prefix) - # Load top-level metadata. - repository_directory = os.path.abspath(repository_directory) - metadata_directory = os.path.join(repository_directory, + + # Locate metadata filepaths and targets filepath. + project_directory = os.path.abspath(project_directory) + metadata_directory = os.path.join(project_directory, METADATA_STAGED_DIRECTORY_NAME) - targets_directory = os.path.join(repository_directory, + targets_directory = os.path.join(project_directory, TARGETS_DIRECTORY_NAME) - # The Repository() object loaded (i.e., containing all the metadata roles - # found) and returned. - repository = Repository(repository_directory, metadata_directory, - targets_directory) + # create a blank project on the target directory + project = Project(project_directory, metadata_directory, targets_directory, + prefix) + + # load the cfg file and update the project. + config_filename = os.path.join(metadata_directory,PROJECT_FILENAME) + try: + fp = open(config_filename,"rt") + except OSError, e: + raise + + project_configuration = json.load(fp) + project.targets.threshold = project_configuration['threshold'] + project.prefix = project_configuration['prefix'] + + # traverse the public keys and add them to the project + keydict = project_configuration['public_keys'] + for keyid in keydict: + if keydict[keyid]['keytype'] == 'rsa': + temp_pubkey = tuf.keys.format_rsakey_from_pem(keydict[keyid]['public']) + elif keydict[keyid]['keytype'] == 'ed25519': + temp_pubkey = {} + temp_pubkey['keytype'] = keydict[keyid]['keytype'] + temp_pubkey['keyval'] = {} + temp_pubkey['keyval']['public'] = keydict[keyid]['public'] + temp_pubkey['keyval']['private'] = '' + else: + temp_pubkey = keydict + project.targets.add_verification_key(temp_pubkey) - filenames = get_metadata_filenames(metadata_directory) - # The Root file is always available without a consistent snapshots digest - # attached to the filename. Store the 'consistent_snapshots' value read the - # loaded Root file so that other metadata files may be located. - # 'consistent_snapshots' value. - consistent_snapshots = False + # load the toplevel metadata + targets_metadata_path = os.path.join(metadata_directory, TARGETS_FILENAME) + signable = tuf.util.load_json_file(targets_metadata_path) + tuf.formats.check_signable_object_format(signable) + targets_metadata = signable['signed'] + + # remove the prefix from the metadata + if project_configuration['prefix'] != '': + unprefixed_targets_metadata = {} + for targets in targets_metadata['targets'].keys(): + unprefixed_target = os.path.relpath(targets, + project_configuration['prefix']) + unprefixed_target = '/' + unprefixed_target + unprefixed_targets_metadata[unprefixed_target] = \ + targets_metadata['targets'][targets] + targets_metadata['targets'] = unprefixed_targets_metadata + for signature in signable['signatures']: + project.targets.add_signature(signature) - # Load the metadata of the top-level roles (i.e., Root, Timestamp, Targets, - # and Release). - repository, consistent_snapshots = _load_top_level_metadata(repository, - filenames) - + # update roledb + roleinfo = tuf.roledb.get_roleinfo('targets') + roleinfo['signatures'].extend(signable['signatures']) + roleinfo['version'] = targets_metadata['version'] + roleinfo['paths'] = targets_metadata['targets'].keys() + roleinfo['delegations'] = targets_metadata['delegations'] + tuf.roledb.update_roleinfo('targets',roleinfo) + + for key_metadata in targets_metadata['delegations']['keys'].values(): + key_object = tuf.keys.format_metadata_to_key(key_metadata) + tuf.keydb.add_key(key_object) + + for role in targets_metadata['delegations']['roles']: + rolename = role['name'] + roleinfo = {'name': role['name'], 'keyids': role['keyids'], + 'threshold': role['threshold'], 'compressions': [''], + 'signing_keyids': [], 'signatures': [], + 'delegations': {'keys':{}, roles:[]} + } + tuf.roledb.add_role(rolename, roleinfo) + # Load delegated targets metadata. # Walk the 'targets/' directory and generate the fileinfo of all the files # listed. This information is stored in the 'meta' field of the release # metadata object. targets_objects = {} loaded_metadata = [] - targets_objects['targets'] = repository.targets + targets_objects['targets'] = project.targets targets_metadata_directory = os.path.join(metadata_directory, TARGETS_DIRECTORY_NAME) if os.path.exists(targets_metadata_directory) and \ @@ -733,12 +856,7 @@ def load_repository(repository_directory): metadata_name = \ metadata_path[len(metadata_directory):].lstrip(os.path.sep) - # Strip the digest if 'consistent_snapshots' is True. - # Example: 'targets/unclaimed/13df98ab0.django.txt' --> - # 'targets/unclaimed/django.txt' - metadata_name, digest_junk = \ - _strip_consistent_snapshots_digest(metadata_name, consistent_snapshots) - + # strip the extension if metadata_name.endswith(METADATA_EXTENSION): extension_length = len(METADATA_EXTENSION) metadata_name = metadata_name[:-extension_length] @@ -769,11 +887,16 @@ def load_repository(repository_directory): if os.path.exists(metadata_path+'.gz'): roleinfo['compressions'].append('gz') - + + _check_if_partial_loaded(metadata_name, signable, roleinfo) + tuf.roledb.update_roleinfo(metadata_name, roleinfo) + + # append to list of elements to avoid reloading repeated metadata loaded_metadata.append(metadata_name) + # add the delegation new_targets_object = Targets(targets_directory, metadata_name, roleinfo) targets_object = \ targets_objects[tuf.roledb.get_parent_rolename(metadata_name)] @@ -800,10 +923,8 @@ def load_repository(repository_directory): 'delegations': {'keys': {}, 'roles': []}} tuf.roledb.add_role(rolename, roleinfo) - - return repository - """ - raise Exception("To be implemented") + + return project if __name__ == '__main__': From 086c3134e854a25d51b54fc2ad37fbaf13185600 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 11 Feb 2014 21:39:14 -0500 Subject: [PATCH 06/51] Fixed some format issues with the docstrings --- tuf/devtools.py | 55 ++++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/tuf/devtools.py b/tuf/devtools.py index 4ad2ea3b..8be7acfc 100644 --- a/tuf/devtools.py +++ b/tuf/devtools.py @@ -6,7 +6,7 @@ Santiago Torres Zane Fisher - Based on the work done by Vladimir Diaz + Based on the work done for the repository tools by Vladimir Diaz January 22, 2014 @@ -93,7 +93,7 @@ TARGETS_DIRECTORY_NAME = 'targets' # The full list of supported TUF metadata extensions. -METADATA_EXTENSIONS = ['.txt', '.txt.gz'] +METADATA_EXTENSIONS = ['.json', '.json.gz'] # The recognized compression extensions. SUPPORTED_COMPRESSION_EXTENSIONS = ['.gz'] @@ -191,13 +191,6 @@ def write(self, write_partial=False): exception if a metadata role cannot be written due to not having enough signatures. - consistent_snapshots: - A boolean indicating whether written metadata and target files should - include a digest in the filename (i.e., root..txt, - targets..txt.gz, README..txt, where is the - file's SHA256 digest. Example: - 'root.1f4e35a60c8f96d439e27e858ce2869c770c1cdd54e1ef76657ceaaf01da18a3.txt' - tuf.Error, if any of the top-level roles do not have a minimum threshold of signatures. @@ -245,6 +238,8 @@ def write(self, write_partial=False): self._targets_directory, self._metadata_directory, prefix=self.prefix) + + #save some other information that is not stored in the project's metadata save_project_configuration(self._metadata_directory, self.targets.keys, self.prefix, self.targets.threshold) @@ -264,8 +259,9 @@ def write_partial(self): None. - - None. + + tuf.Error, if any of the top-level roles do not have a minimum + threshold of signatures. Creates metadata files in the repository's metadata directory. @@ -293,7 +289,8 @@ def status(self): None. - None. + tuf.Error, if any of the top-level roles do not have a minimum + threshold of signatures. Generates and writes temporary metadata files. @@ -455,7 +452,8 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, filenames=None, prefix=''): """ - Non-public function that can generate and write the metadata of the specified + Non-public function that can generate and write the metadata of the + specified top-level 'rolename'. It also increments version numbers if: 1. write_partial==True and the metadata is the first to be written. @@ -675,11 +673,17 @@ def save_project_configuration(metadata_directory, public_keys, prefix, for the load routine - metadata_directory: where the project's metadata is located - public_keys: a list containing the public keys for the toplevel targets - role - prefix: the project's prefix (if any) - threshold: the threshold value for the toplevel targets role + metadata_directory: + where the project's metadata is located + + public_keys: + a list containing the public keys for the toplevel targets role + + prefix: + the project's prefix (if any) + + threshold: + the threshold value for the toplevel targets role Exceptions may rise if the metadata_directory/project.cfg file exists and @@ -735,8 +739,10 @@ def load_project(project_directory, prefix=''): files loaded from the project_directory path - project_directory: The path to the project folder - prefix: the prefix for the metadata + project_directory: + The path to the project's folder + prefix: + the prefix for the metadata tuf.FormatError, if 'project_directory' or any of the metadata files @@ -749,8 +755,6 @@ def load_project(project_directory, prefix=''): libtuf.Repository object. - BEGIN ORIGINAL - return repository """ # Does 'repository_directory' have the correct format? # Raise 'tuf.FormatError' if there is a mismatch. @@ -884,7 +888,7 @@ def load_project(project_directory, prefix=''): roleinfo['expires'] = metadata_object['expires'] roleinfo['paths'] = metadata_object['targets'].keys() roleinfo['delegations'] = metadata_object['delegations'] - + if os.path.exists(metadata_path+'.gz'): roleinfo['compressions'].append('gz') @@ -931,6 +935,5 @@ def load_project(project_directory, prefix=''): # The interactive sessions of the documentation strings can # be tested by running libtuf.py as a standalone module: # $ python libtuf.py. - # import doctest - # doctest.testmod() - print("main") + import doctest + doctest.testmod() From 5e25deb8599a543ea740c61bb36feb67d1926562 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 11 Feb 2014 22:02:43 -0500 Subject: [PATCH 07/51] Added a soft wrapper for the add_target Added the 'project.add_target()' function, that points to the project._targets.add_target() method. --- tuf/devtools.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/tuf/devtools.py b/tuf/devtools.py index 8be7acfc..a9e4d3ed 100644 --- a/tuf/devtools.py +++ b/tuf/devtools.py @@ -170,11 +170,12 @@ def __init__(self, self._targets_directory = targets_directory # Set the top-level role objects. - self.targets = Targets(self._targets_directory, 'targets') + self._targets = Targets(self._targets_directory, 'targets') self.prefix = file_prefix - #TODO: continue where we left off. + + def write(self, write_partial=False): """ @@ -249,7 +250,37 @@ def write(self, write_partial=False): release_signable['signed'], False) - + + def add_target(self,filepath): + """ + + Provide an alternative to project.targets.add_target. using + project.add_target yields a more intuitive and straightforward way of + adding targets to the project. + + + filepath: + The path to the target file. The file must be located under the + projects target's directory + + + tuf.FormatError, if 'filepath' is improperly formatted. + + tuf.Error, if 'filepath' is not under the repository's targets + directory + + + Adds 'filepath' to this role's list of targets. This role's + 'tuf.roledb' is also updated. + + + None + """ + try: + self.targets.add_target(filepath) + except tuf.FormatError, tuf.Error: + raise + def write_partial(self): """ @@ -430,7 +461,25 @@ def get_filepaths_in_directory(self, files_directory, recursive_walk=False, return targets + @property + def targets(self): + """ + + A getter method for the target's role inside the project object. + + None + + + None + + + None + + + The targets contained in this project's instance. + """ + return self._targets From b48741e1d75c310b1069c2baed1ba9002b1b0e6e Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Thu, 13 Feb 2014 17:10:15 -0500 Subject: [PATCH 08/51] Added thin wrappers for the key and delegation management functions --- tuf/devtools.py | 202 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 201 insertions(+), 1 deletion(-) diff --git a/tuf/devtools.py b/tuf/devtools.py index a9e4d3ed..f07b28d7 100644 --- a/tuf/devtools.py +++ b/tuf/devtools.py @@ -281,6 +281,174 @@ def add_target(self,filepath): except tuf.FormatError, tuf.Error: raise + def add_verification_key(self,key): + """ + + Function as a thin wrapper call for the project._targets call + with the same name. This wrapper is only for usability purposes + + + Key: + The role key to be added, conformant to tuf.formats.anykey_schema + Adding a public key to a role means that its corresponding private + key must generate and add its signture to the role. + + + Tuf.FormatError, if the 'key' argument is improperly formatted. + + Tuf.Error, if the project already contains a key + + + The role's entries in 'tuf.keydb.py' and 'tuf.roledb.py' are updated + + + None + """ + ### should check the number of keys for this role. + if len(self._targets.keys()>0): + raise tuf.Error("This project already contains a key") + + try: + self._targets.add_verification_key(key) + except tuf.FormatError: + raise + + + def remove_verification_key(self,key): + """ + + Function as a thin wrapper call for the project._targets call + with the same name. This wrapper is only for usability purposes + + + Key: + The role key to be removed, conformant to tuf.formats.anykey_schema + + + Tuf.FormatError, if the 'key' argument is improperly formatted. + + + The role's entries in 'tuf.roledb.py' are updated + + + None + """ + try: + self._targets.remove_verification_key(key) + except tuf.FormatError: + raise + + def load_signing_key(self,key): + """ + + To function as a thin wrapper call for the project._targets call + with the same name. This wrapper is only for usability purposes. + + + Key: + The key to be used to sign the metadata with. This key is the private + key for the whole project. A project supports only one key. + + + tuf.FormatError, if the 'key' argument is improperly formatted. + + + + The role's entries in 'tuf.keydb.py' and 'tuf.roledb.py' are updated + + + none + """ + try: + self._targets.load_signing_key(key) + except tuf.FormatError: + raise + + + def unload_signing_key(self,key): + """ + + To function as a thin wrapper call for the project._targets call + with the same name. This wrapper is only for usability purposes. + + + Key: + The key to be used to sign the metadata with. This key is the private + key for the whole project. A project supports only one key. + + + tuf.FormatError, if the 'key' argument is improperly formatted. + + + The role's entries in 'tuf.keydb.py' and 'tuf.roledb.py' are updated + + + none + """ + try: + self._targets.unload_signing_key(key) + except tuf.FormatError: + raise + + + + def delegate(self,rolename, public_keys, list_of_targets, threshold=1, + restricted_paths=None, path_hash_prefixes=None): + """ + + To function as a thin wrapper call for the project._targets call + with the same name. This wrapper is only for usability purposes. + + + rolename: + The name of the delegated role (e.g. django, qiime), not the full + rolename + + public_keys: + A list of TUF keys objects in 'ANYKEYLIST_SCHEMA' format. The list + may contain any of the supported key types: RSAKEY_SCHEMA, + ED25519KEY_SCHEMA, etc. + + list_of_targets: + A list of target filepaths that are added to the paths of 'rolename' + 'list_of_targets' is a list of target filepaths, and can be empty. + + threshold: + The threshold number of keys of 'rolename'. + + restricted_paths: + A list of restricted directory or file paths of 'rolename'. Any + targets files added to 'rolenae' must all under one of the + 'restructed' paths. + + path_hash_prefixes: + A list of hash prefixes in 'tuf.formats.PATH_HASH_PREFIXES_SCHEMA' + format, used in hashed bin delegations. Targets may be located and + stored in hashed bins by calculating the target path's hash prefix. + + + tuf.FormatError, if any of the arguments are improperly formatted + + tuf.Error, if the delegated role already exists or if any of the + argument is an invalid path (i.e., not under the repository's targets + directory). + + + A new Target object is created for 'rolename' that is accessible to the + caller (i.e., targets.unclaimed.). The 'tuf.keydb.py' and + 'tuf.roledb.py' stores are updated with 'public_keys' + + + None. + """ + + try: + self._targets.delegate(rolename, public_keys, list_f_targets, + threshold, restricted_paths, path_hash_prefixes) + except tuf.FormatError, tuf.Error: + raise + + def write_partial(self): """ @@ -303,7 +471,39 @@ def write_partial(self): self.write(write_partial=True) - + + def delegations(self, delegation_name): + """ + + To provide a method to access the delegations under this project. This + function is completely analogous to the targets(delegation_name). This + method is also recommended because sanity checks, input format and any + bridge-functions needed to guarantee the correct operation with the + target's object. + + + delegation_name: + The name of the delegation to be accessed, this argument has to match + the one used in the "delegate" method. + + + tuf.FormatError, if any of the arguments are improperly formatted. + + tuf.Error, if the delegated role doesn't exist inside the targets + object. + + + None + + + A targets object with the information for the desired delegation. + """ + try: + delegation = self._targets(delegation_name) + except tuf.FormatError, tuf.Error: + raise + + return delegation def status(self): """ From 9b5a18527da8108d25da241004ff416d71bf3e10 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Mon, 17 Feb 2014 23:50:42 -0500 Subject: [PATCH 09/51] Added the layout_type variable, fixed some stability issues The layout_type flag is used to decide whether a regular directory structure is going to be used. This means, that if the targets folder is under the metadata_directory tree or not. This flag lets the project module decide in which places to locate metadata (i.e. if it needs a metadata subdirectory or not). Also fixed stability issues, in which the delegated target files didn't lose their prefix. --- tuf/devtools.py | 235 +++++++++++++++++++++++++++++------------------- 1 file changed, 141 insertions(+), 94 deletions(-) diff --git a/tuf/devtools.py b/tuf/devtools.py index f07b28d7..937b91a5 100644 --- a/tuf/devtools.py +++ b/tuf/devtools.py @@ -62,6 +62,7 @@ from tuf.repository_tool import generate_targets_metadata from tuf.repository_tool import sign_metadata from tuf.repository_tool import write_metadata_file +from tuf.repository_tool import _check_if_partial_loaded # See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger('tuf.devtools') @@ -149,26 +150,23 @@ class Project(object): roles. """ - def __init__(self, - repository_directory, - metadata_directory, - targets_directory, - file_prefix, - ): + def __init__(self, metadata_directory, targets_directory, file_prefix): # Do the arguments have the correct format? # Ensure the arguments have the appropriate number of objects and object # types, and that all dict keys are properly named. # Raise 'tuf.FormatError' if any are improperly formatted. - tuf.formats.PATH_SCHEMA.check_match(repository_directory) tuf.formats.PATH_SCHEMA.check_match(metadata_directory) tuf.formats.PATH_SCHEMA.check_match(targets_directory) tuf.formats.PATH_SCHEMA.check_match(file_prefix) - self._repository_directory = repository_directory self._metadata_directory = metadata_directory self._targets_directory = targets_directory - + + # layout type defaults to "flat" unless explicitly specified in + # create_new_project + self.layout_type = "flat" + # Set the top-level role objects. self._targets = Targets(self._targets_directory, 'targets') @@ -241,11 +239,13 @@ def write(self, write_partial=False): prefix=self.prefix) #save some other information that is not stored in the project's metadata - save_project_configuration(self._metadata_directory, self.targets.keys, - self.prefix, self.targets.threshold) + save_project_configuration(self._metadata_directory,self._targets_directory, + self.targets.keys, self.prefix, self.targets.threshold, + self.layout_type) # Delete the metadata of roles no longer in 'tuf.roledb'. Obsolete roles # may have been revoked. + # import pdb; pdb.set_trace() _delete_obsolete_metadata(self._metadata_directory, release_signable['signed'], False) @@ -305,7 +305,7 @@ def add_verification_key(self,key): None """ ### should check the number of keys for this role. - if len(self._targets.keys()>0): + if len(self._targets.keys)>0: raise tuf.Error("This project already contains a key") try: @@ -443,7 +443,7 @@ def delegate(self,rolename, public_keys, list_of_targets, threshold=1, """ try: - self._targets.delegate(rolename, public_keys, list_f_targets, + self._targets.delegate(rolename, public_keys, list_of_targets, threshold, restricted_paths, path_hash_prefixes) except tuf.FormatError, tuf.Error: raise @@ -682,7 +682,6 @@ def targets(self): return self._targets - def _print_status(rolename, signable): """ Non-public function prints the number of (good/threshold) signatures of @@ -735,6 +734,7 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, signable = sign_metadata(metadata, roleinfo['signing_keyids'], metadata_filename) + # import pdb; pdb.set_trace() # Check if the version number of 'rolename' may be automatically incremented, # depending on whether if partial metadata is loaded or if the metadata is # written with write() / write_partial(). @@ -758,7 +758,6 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, # Write the metadata to file if contains a threshold of signatures. signable['signatures'].extend(roleinfo['signatures']) - #import pdb; pdb.set_trace() if tuf.sig.verify(signable, rolename) or write_partial: _remove_invalid_and_duplicate_signatures(signable) compressions = roleinfo['compressions'] @@ -811,7 +810,8 @@ def _get_password(prompt='Password: ', confirm=False): print('Mismatch; try again.') -def create_new_project(project_directory,prefix): +def create_new_project(metadata_directory, prefix = '',targets_directory=None, + key=None): """ Create a new project object, instantiate barebones metadata for the @@ -821,77 +821,89 @@ def create_new_project(project_directory,prefix): criteria and then written using the method project.write(). - project_directory: + metadata_directory: The directory that will eventually hold the metadata and target files of the project. - prefix: - a string determining the "upstream" filepath to sign the metadata - appropiately + targets_directory: + An optional argument to point the targets directory somewhere else than + the metadata directory if, for example, a project structure already + exists and the user does not want to move it. + prefix: + An optional argument to hold the "prefix" or the expected location for + the project files in the "upstream" respository. This value is only + used to sign metadata in a way that it matches the future location + of the files. + + key: + The public key to verify the project's metadata. Projects can only + handle one key with a threshold of one. If a project were to modify it's + key it should be removed and updated. + tuf.FormatError, if the arguments are improperly formatted. - - The 'projet_directory' directory is created if it does not exist, - including its metadata and targets sub-directories. + OSError, if the filepaths provided do not have write permissions + tuf.FormatError, if the key is not a valid public key. + + + The 'metadata_directory' and 'targets_directory' directories are created + if they do not exist. + A 'tuf.devtools.Repository' object. """ - # Does 'project_directory' have the correct format? + # Does 'metadata_directory' have the correct format? # Ensure the arguments have the appropriate number of objects and object # types, and that all dict keys are properly named. # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.PATH_SCHEMA.check_match(project_directory) - - # Do the same for the prefix + tuf.formats.PATH_SCHEMA.check_match(metadata_directory) + + # Do the same for the prefix, we first check for it to be something valid tuf.formats.PATH_SCHEMA.check_match(prefix) + # for the targets directory we do the same, but first, let's find out what + # layout the user needs, layout_type is a variable that is usually set to + # 1, which means "flat" (i.e. the cfg file is where the metadata folder is + # located), with a two, the cfg file goes to the "metadata" folder, and a + # new metadata folder is created inside the tree, to separate targets and + # metadata. + layout_type="flat" + if targets_directory is None: + targets_directory = os.path.join(metadata_directory,TARGETS_DIRECTORY_NAME) + metadata_directory = \ + os.path.join(metadata_directory,METADATA_DIRECTORY_NAME) + layout_type="repo-like" + + + tuf.formats.PATH_SCHEMA.check_match(targets_directory); + + if key is not None: + tuf.formats.KEY_SCHEMA.check_match(key) # Set the repository, metadata, and targets directories. These directories # are created if they do not exist. - project_directory = os.path.abspath(project_directory) - metadata_directory = None - targets_directory = None - - # Try to create 'repository_directory' if it does not exist. - try: - message = 'Creating '+repr(project_directory) - logger.info(message) - os.makedirs(project_directory) - - # 'OSError' raised if the leaf directory already exists or cannot be created. - # Check for case where 'repository_directory' has already been created. - except OSError, e: - if e.errno == errno.EEXIST: - pass - else: - raise - - # Set the metadata and targets directories. The metadata directory is a - # staged one so that the "live" repository is not affected. The - # staged metadata changes may be moved over to "live" after all updated - # have been completed. - metadata_directory = \ - os.path.join(project_directory, METADATA_STAGED_DIRECTORY_NAME) - targets_directory = \ - os.path.join(project_directory, TARGETS_DIRECTORY_NAME) + metadata_directory = os.path.abspath(metadata_directory) + targets_directory = os.path.abspath(targets_directory) # Try to create the metadata directory that will hold all of the metadata # files, such as 'root.txt' and 'release.txt'. try: message = 'Creating '+repr(metadata_directory) logger.info(message) - os.mkdir(metadata_directory) + os.makedirs(metadata_directory) # 'OSError' raised if the leaf directory already exists or cannot be created. + # Check for case where 'repository_directory' has already been created. except OSError, e: if e.errno == errno.EEXIST: - pass + # should check if we have wriite permissions here + pass else: raise - + # Try to create the targets directory that will hold all of the target files. try: message = 'Creating '+repr(targets_directory) @@ -906,16 +918,19 @@ def create_new_project(project_directory,prefix): # Create the bare bones repository object, where only the top-level roles # have been set and contain default values (e.g., Root roles has a threshold # of 1, expires 1 year into the future, etc.) - project = Project(project_directory, - metadata_directory, - targets_directory, - prefix - ) + project = Project(metadata_directory, targets_directory, prefix) + + # add the key to the project. + if key is not None: + project.add_verification_key(key); + # save the layout information + project.layout_type = layout_type + return project -def save_project_configuration(metadata_directory, public_keys, prefix, - threshold): +def save_project_configuration(metadata_directory,targets_directory, + public_keys, prefix, threshold, layout_type): """ Persist the project's information in a file to provide the information @@ -933,6 +948,11 @@ def save_project_configuration(metadata_directory, public_keys, prefix, threshold: the threshold value for the toplevel targets role + + layout_type: + the layout type being used by the project, "flat" stands for separated + targets and metadata directories, "repo-like" emulates the layout used + by the repository tools Exceptions may rise if the metadata_directory/project.cfg file exists and @@ -947,15 +967,23 @@ def save_project_configuration(metadata_directory, public_keys, prefix, nothing """ + # import pdb; pdb.set_trace() # schema check for metadata_directory and prefix tuf.formats.PATH_SCHEMA.check_match(metadata_directory) tuf.formats.PATH_SCHEMA.check_match(prefix) + tuf.formats.PATH_SCHEMA.check_match(targets_directory) # get the absolute filepath to our metadata_directory for consistency metadata_directory = os.path.abspath(metadata_directory) + cfg_file_directory = metadata_directory + + # check wheter if the layout type is "flat" or "repo-like" + # if it is, the .cfg file should be saved in the previous directory. + if(layout_type == "repo-like"): + cfg_file_directory = os.path.dirname(metadata_directory) # is the file open-able? open for overwriting - project_filename = os.path.join(metadata_directory,PROJECT_FILENAME) + project_filename = os.path.join(cfg_file_directory,PROJECT_FILENAME) try: fp = open(project_filename,"wt") except OSError, e: @@ -965,8 +993,10 @@ def save_project_configuration(metadata_directory, public_keys, prefix, project_config = {} project_config['prefix'] = prefix project_config['public_keys'] = {} - + project_config['metadata_location'] = metadata_directory + project_config['targets_location'] = targets_directory project_config['threshold'] = threshold + project_config['layout_type'] = layout_type # build a dictionary containing the actual keys for key in public_keys: key_info = tuf.keydb.get_key(key) @@ -991,7 +1021,9 @@ def load_project(project_directory, prefix=''): project_directory: The path to the project's folder prefix: - the prefix for the metadata + the prefix for the metadata, if defined, it will replace the current + prefix, by first removing the existing one (Saved) and setting the new + one in the end. tuf.FormatError, if 'project_directory' or any of the metadata files @@ -1004,7 +1036,8 @@ def load_project(project_directory, prefix=''): libtuf.Repository object. - """ + """ + # import pdb;pdb.set_trace() # Does 'repository_directory' have the correct format? # Raise 'tuf.FormatError' if there is a mismatch. tuf.formats.PATH_SCHEMA.check_match(project_directory) @@ -1014,26 +1047,28 @@ def load_project(project_directory, prefix=''): # Locate metadata filepaths and targets filepath. project_directory = os.path.abspath(project_directory) - metadata_directory = os.path.join(project_directory, - METADATA_STAGED_DIRECTORY_NAME) - targets_directory = os.path.join(project_directory, - TARGETS_DIRECTORY_NAME) - - # create a blank project on the target directory - project = Project(project_directory, metadata_directory, targets_directory, - prefix) - + + # load the cfg file and update the project. - config_filename = os.path.join(metadata_directory,PROJECT_FILENAME) + config_filename = os.path.join(project_directory,PROJECT_FILENAME) try: - fp = open(config_filename,"rt") + project_configuration = tuf.util.load_json_file(config_filename) except OSError, e: raise - project_configuration = json.load(fp) + metadata_directory = project_configuration['metadata_location'] + targets_directory = project_configuration['targets_location'] + if prefix=='': + prefix = project_configuration['prefix'] + + + # create a blank project on the target directory + project = Project(metadata_directory,targets_directory , prefix) + project.targets.threshold = project_configuration['threshold'] project.prefix = project_configuration['prefix'] - + project.layout_type = project_configuration['layout_type'] + # traverse the public keys and add them to the project keydict = project_configuration['public_keys'] for keyid in keydict: @@ -1047,7 +1082,7 @@ def load_project(project_directory, prefix=''): temp_pubkey['keyval']['private'] = '' else: temp_pubkey = keydict - project.targets.add_verification_key(temp_pubkey) + project.add_verification_key(temp_pubkey) # load the toplevel metadata @@ -1057,15 +1092,8 @@ def load_project(project_directory, prefix=''): targets_metadata = signable['signed'] # remove the prefix from the metadata - if project_configuration['prefix'] != '': - unprefixed_targets_metadata = {} - for targets in targets_metadata['targets'].keys(): - unprefixed_target = os.path.relpath(targets, - project_configuration['prefix']) - unprefixed_target = '/' + unprefixed_target - unprefixed_targets_metadata[unprefixed_target] = \ - targets_metadata['targets'][targets] - targets_metadata['targets'] = unprefixed_targets_metadata + targets_metadata = _strip_prefix_from_targets_metadata(targets_metadata, + project_configuration['prefix']) for signature in signable['signatures']: project.targets.add_signature(signature) @@ -1075,8 +1103,12 @@ def load_project(project_directory, prefix=''): roleinfo['version'] = targets_metadata['version'] roleinfo['paths'] = targets_metadata['targets'].keys() roleinfo['delegations'] = targets_metadata['delegations'] + + _check_if_partial_loaded('targets',signable,roleinfo) tuf.roledb.update_roleinfo('targets',roleinfo) + + for key_metadata in targets_metadata['delegations']['keys'].values(): key_object = tuf.keys.format_metadata_to_key(key_metadata) tuf.keydb.add_key(key_object) @@ -1085,8 +1117,8 @@ def load_project(project_directory, prefix=''): rolename = role['name'] roleinfo = {'name': role['name'], 'keyids': role['keyids'], 'threshold': role['threshold'], 'compressions': [''], - 'signing_keyids': [], 'signatures': [], - 'delegations': {'keys':{}, roles:[]} + 'signing_keyids': [], 'signatures': [], 'partial_loaded':False, + 'delegations': {'keys':{}, 'roles':[]} } tuf.roledb.add_role(rolename, roleinfo) @@ -1129,8 +1161,11 @@ def load_project(project_directory, prefix=''): except (ValueError, IOError), e: continue + # strip the extension from the metadata_object = signable['signed'] - + metadata_object = _strip_prefix_from_targets_metadata(metadata_object, + project_configuration['prefix']) + roleinfo = tuf.roledb.get_roleinfo(metadata_name) roleinfo['signatures'].extend(signable['signatures']) roleinfo['version'] = metadata_object['version'] @@ -1143,7 +1178,6 @@ def load_project(project_directory, prefix=''): _check_if_partial_loaded(metadata_name, signable, roleinfo) - tuf.roledb.update_roleinfo(metadata_name, roleinfo) # append to list of elements to avoid reloading repeated metadata @@ -1179,6 +1213,19 @@ def load_project(project_directory, prefix=''): return project +def _strip_prefix_from_targets_metadata(targets_metadata, prefix): + """ non-public method that removes the prefix from the targets metadata + so it can be used again in compliance with the local copies + """ + unprefixed_targets_metadata = {} + for targets in targets_metadata['targets'].keys(): + unprefixed_target = os.path.relpath(targets, prefix) + unprefixed_target = '/' + unprefixed_target + unprefixed_targets_metadata[unprefixed_target] = \ + targets_metadata['targets'][targets] + targets_metadata['targets'] = unprefixed_targets_metadata + + return targets_metadata if __name__ == '__main__': # The interactive sessions of the documentation strings can From 982704cf07422392c6a9579faee2c81d134623cd Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 18 Feb 2014 11:36:23 -0500 Subject: [PATCH 10/51] Clean up of some commented lines --- tuf/devtools.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tuf/devtools.py b/tuf/devtools.py index 937b91a5..45c17ede 100644 --- a/tuf/devtools.py +++ b/tuf/devtools.py @@ -245,7 +245,6 @@ def write(self, write_partial=False): # Delete the metadata of roles no longer in 'tuf.roledb'. Obsolete roles # may have been revoked. - # import pdb; pdb.set_trace() _delete_obsolete_metadata(self._metadata_directory, release_signable['signed'], False) @@ -734,7 +733,6 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, signable = sign_metadata(metadata, roleinfo['signing_keyids'], metadata_filename) - # import pdb; pdb.set_trace() # Check if the version number of 'rolename' may be automatically incremented, # depending on whether if partial metadata is loaded or if the metadata is # written with write() / write_partial(). @@ -967,7 +965,6 @@ def save_project_configuration(metadata_directory,targets_directory, nothing """ - # import pdb; pdb.set_trace() # schema check for metadata_directory and prefix tuf.formats.PATH_SCHEMA.check_match(metadata_directory) tuf.formats.PATH_SCHEMA.check_match(prefix) @@ -1037,7 +1034,6 @@ def load_project(project_directory, prefix=''): libtuf.Repository object. """ - # import pdb;pdb.set_trace() # Does 'repository_directory' have the correct format? # Raise 'tuf.FormatError' if there is a mismatch. tuf.formats.PATH_SCHEMA.check_match(project_directory) From 4f5f2c68b07ae4089dd90f21825a044185651083 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Sun, 16 Mar 2014 15:36:42 -0400 Subject: [PATCH 11/51] Renamed devtools to the appropriate name --- tuf/{devtools.py => developer_tool.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tuf/{devtools.py => developer_tool.py} (100%) diff --git a/tuf/devtools.py b/tuf/developer_tool.py similarity index 100% rename from tuf/devtools.py rename to tuf/developer_tool.py From c3a9d7d025fd58453f6b1faef645495f45bcad70 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Mon, 24 Mar 2014 13:37:48 -0400 Subject: [PATCH 12/51] Added first draft of the developer tools readme --- tuf/README-developer-tools.md | 123 ++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 tuf/README-developer-tools.md diff --git a/tuf/README-developer-tools.md b/tuf/README-developer-tools.md new file mode 100644 index 00000000..e9b21481 --- /dev/null +++ b/tuf/README-developer-tools.md @@ -0,0 +1,123 @@ +# Developing for a TUF repository # + +## Table of Contents ## +- [Overview](#overview) +- [Creating a simple project](#creating_a_simple_project) + - [Generating a key](#generating_a_key) + - [The Project class](#the_project_class) + - [Signing and writing metadata](#signing_and_writing_metadata) +- [Loading an Existing project](#Loading_an_existing_project) +- [Managing keys](#managing_keys) +- [Managing targets](#managing_targets) +- [Delegations](#delegations) + - [Restricted paths](#restricted_paths) + - [Keys and thresholds](#keys_and_thresholds) + + +## Overview ## +The TUF developer tool is a Python library that enables developers to create +and maintain the required metadata for diles hosted in a TUF Repository. This +document has two parts. The first part walks through the creation of a +prototypal TUF project. The second part demonstrates the full capabilities of +the TUF developer tool, which can be userd to expand the project from the first +part to meet the developer''s needs. + + +## Creating a Simple project ## +### Generating a Key ### +First, uou will needd to generate a key to sign the metadata. Keys are generated +in pairs: one public and the other private. the private key is password-protected +and is used to sign metadata. The public key can be shared freely, andi s used +to verify signatures made by the private key. + +The generate_and_write_rsa_keypair function will create two key files in the +path (path/to/) and named "key.pub", which is the public key and "key" +which is the private key. + +``` +>>> from tuf.developer_tool import * + +>>> generate_and_write_rsa_keypair("path/to/key") +Enter a password for the RSA key: +Confirm: +>>> +``` + + +### The project class ### +TUF-dev is built around the Project class, which is used to organize groups of +targets associated with a single set of metadata. Each Project instance keeps +track of which target files ar associated with a single set of metadata. Each +Project instance keeps track of which target files are signed and which need +signing, which keys are used to sign metadata. It also keeps track of delegated +rolse, which are covered later. + +Before creating a project, you must know hwere it be located in the TUF +Repository. In the following example, we will create a project to be hosted as +"repo/example_project" within the repository, and store a local copy of the +metadata at "path/to/metadata". The project will comprise a single target file, +"local/path/to/example_project/target_1" locally, and we will secure it with +the key generated above. + +``` +>>> public_key = import_rsa_publickey_from_file("path/to/key.pub") + +>>> project = create_new_project(metadata_directory="local/path/to/metadata/", +... targets_directory="local/path/to/example_project", +... location_in_repository="repo/example_project", key=public_key) +>>> project.add_target("target_1") +``` + +At this point, the metadata is not valid. We have assigned a key to the project, +but we have not *signed* it with that key. + + +### Signing and writing the metadata ### +In order to signe the metadata, we need to impot the private key corresponding +to the public key we added to the project. One the key is loaded to the project, +it will automatically be used to sign the metadata whenever it is written. + +``` +>>> private_key = import_rsa_privatekey_from_file("path/to/key") +Enter password for the RSA key: +>>> project.load_signing_key(private_key) +>>> project.write() +``` + +When all changes to a project have been written, the Project instance can safely +be deleted. + + +To make changes to existing metadata, we will need the Project again. We can +restore it with the load_project() function. + +``` +>>> from tuf.developer_tool import * +>>> project = load_project("local/path/to/metadata") +>>> +``` +Each time the project is loaded anew, the necessary private keys must also be +loaded in order to sign metadata. + +``` +>>> private_key = import_rsa_privatekey_from_file("path/to/key") +Enter a password for the RSA key: + +>>> project.load_signing_key(private_key) + +>>> project.write() +``` + + +## Managing keys ## +When generating keys, it is possible to specity the length of the key in bits +and its password as parameters: + +``` +>>> generate_and_write_rsa_keypair("path/to/key",bits=2048, password="pw") +``` +The bits parameter defaults to 3072, and values below 2048 will raise an error. +The password parameter is only intended to be used in scripts. + + + From 0b72d275995763b50f0eba86fc319a2345509026 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Mon, 24 Mar 2014 13:41:30 -0400 Subject: [PATCH 13/51] changed the prefix argument in create project to location_in_repository This is intended to make the function more user-friendly by having a more self-explanatory argument name. --- tuf/developer_tool.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 45c17ede..d0fc6d97 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -109,7 +109,8 @@ # Initial 'targets.txt' expiration time of 3 months. TARGETS_EXPIRATION = 7889230 - +# TODO: We should have a method like Describe to show the contents of the +# instance class Project(object): """ @@ -807,9 +808,9 @@ def _get_password(prompt='Password: ', confirm=False): else: print('Mismatch; try again.') - -def create_new_project(metadata_directory, prefix = '',targets_directory=None, - key=None): +# TODO: change prefix to something like location_in_repository. +def create_new_project(metadata_directory, location_in_repository = '', + targets_directory=None, key=None): """ Create a new project object, instantiate barebones metadata for the From a34dfb0bb984eb1b433a21e8e6e8eb6eeb6e74c5 Mon Sep 17 00:00:00 2001 From: SantiagoTorres Date: Mon, 31 Mar 2014 18:17:48 -0400 Subject: [PATCH 14/51] Update README-developer-tools.md Updated part of the documentation, Added a sketch for the delegate method and finished the managing keys section --- tuf/README-developer-tools.md | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tuf/README-developer-tools.md b/tuf/README-developer-tools.md index e9b21481..9ed53be5 100644 --- a/tuf/README-developer-tools.md +++ b/tuf/README-developer-tools.md @@ -109,7 +109,7 @@ Enter a password for the RSA key: ``` -## Managing keys ## +## Managing keys When generating keys, it is possible to specity the length of the key in bits and its password as parameters: @@ -120,4 +120,37 @@ The bits parameter defaults to 3072, and values below 2048 will raise an error. The password parameter is only intended to be used in scripts. +## Managing Targets + +``` + +>>> list_of_targets = project.get_filepaths_in_directory(“path/within/targets/folder”, recursive_walk=False, follow_links=False) +>>> project.add_targets(list_of_targets) +``` + +``` +>>> project.remove_target(“target_1”) +``` + +## Delegations + +The project we created above is secured entirely by one key. If you want to allow someone else to update part of your project independently, you will need to delegate a new role for them. For example, we can + +``` +>>> other_key = import_rsa_publickey_from_file(“sombodys_public_key.pub”) + +>>> project.delegate(“newrole”, [other_key], targets) +``` + +The new role is now an attribute of the Project instance, and contains the same methods as Project. For example, we can add targets in the same way as before: + +``` + +>>> project(“newrole”).add_target(“delegated_1”) + +``` + + + +Recall that we input the other person’s key as part of a list. That list can contain any number of public keys. You can also add keys to the role after creating it using the add_signing_key() method. From a6525ee985493347777df78c7ee17c1029075c23 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Mon, 31 Mar 2014 18:47:56 -0400 Subject: [PATCH 15/51] Updated the developer tool to inherit the Targets object The motivation behind this change is code reutilization and quick updating. Most of the methods called in the old version of the developer tool immediate to the targets counterpart. The calling convention now resembles more objective intended without losing any functionality and with automatic updates from any changes in the repository_tool. The project object used to have a Targets object contained. This did make more sense in a conceptual way (a project could have had more roles than only targets). However, after reviewing the functionality of the tool, it seems clear that the best way to adopt the changes automatically is to inherit the targets object. Soft wrappers are still to be made, we need to override the add_verification_key method to in order to avoid having more than one key on the project. --- tuf/developer_tool.py | 352 ++---------------------------------------- 1 file changed, 9 insertions(+), 343 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index d0fc6d97..f9b9fce1 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -111,7 +111,7 @@ # TODO: We should have a method like Describe to show the contents of the # instance -class Project(object): +class Project(Targets): """ This class works as the abstraction of the developer's files. this module @@ -169,7 +169,7 @@ def __init__(self, metadata_directory, targets_directory, file_prefix): self.layout_type = "flat" # Set the top-level role objects. - self._targets = Targets(self._targets_directory, 'targets') + super(Project, self).__init__(self._targets_directory, 'targets') self.prefix = file_prefix @@ -241,7 +241,7 @@ def write(self, write_partial=False): #save some other information that is not stored in the project's metadata save_project_configuration(self._metadata_directory,self._targets_directory, - self.targets.keys, self.prefix, self.targets.threshold, + self.keys, self.prefix, self.threshold, self.layout_type) # Delete the metadata of roles no longer in 'tuf.roledb'. Obsolete roles @@ -251,259 +251,6 @@ def write(self, write_partial=False): - def add_target(self,filepath): - """ - - Provide an alternative to project.targets.add_target. using - project.add_target yields a more intuitive and straightforward way of - adding targets to the project. - - - filepath: - The path to the target file. The file must be located under the - projects target's directory - - - tuf.FormatError, if 'filepath' is improperly formatted. - - tuf.Error, if 'filepath' is not under the repository's targets - directory - - - Adds 'filepath' to this role's list of targets. This role's - 'tuf.roledb' is also updated. - - - None - """ - try: - self.targets.add_target(filepath) - except tuf.FormatError, tuf.Error: - raise - - def add_verification_key(self,key): - """ - - Function as a thin wrapper call for the project._targets call - with the same name. This wrapper is only for usability purposes - - - Key: - The role key to be added, conformant to tuf.formats.anykey_schema - Adding a public key to a role means that its corresponding private - key must generate and add its signture to the role. - - - Tuf.FormatError, if the 'key' argument is improperly formatted. - - Tuf.Error, if the project already contains a key - - - The role's entries in 'tuf.keydb.py' and 'tuf.roledb.py' are updated - - - None - """ - ### should check the number of keys for this role. - if len(self._targets.keys)>0: - raise tuf.Error("This project already contains a key") - - try: - self._targets.add_verification_key(key) - except tuf.FormatError: - raise - - - def remove_verification_key(self,key): - """ - - Function as a thin wrapper call for the project._targets call - with the same name. This wrapper is only for usability purposes - - - Key: - The role key to be removed, conformant to tuf.formats.anykey_schema - - - Tuf.FormatError, if the 'key' argument is improperly formatted. - - - The role's entries in 'tuf.roledb.py' are updated - - - None - """ - try: - self._targets.remove_verification_key(key) - except tuf.FormatError: - raise - - def load_signing_key(self,key): - """ - - To function as a thin wrapper call for the project._targets call - with the same name. This wrapper is only for usability purposes. - - - Key: - The key to be used to sign the metadata with. This key is the private - key for the whole project. A project supports only one key. - - - tuf.FormatError, if the 'key' argument is improperly formatted. - - - - The role's entries in 'tuf.keydb.py' and 'tuf.roledb.py' are updated - - - none - """ - try: - self._targets.load_signing_key(key) - except tuf.FormatError: - raise - - - def unload_signing_key(self,key): - """ - - To function as a thin wrapper call for the project._targets call - with the same name. This wrapper is only for usability purposes. - - - Key: - The key to be used to sign the metadata with. This key is the private - key for the whole project. A project supports only one key. - - - tuf.FormatError, if the 'key' argument is improperly formatted. - - - The role's entries in 'tuf.keydb.py' and 'tuf.roledb.py' are updated - - - none - """ - try: - self._targets.unload_signing_key(key) - except tuf.FormatError: - raise - - - - def delegate(self,rolename, public_keys, list_of_targets, threshold=1, - restricted_paths=None, path_hash_prefixes=None): - """ - - To function as a thin wrapper call for the project._targets call - with the same name. This wrapper is only for usability purposes. - - - rolename: - The name of the delegated role (e.g. django, qiime), not the full - rolename - - public_keys: - A list of TUF keys objects in 'ANYKEYLIST_SCHEMA' format. The list - may contain any of the supported key types: RSAKEY_SCHEMA, - ED25519KEY_SCHEMA, etc. - - list_of_targets: - A list of target filepaths that are added to the paths of 'rolename' - 'list_of_targets' is a list of target filepaths, and can be empty. - - threshold: - The threshold number of keys of 'rolename'. - - restricted_paths: - A list of restricted directory or file paths of 'rolename'. Any - targets files added to 'rolenae' must all under one of the - 'restructed' paths. - - path_hash_prefixes: - A list of hash prefixes in 'tuf.formats.PATH_HASH_PREFIXES_SCHEMA' - format, used in hashed bin delegations. Targets may be located and - stored in hashed bins by calculating the target path's hash prefix. - - - tuf.FormatError, if any of the arguments are improperly formatted - - tuf.Error, if the delegated role already exists or if any of the - argument is an invalid path (i.e., not under the repository's targets - directory). - - - A new Target object is created for 'rolename' that is accessible to the - caller (i.e., targets.unclaimed.). The 'tuf.keydb.py' and - 'tuf.roledb.py' stores are updated with 'public_keys' - - - None. - """ - - try: - self._targets.delegate(rolename, public_keys, list_of_targets, - threshold, restricted_paths, path_hash_prefixes) - except tuf.FormatError, tuf.Error: - raise - - - def write_partial(self): - """ - - Write all the JSON Metadata objects to their corresponding files, but - allow metadata files to contain an invalid threshold of signatures. - - - None. - - - tuf.Error, if any of the top-level roles do not have a minimum - threshold of signatures. - - - Creates metadata files in the repository's metadata directory. - - - None. - """ - - self.write(write_partial=True) - - - def delegations(self, delegation_name): - """ - - To provide a method to access the delegations under this project. This - function is completely analogous to the targets(delegation_name). This - method is also recommended because sanity checks, input format and any - bridge-functions needed to guarantee the correct operation with the - target's object. - - - delegation_name: - The name of the delegation to be accessed, this argument has to match - the one used in the "delegate" method. - - - tuf.FormatError, if any of the arguments are improperly formatted. - - tuf.Error, if the delegated role doesn't exist inside the targets - object. - - - None - - - A targets object with the information for the desired delegation. - """ - try: - delegation = self._targets(delegation_name) - except tuf.FormatError, tuf.Error: - raise - - return delegation def status(self): """ @@ -599,89 +346,6 @@ def status(self): - def get_filepaths_in_directory(self, files_directory, recursive_walk=False, - followlinks=True): - """ - - Walk the given 'files_directory' and build a list of target files found. - - - files_directory: - The path to a directory of target files. - - recursive_walk: - To recursively walk the directory, set recursive_walk=True. - - followlinks: - To follow symbolic links, set followlinks=True. - - - tuf.FormatError, if the arguments are improperly formatted. - - tuf.Error, if 'file_directory' is not a valid directory. - - Python IO exceptions. - - - None. - - - A list of absolute paths to target files in the given 'files_directory'. - """ - - # Do the arguments have the correct format? - # Ensure the arguments have the appropriate number of objects and object - # types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if any are improperly formatted. - tuf.formats.PATH_SCHEMA.check_match(files_directory) - tuf.formats.BOOLEAN_SCHEMA.check_match(recursive_walk) - tuf.formats.BOOLEAN_SCHEMA.check_match(followlinks) - - # Ensure a valid directory is given. - if not os.path.isdir(files_directory): - message = repr(files_directory)+' is not a directory.' - raise tuf.Error(message) - - # A list of the target filepaths found in 'file_directory'. - targets = [] - - # FIXME: We need a way to tell Python 2, but not Python 3, to return - # filenames in Unicode; see #61 and: - # http://docs.python.org/2/howto/unicode.html#unicode-filenames - for dirpath, dirnames, filenames in os.walk(files_directory, - followlinks=followlinks): - for filename in filenames: - full_target_path = os.path.join(dirpath, filename) - targets.append(full_target_path) - - # Prune the subdirectories to walk right now if we do not wish to - # recursively walk files_directory. - if recursive_walk is False: - del dirnames[:] - - return targets - - @property - def targets(self): - """ - - A getter method for the target's role inside the project object. - - - None - - - None - - - None - - - The targets contained in this project's instance. - """ - return self._targets - - def _print_status(rolename, signable): """ Non-public function prints the number of (good/threshold) signatures of @@ -829,7 +493,7 @@ def create_new_project(metadata_directory, location_in_repository = '', the metadata directory if, for example, a project structure already exists and the user does not want to move it. - prefix: + location_in_repository: An optional argument to hold the "prefix" or the expected location for the project files in the "upstream" respository. This value is only used to sign metadata in a way that it matches the future location @@ -861,8 +525,9 @@ def create_new_project(metadata_directory, location_in_repository = '', # Raise 'tuf.FormatError' if there is a mismatch. tuf.formats.PATH_SCHEMA.check_match(metadata_directory) - # Do the same for the prefix, we first check for it to be something valid - tuf.formats.PATH_SCHEMA.check_match(prefix) + # Do the same for the location in the repo, we first check for it to be + # something valid + tuf.formats.PATH_SCHEMA.check_match(location_in_repository) # for the targets directory we do the same, but first, let's find out what # layout the user needs, layout_type is a variable that is usually set to @@ -917,7 +582,8 @@ def create_new_project(metadata_directory, location_in_repository = '', # Create the bare bones repository object, where only the top-level roles # have been set and contain default values (e.g., Root roles has a threshold # of 1, expires 1 year into the future, etc.) - project = Project(metadata_directory, targets_directory, prefix) + project = Project(metadata_directory, targets_directory, + location_in_repository) # add the key to the project. if key is not None: From cace6820de4f8f7ce3b2121a892e25d32ae66903 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Mon, 31 Mar 2014 19:10:02 -0400 Subject: [PATCH 16/51] Added soft wrappers for the add_verification_key method The add verification key method now prohibits the ability to have more than one key in the Project role, however, delegations do support having more. This is the suggested behavior as of this version, removing these two wapper functions would prevent to have this limitation but would require the extension of the write_project_configuration function --- tuf/developer_tool.py | 60 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index f9b9fce1..77d14f21 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -251,7 +251,65 @@ def write(self, write_partial=False): - + def remove_verification_key(self,key): + """ + + Function as a thin wrapper call for the project._targets call + with the same name. This wrapper is only for usability purposes + + + Key: + The role key to be removed, conformant to tuf.formats.anykey_schema + + + Tuf.FormatError, if the 'key' argument is improperly formatted. + + + The role's entries in 'tuf.roledb.py' are updated + + + None + """ + try: + super(Project, self).remove_verification_key(key) + except tuf.FormatError: + raise + + + def add_verification_key(self,key): + """ + + Function as a thin wrapper call for the project._targets call + with the same name. This wrapper is only for usability purposes + + + Key: + The role key to be added, conformant to tuf.formats.anykey_schema + Adding a public key to a role means that its corresponding private + key must generate and add its signture to the role. + + + Tuf.FormatError, if the 'key' argument is improperly formatted. + + Tuf.Error, if the project already contains a key + + + The role's entries in 'tuf.keydb.py' and 'tuf.roledb.py' are updated + + + None + """ + ### should check the number of keys for this role. + if len(self.keys)>0: + raise tuf.Error("This project already contains a key") + + try: + super(Project, self).add_verification_key(key) + except tuf.FormatError: + raise + + + def status(self): """ From 0c6918b7020ae2c75ab3fb7671ae37848f2b9481 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Mon, 31 Mar 2014 20:44:53 -0400 Subject: [PATCH 17/51] Updated the load project method to include the Targets inherittance The load_project contained statements that called the previously included "targets" instance, since we now inherit Targets, we don't require them, the statements were updated the following way: self._targets to a plain: self --- tuf/developer_tool.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 77d14f21..b7fb7f19 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -382,13 +382,13 @@ def status(self): # Targets role. try: - _check_role_keys(self.targets.rolename) + _check_role_keys(self.rolename) except tuf.InsufficientKeysError, e: print(str(e)) return try: - signable = _generate_and_write_metadata(self.targets.rolename, + signable = _generate_and_write_metadata(self.rolename, filenames['targets'], False, self._targets_directory, self._metadata_directory, @@ -786,7 +786,7 @@ def load_project(project_directory, prefix=''): # create a blank project on the target directory project = Project(metadata_directory,targets_directory , prefix) - project.targets.threshold = project_configuration['threshold'] + project.threshold = project_configuration['threshold'] project.prefix = project_configuration['prefix'] project.layout_type = project_configuration['layout_type'] @@ -816,7 +816,7 @@ def load_project(project_directory, prefix=''): targets_metadata = _strip_prefix_from_targets_metadata(targets_metadata, project_configuration['prefix']) for signature in signable['signatures']: - project.targets.add_signature(signature) + project.add_signature(signature) # update roledb roleinfo = tuf.roledb.get_roleinfo('targets') @@ -849,7 +849,7 @@ def load_project(project_directory, prefix=''): # metadata object. targets_objects = {} loaded_metadata = [] - targets_objects['targets'] = project.targets + targets_objects['targets'] = project targets_metadata_directory = os.path.join(metadata_directory, TARGETS_DIRECTORY_NAME) if os.path.exists(targets_metadata_directory) and \ From ff9fdd833ba3cc21c1a41082cc3eac14bacf2cca Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 1 Apr 2014 12:51:33 -0400 Subject: [PATCH 18/51] The _check_if_partial_loaded method was renamed The repository tool now provides a method called "_metadata_is_partially_loaded" to provide the same functionality, the return type changed so the flow of this module had to adapt to the new function: _check_if_partial_loaded() -> updated the roleinfo data as an effect _metadata_is_partially_loaded -> returns a boolean and forces us to update the roleinfo data. --- tuf/developer_tool.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index b7fb7f19..70cce6e7 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -62,7 +62,7 @@ from tuf.repository_tool import generate_targets_metadata from tuf.repository_tool import sign_metadata from tuf.repository_tool import write_metadata_file -from tuf.repository_tool import _check_if_partial_loaded +from tuf.repository_tool import _metadata_is_partially_loaded # See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger('tuf.devtools') @@ -824,8 +824,15 @@ def load_project(project_directory, prefix=''): roleinfo['version'] = targets_metadata['version'] roleinfo['paths'] = targets_metadata['targets'].keys() roleinfo['delegations'] = targets_metadata['delegations'] + roleinfo['partial_loaded'] = false - _check_if_partial_loaded('targets',signable,roleinfo) + + # check if the loaded metadata was partially written and update the + # flag in the roledb + if _metadata_is_partially_loaded('targets',signable,roleinfo): + roleinfo['partial_loaded'] = True + + tuf.roledb.update_roleinfo('targets',roleinfo) @@ -893,12 +900,16 @@ def load_project(project_directory, prefix=''): roleinfo['expires'] = metadata_object['expires'] roleinfo['paths'] = metadata_object['targets'].keys() roleinfo['delegations'] = metadata_object['delegations'] + roleinfo['partial_loaded'] = False if os.path.exists(metadata_path+'.gz'): roleinfo['compressions'].append('gz') + # if the metadata was partially loaded, update the roleinfo flag. + if _metadata_is_partially_loaded(metadata_name, signable, roleinfo): + roleinfo['partial_loaded'] = True + - _check_if_partial_loaded(metadata_name, signable, roleinfo) tuf.roledb.update_roleinfo(metadata_name, roleinfo) # append to list of elements to avoid reloading repeated metadata From 5495331d817585c9260c1c5e6a3c31ed20d19b6d Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 1 Apr 2014 13:40:40 -0400 Subject: [PATCH 19/51] Updaated the module name in the comments, fixed comment references --- tuf/developer_tool.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 70cce6e7..bf6cc8ff 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -1,6 +1,6 @@ """ - tuf-devtoools.py + developer_tool.py Santiago Torres @@ -15,7 +15,8 @@ See LICENSE for licensing information. - See 'tuf/README' for a complete guide on using 'tuf.devtools.py'. + See 'tuf/README-developer-tools.md' for a complete guide on using + 'developer_tool.py'. """ # Help with Python 3 compatibility, where the print statement is a function, an From ae3b3a01d63d373595512e6556fff7142188256d Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 1 Apr 2014 13:49:41 -0400 Subject: [PATCH 20/51] Removed commented lines and fixed minor comments Some unused methods were commented for testing, they shouldn't appear in upstream --- tuf/developer_tool.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index bf6cc8ff..44fff3d4 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -56,7 +56,6 @@ from tuf.repository_tool import generate_and_write_ed25519_keypair from tuf.repository_tool import import_ed25519_publickey_from_file from tuf.repository_tool import import_ed25519_privatekey_from_file -#from tuf.import _generate_and_write_metadata from tuf.repository_tool import _remove_invalid_and_duplicate_signatures from tuf.repository_tool import _check_role_keys from tuf.repository_tool import _delete_obsolete_metadata @@ -197,7 +196,7 @@ def write(self, write_partial=False): threshold of signatures. - Creates metadata files in the repository's metadata directory. + Creates metadata files in the projects's metadata directory. None. @@ -531,7 +530,11 @@ def _get_password(prompt='Password: ', confirm=False): else: print('Mismatch; try again.') -# TODO: change prefix to something like location_in_repository. + + + + + def create_new_project(metadata_directory, location_in_repository = '', targets_directory=None, key=None): """ @@ -622,7 +625,7 @@ def create_new_project(metadata_directory, location_in_repository = '', # Check for case where 'repository_directory' has already been created. except OSError, e: if e.errno == errno.EEXIST: - # should check if we have wriite permissions here + # should check if we have write permissions here pass else: raise From 036ac692a6809311951accf1dd37696fa9c468b9 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 1 Apr 2014 14:45:56 -0400 Subject: [PATCH 21/51] fixed references to the repository tool in comments. --- tuf/developer_tool.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 44fff3d4..aa87cea8 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -74,8 +74,8 @@ # are the recommended minimum and are good from the present through 2030. DEFAULT_RSA_KEY_BITS = 3072 -# The algorithm used by the repository to generate the hashes of the -# target filepaths. The repository may optionally organize targets into +# The algorithm used by the developer tools to generate the hashes of the +# target filepaths. HASH_FUNCTION = 'sha256' # The extension of TUF metadata. @@ -567,18 +567,17 @@ def create_new_project(metadata_directory, location_in_repository = '', key it should be removed and updated. - tuf.FormatError, if the arguments are improperly formatted. + tuf.FormatError, if the arguments are improperly formatted or if the public + key is not a valid one ( if it's not none ). OSError, if the filepaths provided do not have write permissions - tuf.FormatError, if the key is not a valid public key. - The 'metadata_directory' and 'targets_directory' directories are created if they do not exist. - A 'tuf.devtools.Repository' object. + A 'tuf.developer_tool.Project' object. """ # Does 'metadata_directory' have the correct format? @@ -609,7 +608,8 @@ def create_new_project(metadata_directory, location_in_repository = '', if key is not None: tuf.formats.KEY_SCHEMA.check_match(key) - # Set the repository, metadata, and targets directories. These directories + + # Set the metadata and targets directories. These directories # are created if they do not exist. metadata_directory = os.path.abspath(metadata_directory) targets_directory = os.path.abspath(targets_directory) @@ -760,7 +760,7 @@ def load_project(project_directory, prefix=''): stored in a libtuf.Repository object. - libtuf.Repository object. + A tuf.developer_tool.Project object. """ # Does 'repository_directory' have the correct format? @@ -810,7 +810,7 @@ def load_project(project_directory, prefix=''): project.add_verification_key(temp_pubkey) - # load the toplevel metadata + # load the project's metadata targets_metadata_path = os.path.join(metadata_directory, TARGETS_FILENAME) signable = tuf.util.load_json_file(targets_metadata_path) tuf.formats.check_signable_object_format(signable) From 6337f59a2e422536077ed7e25a89976f29180267 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 1 Apr 2014 14:48:06 -0400 Subject: [PATCH 22/51] Fixed whitespace errors between methods. --- tuf/developer_tool.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index aa87cea8..2dc78d5c 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -109,6 +109,8 @@ # Initial 'targets.txt' expiration time of 3 months. TARGETS_EXPIRATION = 7889230 + + # TODO: We should have a method like Describe to show the contents of the # instance class Project(Targets): @@ -175,6 +177,7 @@ def __init__(self, metadata_directory, targets_directory, file_prefix): + def write(self, write_partial=False): """ @@ -251,6 +254,8 @@ def write(self, write_partial=False): + + def remove_verification_key(self,key): """ @@ -276,6 +281,8 @@ def remove_verification_key(self,key): raise + + def add_verification_key(self,key): """ @@ -310,6 +317,8 @@ def add_verification_key(self,key): + + def status(self): """ @@ -404,6 +413,8 @@ def status(self): + + def _print_status(rolename, signable): """ Non-public function prints the number of (good/threshold) signatures of @@ -417,6 +428,10 @@ def _print_status(rolename, signable): repr(status['threshold'])+' signatures.' print(message) + + + + def _generate_and_write_metadata(rolename, metadata_filename, write_partial, targets_directory, metadata_directory, filenames=None, @@ -498,6 +513,9 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, return signable, filename + + + def _prompt(message, result_type=str): """ Non-public function that prompts the user for input by printing 'message', @@ -656,6 +674,11 @@ def create_new_project(metadata_directory, location_in_repository = '', return project + + + + + def save_project_configuration(metadata_directory,targets_directory, public_keys, prefix, threshold, layout_type): """ @@ -737,6 +760,9 @@ def save_project_configuration(metadata_directory,targets_directory, fp.close() + + + def load_project(project_directory, prefix=''): """ @@ -949,6 +975,10 @@ def load_project(project_directory, prefix=''): return project + + + + def _strip_prefix_from_targets_metadata(targets_metadata, prefix): """ non-public method that removes the prefix from the targets metadata so it can be used again in compliance with the local copies @@ -963,6 +993,10 @@ def _strip_prefix_from_targets_metadata(targets_metadata, prefix): return targets_metadata + + + + if __name__ == '__main__': # The interactive sessions of the documentation strings can # be tested by running libtuf.py as a standalone module: From 4a35d53d2feacd50abee9f6af4e268d85e6159e1 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 1 Apr 2014 14:52:23 -0400 Subject: [PATCH 23/51] Added a comment block describing the project.cfg file --- tuf/developer_tool.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 2dc78d5c..bbdadc3a 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -84,7 +84,33 @@ # The metadata filename for the targets metadata information. TARGETS_FILENAME = 'targets' + METADATA_EXTENSION -# Project configuration filename +# Project configuration filename. This file is intended to hold all of the +# supporting information about the project that's not contained in a usual +# TUF metadata file. The project.cfg file consists of the following fields: +# +# targets_location: the location of the targets folder. +# +# prefix: the directory location to prepend to the metadata so it +# matches the metadata signed in the repository. +# +# metadata_location: the location of the metadata files. +# +# threshold: the threshold for this project object, it is fixed to +# one in the current version +# +# public_keys: a list of the public keys used to verify the metadata +# in this project +# +# layout_type: a field describing the directory layout: +# +# repo-like: matches the layout of the repository tool. +# the targets and metadata folders are +# located under a comon directory for the +# project. +# +# flat: the targets directory and the +# metadata directory are located in different +# paths PROJECT_FILENAME = 'project.cfg' # The targets and metadata directory names. Metadata files are written @@ -127,10 +153,6 @@ class Project(Targets): - project_directory: - The root folder of the project that contains the metadata and targets - sub-directories. - metadata_directory: The metadata sub-directory contains the files of the top-level roles, including all roles delegated from 'targets.txt'. From 1a9f9eb0be2548e41f1c6f23b0ecf0e0b3d9c75d Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 1 Apr 2014 14:53:08 -0400 Subject: [PATCH 24/51] updated method descriptions in the comment blocks --- tuf/developer_tool.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index bbdadc3a..b41bd3a2 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -164,6 +164,11 @@ class Project(Targets): downloaded. Metadata files are similarly referenced in the top-level metadata. + file_prefix: + The path strig that will be prepended to the generated metadata + (e.g. targets/foo -> targets/prefix/foo) so that it matches the actual + location in the upstream repository. + tuf.FormatError, if the arguments are improperly formatted. @@ -681,7 +686,7 @@ def create_new_project(metadata_directory, location_in_repository = '', else: raise - # Create the bare bones repository object, where only the top-level roles + # Create the bare bones project object, where only the top-level roles # have been set and contain default values (e.g., Root roles has a threshold # of 1, expires 1 year into the future, etc.) project = Project(metadata_directory, targets_directory, @@ -710,28 +715,31 @@ def save_project_configuration(metadata_directory,targets_directory, metadata_directory: - where the project's metadata is located + Where the project's metadata is located + + targets_directory: + The location of the target files in this project. public_keys: - a list containing the public keys for the toplevel targets role + A list containing the public keys for the project role. prefix: - the project's prefix (if any) + The project's prefix (if any) threshold: - the threshold value for the toplevel targets role + The threshold value for the project role. layout_type: - the layout type being used by the project, "flat" stands for separated + The layout type being used by the project, "flat" stands for separated targets and metadata directories, "repo-like" emulates the layout used by the repository tools - Exceptions may rise if the metadata_directory/project.cfg file exists and + OSError may rise if the metadata_directory/project.cfg file exists and is non-writeable - Exceptions are also expected if either the prefix or the metadata directory - are malformed + tuf.FormatError are also expected if either the prefix or the metadata + directory are malformed A project.cfg file is created or overwritten From 561e85959bce2d5223c89b8bd66c181ef6dd1033 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 1 Apr 2014 23:43:24 -0400 Subject: [PATCH 25/51] Removed a local definition of the prompt and get_password methods The local _prompt and _get_password methods were deleted in order to use the version contained in the tuf.repository_tools. The main motivation of this change is to provide code reutilization --- tuf/developer_tool.py | 46 ++++++++----------------------------------- 1 file changed, 8 insertions(+), 38 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index b41bd3a2..01f85df7 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -64,9 +64,17 @@ from tuf.repository_tool import write_metadata_file from tuf.repository_tool import _metadata_is_partially_loaded +# import private functions to query for user input. +from tuf.repository_tool import _prompt +from tuf.repository_tool import _get_password + # See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger('tuf.devtools') + + + + # Recommended RSA key sizes: # http://www.emc.com/emc-plus/rsa-labs/historical/twirl-and-rsa-key-size.htm#table1 # According to the document above, revised May 6, 2003, RSA keys of @@ -542,44 +550,6 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, - -def _prompt(message, result_type=str): - """ - Non-public function that prompts the user for input by printing 'message', - converting the input to 'result_type', and returning the value to the - caller. - """ - - return result_type(raw_input(message)) - - - - - -def _get_password(prompt='Password: ', confirm=False): - """ - Non-public function that returns the password entered by the user. If - 'confirm' is True, the user is asked to enter the previously entered - password once again. If they match, the password is returned to the caller. - """ - - while True: - # getpass() prompts the user for a password without echoing - # the user input. - password = getpass.getpass(prompt, sys.stderr) - if not confirm: - return password - password2 = getpass.getpass('Confirm: ', sys.stderr) - if password == password2: - return password - else: - print('Mismatch; try again.') - - - - - - def create_new_project(metadata_directory, location_in_repository = '', targets_directory=None, key=None): """ From 9cb1181bc3996dbf6015e49ee48a8128cb3e5410 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Wed, 2 Apr 2014 23:09:35 -0400 Subject: [PATCH 26/51] Updated constant definitions, they are imported from tuf.developer_tool This makes it easier to propagate design changes through the two different tools. A tuf.constants module would be a good addition to this. --- tuf/developer_tool.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 01f85df7..d24601d8 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -76,21 +76,22 @@ # Recommended RSA key sizes: -# http://www.emc.com/emc-plus/rsa-labs/historical/twirl-and-rsa-key-size.htm#table1 -# According to the document above, revised May 6, 2003, RSA keys of -# size 3072 provide security through 2031 and beyond. 2048-bit keys -# are the recommended minimum and are good from the present through 2030. -DEFAULT_RSA_KEY_BITS = 3072 +# http://www.emc.com/emc-plus/rsa-labs/historical/twirl-and-rsa-key-size.htm#table1 +# According to the document above, revised May 6, 2003, RSA keys of size 3072 +# provide security through 2031 and beyond. 2048-bit keys are the recommended +# minimum and are good from the present through 2030. +from tuf.repository_tool import DEFAULT_RSA_KEY_BITS as DEFAULT_RSA_KEY_BITS # The algorithm used by the developer tools to generate the hashes of the # target filepaths. -HASH_FUNCTION = 'sha256' +from tuf.repository_tool import HASH_FUNCTION as HASH_FUNCTION # The extension of TUF metadata. -METADATA_EXTENSION = '.json' +from tuf.repository_tool import METADATA_EXTENSION as METADATA_EXTENSION + # The metadata filename for the targets metadata information. -TARGETS_FILENAME = 'targets' + METADATA_EXTENSION +from tuf.repository_tool import TARGETS_FILENAME as TARGETS_FILENAME # Project configuration filename. This file is intended to hold all of the # supporting information about the project that's not contained in a usual From 99b2ad40a7a69e17e0669e3e8a5f60aa1ed94b1e Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Wed, 2 Apr 2014 23:11:29 -0400 Subject: [PATCH 27/51] release_signable was renamed to project_signable release_signable didn't make too much sense in such context and was renamed to a more accurate and descriptive variable name. --- tuf/developer_tool.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index d24601d8..ab232e59 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -201,7 +201,7 @@ def __init__(self, metadata_directory, targets_directory, file_prefix): self._metadata_directory = metadata_directory self._targets_directory = targets_directory - + # layout type defaults to "flat" unless explicitly specified in # create_new_project self.layout_type = "flat" @@ -272,7 +272,7 @@ def write(self, write_partial=False): # Generate the 'targets.txt' metadata file. targets_filename = 'targets' + METADATA_EXTENSION targets_filename = os.path.join(self._metadata_directory, targets_filename) - release_signable, targets_filename = \ + project_signable, targets_filename = \ _generate_and_write_metadata('targets', targets_filename, write_partial, self._targets_directory, self._metadata_directory, @@ -286,7 +286,7 @@ def write(self, write_partial=False): # Delete the metadata of roles no longer in 'tuf.roledb'. Obsolete roles # may have been revoked. _delete_obsolete_metadata(self._metadata_directory, - release_signable['signed'], False) + project_signable['signed'], False) @@ -487,7 +487,7 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, # Retrieve the roleinfo of 'rolename' to extract the needed metadata # attributes, such as version number, expiration, etc. roleinfo = tuf.roledb.get_roleinfo(rolename) - #release_compressions = tuf.roledb.get_roleinfo('release')['compressions'] + metadata = generate_targets_metadata(targets_directory, roleinfo['paths'], roleinfo['version'], From a16dd6095ee1a15a47ef44c52836e2217d0bb285 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Wed, 2 Apr 2014 23:12:38 -0400 Subject: [PATCH 28/51] file open is now done with the "with" statement The with statement is a more "native" way to do the same thing, it is done that way not. --- tuf/developer_tool.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index ab232e59..752af11d 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -734,10 +734,6 @@ def save_project_configuration(metadata_directory,targets_directory, # is the file open-able? open for overwriting project_filename = os.path.join(cfg_file_directory,PROJECT_FILENAME) - try: - fp = open(project_filename,"wt") - except OSError, e: - raise # build the data structure project_config = {} @@ -755,7 +751,8 @@ def save_project_configuration(metadata_directory,targets_directory, project_config['public_keys'][key]['public'] = key_info['keyval']['public'] # save the actual data - json.dump(project_config,fp) + with open(project_filename,"wt") as fp: + json.dump(project_config,fp) # clean our mess fp.close() From 1a353c08d4277dbbd8543fae68845dcc8a7609ca Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Thu, 3 Apr 2014 13:56:26 -0400 Subject: [PATCH 29/51] Added support for project names. Project name is now an argument of the project constructor and the create_new project function. The project_name field defines the rolename to add to the roledb, the filename to create and adds a new field to the project.cfg file. --- tuf/developer_tool.py | 96 +++++++++++++++++++++++++++++-------------- 1 file changed, 65 insertions(+), 31 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 752af11d..791b2a70 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -120,6 +120,9 @@ # flat: the targets directory and the # metadata directory are located in different # paths +# +# project_name: The name of the current project, this value is used to +# match the resulting filename with the one in upstream PROJECT_FILENAME = 'project.cfg' # The targets and metadata directory names. Metadata files are written @@ -162,6 +165,10 @@ class Project(Targets): + project_name: + The name of the metadata file as it should be named in the upstream + repository. + metadata_directory: The metadata sub-directory contains the files of the top-level roles, including all roles delegated from 'targets.txt'. @@ -189,7 +196,8 @@ class Project(Targets): roles. """ - def __init__(self, metadata_directory, targets_directory, file_prefix): + def __init__(self, project_name ,metadata_directory, targets_directory, + file_prefix): # Do the arguments have the correct format? # Ensure the arguments have the appropriate number of objects and object @@ -198,16 +206,19 @@ def __init__(self, metadata_directory, targets_directory, file_prefix): tuf.formats.PATH_SCHEMA.check_match(metadata_directory) tuf.formats.PATH_SCHEMA.check_match(targets_directory) tuf.formats.PATH_SCHEMA.check_match(file_prefix) + tuf.formats.RELPATH_SCHEMA.check_match(project_name) self._metadata_directory = metadata_directory self._targets_directory = targets_directory + self._project_name = project_name # layout type defaults to "flat" unless explicitly specified in # create_new_project self.layout_type = "flat" - # Set the top-level role objects. - super(Project, self).__init__(self._targets_directory, 'targets') + # Set the top-level role objects, we set the rolename to be the project's + # name + super(Project, self).__init__(self._targets_directory, project_name) self.prefix = file_prefix @@ -252,7 +263,9 @@ def write(self, write_partial=False): # any of the top-level roles are missing signatures, keys, etc. # Write the metadata files of all the delegated roles. - delegated_rolenames = tuf.roledb.get_delegated_rolenames('targets') + delegated_rolenames = \ + tuf.roledb.get_delegated_rolenames(self._project_name) + for delegated_rolename in delegated_rolenames: roleinfo = tuf.roledb.get_roleinfo(delegated_rolename) delegated_filename = os.path.join(self._metadata_directory, @@ -270,18 +283,17 @@ def write(self, write_partial=False): # Generate the 'targets.txt' metadata file. - targets_filename = 'targets' + METADATA_EXTENSION + targets_filename = self._project_name + METADATA_EXTENSION targets_filename = os.path.join(self._metadata_directory, targets_filename) project_signable, targets_filename = \ - _generate_and_write_metadata('targets', targets_filename, write_partial, - self._targets_directory, - self._metadata_directory, - prefix=self.prefix) + _generate_and_write_metadata( self._project_name, targets_filename, + write_partial, self._targets_directory, + self._metadata_directory, prefix=self.prefix) #save some other information that is not stored in the project's metadata - save_project_configuration(self._metadata_directory,self._targets_directory, - self.keys, self.prefix, self.threshold, - self.layout_type) + _save_project_configuration(self._metadata_directory, + self._targets_directory, self.keys, self.prefix, self.threshold, + self.layout_type, self._project_name) # Delete the metadata of roles no longer in 'tuf.roledb'. Obsolete roles # may have been revoked. @@ -551,7 +563,7 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, -def create_new_project(metadata_directory, location_in_repository = '', +def create_new_project(project_name, metadata_directory, location_in_repository = '', targets_directory=None, key=None): """ @@ -561,7 +573,14 @@ def create_new_project(metadata_directory, location_in_repository = '', The project object returned can be directly modified to meet the designer's criteria and then written using the method project.write(). + The project name provided is the one that will be added to the resulting + metadata file as it should be named in upstream. + + project_name: + The name of the project as it should be called in upstream. For example + targets/unclaimed/django should have its project_name set to "django" + metadata_directory: The directory that will eventually hold the metadata and target files of the project. @@ -577,6 +596,9 @@ def create_new_project(metadata_directory, location_in_repository = '', used to sign metadata in a way that it matches the future location of the files. + For example, targets/unclaimed/django should have its project name set to + "targets/unclaimed" + key: The public key to verify the project's metadata. Projects can only handle one key with a threshold of one. If a project were to modify it's @@ -602,9 +624,10 @@ def create_new_project(metadata_directory, location_in_repository = '', # Raise 'tuf.FormatError' if there is a mismatch. tuf.formats.PATH_SCHEMA.check_match(metadata_directory) - # Do the same for the location in the repo, we first check for it to be - # something valid + # Do the same for the location in the repo and the project name, we must + # ensure they are valid pathnames. tuf.formats.PATH_SCHEMA.check_match(location_in_repository) + tuf.formats.RELPATH_SCHEMA.check_match(project_name) # for the targets directory we do the same, but first, let's find out what # layout the user needs, layout_type is a variable that is usually set to @@ -660,7 +683,7 @@ def create_new_project(metadata_directory, location_in_repository = '', # Create the bare bones project object, where only the top-level roles # have been set and contain default values (e.g., Root roles has a threshold # of 1, expires 1 year into the future, etc.) - project = Project(metadata_directory, targets_directory, + project = Project(project_name, metadata_directory, targets_directory, location_in_repository) # add the key to the project. @@ -677,8 +700,9 @@ def create_new_project(metadata_directory, location_in_repository = '', -def save_project_configuration(metadata_directory,targets_directory, - public_keys, prefix, threshold, layout_type): +def _save_project_configuration(metadata_directory,targets_directory, + public_keys, prefix, threshold, layout_type, + project_name): """ Persist the project's information in a file to provide the information @@ -705,12 +729,15 @@ def save_project_configuration(metadata_directory,targets_directory, targets and metadata directories, "repo-like" emulates the layout used by the repository tools + project_name: + The name given to the project, this sets the metadata filename so it + matches the one stored in upstream. + OSError may rise if the metadata_directory/project.cfg file exists and is non-writeable - tuf.FormatError are also expected if either the prefix or the metadata - directory are malformed + tuf.FormatError are also expected if any of the arguments are malformed. A project.cfg file is created or overwritten @@ -718,10 +745,12 @@ def save_project_configuration(metadata_directory,targets_directory, nothing """ + # schema check for metadata_directory and prefix tuf.formats.PATH_SCHEMA.check_match(metadata_directory) tuf.formats.PATH_SCHEMA.check_match(prefix) tuf.formats.PATH_SCHEMA.check_match(targets_directory) + tuf.formats.RELPATH_SCHEMA.check_match(project_name) # get the absolute filepath to our metadata_directory for consistency metadata_directory = os.path.abspath(metadata_directory) @@ -743,6 +772,8 @@ def save_project_configuration(metadata_directory,targets_directory, project_config['targets_location'] = targets_directory project_config['threshold'] = threshold project_config['layout_type'] = layout_type + project_config['project_name'] = project_name + # build a dictionary containing the actual keys for key in public_keys: key_info = tuf.keydb.get_key(key) @@ -770,6 +801,7 @@ def load_project(project_directory, prefix=''): project_directory: The path to the project's folder + prefix: the prefix for the metadata, if defined, it will replace the current prefix, by first removing the existing one (Saved) and setting the new @@ -809,10 +841,13 @@ def load_project(project_directory, prefix=''): targets_directory = project_configuration['targets_location'] if prefix=='': prefix = project_configuration['prefix'] - + + # load the projects filename. + project_name = project_configuration['project_name'] + project_filename = project_name + METADATA_EXTENSION # create a blank project on the target directory - project = Project(metadata_directory,targets_directory , prefix) + project = Project(project_name, metadata_directory, targets_directory, prefix) project.threshold = project_configuration['threshold'] project.prefix = project_configuration['prefix'] @@ -832,36 +867,35 @@ def load_project(project_directory, prefix=''): else: temp_pubkey = keydict project.add_verification_key(temp_pubkey) - - + # load the project's metadata - targets_metadata_path = os.path.join(metadata_directory, TARGETS_FILENAME) + targets_metadata_path = os.path.join(metadata_directory, project_filename) signable = tuf.util.load_json_file(targets_metadata_path) tuf.formats.check_signable_object_format(signable) targets_metadata = signable['signed'] # remove the prefix from the metadata targets_metadata = _strip_prefix_from_targets_metadata(targets_metadata, - project_configuration['prefix']) + prefix) for signature in signable['signatures']: project.add_signature(signature) # update roledb - roleinfo = tuf.roledb.get_roleinfo('targets') + roleinfo = tuf.roledb.get_roleinfo(project_name) roleinfo['signatures'].extend(signable['signatures']) roleinfo['version'] = targets_metadata['version'] roleinfo['paths'] = targets_metadata['targets'].keys() roleinfo['delegations'] = targets_metadata['delegations'] - roleinfo['partial_loaded'] = false + roleinfo['partial_loaded'] = False # check if the loaded metadata was partially written and update the # flag in the roledb - if _metadata_is_partially_loaded('targets',signable,roleinfo): + if _metadata_is_partially_loaded( project_name, signable, roleinfo): roleinfo['partial_loaded'] = True - tuf.roledb.update_roleinfo('targets',roleinfo) + tuf.roledb.update_roleinfo( project_name, roleinfo) @@ -920,7 +954,7 @@ def load_project(project_directory, prefix=''): # strip the extension from the metadata_object = signable['signed'] metadata_object = _strip_prefix_from_targets_metadata(metadata_object, - project_configuration['prefix']) + prefix) roleinfo = tuf.roledb.get_roleinfo(metadata_name) roleinfo['signatures'].extend(signable['signatures']) From 30b97a8a7a03b687f79298a9f7f4ccf1303131a0 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Fri, 4 Apr 2014 14:08:17 -0400 Subject: [PATCH 30/51] Changed the logger tag tuf.devtools -> tuf.developer_tool --- tuf/developer_tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 791b2a70..b34e2afc 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -69,7 +69,7 @@ from tuf.repository_tool import _get_password # See 'log.py' to learn how logging is handled in TUF. -logger = logging.getLogger('tuf.devtools') +logger = logging.getLogger('tuf.developer_tool') From ab765d6c85644013e04ff1d68d2bde9408b93864 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Fri, 4 Apr 2014 14:36:01 -0400 Subject: [PATCH 31/51] Ran a spellchecker on the readme document Also did a minor proofreading effort. A more thorough proofread will me made after finishing the whole document. --- tuf/README-developer-tools.md | 41 ++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/tuf/README-developer-tools.md b/tuf/README-developer-tools.md index 9ed53be5..f1d55065 100644 --- a/tuf/README-developer-tools.md +++ b/tuf/README-developer-tools.md @@ -14,20 +14,20 @@ - [Keys and thresholds](#keys_and_thresholds) -## Overview ## +## Overview The TUF developer tool is a Python library that enables developers to create -and maintain the required metadata for diles hosted in a TUF Repository. This +and maintain the required metadata for files hosted in a TUF Repository. This document has two parts. The first part walks through the creation of a prototypal TUF project. The second part demonstrates the full capabilities of -the TUF developer tool, which can be userd to expand the project from the first +the TUF developer tool, which can be users to expand the project from the first part to meet the developer''s needs. ## Creating a Simple project ## ### Generating a Key ### -First, uou will needd to generate a key to sign the metadata. Keys are generated -in pairs: one public and the other private. the private key is password-protected -and is used to sign metadata. The public key can be shared freely, andi s used +First, you will need to generate a key to sign the metadata. Keys are generated +in pairs: one public and the other private. The private key is password-protected +and is used to sign metadata. The public key can be shared freely, and is used to verify signatures made by the private key. The generate_and_write_rsa_keypair function will create two key files in the @@ -47,12 +47,12 @@ Confirm: ### The project class ### TUF-dev is built around the Project class, which is used to organize groups of targets associated with a single set of metadata. Each Project instance keeps -track of which target files ar associated with a single set of metadata. Each +track of which target files are associated with a single set of metadata. Each Project instance keeps track of which target files are signed and which need signing, which keys are used to sign metadata. It also keeps track of delegated -rolse, which are covered later. +roles, which are covered later. -Before creating a project, you must know hwere it be located in the TUF +Before creating a project, you must know where it will be located in the TUF Repository. In the following example, we will create a project to be hosted as "repo/example_project" within the repository, and store a local copy of the metadata at "path/to/metadata". The project will comprise a single target file, @@ -60,7 +60,7 @@ metadata at "path/to/metadata". The project will comprise a single target file, the key generated above. ``` ->>> public_key = import_rsa_publickey_from_file("path/to/key.pub") +>>> public_key = import_rsa_publickey_from_file("path/to/keys.pub") >>> project = create_new_project(metadata_directory="local/path/to/metadata/", ... targets_directory="local/path/to/example_project", @@ -73,7 +73,7 @@ but we have not *signed* it with that key. ### Signing and writing the metadata ### -In order to signe the metadata, we need to impot the private key corresponding +In order to sign the metadata, we need to import the private key corresponding to the public key we added to the project. One the key is loaded to the project, it will automatically be used to sign the metadata whenever it is written. @@ -110,7 +110,7 @@ Enter a password for the RSA key: ## Managing keys -When generating keys, it is possible to specity the length of the key in bits +When generating keys, it is possible to specify the length of the key in bits and its password as parameters: ``` @@ -124,8 +124,10 @@ The password parameter is only intended to be used in scripts. ``` ->>> list_of_targets = project.get_filepaths_in_directory(“path/within/targets/folder”, recursive_walk=False, follow_links=False) ->>> project.add_targets(list_of_targets) +>>> list_of_targets = \ +... project.get_filepaths_in_directory(“path/within/targets/folder”, +... recursive_walk=False, follow_links=False) +... project.add_targets(list_of_targets) ``` ``` @@ -134,7 +136,9 @@ The password parameter is only intended to be used in scripts. ## Delegations -The project we created above is secured entirely by one key. If you want to allow someone else to update part of your project independently, you will need to delegate a new role for them. For example, we can +The project we created above is secured entirely by one key. If you want to +allow someone else to update part of your project independently, you will need +to delegate a new role for them. For example, we can ``` >>> other_key = import_rsa_publickey_from_file(“sombodys_public_key.pub”) @@ -142,7 +146,8 @@ The project we created above is secured entirely by one key. If you want to allo >>> project.delegate(“newrole”, [other_key], targets) ``` -The new role is now an attribute of the Project instance, and contains the same methods as Project. For example, we can add targets in the same way as before: +The new role is now an attribute of the Project instance, and contains the same +methods as Project. For example, we can add targets in the same way as before: ``` @@ -152,5 +157,7 @@ The new role is now an attribute of the Project instance, and contains the same -Recall that we input the other person’s key as part of a list. That list can contain any number of public keys. You can also add keys to the role after creating it using the add_signing_key() method. +Recall that we input the other person’s key as part of a list. That list can +contain any number of public keys. You can also add keys to the role after +creating it using the add_signing_key() method. From c35d50e310db7099541275507f9d2ef64449dc0c Mon Sep 17 00:00:00 2001 From: SantiagoTorres Date: Fri, 4 Apr 2014 17:27:47 -0400 Subject: [PATCH 32/51] Update README-developer-tools.md Updated the document with mentions to project names and to fix general redaction issues. --- tuf/README-developer-tools.md | 133 ++++++++++++++++++++++++++++------ 1 file changed, 110 insertions(+), 23 deletions(-) diff --git a/tuf/README-developer-tools.md b/tuf/README-developer-tools.md index f1d55065..802601b5 100644 --- a/tuf/README-developer-tools.md +++ b/tuf/README-developer-tools.md @@ -10,29 +10,37 @@ - [Managing keys](#managing_keys) - [Managing targets](#managing_targets) - [Delegations](#delegations) - - [Restricted paths](#restricted_paths) - - [Keys and thresholds](#keys_and_thresholds) ## Overview The TUF developer tool is a Python library that enables developers to create -and maintain the required metadata for files hosted in a TUF Repository. This -document has two parts. The first part walks through the creation of a +and maintain the required metadata for files hosted in a TUF Repository. The main +concern when generating metadata for a TUF repository is generating information +that matches the future location of the files in the repository. We use +the developer tools to generate valid information so that the project and it's +metadata can be applied to the TUF project transparently. + +This document has two parts. The first part walks through the creation of a prototypal TUF project. The second part demonstrates the full capabilities of -the TUF developer tool, which can be users to expand the project from the first -part to meet the developer''s needs. +the TUF developer tool, which can be used to expand the project from the first +part to meet the developer's needs. + + ## Creating a Simple project ## +The following section describes a very basic example usage of the developer tools with +a one-file project. + ### Generating a Key ### First, you will need to generate a key to sign the metadata. Keys are generated in pairs: one public and the other private. The private key is password-protected and is used to sign metadata. The public key can be shared freely, and is used to verify signatures made by the private key. -The generate_and_write_rsa_keypair function will create two key files in the -path (path/to/) and named "key.pub", which is the public key and "key" -which is the private key. +The generate\_and\_write\_rsa\_keypair function will create two key files +named "path/to/key.pub", which is the public key and "path/to/key", which +is the private key. ``` >>> from tuf.developer_tool import * @@ -43,9 +51,17 @@ Confirm: >>> ``` +We can also use the bits parameter to set a different key length (the default is +3072). We can also provide the password parameter in order to suppress the password +prompt. + +During this example we will be using rsa keys, but ed25519 keys are also supported. + +Now we have a key for our project, we can proceed to create our project. + ### The project class ### -TUF-dev is built around the Project class, which is used to organize groups of +The TUF developer tool is built around the Project class, which is used to organize groups of targets associated with a single set of metadata. Each Project instance keeps track of which target files are associated with a single set of metadata. Each Project instance keeps track of which target files are signed and which need @@ -54,22 +70,49 @@ roles, which are covered later. Before creating a project, you must know where it will be located in the TUF Repository. In the following example, we will create a project to be hosted as -"repo/example_project" within the repository, and store a local copy of the +"repo/unclaimed/example_project" within the repository, and store a local copy of the metadata at "path/to/metadata". The project will comprise a single target file, -"local/path/to/example_project/target_1" locally, and we will secure it with +"local/path/to/example\_project/target\_1" locally, and we will secure it with the key generated above. +First, we must import the generated keys. We can do that by issuing the following: + ``` >>> public_key = import_rsa_publickey_from_file("path/to/keys.pub") +``` ->>> project = create_new_project(metadata_directory="local/path/to/metadata/", -... targets_directory="local/path/to/example_project", -... location_in_repository="repo/example_project", key=public_key) +After importing the key, we can generate a new project with the following command +``` +>>> project = create_new_project(name="example_project", +... metadata_directory="local/path/to/metadata/", +... targets_directory="local/path/to/example_project", +... location_in_repository="repo/unclaimed", key=public_key) +``` +Let's list the arguments and make sense out of this rather long function call: + +- create a project named example_project: the name of the metadata file will match this name +- the metadata will be located in "local/path/to/metadata", this means all of the generated files +for this project will be located here +- the targets are located in local/path/to/example project. If your targets are located in some other +place, you can point the targets directory there. Files must reside under the path local/path/to/example_project or else it won't be possible to add them. +- location\_in\_repository points to repo/unclaimed, this will be prepended to the paths in the generated metadata so the signatures all match. + +Now the project is in memory and we can do different operations on it such as adding and +removing targets, delegating files, changing signatures and keys, etc. For the moment we are +interested in adding our one and only target inside the project. + +To add a target, we issue the following method: +``` >>> project.add_target("target_1") ``` +Have in mind the file "target\_1" should be located in "local/path/to/example\_project" +or else the adding procedure will throw an error. + At this point, the metadata is not valid. We have assigned a key to the project, -but we have not *signed* it with that key. +but we have not *signed* it with that key. Signing is the process of generating +a signature with our private key so it can be verified with the public key by the +server (upon uploading) and by the clients (when updating). ### Signing and writing the metadata ### @@ -85,11 +128,15 @@ Enter password for the RSA key: ``` When all changes to a project have been written, the Project instance can safely -be deleted. +be deleted. + +The project can be loaded later to update changes to the project. The metadata +contains checksums that have to match the actual files or else it won't be accepted +by the upstream repository. To make changes to existing metadata, we will need the Project again. We can -restore it with the load_project() function. +restore it with the load_project() function. ``` >>> from tuf.developer_tool import * @@ -108,8 +155,16 @@ Enter a password for the RSA key: >>> project.write() ``` +Now we have a project properly setup. The rest of this guide contains a more +in-depth description of the functions of the developer\_tool. +=== + ## Managing keys +This section describes the key-related functions and parameters that weren't +mentioned inside the example: + +### Additional parameters for key generation When generating keys, it is possible to specify the length of the key in bits and its password as parameters: @@ -119,9 +174,37 @@ and its password as parameters: The bits parameter defaults to 3072, and values below 2048 will raise an error. The password parameter is only intended to be used in scripts. +### Removing a key from a project/delegation +Removing a verification key is really simple, we should only issue the following +command: + +``` +>>> project.remove_verification_key(key) +>>> +``` + +### Adding a key to a project/delegation +Likewise, it is possible to add a key to a project by issuing the following command: +``` +>>> project.add_verification_key(pubkey) +>>> +``` +Remember that a project can only have one key, so this method will return an error +if there is already a key assigned to it. In order to replace a key we must first +delete the existing one and then add the new one. It is possible to +ommit the key parameter in the create\_new\_project function, and add the key +later. + ## Managing Targets +There are supporting functions of the targets library to make the project +maintenance easier. These functions are described in this section. + +### get\_filepaths\_in\_directory() +This function is specially useful when creating a new project to add all the files +contained in the targets directory. The following code block illustrates the usage +of this function: ``` >>> list_of_targets = \ @@ -130,18 +213,24 @@ The password parameter is only intended to be used in scripts. ... project.add_targets(list_of_targets) ``` + +### deleting targets from a project +It is possible that we want to delete existing targets inside our project. In order +to stop the developer tool to track this file we must issue the following command: ``` >>> project.remove_target(“target_1”) ``` +Now the target file won't be part of the metadata. + ## Delegations The project we created above is secured entirely by one key. If you want to allow someone else to update part of your project independently, you will need -to delegate a new role for them. For example, we can +to delegate a new role for them. For example, we can do the following: ``` ->>> other_key = import_rsa_publickey_from_file(“sombodys_public_key.pub”) +>>> other_key = import_rsa_publickey_from_file(“another_public_key.pub”) >>> project.delegate(“newrole”, [other_key], targets) ``` @@ -155,9 +244,7 @@ methods as Project. For example, we can add targets in the same way as before: ``` - - Recall that we input the other person’s key as part of a list. That list can contain any number of public keys. You can also add keys to the role after -creating it using the add_signing_key() method. +creating it using the add\_verification\_key() method. From 69c88e2328b8ea7e0b89382dd82986d51d9a0b84 Mon Sep 17 00:00:00 2001 From: zanefisher Date: Thu, 17 Apr 2014 15:43:43 -0400 Subject: [PATCH 33/51] fixes to headers and links --- tuf/README-developer-tools.md | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/tuf/README-developer-tools.md b/tuf/README-developer-tools.md index 802601b5..e06db4ec 100644 --- a/tuf/README-developer-tools.md +++ b/tuf/README-developer-tools.md @@ -2,13 +2,13 @@ ## Table of Contents ## - [Overview](#overview) -- [Creating a simple project](#creating_a_simple_project) - - [Generating a key](#generating_a_key) - - [The Project class](#the_project_class) - - [Signing and writing metadata](#signing_and_writing_metadata) -- [Loading an Existing project](#Loading_an_existing_project) -- [Managing keys](#managing_keys) -- [Managing targets](#managing_targets) +- [Creating a Simple Project](#creating_a_simple_project) + - [Generating a Key](#generating_a_key) + - [The Project Class](#the_project_class) + - [Signing and Writing the Metadata](#signing_and_writing_the_metadata) +- [Loading an Existing Project](#loading_an_existing_project) +- [Managing Keys](#managing_keys) +- [Managing Targets](#managing_targets) - [Delegations](#delegations) @@ -28,11 +28,12 @@ part to meet the developer's needs. -## Creating a Simple project ## +## Creating a Simple project The following section describes a very basic example usage of the developer tools with a one-file project. -### Generating a Key ### + +### Generating a Key First, you will need to generate a key to sign the metadata. Keys are generated in pairs: one public and the other private. The private key is password-protected and is used to sign metadata. The public key can be shared freely, and is used @@ -60,7 +61,7 @@ During this example we will be using rsa keys, but ed25519 keys are also support Now we have a key for our project, we can proceed to create our project. -### The project class ### +### The Project Class The TUF developer tool is built around the Project class, which is used to organize groups of targets associated with a single set of metadata. Each Project instance keeps track of which target files are associated with a single set of metadata. Each @@ -115,7 +116,7 @@ a signature with our private key so it can be verified with the public key by th server (upon uploading) and by the clients (when updating). -### Signing and writing the metadata ### +### Signing and Writing the Metadata ### In order to sign the metadata, we need to import the private key corresponding to the public key we added to the project. One the key is loaded to the project, it will automatically be used to sign the metadata whenever it is written. @@ -135,6 +136,7 @@ contains checksums that have to match the actual files or else it won't be accep by the upstream repository. +## Loading an Existing Project To make changes to existing metadata, we will need the Project again. We can restore it with the load_project() function. @@ -157,14 +159,13 @@ Enter a password for the RSA key: Now we have a project properly setup. The rest of this guide contains a more in-depth description of the functions of the developer\_tool. -=== -## Managing keys +## Managing Keys This section describes the key-related functions and parameters that weren't mentioned inside the example: -### Additional parameters for key generation +### Additional Parameters for Key Generation When generating keys, it is possible to specify the length of the key in bits and its password as parameters: @@ -174,7 +175,7 @@ and its password as parameters: The bits parameter defaults to 3072, and values below 2048 will raise an error. The password parameter is only intended to be used in scripts. -### Removing a key from a project/delegation +### Removing a Key from a Project or Delegation Removing a verification key is really simple, we should only issue the following command: @@ -183,7 +184,7 @@ command: >>> ``` -### Adding a key to a project/delegation +### Adding a Key to a Project or Delegation Likewise, it is possible to add a key to a project by issuing the following command: ``` >>> project.add_verification_key(pubkey) @@ -195,7 +196,7 @@ delete the existing one and then add the new one. It is possible to ommit the key parameter in the create\_new\_project function, and add the key later. - + ## Managing Targets There are supporting functions of the targets library to make the project maintenance easier. These functions are described in this section. From b89bb02730c99f9e6fd262271da7a3a3f5a33792 Mon Sep 17 00:00:00 2001 From: zanefisher Date: Thu, 17 Apr 2014 15:51:26 -0400 Subject: [PATCH 34/51] remove whitespace and empty lines from examples --- tuf/README-developer-tools.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tuf/README-developer-tools.md b/tuf/README-developer-tools.md index e06db4ec..2fca8131 100644 --- a/tuf/README-developer-tools.md +++ b/tuf/README-developer-tools.md @@ -45,7 +45,6 @@ is the private key. ``` >>> from tuf.developer_tool import * - >>> generate_and_write_rsa_keypair("path/to/key") Enter a password for the RSA key: Confirm: @@ -143,7 +142,6 @@ restore it with the load_project() function. ``` >>> from tuf.developer_tool import * >>> project = load_project("local/path/to/metadata") ->>> ``` Each time the project is loaded anew, the necessary private keys must also be loaded in order to sign metadata. @@ -151,9 +149,7 @@ loaded in order to sign metadata. ``` >>> private_key = import_rsa_privatekey_from_file("path/to/key") Enter a password for the RSA key: - >>> project.load_signing_key(private_key) - >>> project.write() ``` @@ -181,14 +177,12 @@ command: ``` >>> project.remove_verification_key(key) ->>> ``` ### Adding a Key to a Project or Delegation Likewise, it is possible to add a key to a project by issuing the following command: ``` >>> project.add_verification_key(pubkey) ->>> ``` Remember that a project can only have one key, so this method will return an error if there is already a key assigned to it. In order to replace a key we must first @@ -232,7 +226,6 @@ to delegate a new role for them. For example, we can do the following: ``` >>> other_key = import_rsa_publickey_from_file(“another_public_key.pub”) - >>> project.delegate(“newrole”, [other_key], targets) ``` @@ -240,9 +233,7 @@ The new role is now an attribute of the Project instance, and contains the same methods as Project. For example, we can add targets in the same way as before: ``` - >>> project(“newrole”).add_target(“delegated_1”) - ``` Recall that we input the other person’s key as part of a list. That list can From a4c483f5c67cc12cfb417876cd99a52bec43043a Mon Sep 17 00:00:00 2001 From: zanefisher Date: Thu, 17 Apr 2014 16:19:20 -0400 Subject: [PATCH 35/51] Add headers for sections to be added. --- tuf/README-developer-tools.md | 137 ++++++++++++++++++---------------- 1 file changed, 74 insertions(+), 63 deletions(-) diff --git a/tuf/README-developer-tools.md b/tuf/README-developer-tools.md index 2fca8131..d949e95e 100644 --- a/tuf/README-developer-tools.md +++ b/tuf/README-developer-tools.md @@ -7,9 +7,9 @@ - [The Project Class](#the_project_class) - [Signing and Writing the Metadata](#signing_and_writing_the_metadata) - [Loading an Existing Project](#loading_an_existing_project) +- [Delegations](#delegations) - [Managing Keys](#managing_keys) - [Managing Targets](#managing_targets) -- [Delegations](#delegations) ## Overview @@ -156,68 +156,7 @@ Enter a password for the RSA key: Now we have a project properly setup. The rest of this guide contains a more in-depth description of the functions of the developer\_tool. - -## Managing Keys -This section describes the key-related functions and parameters that weren't -mentioned inside the example: - -### Additional Parameters for Key Generation -When generating keys, it is possible to specify the length of the key in bits -and its password as parameters: - -``` ->>> generate_and_write_rsa_keypair("path/to/key",bits=2048, password="pw") -``` -The bits parameter defaults to 3072, and values below 2048 will raise an error. -The password parameter is only intended to be used in scripts. - -### Removing a Key from a Project or Delegation -Removing a verification key is really simple, we should only issue the following -command: - -``` ->>> project.remove_verification_key(key) -``` - -### Adding a Key to a Project or Delegation -Likewise, it is possible to add a key to a project by issuing the following command: -``` ->>> project.add_verification_key(pubkey) -``` -Remember that a project can only have one key, so this method will return an error -if there is already a key assigned to it. In order to replace a key we must first -delete the existing one and then add the new one. It is possible to -ommit the key parameter in the create\_new\_project function, and add the key -later. - - -## Managing Targets -There are supporting functions of the targets library to make the project -maintenance easier. These functions are described in this section. - - -### get\_filepaths\_in\_directory() -This function is specially useful when creating a new project to add all the files -contained in the targets directory. The following code block illustrates the usage -of this function: -``` - ->>> list_of_targets = \ -... project.get_filepaths_in_directory(“path/within/targets/folder”, -... recursive_walk=False, follow_links=False) -... project.add_targets(list_of_targets) -``` - - -### deleting targets from a project -It is possible that we want to delete existing targets inside our project. In order -to stop the developer tool to track this file we must issue the following command: -``` ->>> project.remove_target(“target_1”) -``` -Now the target file won't be part of the metadata. - - + ## Delegations The project we created above is secured entirely by one key. If you want to @@ -240,3 +179,75 @@ Recall that we input the other person’s key as part of a list. That list can contain any number of public keys. You can also add keys to the role after creating it using the add\_verification\_key() method. +### Restricted Paths + +### Nested Delegations + +### Revoking Delegations + + +## Managing Keys +This section describes the key-related functions and parameters that weren't +mentioned inside the example: + +### Additional Parameters for Key Generation +When generating keys, it is possible to specify the length of the key in bits +and its password as parameters: + +``` +>>> generate_and_write_rsa_keypair("path/to/key",bits=2048, password="pw") +``` +The bits parameter defaults to 3072, and values below 2048 will raise an error. +The password parameter is only intended to be used in scripts. + +### Adding a Key to a Delegation +Likewise, it is possible to add a key to a project by issuing the following command: +``` +>>> project.add_verification_key(pubkey) +``` + +### Removing a Key from a Delegation +Removing a verification key is really simple, we should only issue the following +command: + +``` +>>> project.remove_verification_key(key) +``` + +Remember that a project can only have one key, so this method will return an error +if there is already a key assigned to it. In order to replace a key we must first +delete the existing one and then add the new one. It is possible to +ommit the key parameter in the create\_new\_project function, and add the key +later. + +### Changing the Project Key + +### Delegation Thresholds + + +## Managing Targets +There are supporting functions of the targets library to make the project +maintenance easier. These functions are described in this section. + + +### Adding Targets by Directory +This function is specially useful when creating a new project to add all the files +contained in the targets directory. The following code block illustrates the usage +of this function: +``` + +>>> list_of_targets = \ +... project.get_filepaths_in_directory(“path/within/targets/folder”, +... recursive_walk=False, follow_links=False) +>>> project.add_targets(list_of_targets) +``` + + +### Deleting Targets from a Project +It is possible that we want to delete existing targets inside our project. In order +to stop the developer tool to track this file we must issue the following command: +``` +>>> project.remove_target(“target_1”) +``` +Now the target file won't be part of the metadata. + From c29b362df71a980f50232d4df54d9ec6e3d81174 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 22 Apr 2014 14:24:13 -0400 Subject: [PATCH 36/51] Updated function import for get_metadata_fileinfo get_metadata_fileinfo was originally called get_metadata_file_info. This was fixed to match the new repository tool. --- tuf/developer_tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index b34e2afc..94374762 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -48,7 +48,7 @@ import tuf.repository_tool from tuf.repository_tool import Targets -from tuf.repository_tool import get_metadata_file_info +from tuf.repository_tool import get_metadata_fileinfo from tuf.repository_tool import get_metadata_filenames from tuf.repository_tool import generate_and_write_rsa_keypair from tuf.repository_tool import import_rsa_publickey_from_file From 7153c34eed92fa1f60cfa9c1fde394a59c2d0ccd Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Thu, 24 Apr 2014 18:48:20 -0400 Subject: [PATCH 37/51] Added roledb and keydb clear statements upon load To avoid conflicts in the load_project function, we clear the roledb and the keydb modules. This ensures that there are no repeated entries from previous runs. --- tuf/developer_tool.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 94374762..87ca5c25 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -825,6 +825,9 @@ def load_project(project_directory, prefix=''): # do the same for the prefix tuf.formats.PATH_SCHEMA.check_match(prefix) + # clear the role and keydbs + tuf.roledb.clear_roledb() + tuf.keydb.clear_keydb() # Locate metadata filepaths and targets filepath. project_directory = os.path.abspath(project_directory) From 559f8d796d0aef0a8165edd7c325b0f59fcfeb96 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Thu, 24 Apr 2014 18:49:34 -0400 Subject: [PATCH 38/51] Updated config.cfg file structure The public key for the whole project used to be stored in a non-standard way (this means, not as a keydict schema). I migrated the data structure to a valid keydict schema with the aim of improving code-consistency in the developer tools. This will also make schema checks easier --- tuf/developer_tool.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 87ca5c25..1e90e449 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -47,6 +47,9 @@ import tuf.conf import tuf.repository_tool +from tuf.keys import format_keyval_to_metadata +from tuf.keys import format_metadata_to_key + from tuf.repository_tool import Targets from tuf.repository_tool import get_metadata_fileinfo from tuf.repository_tool import get_metadata_filenames @@ -777,9 +780,9 @@ def _save_project_configuration(metadata_directory,targets_directory, # build a dictionary containing the actual keys for key in public_keys: key_info = tuf.keydb.get_key(key) - project_config['public_keys'][key] = {} - project_config['public_keys'][key]['keytype'] = key_info['keytype'] - project_config['public_keys'][key]['public'] = key_info['keyval']['public'] + key_metadata = format_keyval_to_metadata(key_info['keytype'], + key_info['keyval']) + project_config['public_keys'][key] = key_metadata # save the actual data with open(project_filename,"wt") as fp: @@ -859,17 +862,8 @@ def load_project(project_directory, prefix=''): # traverse the public keys and add them to the project keydict = project_configuration['public_keys'] for keyid in keydict: - if keydict[keyid]['keytype'] == 'rsa': - temp_pubkey = tuf.keys.format_rsakey_from_pem(keydict[keyid]['public']) - elif keydict[keyid]['keytype'] == 'ed25519': - temp_pubkey = {} - temp_pubkey['keytype'] = keydict[keyid]['keytype'] - temp_pubkey['keyval'] = {} - temp_pubkey['keyval']['public'] = keydict[keyid]['public'] - temp_pubkey['keyval']['private'] = '' - else: - temp_pubkey = keydict - project.add_verification_key(temp_pubkey) + key = format_metadata_to_key(keydict[keyid]) + project.add_verification_key(key) # load the project's metadata targets_metadata_path = os.path.join(metadata_directory, project_filename) From 36c3f2c1d2766f3d9fb25616d8647ed969d0dd00 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Thu, 24 Apr 2014 18:51:35 -0400 Subject: [PATCH 39/51] Added schema check and project.cfg schema There is a new entry in tuf.formats.py that represents the layout of the project.cfg file. This new schema will be used to check for sanity upon loading the .cfg file in the load_project function --- tuf/developer_tool.py | 2 ++ tuf/formats.py | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 1e90e449..e78821a9 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -840,6 +840,8 @@ def load_project(project_directory, prefix=''): config_filename = os.path.join(project_directory,PROJECT_FILENAME) try: project_configuration = tuf.util.load_json_file(config_filename) + tuf.formats.PROJECT_CFG_SCHEMA.check_match(project_configuration) + except OSError, e: raise diff --git a/tuf/formats.py b/tuf/formats.py index 2544312b..fbbfa8fa 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -428,6 +428,18 @@ expires = ISO8601_DATETIME_SCHEMA, meta = FILEDICT_SCHEMA) +# project.cfg file: stores information about the project in a json dictionary +PROJECT_CFG_SCHEMA = SCHEMA.Object( + object_name = 'PROJECT_CFG_SCHEMA', + project_name = SCHEMA.AnyString(), + layout_type = SCHEMA.OneOf([SCHEMA.String('repo-like'), SCHEMA.String('flat')]), + targets_location = PATH_SCHEMA, + metadata_location = PATH_SCHEMA, + prefix = PATH_SCHEMA, + public_keys = KEYDICT_SCHEMA, + threshold = SCHEMA.Integer(lo = 0, hi = 2) + ) + # A schema containing information a repository mirror may require, # such as a url, the path of the directory metadata files, etc. MIRROR_SCHEMA = SCHEMA.Object( From f50955ee79920bc71af3e60f6e40c4c533c914be Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Fri, 25 Apr 2014 23:55:00 -0400 Subject: [PATCH 40/51] Added import statements for constant definitions Metadata and targets directory names were redeclared in a local variable. They are now imported from the repository_tool. I also removed an unnecessary declaration for the targets.json file expiration since it didn't have anything to do with the developer tools --- tuf/developer_tool.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index e78821a9..6e26a26d 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -130,30 +130,21 @@ # The targets and metadata directory names. Metadata files are written # to the staged metadata directory instead of the "live" one. -METADATA_STAGED_DIRECTORY_NAME = 'metadata.staged' -METADATA_DIRECTORY_NAME = 'metadata' -TARGETS_DIRECTORY_NAME = 'targets' +from tuf.repository_tool import METADATA_STAGED_DIRECTORY_NAME +from tuf.repository_tool import METADATA_DIRECTORY_NAME +from tuf.repository_tool import TARGETS_DIRECTORY_NAME # The full list of supported TUF metadata extensions. -METADATA_EXTENSIONS = ['.json', '.json.gz'] +from tuf.repository_tool import METADATA_EXTENSIONS # The recognized compression extensions. -SUPPORTED_COMPRESSION_EXTENSIONS = ['.gz'] +from tuf.repository_tool import SUPPORTED_COMPRESSION_EXTENSIONS + # Supported key types. -SUPPORTED_KEY_TYPES = ['rsa', 'ed25519'] - -# Expiration date delta, in seconds, of the top-level roles. A metadata -# expiration date is set by taking the current time and adding the expiration -# seconds listed below. - -# Initial 'targets.txt' expiration time of 3 months. -TARGETS_EXPIRATION = 7889230 +from tuf.repository_tool import SUPPORTED_KEY_TYPES - -# TODO: We should have a method like Describe to show the contents of the -# instance class Project(Targets): """ From be320acf300a031d06d6f6fa81a629c489093111 Mon Sep 17 00:00:00 2001 From: zanefisher Date: Wed, 30 Apr 2014 16:44:15 -0400 Subject: [PATCH 41/51] Expand Delegations section. --- tuf/README-developer-tools.md | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tuf/README-developer-tools.md b/tuf/README-developer-tools.md index d949e95e..c8020baa 100644 --- a/tuf/README-developer-tools.md +++ b/tuf/README-developer-tools.md @@ -25,8 +25,6 @@ prototypal TUF project. The second part demonstrates the full capabilities of the TUF developer tool, which can be used to expand the project from the first part to meet the developer's needs. - - ## Creating a Simple project The following section describes a very basic example usage of the developer tools with @@ -177,12 +175,30 @@ methods as Project. For example, we can add targets in the same way as before: Recall that we input the other person’s key as part of a list. That list can contain any number of public keys. You can also add keys to the role after -creating it using the add\_verification\_key() method. +creating it using the [add\_verification\_key()](#adding_a_key_to_a_delegation) method. ### Restricted Paths +By default, a delegated role is permitted to add and modify targets anywhere in the +Project's targets directory. We can assign restricted paths to a delegated role to +limit this permission. + +``` +>>> project.add_restricted_paths(["restricted/filepath"], "newrole") +``` + +This will prevent the delegated role from signing targets whose local filepaths do not +begin with "restricted/filepath". We can assign several restricted filepaths to a role +by adding them to the list in the first parameter, or by invoking the method again. A +role with multiple restricted paths can add targets to any of them. + +Note that this method is invoked the parent role (in this case, the project) and takes +the delegated role name as an argument. + ### Nested Delegations + + ### Revoking Delegations @@ -200,6 +216,7 @@ and its password as parameters: The bits parameter defaults to 3072, and values below 2048 will raise an error. The password parameter is only intended to be used in scripts. + ### Adding a Key to a Delegation Likewise, it is possible to add a key to a project by issuing the following command: ``` @@ -229,7 +246,6 @@ later. There are supporting functions of the targets library to make the project maintenance easier. These functions are described in this section. - ### Adding Targets by Directory This function is specially useful when creating a new project to add all the files contained in the targets directory. The following code block illustrates the usage @@ -242,7 +258,6 @@ of this function: >>> project.add_targets(list_of_targets) ``` - ### Deleting Targets from a Project It is possible that we want to delete existing targets inside our project. In order to stop the developer tool to track this file we must issue the following command: From 89b1b54f46171ba6cfd9e83828878a6a272917af Mon Sep 17 00:00:00 2001 From: zanefisher Date: Tue, 3 Jun 2014 15:23:25 -0400 Subject: [PATCH 42/51] complete first draft to developer tool document --- tuf/README-developer-tools.md | 214 +++++++++++++++++++++------------- 1 file changed, 136 insertions(+), 78 deletions(-) diff --git a/tuf/README-developer-tools.md b/tuf/README-developer-tools.md index c8020baa..43d9c493 100644 --- a/tuf/README-developer-tools.md +++ b/tuf/README-developer-tools.md @@ -13,32 +13,32 @@ ## Overview -The TUF developer tool is a Python library that enables developers to create -and maintain the required metadata for files hosted in a TUF Repository. The main -concern when generating metadata for a TUF repository is generating information -that matches the future location of the files in the repository. We use -the developer tools to generate valid information so that the project and it's -metadata can be applied to the TUF project transparently. +The TUF developer tool is a Python library that enables developers to create +and maintain the required metadata for files hosted in a TUF Repository. The +main concern when generating metadata for a TUF repository is generating +information that matches the future location of the files in the repository. We +use the developer tools to generate valid information so that the project and +its metadata can be applied to the TUF project transparently. This document has two parts. The first part walks through the creation of a -prototypal TUF project. The second part demonstrates the full capabilities of +prototypal TUF project. The second part demonstrates the full capabilities of the TUF developer tool, which can be used to expand the project from the first part to meet the developer's needs. ## Creating a Simple project -The following section describes a very basic example usage of the developer tools with -a one-file project. +The following section describes a very basic example usage of the developer +tools with a one-file project. ### Generating a Key -First, you will need to generate a key to sign the metadata. Keys are generated -in pairs: one public and the other private. The private key is password-protected -and is used to sign metadata. The public key can be shared freely, and is used -to verify signatures made by the private key. +First, we will need to generate a key to sign the metadata. Keys are generated +in pairs: one public and the other private. The private key is +password-protected and is used to sign metadata. The public key can be shared +freely, and is used to verify signatures made by the private key. -The generate\_and\_write\_rsa\_keypair function will create two key files -named "path/to/key.pub", which is the public key and "path/to/key", which +The generate\_and\_write\_rsa\_keypair function will create two key files named +"path/to/key.pub", which is the public key and "path/to/key", which is the private key. ``` @@ -49,43 +49,47 @@ Confirm: >>> ``` -We can also use the bits parameter to set a different key length (the default is -3072). We can also provide the password parameter in order to suppress the password -prompt. +We can also use the bits parameter to set a different key length (the default +is 3072). We can also provide the password parameter in order to suppress the +password prompt. -During this example we will be using rsa keys, but ed25519 keys are also supported. +In this example we will be using rsa keys, but ed25519 keys are also supported. -Now we have a key for our project, we can proceed to create our project. +Now we have a key for our project, we can proceed to create our project. ### The Project Class -The TUF developer tool is built around the Project class, which is used to organize groups of -targets associated with a single set of metadata. Each Project instance keeps -track of which target files are associated with a single set of metadata. Each -Project instance keeps track of which target files are signed and which need -signing, which keys are used to sign metadata. It also keeps track of delegated -roles, which are covered later. +The TUF developer tool is built around the Project class, which is used to +organize groups of targets associated with a single set of metadata. Each +Project instance keeps track of which target files are associated with a single +set of metadata. Each Project instance keeps track of which target files are +signed and which need signing, which keys are used to sign metadata. It also +keeps track of delegated roles, which are covered later. -Before creating a project, you must know where it will be located in the TUF +Before creating a project, you must know where it will be located in the TUF Repository. In the following example, we will create a project to be hosted as -"repo/unclaimed/example_project" within the repository, and store a local copy of the -metadata at "path/to/metadata". The project will comprise a single target file, -"local/path/to/example\_project/target\_1" locally, and we will secure it with -the key generated above. +"repo/unclaimed/example_project" within the repository, and store a local copy +of the metadata at "path/to/metadata". The project will comprise a single +target file, "local/path/to/example\_project/target\_1" locally, and we will +secure it with the key generated above. -First, we must import the generated keys. We can do that by issuing the following: +First, we must import the generated keys. We can do that by issuing the +following: ``` >>> public_key = import_rsa_publickey_from_file("path/to/keys.pub") ``` -After importing the key, we can generate a new project with the following command +After importing the key, we can generate a new project with the following +command: + ``` >>> project = create_new_project(name="example_project", ... metadata_directory="local/path/to/metadata/", ... targets_directory="local/path/to/example_project", ... location_in_repository="repo/unclaimed", key=public_key) ``` + Let's list the arguments and make sense out of this rather long function call: - create a project named example_project: the name of the metadata file will match this name @@ -95,22 +99,25 @@ for this project will be located here place, you can point the targets directory there. Files must reside under the path local/path/to/example_project or else it won't be possible to add them. - location\_in\_repository points to repo/unclaimed, this will be prepended to the paths in the generated metadata so the signatures all match. -Now the project is in memory and we can do different operations on it such as adding and -removing targets, delegating files, changing signatures and keys, etc. For the moment we are -interested in adding our one and only target inside the project. +Now the project is in memory and we can do different operations on it such as +adding and removing targets, delegating files, changing signatures and keys, +etc. For the moment we are interested in adding our one and only target inside +the project. To add a target, we issue the following method: + ``` >>> project.add_target("target_1") ``` -Have in mind the file "target\_1" should be located in "local/path/to/example\_project" -or else the adding procedure will throw an error. +Have in mind the file "target\_1" should be located in +"local/path/to/example\_project" or else the adding procedure will throw an +error. -At this point, the metadata is not valid. We have assigned a key to the project, -but we have not *signed* it with that key. Signing is the process of generating -a signature with our private key so it can be verified with the public key by the -server (upon uploading) and by the clients (when updating). +At this point, the metadata is not valid. We have assigned a key to the +project, but we have not *signed* it with that key. Signing is the process of +generating a signature with our private key so it can be verified with the +public key by the server (upon uploading) and by the clients (when updating). ### Signing and Writing the Metadata ### @@ -125,12 +132,20 @@ Enter password for the RSA key: >>> project.write() ``` -When all changes to a project have been written, the Project instance can safely -be deleted. +When all changes to a project have been written, the Project instance can +safely be deleted. The project can be loaded later to update changes to the project. The metadata -contains checksums that have to match the actual files or else it won't be accepted -by the upstream repository. +contains checksums that have to match the actual files or else it won't be +accepted by the upstream repository. + +At this point, if you have followed all the steps in this document so far +(substituting appropriate names and filepaths) you will have created a basic +TUF project, which can be expanded as needed. The simplest way to get your +project secured is to add all your files using add\_target() (or see [Managing +Keys](#managing_keys) on how to add whole directories). If your project has +several contributors, you may want to consider adding +[delegations](#delegations) to your project. ## Loading an Existing Project @@ -151,9 +166,6 @@ Enter a password for the RSA key: >>> project.write() ``` -Now we have a project properly setup. The rest of this guide contains a more -in-depth description of the functions of the developer\_tool. - ## Delegations @@ -174,37 +186,56 @@ methods as Project. For example, we can add targets in the same way as before: ``` Recall that we input the other person’s key as part of a list. That list can -contain any number of public keys. You can also add keys to the role after -creating it using the [add\_verification\_key()](#adding_a_key_to_a_delegation) method. +contain any number of public keys. We can also add keys to the role after +creating it using the [add\_verification\_key()](#adding_a_key_to_a_delegation) +method. ### Restricted Paths -By default, a delegated role is permitted to add and modify targets anywhere in the -Project's targets directory. We can assign restricted paths to a delegated role to -limit this permission. +By default, a delegated role is permitted to add and modify targets anywhere in +the Project's targets directory. We can assign restricted paths to a delegated +role to limit this permission. ``` >>> project.add_restricted_paths(["restricted/filepath"], "newrole") ``` -This will prevent the delegated role from signing targets whose local filepaths do not -begin with "restricted/filepath". We can assign several restricted filepaths to a role -by adding them to the list in the first parameter, or by invoking the method again. A -role with multiple restricted paths can add targets to any of them. +This will prevent the delegated role from signing targets whose local filepaths +do not begin with "restricted/filepath". We can assign several restricted +filepaths to a role by adding them to the list in the first parameter, or by +invoking the method again. A role with multiple restricted paths can add +targets to any of them. -Note that this method is invoked the parent role (in this case, the project) and takes -the delegated role name as an argument. +Note that this method is invoked the parent role (in this case, the Project) +and takes the delegated role name as an argument. ### Nested Delegations +It is possible for a delegated role to have delegations of its own. We can do +this by calling delegate() on a delegated role: +``` +>>> project("newrole").delegate(“nestedrole”, [key], targets) +``` + +Nested delegations function no differently than first-order delegations. to +demonstrate, adding a target to nested delegation looks like this: + +``` +>>> project("newrole")("nestedrole").add_target("foo") +``` ### Revoking Delegations +Delegations can be revoked, removing the delegated role from the project. + +``` +>>> project.revoke("newrole") +``` ## Managing Keys -This section describes the key-related functions and parameters that weren't -mentioned inside the example: +This section describes the key-related functions and parameters not covered in +the [Creating a Simple Project](#creating_a_simple_project) section. ### Additional Parameters for Key Generation When generating keys, it is possible to specify the length of the key in bits @@ -218,40 +249,65 @@ The password parameter is only intended to be used in scripts. ### Adding a Key to a Delegation -Likewise, it is possible to add a key to a project by issuing the following command: +New verifications keys can be added to an existing delegation using +add\_verification\_key(): + ``` ->>> project.add_verification_key(pubkey) +>>> project("rolename").add_verification_key(pubkey) ``` +A delegation can have several verification keys at once. By default, a +delegated role with multiple keys can be written using any one of their +corresponding signing keys. To modify this behavior, you can change the +delegated role's [threshold](#delegation_thrsholds). + ### Removing a Key from a Delegation -Removing a verification key is really simple, we should only issue the following -command: +Verification keys can also be removed, like this: ``` ->>> project.remove_verification_key(key) +>>> project("rolename").remove_verification_key(pubkey) ``` -Remember that a project can only have one key, so this method will return an error -if there is already a key assigned to it. In order to replace a key we must first -delete the existing one and then add the new one. It is possible to -ommit the key parameter in the create\_new\_project function, and add the key +Remember that a project can only have one key, so this method will return an +error if there is already a key assigned to it. In order to replace a key we +must first delete the existing one and then add the new one. It is possible to +omit the key parameter in the create\_new\_project() function, and add the key later. ### Changing the Project Key +Each Project instance can only have one verification key. This key can be +replaced by removing it and adding a new key, in that order. +``` +>>> project.remove_verification_key(oldkey) +>>> project.add_verification_key(new) +``` + + ### Delegation Thresholds +Every delegated role has a threshold, which determines how many of its signing +keys need to be loaded to write the role. The threshold defaults to 1, and +should not exceed the number of verification keys assigned to the role. The +threshold can be accessed as a property of a delegated role. + +``` +>>> project("rolename").threshold = 2 +``` + +The above line will set the "rolename" role's threshold to 2. + ## Managing Targets There are supporting functions of the targets library to make the project maintenance easier. These functions are described in this section. ### Adding Targets by Directory -This function is specially useful when creating a new project to add all the files -contained in the targets directory. The following code block illustrates the usage -of this function: -``` +This function is especially useful when creating a new project to add all the +files contained in the targets directory. The following code block illustrates +the usage of this function: +``` >>> list_of_targets = \ ... project.get_filepaths_in_directory(“path/within/targets/folder”, ... recursive_walk=False, follow_links=False) @@ -259,10 +315,12 @@ of this function: ``` ### Deleting Targets from a Project -It is possible that we want to delete existing targets inside our project. In order -to stop the developer tool to track this file we must issue the following command: +It is possible that we want to delete existing targets inside our project. To +stop the developer tool from tracking this file we can issue the following +command: + ``` >>> project.remove_target(“target_1”) ``` -Now the target file won't be part of the metadata. +Now the target file won't be part of the metadata. From 07e8ef0be78934da6790c4c4eabcb800289e2b40 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Fri, 20 Jun 2014 19:36:42 -0400 Subject: [PATCH 43/51] Added the generate_repository_data.py script and its produce The generate_repository_data script works in the exact same way as the generate script and it generates a fresh batch of pre-signed metadata to test the load_project function. --- .../repository_data/generate_project_data.py | 160 ++++++++++++++++++ .../repository_data/project/targets/file1.txt | 1 + .../repository_data/project/targets/file2.txt | 1 + .../repository_data/project/targets/file3.txt | 1 + .../project/test-flat/project.cfg | 1 + .../project/test-flat/test-flat.json | Bin 0 -> 1974 bytes .../project/test-flat/test-flat/role1.json | Bin 0 -> 980 bytes .../test-repo/metadata/test-repo-like.json | Bin 0 -> 1979 bytes .../metadata/test-repo-like/role1.json | Bin 0 -> 980 bytes .../project/test-repo/project.cfg | 1 + .../project/test-repo/targets/file1.txt | 1 + .../project/test-repo/targets/file2.txt | 1 + .../project/test-repo/targets/file3.txt | 1 + 13 files changed, 168 insertions(+) create mode 100755 tests/repository_data/generate_project_data.py create mode 100644 tests/repository_data/project/targets/file1.txt create mode 100644 tests/repository_data/project/targets/file2.txt create mode 100644 tests/repository_data/project/targets/file3.txt create mode 100644 tests/repository_data/project/test-flat/project.cfg create mode 100644 tests/repository_data/project/test-flat/test-flat.json create mode 100644 tests/repository_data/project/test-flat/test-flat/role1.json create mode 100644 tests/repository_data/project/test-repo/metadata/test-repo-like.json create mode 100644 tests/repository_data/project/test-repo/metadata/test-repo-like/role1.json create mode 100644 tests/repository_data/project/test-repo/project.cfg create mode 100644 tests/repository_data/project/test-repo/targets/file1.txt create mode 100644 tests/repository_data/project/test-repo/targets/file2.txt create mode 100644 tests/repository_data/project/test-repo/targets/file3.txt diff --git a/tests/repository_data/generate_project_data.py b/tests/repository_data/generate_project_data.py new file mode 100755 index 00000000..19d6a8db --- /dev/null +++ b/tests/repository_data/generate_project_data.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python + +""" + + generate_project_data.py + + + Santiago Torres + + + + See LICENSE for licensing information. + + + Generate a pre-fabricated set of metadata files to use for the unit testing. +""" + +import shutil +import datetime +import optparse +import os + +from tuf.developer_tool import * +import tuf.util + + +parser = optparse.OptionParser() + +parser.add_option("-d","--dry-run", action='store_true', dest="dry_run", + help="Do not write the files, just run", default=False) +(options, args) = parser.parse_args() + + +project_key_file = 'keystore/root_key' +targets_key_file = 'keystore/targets_key' +delegation_key_file = 'keystore/delegation_key' + +# the files we use for signing in the unit tests should exist, if they are not +# populated, run generate.py +assert os.path.exists(project_key_file) +assert os.path.exists(targets_key_file) +assert os.path.exists(delegation_key_file) + +# Import the public keys. These keys are needed so that metadata roles are +# assigned verification keys, which clients use to verify the signatures created +# by the corresponding private keys. +project_public = import_rsa_publickey_from_file(project_key_file+'.pub') +targets_public = import_rsa_publickey_from_file(targets_key_file+'.pub') +delegation_public = import_rsa_publickey_from_file(delegation_key_file+'.pub') + +# Import the private keys. These private keys are needed to generate the +# signatures included in metadata. +project_private = import_rsa_privatekey_from_file(project_key_file, 'password') +targets_private = import_rsa_privatekey_from_file(targets_key_file, 'password') +delegation_private = import_rsa_privatekey_from_file(delegation_key_file, 'password') + +os.mkdir("project") +os.mkdir("project/targets") + +# Create the target files (downloaded by clients) whose file size and digest +# are specified in the 'targets.json' file. +target1_filepath = 'project/targets/file1.txt' +tuf.util.ensure_parent_dir(target1_filepath) +target2_filepath = 'project/targets/file2.txt' +tuf.util.ensure_parent_dir(target2_filepath) +target3_filepath = 'project/targets/file3.txt' +tuf.util.ensure_parent_dir(target2_filepath) + +if not options.dry_run: + with open(target1_filepath, 'wt') as file_object: + file_object.write('This is an example target file.') + + with open(target2_filepath, 'wt') as file_object: + file_object.write('This is an another example target file.') + + with open(target3_filepath, 'wt') as file_object: + file_object.write('This is role1\'s target file.') + + +project = create_new_project("test-flat", 'project/test-flat', 'prefix', + 'project/targets') + +# Add target files to the top-level 'targets.json' role. These target files +# should already exist. +project.add_target(target1_filepath) +project.add_target(target2_filepath) + +# add keys to the project +project.add_verification_key(project_public) +project.load_signing_key(project_private) + +project.delegate('role1', [delegation_public], [target3_filepath]) +project('role1').load_signing_key(delegation_private) + +# Set the top-level expiration times far into the future so that +# they do not expire anytime soon, or else the tests fail. Unit tests may +# modify the expiration datetimes (of the copied files), if they wish. +project.expiration = datetime.datetime(2030, 1, 1, 0, 0) +project('role1').expiration = datetime.datetime(2030, 1, 1, 0, 0) + +# Compress the 'targets.json' role so that the unit tests have a pre-generated +# example of compressed metadata. +project.compressions = ['gz'] + +# Create the actual metadata files, which are saved to 'metadata.staged'. +if not options.dry_run: + project.write() + +tuf.roledb.clear_roledb() +tuf.keydb.clear_keydb() + +project = create_new_project("test-repo-like", "project/test-repo", 'prefix') + +# Create the target files (downloaded by clients) whose file size and digest +# are specified in the 'targets.json' file. +target1_filepath = 'project/test-repo/targets/file1.txt' +tuf.util.ensure_parent_dir(target1_filepath) +target2_filepath = 'project/test-repo/targets/file2.txt' +tuf.util.ensure_parent_dir(target2_filepath) +target3_filepath = 'project/test-repo/targets/file3.txt' +tuf.util.ensure_parent_dir(target2_filepath) + +if not options.dry_run: + with open(target1_filepath, 'wt') as file_object: + file_object.write('This is an example target file.') + + with open(target2_filepath, 'wt') as file_object: + file_object.write('This is an another example target file.') + + with open(target3_filepath, 'wt') as file_object: + file_object.write('This is role1\'s target file.') + + +# Add target files to the top-level 'targets.json' role. These target files +# should already exist. +project.add_target(target1_filepath) +project.add_target(target2_filepath) + +# add keys to the project +project.add_verification_key(project_public) +project.load_signing_key(project_private) + +project.delegate('role1', [delegation_public], [target3_filepath]) +project('role1').load_signing_key(delegation_private) + +# Set the top-level expiration times far into the future so that +# they do not expire anytime soon, or else the tests fail. Unit tests may +# modify the expiration datetimes (of the copied files), if they wish. +project.expiration = datetime.datetime(2030, 1, 1, 0, 0) +project('role1').expiration = datetime.datetime(2030, 1, 1, 0, 0) + +# Compress the 'targets.json' role so that the unit tests have a pre-generated +# example of compressed metadata. +project.compressions = ['gz'] + +# Create the actual metadata files, which are saved to 'metadata.staged'. +if not options.dry_run: + project.write() + + diff --git a/tests/repository_data/project/targets/file1.txt b/tests/repository_data/project/targets/file1.txt new file mode 100644 index 00000000..7bf3499f --- /dev/null +++ b/tests/repository_data/project/targets/file1.txt @@ -0,0 +1 @@ +This is an example target file. \ No newline at end of file diff --git a/tests/repository_data/project/targets/file2.txt b/tests/repository_data/project/targets/file2.txt new file mode 100644 index 00000000..606f18ef --- /dev/null +++ b/tests/repository_data/project/targets/file2.txt @@ -0,0 +1 @@ +This is an another example target file. \ No newline at end of file diff --git a/tests/repository_data/project/targets/file3.txt b/tests/repository_data/project/targets/file3.txt new file mode 100644 index 00000000..60464604 --- /dev/null +++ b/tests/repository_data/project/targets/file3.txt @@ -0,0 +1 @@ +This is role1's target file. \ No newline at end of file diff --git a/tests/repository_data/project/test-flat/project.cfg b/tests/repository_data/project/test-flat/project.cfg new file mode 100644 index 00000000..61b14806 --- /dev/null +++ b/tests/repository_data/project/test-flat/project.cfg @@ -0,0 +1 @@ +{"project_name": "test-flat", "targets_location": "/home/santiago/Documents/v2014/TUF/tuf/tests/repository_data/project/targets", "prefix": "prefix", "metadata_location": "/home/santiago/Documents/v2014/TUF/tuf/tests/repository_data/project/test-flat", "threshold": 1, "public_keys": {"6986b667c736a3b37471e030cf4ce7aa6c7e0d530325e64c2660276b77be3754": {"keytype": "rsa", "keyval": {"public": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7J15ZaeDQPrhQsRj29wB\nPhibH+Do59xsT2396L+uCg793gZlar5wZN2eHSh725cNQWyTAa9LwG+lXaKMukQ+\n8176CKR2J5sv3DezrGVu3x8V1qhyJyy79FlNZRVYTVqNaYzvJzxsVnFPpg7f8B7C\nffiqWJr9XkpqwRlCpxooXm4hplZ7uek5Ku21CzQ4OWg7hbuc+ZjCGzpXfm8NuosU\n7TipnKGpEt0Agiph5g6TB2/scoeFar1CKMONIl80maxzAQk+xkWgiJ00+Z2qFCsx\nESfis/YkILS6RMFyZz7oa1WwMtUjYmrsRuz+jlFcbNuxZpIkaISiG9a2YdGcJ1Aj\n3QIDAQAB\n-----END PUBLIC KEY-----"}}}, "layout_type": "flat"} \ No newline at end of file diff --git a/tests/repository_data/project/test-flat/test-flat.json b/tests/repository_data/project/test-flat/test-flat.json new file mode 100644 index 0000000000000000000000000000000000000000..147417248d707c8c92e95f3f7784a2c926689368 GIT binary patch literal 1974 zcmbtV+ioI95PkPoAYM0)`+nKC!C(h-u?+@a;#Jg_YQV+=hB263`QKAB*xn>kq)2HE zqoI4I`c$1$r+*yO>g(Agcg4#J*7dKo-w$fFANZ@)A7M9(I8#^FD6fTB`+EABiR$j3XG(2NfL`k~`rwB~BYllu*=Z9ZXV81|mhW#s`^n^u$Wb;W&zMz)1s{dBK}Jcna*|6bl%@_irIIi!5VTRk8{9>4!IG08C09aw z8z}M2b7zy}7E&ZGTF$^k<8(;LQ-L|eg!PCaStP>=PK*hVG}6J6FviKKOuaVjLBa0>com)Ktp^8ViRJf&)Q-%JO?|9g-lx zSQnz_++vnoa1dHVB;6aqm5#zwV-Y*ZBy0_g(@;GCeiKleYQ2*NXCgiQu4B|2E3n4yrtKtTCAPMBiKB|~KT2)l2G4-l)h+Y5I;y>auO@73zd z7eAkc66|3)8m-fIzjiTdblbcBB1}^&6AN=jr3@)NV;5yKJ3g+4*sH zX|i(tc+$b=R`%S9SA%iRpSqVPqeR$X`7*iBx8v4G?9K`Z({{J(Ro>HIURG(_1(kGb z-9K)gVBo6@z3IGM?zXXeyP9tAI=6Sd=93%e?fd*$#G|6o-T0%2)7zu=#mRVc?X&ai zVtak6-Ex+ltiqDF=ZJBv7f{Jxyr(6(R2XU!X< zdGeG^8%KO+1Y~4vf_MD zwAyrcb-X!>=NE2yGy}f@IDW{{zJ7b`!S}O891hG4U-l-wPH((>onjV~ZM&IWW{vy2 zBB#}F{*$j#nK)3KANRMtsjU|CPuqMqP85ZA;jviZ4OLLAUZ8q+FP2&SaM?1{{}<>U zSlKdFyxcuieNw=>I81X_9F+*^r(Bn=nEnmGQJT$x|605j?>T;kDyDcMOc(RA3F*(o zzu{w*d>vkwv;Cn_XLuYO5{lOqAz!QOZr`KDd&7S8=W+#U_FC;qt6kq0YW1mGPpev1 zjf}NfPtjmi*+@#3R;`nyXi(_p=){sFL7YPq=2S-`g(U=ylL%v_Q0UX>NJitlKy%-x zwx_ICpF=(=rWlu3Y3--$e*wn+GcX}p0D~4R35kY75`qbIYfwZnE-j;YsIY`f8l1(` jkbqPaEtfhGoVEW3%$D^o)wZwn23BiyxjhkY2XB7@#;hCA literal 0 HcmV?d00001 diff --git a/tests/repository_data/project/test-flat/test-flat/role1.json b/tests/repository_data/project/test-flat/test-flat/role1.json new file mode 100644 index 0000000000000000000000000000000000000000..4e38e30f124edd9b78248a88d834468ea0145b4b GIT binary patch literal 980 zcmX|=(QXtm3`O7j6{~rs5Ic_J@Wvk?wof30?8ND=C~dVnpd!?N$FnU+8EMB}-Fxkv z`E#{f&d1w(U4NeZyu9Cix!Udi#Iswz`|sn3%A%BL5L+TtbZau`nElz|Ej(=<3wDJreZA*p4l+SE%o<_esW zdj+V?m8%!5WSz}1CSq+kB9L&ju@BFXGie3OHG`x~)e1&NVl^dBZKfEDTOMsVA61x#ElZ#$Zpv| z%|?kV5j%DRB$3%m#hiGe+AW#%`*JXZ*@IB+;8Om zr}pyl`~uzS;qC?NubcK(Tm0$q_`>>vDW~fMac)w2e>tCDq}NxUf3lA!pU07yr;d9~293VIl6IQVK+?1hQQVnP>@`5W7Xoi8_L|CR0&GQ&H89 nJ*D`#*tUJVrViaZzrS4%p^LKBV^OaI{qoZ}b`Tx7it_Vcg69Ja literal 0 HcmV?d00001 diff --git a/tests/repository_data/project/test-repo/metadata/test-repo-like.json b/tests/repository_data/project/test-repo/metadata/test-repo-like.json new file mode 100644 index 0000000000000000000000000000000000000000..cad9c4c1ba6b25a216eb24187c965e8beda05de3 GIT binary patch literal 1979 zcmbtVYj5I46#brGf%v*jGw;WKGl8UpM-m9+(X6U@+yN3|2sjXSmH*yrlVsaUl`3^) zTlUQO&bepKJ@ezBR$tF1xhq~)u&#ft{eDoZ{lL3ce}vsEVozOJqr6f&Xs#UhTnkNs za1xRTpq*1e1BsFo&Lk)im{Npk<+b*}wG{PVs(4ROOczza!7v*Rv%`zw@O=molnaGm zU6M3t5-jsBMMZ^U8U(WUPr3pb>rxY_F2q8E>fFzVs4?r~% z>n3RC10@&muSKJV#feVUzW_|UmQ9@e?l3|C_n5|YW678sulpAeKAk;WZ zC~=GgN`s3*MIQmdC7dRbr%qchEXBl()S5fbqEI3N^23>>k_ZwtCmOu;Rw%7ZsH0Ba z4r<>}FO?{vR7)ww`X9w^38h4?+-d^Fy3$QO!W<^9m@V>Bxu2^2{HWEPAOe>~Sj;i- zmL(E`@JOeyC@7>|2Mg3l-1J}|pnM%COflt>Au@f0-Iv1$h}GKdg}d+GxcSd{wfgeK z&u5_odswzc>$KgkU5pyt_HnJ#x~)3z^Ip5%Xg_5A#^mvN`ZznaThhobTPIm|ewB_>#}o>-Fu~eICZ0MVvo=Urugl+pprY=8e%j zd!4o}`^{|Blm7eeY;;W;BXK@_7%fgZ$CF?h!^eIYcDFa@_xWJCd(O7a<;C@^t4yCZ z!&DFKOH-Y;tQ)j$?Xa`B#bh?4^Ob9dClKlEj`yElmwi4Q=QsCx0n5egY{6z(aXu(o zZMwTU-WhQXp?N5$6!!zNKP`s`P`C46f`$8?=tM;Qvmn%rK*J@Q-t@=Jtt54l} zS`k`RGZM6(IwnQwghaz33Bk~#4Je`*mzLpigU*9P m8l1%=k$_YbN{ixjoYxSgq0O_8akb@b)KMFC6Cp literal 0 HcmV?d00001 diff --git a/tests/repository_data/project/test-repo/metadata/test-repo-like/role1.json b/tests/repository_data/project/test-repo/metadata/test-repo-like/role1.json new file mode 100644 index 0000000000000000000000000000000000000000..caac5a0dcd78aa315017fb14ee00f9942eafecf7 GIT binary patch literal 980 zcmX|9!EO{W488X&R&%BhJ8|rA;|~zq69^$EvAZiuTkQ_02=(9bYzw2zA&&Fj%k%ql zwOh`|+k0Jqp6tB5-+j5-?f%5STfW=xU z4N}R|t0v{Fz2pHrF)P(5MU9GkZV3f2Zq<-hm`zyJIRMprO^{}>DUDm@s*<>OMwh}+ zYU?A9xSyvXH63H9a7_(t0BEa_rFZDHPoe6gC#xhaH`ggV7>JwZk^{>v2xs(=cSTnv zs2ws8iDRmT59A68{u%WnNtakk^^;Q zlV}5+g^_59oIGFczD8ayp|mY6TNsyb>+eU~B66drTU*bU+$_`X>{i#~!~K@r|Eaxf zo?k+DdboRu_16u1t1b5QczntFf+?r#1hH;XdVg7;U!d1lo`0~9C-dXeJ3ro8e!D)c z|M+&xLC=Q^#f3lc&j&@Sn^-8|VTDs>feAyYbWut%0vtV#tdxO~t|lIX5ly5qV;m`R kEVXZbys8e|JG;MK4}pua)x)UQ2mP|sIXs9HxQg=gU$-3s_W%F@ literal 0 HcmV?d00001 diff --git a/tests/repository_data/project/test-repo/project.cfg b/tests/repository_data/project/test-repo/project.cfg new file mode 100644 index 00000000..e3a93c8d --- /dev/null +++ b/tests/repository_data/project/test-repo/project.cfg @@ -0,0 +1 @@ +{"project_name": "test-repo-like", "targets_location": "/home/santiago/Documents/v2014/TUF/tuf/tests/repository_data/project/test-repo/targets", "prefix": "prefix", "metadata_location": "/home/santiago/Documents/v2014/TUF/tuf/tests/repository_data/project/test-repo/metadata", "threshold": 1, "public_keys": {"6986b667c736a3b37471e030cf4ce7aa6c7e0d530325e64c2660276b77be3754": {"keytype": "rsa", "keyval": {"public": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7J15ZaeDQPrhQsRj29wB\nPhibH+Do59xsT2396L+uCg793gZlar5wZN2eHSh725cNQWyTAa9LwG+lXaKMukQ+\n8176CKR2J5sv3DezrGVu3x8V1qhyJyy79FlNZRVYTVqNaYzvJzxsVnFPpg7f8B7C\nffiqWJr9XkpqwRlCpxooXm4hplZ7uek5Ku21CzQ4OWg7hbuc+ZjCGzpXfm8NuosU\n7TipnKGpEt0Agiph5g6TB2/scoeFar1CKMONIl80maxzAQk+xkWgiJ00+Z2qFCsx\nESfis/YkILS6RMFyZz7oa1WwMtUjYmrsRuz+jlFcbNuxZpIkaISiG9a2YdGcJ1Aj\n3QIDAQAB\n-----END PUBLIC KEY-----"}}}, "layout_type": "repo-like"} \ No newline at end of file diff --git a/tests/repository_data/project/test-repo/targets/file1.txt b/tests/repository_data/project/test-repo/targets/file1.txt new file mode 100644 index 00000000..7bf3499f --- /dev/null +++ b/tests/repository_data/project/test-repo/targets/file1.txt @@ -0,0 +1 @@ +This is an example target file. \ No newline at end of file diff --git a/tests/repository_data/project/test-repo/targets/file2.txt b/tests/repository_data/project/test-repo/targets/file2.txt new file mode 100644 index 00000000..606f18ef --- /dev/null +++ b/tests/repository_data/project/test-repo/targets/file2.txt @@ -0,0 +1 @@ +This is an another example target file. \ No newline at end of file diff --git a/tests/repository_data/project/test-repo/targets/file3.txt b/tests/repository_data/project/test-repo/targets/file3.txt new file mode 100644 index 00000000..60464604 --- /dev/null +++ b/tests/repository_data/project/test-repo/targets/file3.txt @@ -0,0 +1 @@ +This is role1's target file. \ No newline at end of file From 91579e9c1534b3a1e64c704658c727ed1423b643 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Fri, 20 Jun 2014 19:38:13 -0400 Subject: [PATCH 44/51] Fixed developer tools to match the new repository_lib file As of the current version, some functions that were provided by the repository_tool have been moved to the repository_lib module. This commit fixes those conflicts --- tuf/developer_tool.py | 166 ++++++++++++++++++------------------------ 1 file changed, 69 insertions(+), 97 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 6e26a26d..52b73b59 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -51,50 +51,44 @@ from tuf.keys import format_metadata_to_key from tuf.repository_tool import Targets -from tuf.repository_tool import get_metadata_fileinfo -from tuf.repository_tool import get_metadata_filenames +from tuf.repository_lib import get_metadata_fileinfo +from tuf.repository_lib import get_metadata_filenames from tuf.repository_tool import generate_and_write_rsa_keypair from tuf.repository_tool import import_rsa_publickey_from_file from tuf.repository_tool import import_rsa_privatekey_from_file from tuf.repository_tool import generate_and_write_ed25519_keypair from tuf.repository_tool import import_ed25519_publickey_from_file from tuf.repository_tool import import_ed25519_privatekey_from_file -from tuf.repository_tool import _remove_invalid_and_duplicate_signatures -from tuf.repository_tool import _check_role_keys -from tuf.repository_tool import _delete_obsolete_metadata -from tuf.repository_tool import generate_targets_metadata -from tuf.repository_tool import sign_metadata -from tuf.repository_tool import write_metadata_file -from tuf.repository_tool import _metadata_is_partially_loaded +from tuf.repository_lib import _remove_invalid_and_duplicate_signatures +from tuf.repository_lib import _check_role_keys +from tuf.repository_lib import _delete_obsolete_metadata +from tuf.repository_lib import generate_targets_metadata +from tuf.repository_lib import sign_metadata +from tuf.repository_lib import write_metadata_file +from tuf.repository_lib import _metadata_is_partially_loaded -# import private functions to query for user input. -from tuf.repository_tool import _prompt -from tuf.repository_tool import _get_password # See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger('tuf.developer_tool') - - - # Recommended RSA key sizes: # http://www.emc.com/emc-plus/rsa-labs/historical/twirl-and-rsa-key-size.htm#table1 # According to the document above, revised May 6, 2003, RSA keys of size 3072 # provide security through 2031 and beyond. 2048-bit keys are the recommended # minimum and are good from the present through 2030. -from tuf.repository_tool import DEFAULT_RSA_KEY_BITS as DEFAULT_RSA_KEY_BITS +from tuf.repository_lib import DEFAULT_RSA_KEY_BITS as DEFAULT_RSA_KEY_BITS # The algorithm used by the developer tools to generate the hashes of the # target filepaths. from tuf.repository_tool import HASH_FUNCTION as HASH_FUNCTION # The extension of TUF metadata. -from tuf.repository_tool import METADATA_EXTENSION as METADATA_EXTENSION +from tuf.repository_lib import METADATA_EXTENSION as METADATA_EXTENSION # The metadata filename for the targets metadata information. -from tuf.repository_tool import TARGETS_FILENAME as TARGETS_FILENAME +from tuf.repository_lib import TARGETS_FILENAME as TARGETS_FILENAME # Project configuration filename. This file is intended to hold all of the # supporting information about the project that's not contained in a usual @@ -135,14 +129,14 @@ from tuf.repository_tool import TARGETS_DIRECTORY_NAME # The full list of supported TUF metadata extensions. -from tuf.repository_tool import METADATA_EXTENSIONS +from tuf.repository_lib import METADATA_EXTENSIONS # The recognized compression extensions. -from tuf.repository_tool import SUPPORTED_COMPRESSION_EXTENSIONS +from tuf.repository_lib import SUPPORTED_COMPRESSION_EXTENSIONS # Supported key types. -from tuf.repository_tool import SUPPORTED_KEY_TYPES +from tuf.repository_lib import SUPPORTED_KEY_TYPES class Project(Targets): @@ -276,7 +270,7 @@ def write(self, write_partial=False): prefix=self.prefix) - # Generate the 'targets.txt' metadata file. + # Generate the 'project_name' metadata file. targets_filename = self._project_name + METADATA_EXTENSION targets_filename = os.path.join(self._metadata_directory, targets_filename) project_signable, targets_filename = \ @@ -298,32 +292,6 @@ def write(self, write_partial=False): - def remove_verification_key(self,key): - """ - - Function as a thin wrapper call for the project._targets call - with the same name. This wrapper is only for usability purposes - - - Key: - The role key to be removed, conformant to tuf.formats.anykey_schema - - - Tuf.FormatError, if the 'key' argument is improperly formatted. - - - The role's entries in 'tuf.roledb.py' are updated - - - None - """ - try: - super(Project, self).remove_verification_key(key) - except tuf.FormatError: - raise - - - def add_verification_key(self,key): """ @@ -393,61 +361,60 @@ def status(self): METADATA_STAGED_DIRECTORY_NAME) os.mkdir(metadata_directory) - #filenames = get_metadata_filenames(metadata_directory)A + #filenames = get_metadata_filenames(metadata_directory) # we should do the schema check filenames = {} - filenames['targets'] = os.path.join(metadata_directory,TARGETS_FILENAME) + filenames['targets'] = os.path.join(metadata_directory,self._project_name) # Delegated roles. - delegated_roles = tuf.roledb.get_delegated_rolenames('targets') + delegated_roles = tuf.roledb.get_delegated_rolenames(self._project_name) insufficient_keys = [] insufficient_signatures = [] for delegated_role in delegated_roles: try: _check_role_keys(delegated_role) - except tuf.InsufficientKeysError, e: + except tuf.InsufficientKeysError: insufficient_keys.append(delegated_role) continue roleinfo = tuf.roledb.get_roleinfo(delegated_role) try: - write_delegated_metadata_file(temp_project_directory, - self._targets_directory, - delegated_role, roleinfo, - write_partial=False) - except tuf.Error, e: + signable = _generate_and_write_metadata(delegated_role, + filenames['targets'], False, + self._targets_directory, + self._metadata_directory, + False) + self._print_status(delegated_role, signable) + except tuf.Error: insufficient_signatures.append(delegated_role) if len(insufficient_keys): message = 'Delegated roles with insufficient keys: '+ \ repr(insufficient_keys) print(message) - return if len(insufficient_signatures): message = 'Delegated roles with insufficient signatures: '+ \ repr(insufficient_signatures) print(message) - return # Targets role. try: _check_role_keys(self.rolename) - except tuf.InsufficientKeysError, e: + except tuf.InsufficientKeysError as e: print(str(e)) - return try: - signable = _generate_and_write_metadata(self.rolename, + signable, filename = _generate_and_write_metadata(self._project_name, filenames['targets'], False, self._targets_directory, self._metadata_directory, False) - #_print_status(self.targets.rolename, signable) - except tuf.Error, e: + self._print_status(self._project_name, signable) + except tuf.Error as e: signable = e[1] - #_print_status(self.targets.rolename, signable) + self._print_status(self._project_name, signable) return finally: @@ -457,18 +424,18 @@ def status(self): -def _print_status(rolename, signable): - """ - Non-public function prints the number of (good/threshold) signatures of - 'rolename'. - """ + def _print_status(self, rolename, signable): + """ + Non-public function prints the number of (good/threshold) signatures of + 'rolename'. + """ - status = tuf.sig.get_signature_status(signable, rolename) - - message = repr(rolename)+' role contains '+ \ - repr(len(status['good_sigs']))+' / '+ \ - repr(status['threshold'])+' signatures.' - print(message) + status = tuf.sig.get_signature_status(signable, rolename) + + message = repr(rolename)+' role contains '+ \ + repr(len(status['good_sigs']))+' / '+ \ + repr(status['threshold'])+' signatures.' + print(message) @@ -503,7 +470,8 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, # preprend the prefix to the project's filepath to avoid signature errors # in upstream - for element in metadata['targets'].keys(): + target_filepaths = metadata['targets'].items() + for element in list(metadata['targets']): junk_path, relative_target = os.path.split(element) prefixed_path = os.path.join(prefix,relative_target) metadata['targets'][prefixed_path] = metadata['targets'][element] @@ -528,7 +496,7 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, # non-partial write() else: - if tuf.sig.verify(signable, rolename) and not roleinfo['partial_loaded']: + if tuf.sig.verify(signable, rolename): #and not roleinfo['partial_loaded']: metadata['version'] = metadata['version'] + 1 signable = sign_metadata(metadata, roleinfo['signing_keyids'], metadata_filename) @@ -656,7 +624,7 @@ def create_new_project(project_name, metadata_directory, location_in_repository # 'OSError' raised if the leaf directory already exists or cannot be created. # Check for case where 'repository_directory' has already been created. - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST: # should check if we have write permissions here pass @@ -668,7 +636,7 @@ def create_new_project(project_name, metadata_directory, location_in_repository message = 'Creating '+repr(targets_directory) logger.info(message) os.mkdir(targets_directory) - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST: pass else: @@ -827,19 +795,22 @@ def load_project(project_directory, prefix=''): project_directory = os.path.abspath(project_directory) - # load the cfg file and update the project. + # load the cfg file and the project. config_filename = os.path.join(project_directory,PROJECT_FILENAME) try: project_configuration = tuf.util.load_json_file(config_filename) - tuf.formats.PROJECT_CFG_SCHEMA.check_match(project_configuration) - - except OSError, e: + tuf.formats.PROJECT_CFG_SCHEMA.check_match(project_configuration) + except (OSError, IOError) as e: raise metadata_directory = project_configuration['metadata_location'] targets_directory = project_configuration['targets_location'] - if prefix=='': - prefix = project_configuration['prefix'] + + new_prefix = None + if prefix != '': + new_prefix = prefix + + prefix = project_configuration['prefix'] # load the projects filename. project_name = project_configuration['project_name'] @@ -874,18 +845,17 @@ def load_project(project_directory, prefix=''): roleinfo = tuf.roledb.get_roleinfo(project_name) roleinfo['signatures'].extend(signable['signatures']) roleinfo['version'] = targets_metadata['version'] - roleinfo['paths'] = targets_metadata['targets'].keys() + roleinfo['paths'] = list(targets_metadata['targets']) roleinfo['delegations'] = targets_metadata['delegations'] roleinfo['partial_loaded'] = False # check if the loaded metadata was partially written and update the # flag in the roledb - if _metadata_is_partially_loaded( project_name, signable, roleinfo): + if _metadata_is_partially_loaded(project_name, signable, roleinfo): roleinfo['partial_loaded'] = True - - tuf.roledb.update_roleinfo( project_name, roleinfo) + tuf.roledb.update_roleinfo(project_name, roleinfo) @@ -908,9 +878,9 @@ def load_project(project_directory, prefix=''): # metadata object. targets_objects = {} loaded_metadata = [] - targets_objects['targets'] = project + targets_objects[project_name] = project targets_metadata_directory = os.path.join(metadata_directory, - TARGETS_DIRECTORY_NAME) + project_name) if os.path.exists(targets_metadata_directory) and \ os.path.isdir(targets_metadata_directory): for root, directories, files in os.walk(targets_metadata_directory): @@ -938,8 +908,8 @@ def load_project(project_directory, prefix=''): signable = None try: signable = tuf.util.load_json_file(metadata_path) - except (ValueError, IOError), e: - continue + except (ValueError, IOError): + raise # strip the extension from the metadata_object = signable['signed'] @@ -950,7 +920,7 @@ def load_project(project_directory, prefix=''): roleinfo['signatures'].extend(signable['signatures']) roleinfo['version'] = metadata_object['version'] roleinfo['expires'] = metadata_object['expires'] - roleinfo['paths'] = metadata_object['targets'].keys() + roleinfo['paths'] = list(metadata_object['targets']) roleinfo['delegations'] = metadata_object['delegations'] roleinfo['partial_loaded'] = False @@ -981,7 +951,7 @@ def load_project(project_directory, prefix=''): key_object = tuf.keys.format_metadata_to_key(key_metadata) try: tuf.keydb.add_key(key_object) - except tuf.KeyAlreadyExistsError, e: + except tuf.KeyAlreadyExistsError: pass for role in metadata_object['delegations']['roles']: @@ -994,7 +964,9 @@ def load_project(project_directory, prefix=''): 'delegations': {'keys': {}, 'roles': []}} tuf.roledb.add_role(rolename, roleinfo) - + + if new_prefix: + project.prefix = new_prefix return project From f9ce4fcb363f0fc301994a58268071b7e8eb5b2f Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Fri, 20 Jun 2014 19:39:11 -0400 Subject: [PATCH 45/51] Added the test_developer_tool module All of the neccessary test cases for the developer tool --- tests/test_developer_tool.py | 406 +++++++++++++++++++++++++++++++++++ 1 file changed, 406 insertions(+) create mode 100755 tests/test_developer_tool.py diff --git a/tests/test_developer_tool.py b/tests/test_developer_tool.py new file mode 100755 index 00000000..67853317 --- /dev/null +++ b/tests/test_developer_tool.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python + +""" + + test_developer_tool.py + + + Santiago Torres Arias + + + See LICENSE for licensing inforation + + + Unit tests for the 'developer_tool' module +""" + +import os +import time +import datetime +import unittest +import logging +import tempfile +import shutil + +import tuf +import tuf.log +import tuf.formats +import tuf.roledb +import tuf.keydb +import tuf.developer_tool as developer_tool + +from tuf.developer_tool import METADATA_DIRECTORY_NAME +from tuf.developer_tool import TARGETS_DIRECTORY_NAME + +logger = logging.getLogger("tuf.test_developer_tool") + +class TestProject(unittest.TestCase): + + tmp_dir = None + + @classmethod + def setUpClass(cls): + cls.tmp_dir = tempfile.mkdtemp(dir = os.getcwd()) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmp_dir) + + def setUp(self): + # called before every test case + pass + + def tearDown(self): + # called after every test case + tuf.roledb.clear_roledb() + tuf.keydb.clear_keydb() + pass + + def test_emtpy(self): + return + + def test_create_new_project(self): + # test cases for the create_new_project function. In this test we will + # check input, correct file creation and format. We also check + # that a proper object is generated. We will use the normal layout for this + # test suite. + + # create a local subfolder for this test. + local_tmp = tempfile.mkdtemp(dir = self.tmp_dir) + + # these are the usual values we will be throwing to the function, however + # we will swap these for nulls or malformed values every now and then to + # test input + project_name = "test_suite" + metadata_directory = local_tmp + location_in_repository = '/prefix' + targets_directory = None + key = None + + # create a blank project + project = developer_tool.create_new_project(project_name, metadata_directory, + location_in_repository) + + self.assertTrue(isinstance(project, developer_tool.Project)) + self.assertTrue(project.layout_type == 'repo-like') + self.assertTrue(project.prefix == location_in_repository) + self.assertTrue(project._project_name == project_name) + self.assertTrue(project._metadata_directory == + os.path.join(metadata_directory,METADATA_DIRECTORY_NAME)) + self.assertTrue(project._targets_directory == + os.path.join(metadata_directory,TARGETS_DIRECTORY_NAME)) + + # create a blank project without a prefix + project = developer_tool.create_new_project(project_name, metadata_directory) + self.assertTrue(isinstance(project, developer_tool.Project)) + self.assertTrue(project.layout_type == 'repo-like') + self.assertTrue(project.prefix == '') + self.assertTrue(project._project_name == project_name) + self.assertTrue(project._metadata_directory == + os.path.join(metadata_directory,METADATA_DIRECTORY_NAME)) + self.assertTrue(project._targets_directory == + os.path.join(metadata_directory,TARGETS_DIRECTORY_NAME)) + + # create a blank project without a valid metadata directory + self.assertRaises(tuf.FormatError, developer_tool.create_new_project, + 0, metadata_directory, location_in_repository) + self.assertRaises(tuf.FormatError, developer_tool.create_new_project, + project_name, 0, location_in_repository) + self.assertRaises(tuf.FormatError, developer_tool.create_new_project, + project_name, metadata_directory, 0) + + + # create a new project with a flat layout + targets_directory = tempfile.mkdtemp(dir = local_tmp) + metadata_directory = tempfile.mkdtemp(dir = local_tmp) + project = developer_tool.create_new_project(project_name, metadata_directory, + location_in_repository, targets_directory) + self.assertTrue(isinstance(project, developer_tool.Project)) + self.assertTrue(project.layout_type == 'flat') + self.assertTrue(project.prefix == location_in_repository) + self.assertTrue(project._project_name == project_name) + self.assertTrue(project._metadata_directory == metadata_directory) + self.assertTrue(project._targets_directory == targets_directory) + + # finally, check that if targets_directory is set, it is valid + self.assertRaises(tuf.FormatError, developer_tool.create_new_project, + project_name, metadata_directory, location_in_repository, 0) + + # copy a key to our workspace and create a new project with it. + keystore_path = os.path.join('repository_data','keystore') + + # I will use the same key as the one provided in the repository + # tool tests for the root role, but this is not a root role... + root_key_path = os.path.join(keystore_path,'root_key.pub') + project_key = developer_tool.import_rsa_publickey_from_file(root_key_path) + + # test create new project with a key added by default + project = developer_tool.create_new_project(project_name, metadata_directory, + location_in_repository, targets_directory, project_key) + + self.assertTrue(isinstance(project, developer_tool.Project)) + self.assertTrue(project.layout_type == 'flat') + self.assertTrue(project.prefix == location_in_repository) + self.assertTrue(project._project_name == project_name) + self.assertTrue(project._metadata_directory == metadata_directory) + self.assertTrue(project._targets_directory == targets_directory) + self.assertTrue(len(project.keys) == 1) + self.assertTrue(project.keys[0] == project_key['keyid']) + + shutil.rmtree(local_tmp) + + + def test_load_project(self): + # this test case will try to load an existing project and test for the result + # then we will try to load a nonexisting project and expect a correct error + # handler + # finally, we will try to overwrite the existing prefix on the loaded project + + # create a local subfolder for this test. + local_tmp = tempfile.mkdtemp(dir = self.tmp_dir) + + # test inexisting project filepath + nonexistent_path = os.path.join(local_tmp, "nonexistent") + self.assertRaises(IOError, developer_tool.load_project, nonexistent_path) + + # copy the pregenerated metadata + project_data_filepath = os.path.join('repository_data', 'project') + target_project_data_filepath = os.path.join(local_tmp, 'project') + shutil.copytree("repository_data/project", target_project_data_filepath) + + # load a project properly... + repo_filepath = os.path.join(local_tmp, 'project', 'test-repo') + project = developer_tool.load_project(repo_filepath) + + self.assertTrue(project.layout_type == "repo-like") + + repo_filepath = os.path.join(local_tmp, 'project', 'test-flat') + project = developer_tool.load_project(repo_filepath) + + self.assertTrue(project.layout_type == 'flat') + + + # load a project ovrwriting the prefix + project = developer_tool.load_project(repo_filepath, prefix='new') + self.assertTrue(project.prefix == 'new') + + + def test_add_verification_keys(self): + + # maybe we need to clear the roledb and keydb + + # create a new project instance + project = developer_tool.Project("test_verification_keys", "somepath", + "someotherpath", "prefix") + + # add invalid verification key + self.assertRaises(tuf.FormatError, project.add_verification_key, "invalid") + + # add verification key + # - load it first + keystore_path = os.path.join('repository_data','keystore') + first_verification_key_path = os.path.join(keystore_path,'root_key.pub') + first_verification_key = \ + developer_tool.import_rsa_publickey_from_file(first_verification_key_path) + + project.add_verification_key(first_verification_key) + + + # add another verification key (should expect exception) + second_verification_key_path = os.path.join(keystore_path,'snapshot_key.pub') + second_verification_key = \ + developer_tool.import_rsa_publickey_from_file(second_verification_key_path) + + self.assertRaises(tuf.Error, + project.add_verification_key,(second_verification_key)) + + + + # add a verification key for the delegation + project.delegate("somedelegation", [], []) + project("somedelegation").add_verification_key(first_verification_key) + project("somedelegation").add_verification_key(second_verification_key) + + + # add another delegation of the delegation + project("somedelegation").delegate("somesubdelegation", [], []) + project("somedelegation")("somesubdelegation").add_verification_key( + first_verification_key) + project("somedelegation")("somesubdelegation").add_verification_key( + second_verification_key) + + + def test_write(self): + + # create tmp directory + local_tmp = tempfile.mkdtemp(dir=self.tmp_dir) + + # create new project inside tmp directory + project = developer_tool.create_new_project("test_write", local_tmp, + "prefix"); + + # create some target files inside the tmp directory + target_filepath = os.path.join(local_tmp, "targets", "test_target") + with open(target_filepath, "wt") as fp: + fp.write("testing file") + + + # add the targets + project.add_target(target_filepath) + + # add verification keys + keystore_path = os.path.join('repository_data','keystore') + project_key_path = os.path.join(keystore_path,'root_key.pub') + project_key = \ + developer_tool.import_rsa_publickey_from_file(project_key_path) + + + # call status (for the sake of doing it) + project.status() + + project.add_verification_key(project_key) + + + # add another verification key (should expect exception) + delegation_key_path = os.path.join(keystore_path,'snapshot_key.pub') + delegation_key = \ + developer_tool.import_rsa_publickey_from_file(delegation_key_path) + + # add a subdelegation + subdelegation_key_path = os.path.join(keystore_path,'timestamp_key.pub') + subdelegation_key = \ + developer_tool.import_rsa_publickey_from_file(subdelegation_key_path) + + # add a delegation + project.delegate("delegation", [delegation_key], []) + project("delegation").delegate("subdelegation", [subdelegation_key], []) + + # call write (except) + self.assertRaises(tuf.Error, project.write, ()) + + # call status (for the sake of doing it) + project.status() + + # load private keys + project_private_key_path = os.path.join(keystore_path, 'root_key') + project_private_key = \ + developer_tool.import_rsa_privatekey_from_file(project_private_key_path, + 'password') + + delegation_private_key_path = os.path.join(keystore_path, 'snapshot_key') + delegation_private_key = \ + developer_tool.import_rsa_privatekey_from_file(delegation_private_key_path, + 'password') + + subdelegation_private_key_path = \ + os.path.join(keystore_path, 'timestamp_key') + subdelegation_private_key = \ + developer_tool.import_rsa_privatekey_from_file(subdelegation_private_key_path, + 'password') + + # test partial write + # backup everything (again) + # + backup targets: + targets_backup = project.target_files + + # + backup delegations + delegations_backup = \ + tuf.roledb.get_delegated_rolenames(project._project_name) + + # + backup layout type + layout_type_backup = project.layout_type + + # + backup keyid's + keys_backup = project.keys + delegation_keys_backup = project("delegation").keys + + # + backup the prefix + prefix_backup = project.prefix + + # + backup the name + name_backup = project._project_name + + # set the compressions, we will be checking this part here too + project.compressions = ['gz'] + project('delegation').compressions = project.compressions + + # write an reload + project.write(write_partial=True) +# import pdb; pdb.set_trace() + project = developer_tool.load_project(local_tmp) + + # check against backup + self.assertEquals(project.target_files, targets_backup) + new_delegations = tuf.roledb.get_delegated_rolenames(project._project_name) + self.assertEquals(new_delegations, delegations_backup) + self.assertEquals(project.layout_type, layout_type_backup) + self.assertEquals(project.keys, keys_backup) + self.assertEquals(project("delegation").keys, delegation_keys_backup) + self.assertEquals(project.prefix, prefix_backup) + self.assertEquals(project._project_name, name_backup) + + + + roleinfo = tuf.roledb.get_roleinfo(project._project_name) + #import pdb;pdb.set_trace() + self.assertEquals(roleinfo['partial_loaded'], True) + + + + # load_signing_keys + project.load_signing_key(project_private_key) + project("delegation").load_signing_key(delegation_private_key) + project("delegation")("subdelegation").load_signing_key( + subdelegation_private_key) + + project.status() + + # backup everything + # + backup targets: + targets_backup = project.target_files + + # + backup delegations + delegations_backup = \ + tuf.roledb.get_delegated_rolenames(project._project_name) + + # + backup layout type + layout_type_backup = project.layout_type + + # + backup keyid's + keys_backup = project.keys + delegation_keys_backup = project("delegation").keys + + # + backup the prefix + prefix_backup = project.prefix + + # + backup the name + name_backup = project._project_name + + # call status (for the sake of doing it) + # project.status() + + # call write + project.write() + + # call load + project = developer_tool.load_project(local_tmp) + + + # check against backup + self.assertEquals(project.target_files, targets_backup) + + new_delegations = tuf.roledb.get_delegated_rolenames(project._project_name) + self.assertEquals(new_delegations, delegations_backup) + self.assertEquals(project.layout_type, layout_type_backup) + self.assertEquals(project.keys, keys_backup) + self.assertEquals(project("delegation").keys, delegation_keys_backup) + self.assertEquals(project.prefix, prefix_backup) + self.assertEquals(project._project_name, name_backup) + + + + + +if __name__ == '__main__': + unittest.main() From 26eb5b7e354545539f5dbc380347d005bf057ea1 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Sat, 21 Jun 2014 19:07:34 -0400 Subject: [PATCH 46/51] Added relative filepath support and increased coverage After seeing the coveralls report I realized the projects were not relocatable. While this might be convenient for some uses, moving a project from one place to another might not be optimal. This has been changed now. The only place where absolute filepaths are handled now is with flat project layouts. However, it is possible to overwrite this filepath if the targets folder is to be changed. --- .../project/test-flat/project.cfg | 2 +- .../project/test-flat/test-flat.json | Bin 1974 -> 1974 bytes .../project/test-flat/test-flat/role1.json | Bin 980 -> 980 bytes .../test-repo/metadata/test-repo-like.json | Bin 1979 -> 1979 bytes .../metadata/test-repo-like/role1.json | Bin 980 -> 980 bytes .../project/test-repo/project.cfg | 2 +- tests/test_developer_tool.py | 57 ++++++++++++-- tuf/developer_tool.py | 71 +++++++++++------- 8 files changed, 96 insertions(+), 36 deletions(-) diff --git a/tests/repository_data/project/test-flat/project.cfg b/tests/repository_data/project/test-flat/project.cfg index 61b14806..135cbaed 100644 --- a/tests/repository_data/project/test-flat/project.cfg +++ b/tests/repository_data/project/test-flat/project.cfg @@ -1 +1 @@ -{"project_name": "test-flat", "targets_location": "/home/santiago/Documents/v2014/TUF/tuf/tests/repository_data/project/targets", "prefix": "prefix", "metadata_location": "/home/santiago/Documents/v2014/TUF/tuf/tests/repository_data/project/test-flat", "threshold": 1, "public_keys": {"6986b667c736a3b37471e030cf4ce7aa6c7e0d530325e64c2660276b77be3754": {"keytype": "rsa", "keyval": {"public": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7J15ZaeDQPrhQsRj29wB\nPhibH+Do59xsT2396L+uCg793gZlar5wZN2eHSh725cNQWyTAa9LwG+lXaKMukQ+\n8176CKR2J5sv3DezrGVu3x8V1qhyJyy79FlNZRVYTVqNaYzvJzxsVnFPpg7f8B7C\nffiqWJr9XkpqwRlCpxooXm4hplZ7uek5Ku21CzQ4OWg7hbuc+ZjCGzpXfm8NuosU\n7TipnKGpEt0Agiph5g6TB2/scoeFar1CKMONIl80maxzAQk+xkWgiJ00+Z2qFCsx\nESfis/YkILS6RMFyZz7oa1WwMtUjYmrsRuz+jlFcbNuxZpIkaISiG9a2YdGcJ1Aj\n3QIDAQAB\n-----END PUBLIC KEY-----"}}}, "layout_type": "flat"} \ No newline at end of file +{"project_name": "test-flat", "targets_location": "/home/santiago/Documents/v2014/TUF/tuf/tests/repository_data/project/targets", "prefix": "prefix", "metadata_location": "test-flat", "threshold": 1, "public_keys": {"6986b667c736a3b37471e030cf4ce7aa6c7e0d530325e64c2660276b77be3754": {"keytype": "rsa", "keyval": {"public": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7J15ZaeDQPrhQsRj29wB\nPhibH+Do59xsT2396L+uCg793gZlar5wZN2eHSh725cNQWyTAa9LwG+lXaKMukQ+\n8176CKR2J5sv3DezrGVu3x8V1qhyJyy79FlNZRVYTVqNaYzvJzxsVnFPpg7f8B7C\nffiqWJr9XkpqwRlCpxooXm4hplZ7uek5Ku21CzQ4OWg7hbuc+ZjCGzpXfm8NuosU\n7TipnKGpEt0Agiph5g6TB2/scoeFar1CKMONIl80maxzAQk+xkWgiJ00+Z2qFCsx\nESfis/YkILS6RMFyZz7oa1WwMtUjYmrsRuz+jlFcbNuxZpIkaISiG9a2YdGcJ1Aj\n3QIDAQAB\n-----END PUBLIC KEY-----"}}}, "layout_type": "flat"} \ No newline at end of file diff --git a/tests/repository_data/project/test-flat/test-flat.json b/tests/repository_data/project/test-flat/test-flat.json index 147417248d707c8c92e95f3f7784a2c926689368..98c5665e27f88737c5462ad30cc6780e8f3ec83b 100644 GIT binary patch delta 528 zcmWlW%Z(T?2t=hQId2`gfib^wz(5;Y!Aj{RQtAtPGnQuF4E+1}_wncJ^FCL1@i~pg zknIA+eXDNLy^|3vqrop9KsB|G4#_@lHmPYdzfN^9LT4_2f+VCeWEA?+=9twj8^jx# zm8-WhfqtJ&askx>$5Z(n3=hIS`sN|iP`%NM>ru z0GD;gRdrfU8SW71GwQ}Mwsq^k=ON0Omx7FwOY*M9cq@_Tt7Xipuv)`7agJMS_y8WJVP(fwev%to10_Ot!{@%{1V>(d=lT$S8iX*?Sg za4bOJW=RJUV^HdWP;(HH6RKq^mkXHIEsIw>!8y~rT~1*eB*&e;daH2K1rRH(1SQUa zjzj%{=tLACr8EK JuOGjE{R3a7k}m)N diff --git a/tests/repository_data/project/test-flat/test-flat/role1.json b/tests/repository_data/project/test-flat/test-flat/role1.json index 4e38e30f124edd9b78248a88d834468ea0145b4b..20e9f7cf80003463222e17be014afc9b483aae2d 100644 GIT binary patch delta 528 zcmWlV+l?472n3ZV(zV0GfTGv6QIAAi0+*-~v}GQ?d@ zVXNI7!YYrt2aQ$qamg!5%_EoL;Q&15qVh+Aji)+zV2nDlJ8pvjnwNH`ZKm%9lhZU7 zeNcd-AIm41Is`t0@KB3DIdD@Cfzq1+8Ek(lHQ{LG4u>{D+M87)#%q&K#=CfUkDiVb zkz1j z1{aPJu6WDRd0!pkrd~Y~bjerbj->et=PJ_lF^|^|wzwf)DB^7|zIz0*4k-JEO-lb- M6ZrS*=kFi?0Qb_ArvLx| delta 528 zcmWNO+ld`83`9u?Y&+Ilsvvbpmet0sU?IJvl@t_#Ka6ZNXJ-C={rmd!{n4pr!s27$ z(uPMK3MKa>(-Q}nJJSGr<|P%p5qj*AZpZE=0h9}gVI5WjUai9Db7n%tpE4HI#T1gT z=&Q`cWoY!dES)Zqs9FHJ%EnZ6^yC<@daLzIh)4w%?#k0j+ujoNmwJgkn zZ#uxf)K3iQ=wMOUwp5!`Nsj1F)J~;)AI4~p7Nw0XXzSTuJn!l|x9u_l@2)DprWi`U MbX)oR{qy&a|DW5C9smFU diff --git a/tests/repository_data/project/test-repo/metadata/test-repo-like.json b/tests/repository_data/project/test-repo/metadata/test-repo-like.json index cad9c4c1ba6b25a216eb24187c965e8beda05de3..10caafc6086b39ffb415eb949d95f16c66b87ef4 100644 GIT binary patch delta 528 zcmWkq$!!=w2$Z7a?M|ZQSdMa*s|{CRDZS+B=0biThPl2!zCZqaeWFD?OccC)(%egn z-CX9S5i9eI(=`Xyn@@L!8!=;qX;;-+kcZJUUV*3Y$~HQ1ywC%&pi|kD8lGF54D+Fb zZE-%$mHlMj7^Ku@0+>}hN_M=bLSmu9Fpu-}GCnnaBEgQ$`(k?0O`O9_?tPtbnkSY@ zNc>d8DZ*EstZv*VRPM98K|jm%SZ+!HC5T4uaSgBAOE~5KLnGs3s#1Y{v$;?;Av!`} zNdw!-(4prmv1Z=lSUA^jow>)f2K3?H>&2G+8i;OGeew-xkHv*4Q2lqno$LAt49w1U zz)Ps|3MDQTE}Y_7x`U%|t=|zzVe8zAr)ZOoAE(#$Qp#V%2BN-tFsW?(t{}`$YURIQ IKYsuE2dqYs^#A|> delta 528 zcmW-dNsbUe2t^a4>%NI$Du&}Q6xZkzbmCoHdO1tk0U!9hKOcWSetms*NAAHk=%C6S zMM<(N6w>SW9&TOF2?ksS=ownq1B*>BVwi-|CDCqZsm5wv-9zYcvQ0U2FJipTnQ>F2 zc3!DG?>?oLw455fQ@0Sg!q+Qqf(?&f+#+USNSA9{tf_pfOV zSYi#bn|lL#1|TFsBX&;f)ZFK8pG>weElL94Yktf)i$+xDK0-`^2L#3#?AECe{ch_0 K`|I1!AAbQ_`H^%0 diff --git a/tests/repository_data/project/test-repo/metadata/test-repo-like/role1.json b/tests/repository_data/project/test-repo/metadata/test-repo-like/role1.json index caac5a0dcd78aa315017fb14ee00f9942eafecf7..588d8f6e5dc59551b97163f92e3a2a9e4bedd46a 100644 GIT binary patch delta 528 zcmWlV*=-O&3w%RLMucTd_(cv4Yo4ayo6``v}$RS zAr_hKr)%s{WJ=bXU3A!9f=ZmU>Y35TGS$;d=y(|eB7*W!=s-iTj~cv>UCvTkQkNCJ z*Viv9S`M*V(?IT_DRnB)4Xn@L?YXbRfJ);@IeC?9zMatP>5k51ni)g%XaYUr1V;R% zxZkUqeG*&4PC4-e+H;v)fBXkI{*Y+^ delta 528 zcmWMi*^M1Q2ved++s<%91*wC9p^aC;O6eu7q@chbma&chef|6T^ZgNT8%r&t(oqaL z-T?Ck=fPp>(L8CCm#M%SDMc4rOn8RkpabreItQCQH_x#Z%(;5tG{5$+P}B5d_Zp6~ z0c+g)P^|#<{9 diff --git a/tests/repository_data/project/test-repo/project.cfg b/tests/repository_data/project/test-repo/project.cfg index e3a93c8d..7cb30410 100644 --- a/tests/repository_data/project/test-repo/project.cfg +++ b/tests/repository_data/project/test-repo/project.cfg @@ -1 +1 @@ -{"project_name": "test-repo-like", "targets_location": "/home/santiago/Documents/v2014/TUF/tuf/tests/repository_data/project/test-repo/targets", "prefix": "prefix", "metadata_location": "/home/santiago/Documents/v2014/TUF/tuf/tests/repository_data/project/test-repo/metadata", "threshold": 1, "public_keys": {"6986b667c736a3b37471e030cf4ce7aa6c7e0d530325e64c2660276b77be3754": {"keytype": "rsa", "keyval": {"public": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7J15ZaeDQPrhQsRj29wB\nPhibH+Do59xsT2396L+uCg793gZlar5wZN2eHSh725cNQWyTAa9LwG+lXaKMukQ+\n8176CKR2J5sv3DezrGVu3x8V1qhyJyy79FlNZRVYTVqNaYzvJzxsVnFPpg7f8B7C\nffiqWJr9XkpqwRlCpxooXm4hplZ7uek5Ku21CzQ4OWg7hbuc+ZjCGzpXfm8NuosU\n7TipnKGpEt0Agiph5g6TB2/scoeFar1CKMONIl80maxzAQk+xkWgiJ00+Z2qFCsx\nESfis/YkILS6RMFyZz7oa1WwMtUjYmrsRuz+jlFcbNuxZpIkaISiG9a2YdGcJ1Aj\n3QIDAQAB\n-----END PUBLIC KEY-----"}}}, "layout_type": "repo-like"} \ No newline at end of file +{"project_name": "test-repo-like", "targets_location": "targets", "prefix": "prefix", "metadata_location": "metadata", "threshold": 1, "public_keys": {"6986b667c736a3b37471e030cf4ce7aa6c7e0d530325e64c2660276b77be3754": {"keytype": "rsa", "keyval": {"public": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7J15ZaeDQPrhQsRj29wB\nPhibH+Do59xsT2396L+uCg793gZlar5wZN2eHSh725cNQWyTAa9LwG+lXaKMukQ+\n8176CKR2J5sv3DezrGVu3x8V1qhyJyy79FlNZRVYTVqNaYzvJzxsVnFPpg7f8B7C\nffiqWJr9XkpqwRlCpxooXm4hplZ7uek5Ku21CzQ4OWg7hbuc+ZjCGzpXfm8NuosU\n7TipnKGpEt0Agiph5g6TB2/scoeFar1CKMONIl80maxzAQk+xkWgiJ00+Z2qFCsx\nESfis/YkILS6RMFyZz7oa1WwMtUjYmrsRuz+jlFcbNuxZpIkaISiG9a2YdGcJ1Aj\n3QIDAQAB\n-----END PUBLIC KEY-----"}}}, "layout_type": "repo-like"} \ No newline at end of file diff --git a/tests/test_developer_tool.py b/tests/test_developer_tool.py index 67853317..afc802fd 100755 --- a/tests/test_developer_tool.py +++ b/tests/test_developer_tool.py @@ -148,7 +148,31 @@ def test_create_new_project(self): self.assertTrue(len(project.keys) == 1) self.assertTrue(project.keys[0] == project_key['keyid']) + # set as readonly and try to write a repo + shutil.rmtree(targets_directory) + os.chmod(local_tmp, 0o0555) + + tuf.roledb.clear_roledb() + tuf.keydb.clear_keydb() + self.assertRaises(OSError, developer_tool.create_new_project ,project_name, + metadata_directory, location_in_repository, targets_directory, + project_key) + + os.chmod(local_tmp, 0o0777) + + shutil.rmtree(metadata_directory) + os.chmod(local_tmp, 0o0555) + + tuf.roledb.clear_roledb() + tuf.keydb.clear_keydb() + self.assertRaises(OSError, developer_tool.create_new_project ,project_name, + metadata_directory, location_in_repository, targets_directory, + project_key) + + + os.chmod(local_tmp, 0o0777) shutil.rmtree(local_tmp) + def test_load_project(self): @@ -176,8 +200,10 @@ def test_load_project(self): self.assertTrue(project.layout_type == "repo-like") repo_filepath = os.path.join(local_tmp, 'project', 'test-flat') - project = developer_tool.load_project(repo_filepath) - + new_targets_path = os.path.join(local_tmp, 'project', 'targets') + project = developer_tool.load_project(repo_filepath, + new_targets_location = new_targets_path) + self.assertTrue(project._targets_directory == new_targets_path) self.assertTrue(project.layout_type == 'flat') @@ -185,6 +211,15 @@ def test_load_project(self): project = developer_tool.load_project(repo_filepath, prefix='new') self.assertTrue(project.prefix == 'new') + # load a project with a file missing + file_to_corrupt = os.path.join(repo_filepath, 'test-flat','role1.json') + with open(file_to_corrupt, 'wt') as fp: + fp.write("this is not a json file") + + self.assertRaises(tuf.Error, developer_tool.load_project, repo_filepath) + + + def test_add_verification_keys(self): @@ -325,9 +360,10 @@ def test_write(self): project.compressions = ['gz'] project('delegation').compressions = project.compressions - # write an reload + # write and reload + self.assertRaises(tuf.Error, project.write) project.write(write_partial=True) -# import pdb; pdb.set_trace() + project = developer_tool.load_project(local_tmp) # check against backup @@ -343,19 +379,24 @@ def test_write(self): roleinfo = tuf.roledb.get_roleinfo(project._project_name) - #import pdb;pdb.set_trace() + self.assertEquals(roleinfo['partial_loaded'], True) # load_signing_keys - project.load_signing_key(project_private_key) project("delegation").load_signing_key(delegation_private_key) + + project.status() + project("delegation")("subdelegation").load_signing_key( subdelegation_private_key) +# import pdb; pdb.set_trace() project.status() + project.load_signing_key(project_private_key) + # backup everything # + backup targets: targets_backup = project.target_files @@ -376,9 +417,9 @@ def test_write(self): # + backup the name name_backup = project._project_name - + # call status (for the sake of doing it) - # project.status() + project.status() # call write project.write() diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 52b73b59..33c55ae2 100644 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -358,8 +358,15 @@ def status(self): try: temp_project_directory = tempfile.mkdtemp() metadata_directory = os.path.join(temp_project_directory, - METADATA_STAGED_DIRECTORY_NAME) - os.mkdir(metadata_directory) + self._metadata_directory[1:]) + + #targets_directory = os.path.join(temp_project_directory, + # self._targets_directory[1:]) + targets_directory = self._targets_directory + + + os.makedirs(metadata_directory) + # os.makedirs(targets_directory) #filenames = get_metadata_filenames(metadata_directory) # we should do the schema check @@ -382,10 +389,10 @@ def status(self): try: signable = _generate_and_write_metadata(delegated_role, filenames['targets'], False, - self._targets_directory, - self._metadata_directory, + targets_directory, + metadata_directory, False) - self._print_status(delegated_role, signable) + self._print_status(delegated_role, signable[0]) except tuf.Error: insufficient_signatures.append(delegated_role) @@ -393,23 +400,26 @@ def status(self): message = 'Delegated roles with insufficient keys: '+ \ repr(insufficient_keys) print(message) + return if len(insufficient_signatures): message = 'Delegated roles with insufficient signatures: '+ \ repr(insufficient_signatures) print(message) + return # Targets role. try: _check_role_keys(self.rolename) except tuf.InsufficientKeysError as e: print(str(e)) + return try: signable, filename = _generate_and_write_metadata(self._project_name, filenames['targets'], False, - self._targets_directory, - self._metadata_directory, + targets_directory, + metadata_directory, False) self._print_status(self._project_name, signable) except tuf.Error as e: @@ -715,14 +725,16 @@ def _save_project_configuration(metadata_directory,targets_directory, tuf.formats.RELPATH_SCHEMA.check_match(project_name) # get the absolute filepath to our metadata_directory for consistency - metadata_directory = os.path.abspath(metadata_directory) + #metadata_directory = os.path.abspath(metadata_directory) cfg_file_directory = metadata_directory # check wheter if the layout type is "flat" or "repo-like" # if it is, the .cfg file should be saved in the previous directory. if(layout_type == "repo-like"): cfg_file_directory = os.path.dirname(metadata_directory) + absolute_location, targets_directory = os.path.split(targets_directory) + absolute_location, metadata_directory = os.path.split(metadata_directory) # is the file open-able? open for overwriting project_filename = os.path.join(cfg_file_directory,PROJECT_FILENAME) @@ -747,14 +759,12 @@ def _save_project_configuration(metadata_directory,targets_directory, with open(project_filename,"wt") as fp: json.dump(project_config,fp) - # clean our mess - fp.close() -def load_project(project_directory, prefix=''): +def load_project(project_directory, prefix='', new_targets_location=None): """ Return a project object initialized with the contents of the metadata @@ -769,6 +779,11 @@ def load_project(project_directory, prefix=''): prefix, by first removing the existing one (Saved) and setting the new one in the end. + new_targets_location: + For flat project configurations, you might want to reload the project + with a new location for the target files. This overwrites the previous + path to search for the target files. + tuf.FormatError, if 'project_directory' or any of the metadata files are improperly formatted. @@ -802,10 +817,19 @@ def load_project(project_directory, prefix=''): tuf.formats.PROJECT_CFG_SCHEMA.check_match(project_configuration) except (OSError, IOError) as e: raise + + targets_directory = os.path.join(project_directory, + project_configuration['targets_location']) - metadata_directory = project_configuration['metadata_location'] - targets_directory = project_configuration['targets_location'] - + if project_configuration['layout_type'] == 'flat': + project_directory, relative_junk = os.path.split(project_directory) + targets_directory = project_configuration['targets_location'] + if new_targets_location is not None: + targets_directory = new_targets_location + + metadata_directory = os.path.join(project_directory, + project_configuration['metadata_location']) + new_prefix = None if prefix != '': new_prefix = prefix @@ -830,7 +854,8 @@ def load_project(project_directory, prefix=''): project.add_verification_key(key) # load the project's metadata - targets_metadata_path = os.path.join(metadata_directory, project_filename) + targets_metadata_path = os.path.join(project_directory, metadata_directory, + project_filename) signable = tuf.util.load_json_file(targets_metadata_path) tuf.formats.check_signable_object_format(signable) targets_metadata = signable['signed'] @@ -879,8 +904,8 @@ def load_project(project_directory, prefix=''): targets_objects = {} loaded_metadata = [] targets_objects[project_name] = project - targets_metadata_directory = os.path.join(metadata_directory, - project_name) + metadata_directory = os.path.join(project_directory, metadata_directory) + targets_metadata_directory = os.path.join(metadata_directory, project_name) if os.path.exists(targets_metadata_directory) and \ os.path.isdir(targets_metadata_directory): for root, directories, files in os.walk(targets_metadata_directory): @@ -897,21 +922,15 @@ def load_project(project_directory, prefix=''): metadata_name = metadata_name[:-extension_length] else: continue - - # Keep a store metadata previously loaded metadata to prevent - # re-loading duplicate versions. Duplicate versions may occur with - # consistent_snapshots, where the same metadata may be available in - # multiples files (the different hash is included in each filename. - if metadata_name in loaded_metadata: - continue signable = None try: signable = tuf.util.load_json_file(metadata_path) - except (ValueError, IOError): + except (ValueError, IOError, tuf.Error): raise - # strip the extension from the + # strip the prefix from the local working copy, it will be added again + # at the time of writing. metadata_object = signable['signed'] metadata_object = _strip_prefix_from_targets_metadata(metadata_object, prefix) From 00b4c3500d45a535f08a7759e77cf7c1d400eb5a Mon Sep 17 00:00:00 2001 From: vladdd Date: Fri, 27 Jun 2014 20:24:07 -0400 Subject: [PATCH 47/51] Add and fix links, move diagram to end, and rename heading. --- tuf/README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tuf/README.md b/tuf/README.md index 5a8828bd..e0d3ecf9 100644 --- a/tuf/README.md +++ b/tuf/README.md @@ -1,8 +1,7 @@ # Repository Management # ## Table of Contents ## -- [Repository Tool Diagram](#repository-tool-diagram) -- [TUF Repository](#create-tuf-repository) +- [The Files of a TUF Repository](#the-files-of-a-tuf-repository) - [Purpose](#purpose) - [Keys](#keys) - [Create RSA Keys](#create-rsa-keys) @@ -21,19 +20,16 @@ - [Client Setup and Repository Trial](#client-setup-and-repository-trial) - [Using TUF Within an Example Client Updater](#using-tuf-within-an-example-client-updater) - [Test TUF Locally](#test-tuf-locally) +- [Repository Tool Diagram](#repository-tool-diagram) -## Repository Tool Diagram ## -![Repo Tools Diagram 1](../docs/images/repository_tool-diagram.png) - - -## TUF Repository ## +## The Files of a TUF Repository ## ### Purpose ### -The **tuf.repository_tool** module can be used to create a TUF repository. -It may either be imported into a Python module or used with the Python -interpreter in interactive mode. +The [tuf.repository_tool](tuf/repository_tool.py) module can be used to create a +TUF repository. It may either be imported into a Python module or used with the +Python interpreter in interactive mode. ```Bash $ python @@ -43,8 +39,9 @@ Type "help", "copyright", "credits" or "license" for more information. >>> from tuf.repository_tool import * >>> repository = load_repository("path/to/repository") ``` -Note that **tuf.repository_tool.py** is not used in TUF integrations. The -[tuf.interposition](/interposition/README.md)** package and +Note that [tuf.repository_tool.py](tuf/repository_tool.py) is not used in TUF +integrations. The +[tuf.interposition](/interposition/README.md) package and [tuf.client.updater](/client/README.md) module assist in integrating TUF with a software updater. @@ -433,3 +430,6 @@ django file1.txt file2.txt targets/django/: file4.txt ``` + +## Repository Tool Diagram ## +![Repo Tools Diagram 1](../docs/images/repository_tool-diagram.png) From 7be31965e716b2e7e097a4d22ce1fc152aa1bf3f Mon Sep 17 00:00:00 2001 From: vladdd Date: Sun, 29 Jun 2014 21:33:22 -0400 Subject: [PATCH 48/51] Minor edits. --- tuf/keys.py | 24 ++++++++++++------------ tuf/pycrypto_keys.py | 15 +++++++-------- tuf/repository_lib.py | 5 +++-- tuf/repository_tool.py | 6 +++--- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tuf/keys.py b/tuf/keys.py index 7a50736b..c3af4d66 100755 --- a/tuf/keys.py +++ b/tuf/keys.py @@ -224,7 +224,7 @@ def generate_rsa_key(bits=_DEFAULT_RSA_KEY_BITS): public, private = tuf.pycrypto_keys.generate_rsa_public_and_private(bits) else: # pragma: no cover - message = 'Invalid crypto library: '+repr(_RSA_CRYPTO_LIBRARY)+'.' + message = 'Invalid crypto library: ' + repr(_RSA_CRYPTO_LIBRARY) + '.' raise tuf.UnsupportedLibraryError(message) # Generate the keyid of the RSA key. 'key_value' corresponds to the @@ -551,9 +551,9 @@ def check_crypto_libraries(required_libraries): if 'rsa' in required_libraries and _RSA_CRYPTO_LIBRARY not in \ _SUPPORTED_RSA_CRYPTO_LIBRARIES: - message = 'The '+repr(_RSA_CRYPTO_LIBRARY)+' crypto library specified'+ \ - ' in "tuf.conf.RSA_CRYPTO_LIBRARY" is not supported.\n'+ \ - 'Supported crypto libraries: '+repr(_SUPPORTED_RSA_CRYPTO_LIBRARIES)+'.' + message = 'The ' + repr(_RSA_CRYPTO_LIBRARY) + ' crypto library specified' +\ + ' in "tuf.conf.RSA_CRYPTO_LIBRARY" is not supported.\n' +\ + 'Supported crypto libraries: ' + repr(_SUPPORTED_RSA_CRYPTO_LIBRARIES) + '.' raise tuf.UnsupportedLibraryError(message) if 'ed25519' in required_libraries and _ED25519_CRYPTO_LIBRARY not in \ @@ -697,8 +697,8 @@ def create_signature(key_dict, data): sig, method = tuf.pycrypto_keys.create_rsa_signature(private, data.encode('utf-8')) else: # pragma: no cover - message = 'Unsupported "tuf.conf.RSA_CRYPTO_LIBRARY": '+\ - repr(_RSA_CRYPTO_LIBRARY)+'.' + message = 'Unsupported "tuf.conf.RSA_CRYPTO_LIBRARY": ' +\ + repr(_RSA_CRYPTO_LIBRARY) + '.' raise tuf.UnsupportedLibraryError(message) elif keytype == 'ed25519': @@ -824,7 +824,7 @@ def verify_signature(key_dict, signature, data): if keytype == 'rsa': if _RSA_CRYPTO_LIBRARY == 'pycrypto': if 'pycrypto' not in _available_crypto_libraries: # pragma: no cover - message = 'Metadata downloaded from the remote repository specified'+\ + message = 'Metadata downloaded from the remote repository specified' +\ ' an RSA signature. Verifying RSA signatures requires PyCrypto.' +\ '\n$ pip install PyCrypto, or pip install tuf[tools].' raise tuf.UnsupportedLibraryError(message) @@ -833,7 +833,7 @@ def verify_signature(key_dict, signature, data): valid_signature = tuf.pycrypto_keys.verify_rsa_signature(sig, method, public, data) else: # pragma: no cover - message = 'Unsupported "tuf.conf.RSA_CRYPTO_LIBRARY": '+\ + message = 'Unsupported "tuf.conf.RSA_CRYPTO_LIBRARY": ' +\ repr(_RSA_CRYPTO_LIBRARY)+'.' raise tuf.UnsupportedLibraryError(message) @@ -944,7 +944,7 @@ def import_rsakey_from_encrypted_pem(encrypted_pem, password): tuf.pycrypto_keys.create_rsa_public_and_private_from_encrypted_pem(encrypted_pem, password) else: #pragma: no cover - message = 'Invalid crypto library: '+repr(_RSA_CRYPTO_LIBRARY)+'.' + message = 'Invalid crypto library: ' + repr(_RSA_CRYPTO_LIBRARY) + '.' raise tuf.UnsupportedLibraryError(message) # Generate the keyid of the RSA key. 'key_value' corresponds to the @@ -1120,7 +1120,7 @@ def encrypt_key(key_object, password): # check_crypto_libraries() should have fully verified _GENERAL_CRYPTO_LIBRARY. else: # pragma: no cover - message = 'Invalid crypto library: '+repr(_GENERAL_CRYPTO_LIBRARY)+'.' + message = 'Invalid crypto library: ' + repr(_GENERAL_CRYPTO_LIBRARY) + '.' raise tuf.UnsupportedLibraryError(message) return encrypted_key @@ -1218,7 +1218,7 @@ def decrypt_key(encrypted_key, passphrase): # check_crypto_libraries() should have fully verified _GENERAL_CRYPTO_LIBRARY. else: # pragma: no cover - message = 'Invalid crypto library: '+repr(_GENERAL_CRYPTO_LIBRARY)+'.' + message = 'Invalid crypto library: ' + repr(_GENERAL_CRYPTO_LIBRARY) + '.' raise tuf.UnsupportedLibraryError(message) # The corresponding encrypt_key() encrypts and stores key objects in @@ -1301,7 +1301,7 @@ def create_rsa_encrypted_pem(private_key, passphrase): # check_crypto_libraries() should have fully verified _RSA_CRYPTO_LIBRARY. else: # pragma: no cover - message = 'Invalid crypto library: '+repr(_RSA_CRYPTO_LIBRARY)+'.' + message = 'Invalid crypto library: ' + repr(_RSA_CRYPTO_LIBRARY) + '.' raise tuf.UnsupportedLibraryError(message) return encrypted_pem diff --git a/tuf/pycrypto_keys.py b/tuf/pycrypto_keys.py index 72c7e8b7..f3ac2382 100755 --- a/tuf/pycrypto_keys.py +++ b/tuf/pycrypto_keys.py @@ -295,7 +295,7 @@ def create_rsa_signature(private_key, data): rsa_key_object = Crypto.PublicKey.RSA.importKey(private_key) except (ValueError, IndexError, TypeError) as e: - message = 'Invalid private key or hash data: '+str(e) + message = 'Invalid private key or hash data: ' + str(e) raise tuf.CryptoError(message) # Generate RSSA-PSS signature. Raise 'tuf.CryptoError' for the expected @@ -311,7 +311,7 @@ def create_rsa_signature(private_key, data): raise tuf.CryptoError('Missing required RSA private key.') except IndexError: - message = 'An RSA signature cannot be generated: '+str(e) + message = 'An RSA signature cannot be generated: ' + str(e) raise tuf.CryptoError(message) else: @@ -474,7 +474,7 @@ def create_rsa_encrypted_pem(private_key, passphrase): passphrase=passphrase) except (ValueError, IndexError, TypeError) as e: - message = 'An encrypted RSA key in PEM format cannot be generated: '+str(e) + message = 'An encrypted RSA key in PEM format cannot be generated: ' + str(e) raise tuf.CryptoError(message) else: @@ -570,8 +570,8 @@ def create_rsa_public_and_private_from_encrypted_pem(encrypted_pem, passphrase): # If the passphrase is incorrect, PyCrypto returns: "RSA key format is not # supported". except (ValueError, IndexError, TypeError) as e: - message = 'RSA (public, private) tuple cannot be generated from the'+\ - ' encrypted PEM string: '+str(e) + message = 'RSA (public, private) tuple cannot be generated from the' +\ + ' encrypted PEM string: ' + str(e) # Raise 'tuf.CryptoError' and PyCrypto's exception message. Avoid # propogating PyCrypto's exception trace to avoid revealing sensitive error. raise tuf.CryptoError(message) @@ -692,7 +692,6 @@ def encrypt_key(key_object, password): def decrypt_key(encrypted_key, password): """ - Return a string containing 'encrypted_key' in non-encrypted form. The decrypt_key() function can be applied to the encrypted string to restore the original key object, a TUF key (e.g., RSAKEY_SCHEMA, ED25519KEY_SCHEMA). @@ -862,7 +861,7 @@ def _encrypt(key_data, derived_key_information): # checking for exceptions. Avoid propogating the exception trace and only # raise 'tuf.CryptoError', along with the cause of encryption failure. except (ValueError, IndexError, TypeError) as e: - message = 'The key data cannot be encrypted: '+str(e) + message = 'The key data cannot be encrypted: ' + str(e) raise tuf.CryptoError(message) # Generate the hmac of the ciphertext to ensure it has not been modified. @@ -951,7 +950,7 @@ def _decrypt(file_contents, password): # Note: decryption failure, due to malicious ciphertext, should not occur here # if the hmac check above passed. except (ValueError, IndexError, TypeError) as e: # pragma: no cover - raise tuf.CryptoError('Decryption failed: '+str(e)) + raise tuf.CryptoError('Decryption failed: ' + str(e)) return key_plaintext diff --git a/tuf/repository_lib.py b/tuf/repository_lib.py index fb747695..62efd001 100755 --- a/tuf/repository_lib.py +++ b/tuf/repository_lib.py @@ -1935,6 +1935,7 @@ def write_metadata_file(metadata, filename, compressions, consistent_snapshot): gzip_object = gzip.GzipFile(fileobj=file_object, mode='wb') try: gzip_object.write(file_content) + finally: gzip_object.close() @@ -2188,8 +2189,8 @@ def create_tuf_client_directory(repository_directory, client_directory): except OSError as e: if e.errno == errno.EEXIST: - message = 'Cannot create a fresh client metadata directory: '+ \ - repr(client_metadata_directory)+'. Already exists.' + message = 'Cannot create a fresh client metadata directory: ' +\ + repr(client_metadata_directory) + '. Already exists.' raise tuf.RepositoryError(message) else: diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index 1fb032e2..3210729c 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -469,7 +469,7 @@ def get_filepaths_in_directory(files_directory, recursive_walk=False, # Ensure a valid directory is given. if not os.path.isdir(files_directory): - message = repr(files_directory)+' is not a directory.' + message = repr(files_directory) + ' is not a directory.' raise tuf.Error(message) # A list of the target filepaths found in 'files_directory'. @@ -2271,8 +2271,8 @@ def delegate_hashed_bins(self, list_of_targets, keys_of_hashed_bins, for target_path in list_of_targets: target_path = os.path.abspath(target_path) if not target_path.startswith(self._targets_directory+os.sep): - message = 'A path in the list of targets argument is not '+\ - 'under the repository\'s targets directory: '+repr(target_path) + message = 'A path in the list of targets argument is not ' +\ + 'under the repository\'s targets directory: ' + repr(target_path) raise tuf.Error(message) # Determine the hash prefix of 'target_path' by computing the digest of From 2f520dd1409b6d59c856ec670192d8381cb4f48f Mon Sep 17 00:00:00 2001 From: vladdd Date: Mon, 30 Jun 2014 14:04:01 -0400 Subject: [PATCH 49/51] Review and update pull request #188. --- .../repository_data/generate_project_data.py | 85 +--- tests/test_developer_tool.py | 143 ++++--- tuf/developer_tool.py | 374 +++++++++--------- 3 files changed, 280 insertions(+), 322 deletions(-) mode change 100644 => 100755 tuf/developer_tool.py diff --git a/tests/repository_data/generate_project_data.py b/tests/repository_data/generate_project_data.py index 19d6a8db..6e647487 100755 --- a/tests/repository_data/generate_project_data.py +++ b/tests/repository_data/generate_project_data.py @@ -7,12 +7,12 @@ Santiago Torres - See LICENSE for licensing information. - Generate a pre-fabricated set of metadata files to use for the unit testing. + Generate a pre-fabricated set of metadata files for 'test_developer_tool.py' + test cases. """ import shutil @@ -20,9 +20,8 @@ import optparse import os -from tuf.developer_tool import * import tuf.util - +from tuf.developer_tool import * parser = optparse.OptionParser() @@ -35,8 +34,8 @@ targets_key_file = 'keystore/targets_key' delegation_key_file = 'keystore/delegation_key' -# the files we use for signing in the unit tests should exist, if they are not -# populated, run generate.py +# The files we use for signing in the unit tests should exist, if they are not +# populated, run 'generate.py'. assert os.path.exists(project_key_file) assert os.path.exists(targets_key_file) assert os.path.exists(delegation_key_file) @@ -44,9 +43,9 @@ # Import the public keys. These keys are needed so that metadata roles are # assigned verification keys, which clients use to verify the signatures created # by the corresponding private keys. -project_public = import_rsa_publickey_from_file(project_key_file+'.pub') -targets_public = import_rsa_publickey_from_file(targets_key_file+'.pub') -delegation_public = import_rsa_publickey_from_file(delegation_key_file+'.pub') +project_public = import_rsa_publickey_from_file(project_key_file + '.pub') +targets_public = import_rsa_publickey_from_file(targets_key_file + '.pub') +delegation_public = import_rsa_publickey_from_file(delegation_key_file + '.pub') # Import the private keys. These private keys are needed to generate the # signatures included in metadata. @@ -80,81 +79,29 @@ project = create_new_project("test-flat", 'project/test-flat', 'prefix', 'project/targets') -# Add target files to the top-level 'targets.json' role. These target files -# should already exist. +# Add target files to the top-level projects role. These target files should +# already exist. project.add_target(target1_filepath) project.add_target(target2_filepath) -# add keys to the project +# Add one key to the project. project.add_verification_key(project_public) project.load_signing_key(project_private) +# Add the delegated role keys. project.delegate('role1', [delegation_public], [target3_filepath]) project('role1').load_signing_key(delegation_private) -# Set the top-level expiration times far into the future so that -# they do not expire anytime soon, or else the tests fail. Unit tests may -# modify the expiration datetimes (of the copied files), if they wish. +# Set the project expiration time far into the future so that its metadata does +# not expire anytime soon, or else the tests fail. Unit tests may modify the +# expiration datetimes (of the copied files), if they wish. project.expiration = datetime.datetime(2030, 1, 1, 0, 0) project('role1').expiration = datetime.datetime(2030, 1, 1, 0, 0) -# Compress the 'targets.json' role so that the unit tests have a pre-generated +# Compress the project role metadata so that the unit tests have a pre-generated # example of compressed metadata. project.compressions = ['gz'] # Create the actual metadata files, which are saved to 'metadata.staged'. if not options.dry_run: project.write() - -tuf.roledb.clear_roledb() -tuf.keydb.clear_keydb() - -project = create_new_project("test-repo-like", "project/test-repo", 'prefix') - -# Create the target files (downloaded by clients) whose file size and digest -# are specified in the 'targets.json' file. -target1_filepath = 'project/test-repo/targets/file1.txt' -tuf.util.ensure_parent_dir(target1_filepath) -target2_filepath = 'project/test-repo/targets/file2.txt' -tuf.util.ensure_parent_dir(target2_filepath) -target3_filepath = 'project/test-repo/targets/file3.txt' -tuf.util.ensure_parent_dir(target2_filepath) - -if not options.dry_run: - with open(target1_filepath, 'wt') as file_object: - file_object.write('This is an example target file.') - - with open(target2_filepath, 'wt') as file_object: - file_object.write('This is an another example target file.') - - with open(target3_filepath, 'wt') as file_object: - file_object.write('This is role1\'s target file.') - - -# Add target files to the top-level 'targets.json' role. These target files -# should already exist. -project.add_target(target1_filepath) -project.add_target(target2_filepath) - -# add keys to the project -project.add_verification_key(project_public) -project.load_signing_key(project_private) - -project.delegate('role1', [delegation_public], [target3_filepath]) -project('role1').load_signing_key(delegation_private) - -# Set the top-level expiration times far into the future so that -# they do not expire anytime soon, or else the tests fail. Unit tests may -# modify the expiration datetimes (of the copied files), if they wish. -project.expiration = datetime.datetime(2030, 1, 1, 0, 0) -project('role1').expiration = datetime.datetime(2030, 1, 1, 0, 0) - -# Compress the 'targets.json' role so that the unit tests have a pre-generated -# example of compressed metadata. -project.compressions = ['gz'] - -# Create the actual metadata files, which are saved to 'metadata.staged'. -if not options.dry_run: - project.write() - - diff --git a/tests/test_developer_tool.py b/tests/test_developer_tool.py index afc802fd..bedcccf6 100755 --- a/tests/test_developer_tool.py +++ b/tests/test_developer_tool.py @@ -2,17 +2,17 @@ """ - test_developer_tool.py + test_developer_tool.py. Santiago Torres Arias - See LICENSE for licensing inforation + See LICENSE for licensing inforation. - Unit tests for the 'developer_tool' module + Unit test for the 'developer_tool.py' module. """ import os @@ -35,6 +35,8 @@ logger = logging.getLogger("tuf.test_developer_tool") +developer_tool.disable_console_log_messages() + class TestProject(unittest.TestCase): tmp_dir = None @@ -57,52 +59,49 @@ def tearDown(self): tuf.keydb.clear_keydb() pass - def test_emtpy(self): - return - def test_create_new_project(self): - # test cases for the create_new_project function. In this test we will + # Test cases for the create_new_project function. In this test we will # check input, correct file creation and format. We also check # that a proper object is generated. We will use the normal layout for this # test suite. - # create a local subfolder for this test. + # Create a local subfolder for this test. local_tmp = tempfile.mkdtemp(dir = self.tmp_dir) - # these are the usual values we will be throwing to the function, however + # These are the usual values we will be throwing to the function, however # we will swap these for nulls or malformed values every now and then to - # test input + # test input. project_name = "test_suite" metadata_directory = local_tmp location_in_repository = '/prefix' targets_directory = None key = None - # create a blank project + # Create a blank project. project = developer_tool.create_new_project(project_name, metadata_directory, location_in_repository) self.assertTrue(isinstance(project, developer_tool.Project)) self.assertTrue(project.layout_type == 'repo-like') - self.assertTrue(project.prefix == location_in_repository) + self.assertTrue(project._prefix == location_in_repository) self.assertTrue(project._project_name == project_name) self.assertTrue(project._metadata_directory == os.path.join(metadata_directory,METADATA_DIRECTORY_NAME)) self.assertTrue(project._targets_directory == os.path.join(metadata_directory,TARGETS_DIRECTORY_NAME)) - # create a blank project without a prefix + # Create a blank project without a prefix. project = developer_tool.create_new_project(project_name, metadata_directory) self.assertTrue(isinstance(project, developer_tool.Project)) self.assertTrue(project.layout_type == 'repo-like') - self.assertTrue(project.prefix == '') + self.assertTrue(project._prefix == '') self.assertTrue(project._project_name == project_name) self.assertTrue(project._metadata_directory == os.path.join(metadata_directory,METADATA_DIRECTORY_NAME)) self.assertTrue(project._targets_directory == os.path.join(metadata_directory,TARGETS_DIRECTORY_NAME)) - # create a blank project without a valid metadata directory + # Create a blank project without a valid metadata directory. self.assertRaises(tuf.FormatError, developer_tool.create_new_project, 0, metadata_directory, location_in_repository) self.assertRaises(tuf.FormatError, developer_tool.create_new_project, @@ -111,23 +110,23 @@ def test_create_new_project(self): project_name, metadata_directory, 0) - # create a new project with a flat layout + # Create a new project with a flat layout. targets_directory = tempfile.mkdtemp(dir = local_tmp) metadata_directory = tempfile.mkdtemp(dir = local_tmp) project = developer_tool.create_new_project(project_name, metadata_directory, location_in_repository, targets_directory) self.assertTrue(isinstance(project, developer_tool.Project)) self.assertTrue(project.layout_type == 'flat') - self.assertTrue(project.prefix == location_in_repository) + self.assertTrue(project._prefix == location_in_repository) self.assertTrue(project._project_name == project_name) self.assertTrue(project._metadata_directory == metadata_directory) self.assertTrue(project._targets_directory == targets_directory) - # finally, check that if targets_directory is set, it is valid + # Finally, check that if targets_directory is set, it is valid. self.assertRaises(tuf.FormatError, developer_tool.create_new_project, project_name, metadata_directory, location_in_repository, 0) - # copy a key to our workspace and create a new project with it. + # Copy a key to our workspace and create a new project with it. keystore_path = os.path.join('repository_data','keystore') # I will use the same key as the one provided in the repository @@ -135,20 +134,20 @@ def test_create_new_project(self): root_key_path = os.path.join(keystore_path,'root_key.pub') project_key = developer_tool.import_rsa_publickey_from_file(root_key_path) - # test create new project with a key added by default + # Test create new project with a key added by default. project = developer_tool.create_new_project(project_name, metadata_directory, location_in_repository, targets_directory, project_key) self.assertTrue(isinstance(project, developer_tool.Project)) self.assertTrue(project.layout_type == 'flat') - self.assertTrue(project.prefix == location_in_repository) + self.assertTrue(project._prefix == location_in_repository) self.assertTrue(project._project_name == project_name) self.assertTrue(project._metadata_directory == metadata_directory) self.assertTrue(project._targets_directory == targets_directory) self.assertTrue(len(project.keys) == 1) self.assertTrue(project.keys[0] == project_key['keyid']) - # set as readonly and try to write a repo + # Set as readonly and try to write a repo. shutil.rmtree(targets_directory) os.chmod(local_tmp, 0o0555) @@ -176,24 +175,24 @@ def test_create_new_project(self): def test_load_project(self): - # this test case will try to load an existing project and test for the result - # then we will try to load a nonexisting project and expect a correct error - # handler - # finally, we will try to overwrite the existing prefix on the loaded project + # This test case will first try to load an existing project and test for + # verify the loaded object. It will next try to load a nonexisting project + # and expect a correct error handler. Finally, it will try to overwrite the + # existing prefix of the loaded project. - # create a local subfolder for this test. + # Create a local subfolder for this test. local_tmp = tempfile.mkdtemp(dir = self.tmp_dir) - # test inexisting project filepath + # Test non-existent project filepath. nonexistent_path = os.path.join(local_tmp, "nonexistent") self.assertRaises(IOError, developer_tool.load_project, nonexistent_path) - # copy the pregenerated metadata + # Copy the pregenerated metadata. project_data_filepath = os.path.join('repository_data', 'project') target_project_data_filepath = os.path.join(local_tmp, 'project') shutil.copytree("repository_data/project", target_project_data_filepath) - # load a project properly... + # Properly load a project. repo_filepath = os.path.join(local_tmp, 'project', 'test-repo') project = developer_tool.load_project(repo_filepath) @@ -207,12 +206,12 @@ def test_load_project(self): self.assertTrue(project.layout_type == 'flat') - # load a project ovrwriting the prefix + # Load a project overwriting the prefix. project = developer_tool.load_project(repo_filepath, prefix='new') - self.assertTrue(project.prefix == 'new') + self.assertTrue(project._prefix == 'new') - # load a project with a file missing - file_to_corrupt = os.path.join(repo_filepath, 'test-flat','role1.json') + # Load a project with a file missing. + file_to_corrupt = os.path.join(repo_filepath, 'test-flat', 'role1.json') with open(file_to_corrupt, 'wt') as fp: fp.write("this is not a json file") @@ -222,9 +221,6 @@ def test_load_project(self): def test_add_verification_keys(self): - - # maybe we need to clear the roledb and keydb - # create a new project instance project = developer_tool.Project("test_verification_keys", "somepath", "someotherpath", "prefix") @@ -334,7 +330,7 @@ def test_write(self): developer_tool.import_rsa_privatekey_from_file(subdelegation_private_key_path, 'password') - # test partial write + # Test partial write # backup everything (again) # + backup targets: targets_backup = project.target_files @@ -346,45 +342,45 @@ def test_write(self): # + backup layout type layout_type_backup = project.layout_type - # + backup keyid's + # + backup keyids keys_backup = project.keys delegation_keys_backup = project("delegation").keys - # + backup the prefix - prefix_backup = project.prefix + # + backup the prefix. + prefix_backup = project._prefix - # + backup the name + # + backup the name. name_backup = project._project_name - # set the compressions, we will be checking this part here too + # Set the compressions, we will be checking this part here too project.compressions = ['gz'] project('delegation').compressions = project.compressions - # write and reload + # Write and reload. self.assertRaises(tuf.Error, project.write) project.write(write_partial=True) project = developer_tool.load_project(local_tmp) - # check against backup - self.assertEquals(project.target_files, targets_backup) + # Check against backup. + self.assertEqual(project.target_files, list(targets_backup.keys())) new_delegations = tuf.roledb.get_delegated_rolenames(project._project_name) - self.assertEquals(new_delegations, delegations_backup) - self.assertEquals(project.layout_type, layout_type_backup) - self.assertEquals(project.keys, keys_backup) - self.assertEquals(project("delegation").keys, delegation_keys_backup) - self.assertEquals(project.prefix, prefix_backup) - self.assertEquals(project._project_name, name_backup) + self.assertEqual(new_delegations, delegations_backup) + self.assertEqual(project.layout_type, layout_type_backup) + self.assertEqual(project.keys, keys_backup) + self.assertEqual(project("delegation").keys, delegation_keys_backup) + self.assertEqual(project._prefix, prefix_backup) + self.assertEqual(project._project_name, name_backup) roleinfo = tuf.roledb.get_roleinfo(project._project_name) - self.assertEquals(roleinfo['partial_loaded'], True) + self.assertEqual(roleinfo['partial_loaded'], True) - # load_signing_keys + # Load_signing_keys. project("delegation").load_signing_key(delegation_private_key) project.status() @@ -392,52 +388,51 @@ def test_write(self): project("delegation")("subdelegation").load_signing_key( subdelegation_private_key) -# import pdb; pdb.set_trace() project.status() project.load_signing_key(project_private_key) - # backup everything - # + backup targets: + # Backup everything. + # + backup targets. targets_backup = project.target_files - # + backup delegations + # + backup delegations. delegations_backup = \ tuf.roledb.get_delegated_rolenames(project._project_name) - # + backup layout type + # + backup layout type. layout_type_backup = project.layout_type - # + backup keyid's + # + backup keyids keys_backup = project.keys delegation_keys_backup = project("delegation").keys - # + backup the prefix - prefix_backup = project.prefix + # + backup the prefix. + prefix_backup = project._prefix - # + backup the name + # + backup the name. name_backup = project._project_name - # call status (for the sake of doing it) + # Call status (for the sake of doing it.) project.status() - # call write + # Call write. project.write() - # call load + # Call load. project = developer_tool.load_project(local_tmp) - # check against backup - self.assertEquals(project.target_files, targets_backup) + # Check against backup. + self.assertEqual(project.target_files, targets_backup) new_delegations = tuf.roledb.get_delegated_rolenames(project._project_name) - self.assertEquals(new_delegations, delegations_backup) - self.assertEquals(project.layout_type, layout_type_backup) - self.assertEquals(project.keys, keys_backup) - self.assertEquals(project("delegation").keys, delegation_keys_backup) - self.assertEquals(project.prefix, prefix_backup) - self.assertEquals(project._project_name, name_backup) + self.assertEqual(new_delegations, delegations_backup) + self.assertEqual(project.layout_type, layout_type_backup) + self.assertEqual(project.keys, keys_backup) + self.assertEqual(project("delegation").keys, delegation_keys_backup) + self.assertEqual(project._prefix, prefix_backup) + self.assertEqual(project._project_name, name_backup) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py old mode 100644 new mode 100755 index 33c55ae2..e94843b9 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ developer_tool.py @@ -6,10 +8,10 @@ Santiago Torres Zane Fisher - Based on the work done for the repository tools by Vladimir Diaz + Based on the work done for 'repository_tool.py' by Vladimir Diaz. - January 22, 2014 + January 22, 2014. See LICENSE for licensing information. @@ -26,8 +28,6 @@ from __future__ import absolute_import from __future__ import division -#this import is the interface to the whole tuf module, since the imports are -# made there. import os import errno import sys @@ -47,6 +47,8 @@ import tuf.conf import tuf.repository_tool +# These imports provide the interface for 'developer_tool.py', since the imports +# are made there. from tuf.keys import format_keyval_to_metadata from tuf.keys import format_metadata_to_key @@ -60,6 +62,7 @@ from tuf.repository_tool import import_ed25519_publickey_from_file from tuf.repository_tool import import_ed25519_privatekey_from_file from tuf.repository_lib import _remove_invalid_and_duplicate_signatures +from tuf.repository_lib import disable_console_log_messages from tuf.repository_lib import _check_role_keys from tuf.repository_lib import _delete_obsolete_metadata from tuf.repository_lib import generate_targets_metadata @@ -67,11 +70,9 @@ from tuf.repository_lib import write_metadata_file from tuf.repository_lib import _metadata_is_partially_loaded - # See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger('tuf.developer_tool') - # Recommended RSA key sizes: # http://www.emc.com/emc-plus/rsa-labs/historical/twirl-and-rsa-key-size.htm#table1 # According to the document above, revised May 6, 2003, RSA keys of size 3072 @@ -86,13 +87,12 @@ # The extension of TUF metadata. from tuf.repository_lib import METADATA_EXTENSION as METADATA_EXTENSION - # The metadata filename for the targets metadata information. from tuf.repository_lib import TARGETS_FILENAME as TARGETS_FILENAME # Project configuration filename. This file is intended to hold all of the # supporting information about the project that's not contained in a usual -# TUF metadata file. The project.cfg file consists of the following fields: +# TUF metadata file. 'project.cfg' consists of the following fields: # # targets_location: the location of the targets folder. # @@ -102,24 +102,24 @@ # metadata_location: the location of the metadata files. # # threshold: the threshold for this project object, it is fixed to -# one in the current version +# one in the current version. # # public_keys: a list of the public keys used to verify the metadata -# in this project +# in this project. # # layout_type: a field describing the directory layout: # # repo-like: matches the layout of the repository tool. # the targets and metadata folders are -# located under a comon directory for the +# located under a common directory for the # project. # # flat: the targets directory and the # metadata directory are located in different -# paths +# paths. # -# project_name: The name of the current project, this value is used to -# match the resulting filename with the one in upstream +# project_name: The name of the current project, this value is used to +# match the resulting filename with the one in upstream. PROJECT_FILENAME = 'project.cfg' # The targets and metadata directory names. Metadata files are written @@ -134,7 +134,6 @@ # The recognized compression extensions. from tuf.repository_lib import SUPPORTED_COMPRESSION_EXTENSIONS - # Supported key types. from tuf.repository_lib import SUPPORTED_KEY_TYPES @@ -142,15 +141,18 @@ class Project(Targets): """ - This class works as the abstraction of the developer's files. this module - was created with the objective of simplifying the publishing process using - TUF by taking care of all of the bookkeeping, signature handling and - metadata integrity verification. - - This class is the direct representation of a metadata file* with the - intention to provide the ability to modify this data in an OOP manner - without messing with syntax and sanity-checking. + Simplify the publishing process of third-party projects by handling all of + the bookkeeping, signature handling, and integrity checks of delegated TUF + metadata. 'repository_tool.py' is responsible for publishing and + maintaining metadata of the top-level roles, and 'developer_tool.py' is used + by projects that have been delegated responsibility for a delegated projects + role. Metadata created by this module may then be added to other metadata + available in a TUF repository. + Project() is the representation of a project's metadata file(s), with the + ability to modify this data in an OOP manner. Project owners do not have to + manually verify that metadata files are properly formatted or that they + contain valid data. project_name: @@ -158,57 +160,57 @@ class Project(Targets): repository. metadata_directory: - The metadata sub-directory contains the files of the top-level - roles, including all roles delegated from 'targets.txt'. + The metadata sub-directory contains the metadata file(s) of this project, + including any of its delegated roles. targets_directory: - The targets sub-directory contains all the target files that are - downloaded by clients and are referenced in TUF Metadata. The hashes and + The targets sub-directory contains the project's target files that are + downloaded by clients and are referenced in its metadata. The hashes and file lengths are listed in Metadata files so that they are securely downloaded. Metadata files are similarly referenced in the top-level metadata. file_prefix: - The path strig that will be prepended to the generated metadata - (e.g. targets/foo -> targets/prefix/foo) so that it matches the actual - location in the upstream repository. + The path string that will be prepended to the generated metadata + (e.g., targets/foo -> targets/prefix/foo) so that it matches the actual + targets location in the upstream repository. tuf.FormatError, if the arguments are improperly formatted. - Creates top-level role objects and assigns them as attributes. + Creates a project Targets role object, with the same object attributes of + the top-level targets role. - A project object that contains default Metadata objects for the top-level - roles. + None. """ - def __init__(self, project_name ,metadata_directory, targets_directory, + def __init__(self, project_name, metadata_directory, targets_directory, file_prefix): # Do the arguments have the correct format? # Ensure the arguments have the appropriate number of objects and object # types, and that all dict keys are properly named. # Raise 'tuf.FormatError' if any are improperly formatted. + tuf.formats.NAME_SCHEMA.check_match(project_name) tuf.formats.PATH_SCHEMA.check_match(metadata_directory) tuf.formats.PATH_SCHEMA.check_match(targets_directory) tuf.formats.PATH_SCHEMA.check_match(file_prefix) - tuf.formats.RELPATH_SCHEMA.check_match(project_name) self._metadata_directory = metadata_directory self._targets_directory = targets_directory self._project_name = project_name + self._prefix = file_prefix - # layout type defaults to "flat" unless explicitly specified in - # create_new_project - self.layout_type = "flat" + # Layout type defaults to "flat" unless explicitly specified in + # create_new_project(). + self.layout_type = 'flat' - # Set the top-level role objects, we set the rolename to be the project's - # name + # Set the top-level Targets object. Set the rolename to be the project's + # name. super(Project, self).__init__(self._targets_directory, project_name) - self.prefix = file_prefix @@ -230,11 +232,11 @@ def write(self, write_partial=False): signatures. - tuf.Error, if any of the top-level roles do not have a minimum - threshold of signatures. + tuf.Error, if any of the project roles do not have a minimum threshold of + signatures. - Creates metadata files in the projects's metadata directory. + Creates metadata files in the project's metadata directory. None. @@ -248,9 +250,9 @@ def write(self, write_partial=False): # At this point the tuf.keydb and tuf.roledb stores must be fully # populated, otherwise write() throwns a 'tuf.Repository' exception if - # any of the top-level roles are missing signatures, keys, etc. + # any of the project roles are missing signatures, keys, etc. - # Write the metadata files of all the delegated roles. + # Write the metadata files of all the delegated roles of the project. delegated_rolenames = \ tuf.roledb.get_delegated_rolenames(self._project_name) @@ -267,21 +269,23 @@ def write(self, write_partial=False): _generate_and_write_metadata(delegated_rolename, delegated_filename, write_partial, self._targets_directory, self._metadata_directory, - prefix=self.prefix) + prefix=self._prefix) # Generate the 'project_name' metadata file. targets_filename = self._project_name + METADATA_EXTENSION targets_filename = os.path.join(self._metadata_directory, targets_filename) project_signable, targets_filename = \ - _generate_and_write_metadata( self._project_name, targets_filename, + _generate_and_write_metadata(self._project_name, targets_filename, write_partial, self._targets_directory, - self._metadata_directory, prefix=self.prefix) + self._metadata_directory, prefix=self._prefix) - #save some other information that is not stored in the project's metadata + # Save configuration information that is not stored in the project's + # metadata _save_project_configuration(self._metadata_directory, - self._targets_directory, self.keys, self.prefix, self.threshold, - self.layout_type, self._project_name) + self._targets_directory, self.keys, self._prefix, + self.threshold, self.layout_type, + self._project_name) # Delete the metadata of roles no longer in 'tuf.roledb'. Obsolete roles # may have been revoked. @@ -292,36 +296,40 @@ def write(self, write_partial=False): - - def add_verification_key(self,key): + def add_verification_key(self, key): """ Function as a thin wrapper call for the project._targets call - with the same name. This wrapper is only for usability purposes + with the same name. This wrapper is only for usability purposes. - Key: - The role key to be added, conformant to tuf.formats.anykey_schema + key: + The role key to be added, conformant to 'tuf.formats.ANYKEY_SCHEMA'. Adding a public key to a role means that its corresponding private key must generate and add its signture to the role. - Tuf.FormatError, if the 'key' argument is improperly formatted. + tuf.FormatError, if the 'key' argument is improperly formatted. - Tuf.Error, if the project already contains a key + tuf.Error, if the project already contains a key. - The role's entries in 'tuf.keydb.py' and 'tuf.roledb.py' are updated + The role's entries in 'tuf.keydb.py' and 'tuf.roledb.py' are updated. None """ - ### should check the number of keys for this role. - if len(self.keys)>0: - raise tuf.Error("This project already contains a key") + + # Verify that this role does not already contain a key. The parent project + # role is restricted to one key. Any of its delegated roles may have + # more than one key. + # TODO: Add condition check for the requirement stated above. + if len(self.keys) > 0: + raise tuf.Error("This project already contains a key.") try: super(Project, self).add_verification_key(key) + except tuf.FormatError: raise @@ -332,20 +340,20 @@ def add_verification_key(self,key): def status(self): """ - Determine the status of the top-level roles, including those delegated. + Determine the status of the project, including its delegated roles. status() checks if each role provides sufficient public keys, signatures, and that a valid metadata file is generated if write() were to be called. - Metadata files are temporary written to check that proper metadata files - are written, where file hashes and lengths are calculated and referenced - by the top-level roles. status() does not do a simple check for number - of threshold keys and signatures. + Metadata files are temporarily written to check that proper metadata files + is written, where file hashes and lengths are calculated and referenced + by the project. status() does not do a simple check for number of + threshold keys and signatures. None. - tuf.Error, if any of the top-level roles do not have a minimum - threshold of signatures. + tuf.Error, if the project, or any of its delegated roles, do not have a + minimum threshold of signatures. Generates and writes temporary metadata files. @@ -353,6 +361,7 @@ def status(self): None. """ + temp_project_directory = None try: @@ -360,18 +369,13 @@ def status(self): metadata_directory = os.path.join(temp_project_directory, self._metadata_directory[1:]) - #targets_directory = os.path.join(temp_project_directory, - # self._targets_directory[1:]) targets_directory = self._targets_directory - os.makedirs(metadata_directory) - # os.makedirs(targets_directory) - #filenames = get_metadata_filenames(metadata_directory) - # we should do the schema check + # TODO: We should do the schema check. filenames = {} - filenames['targets'] = os.path.join(metadata_directory,self._project_name) + filenames['targets'] = os.path.join(metadata_directory, self._project_name) # Delegated roles. delegated_roles = tuf.roledb.get_delegated_rolenames(self._project_name) @@ -381,6 +385,7 @@ def status(self): for delegated_role in delegated_roles: try: _check_role_keys(delegated_role) + except tuf.InsufficientKeysError: insufficient_keys.append(delegated_role) continue @@ -393,17 +398,18 @@ def status(self): metadata_directory, False) self._print_status(delegated_role, signable[0]) + except tuf.Error: insufficient_signatures.append(delegated_role) if len(insufficient_keys): - message = 'Delegated roles with insufficient keys: '+ \ + message = 'Delegated roles with insufficient keys: ' +\ repr(insufficient_keys) print(message) return if len(insufficient_signatures): - message = 'Delegated roles with insufficient signatures: '+ \ + message = 'Delegated roles with insufficient signatures: ' +\ repr(insufficient_signatures) print(message) return @@ -411,6 +417,7 @@ def status(self): # Targets role. try: _check_role_keys(self.rolename) + except tuf.InsufficientKeysError as e: print(str(e)) return @@ -422,6 +429,7 @@ def status(self): metadata_directory, False) self._print_status(self._project_name, signable) + except tuf.Error as e: signable = e[1] self._print_status(self._project_name, signable) @@ -442,9 +450,9 @@ def _print_status(self, rolename, signable): status = tuf.sig.get_signature_status(signable, rolename) - message = repr(rolename)+' role contains '+ \ - repr(len(status['good_sigs']))+' / '+ \ - repr(status['threshold'])+' signatures.' + message = repr(rolename) + ' role contains ' +\ + repr(len(status['good_sigs'])) + ' / ' + repr(status['threshold']) +\ + ' signatures.' print(message) @@ -457,8 +465,7 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, prefix=''): """ Non-public function that can generate and write the metadata of the - specified - top-level 'rolename'. It also increments version numbers if: + specified 'rolename'. It also increments version numbers if: 1. write_partial==True and the metadata is the first to be written. @@ -467,6 +474,7 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, """ metadata = None + # Retrieve the roleinfo of 'rolename' to extract the needed metadata # attributes, such as version number, expiration, etc. roleinfo = tuf.roledb.get_roleinfo(rolename) @@ -478,8 +486,8 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, roleinfo['delegations'], False) - # preprend the prefix to the project's filepath to avoid signature errors - # in upstream + # Preprend the prefix to the project's filepath to avoid signature errors + # in upstream. target_filepaths = metadata['targets'].items() for element in list(metadata['targets']): junk_path, relative_target = os.path.split(element) @@ -520,23 +528,19 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, filename = write_metadata_file(signable, metadata_filename, compressions, False) - # 'signable' contains an invalid threshold of signatures. else: - message = 'Not enough signatures for '+repr(metadata_filename) + message = 'Not enough signatures for ' + repr(metadata_filename) raise tuf.Error(message, signable) - - # The root and timestamp files should also be written without a digest if - # 'consistent_snaptshots' is True. Client may request a timestamp and root - # file without knowing its digest and file size. return signable, filename -def create_new_project(project_name, metadata_directory, location_in_repository = '', - targets_directory=None, key=None): +def create_new_project(project_name, metadata_directory, + location_in_repository = '', targets_directory=None, + key=None): """ Create a new project object, instantiate barebones metadata for the @@ -556,12 +560,7 @@ def create_new_project(project_name, metadata_directory, location_in_repository metadata_directory: The directory that will eventually hold the metadata and target files of the project. - - targets_directory: - An optional argument to point the targets directory somewhere else than - the metadata directory if, for example, a project structure already - exists and the user does not want to move it. - + location_in_repository: An optional argument to hold the "prefix" or the expected location for the project files in the "upstream" respository. This value is only @@ -571,6 +570,11 @@ def create_new_project(project_name, metadata_directory, location_in_repository For example, targets/unclaimed/django should have its project name set to "targets/unclaimed" + targets_directory: + An optional argument to point the targets directory somewhere else than + the metadata directory if, for example, a project structure already + exists and the user does not want to move it. + key: The public key to verify the project's metadata. Projects can only handle one key with a threshold of one. If a project were to modify it's @@ -578,9 +582,9 @@ def create_new_project(project_name, metadata_directory, location_in_repository tuf.FormatError, if the arguments are improperly formatted or if the public - key is not a valid one ( if it's not none ). + key is not a valid one (if it's not none.) - OSError, if the filepaths provided do not have write permissions + OSError, if the filepaths provided do not have write permissions. The 'metadata_directory' and 'targets_directory' directories are created @@ -598,8 +602,8 @@ def create_new_project(project_name, metadata_directory, location_in_repository # Do the same for the location in the repo and the project name, we must # ensure they are valid pathnames. + tuf.formats.NAME_SCHEMA.check_match(project_name) tuf.formats.PATH_SCHEMA.check_match(location_in_repository) - tuf.formats.RELPATH_SCHEMA.check_match(project_name) # for the targets directory we do the same, but first, let's find out what # layout the user needs, layout_type is a variable that is usually set to @@ -607,15 +611,15 @@ def create_new_project(project_name, metadata_directory, location_in_repository # located), with a two, the cfg file goes to the "metadata" folder, and a # new metadata folder is created inside the tree, to separate targets and # metadata. - layout_type="flat" + layout_type = 'flat' if targets_directory is None: - targets_directory = os.path.join(metadata_directory,TARGETS_DIRECTORY_NAME) + targets_directory = os.path.join(metadata_directory, TARGETS_DIRECTORY_NAME) metadata_directory = \ - os.path.join(metadata_directory,METADATA_DIRECTORY_NAME) - layout_type="repo-like" - + os.path.join(metadata_directory, METADATA_DIRECTORY_NAME) + layout_type = 'repo-like' - tuf.formats.PATH_SCHEMA.check_match(targets_directory); + if targets_directory is not None: + tuf.formats.PATH_SCHEMA.check_match(targets_directory); if key is not None: tuf.formats.KEY_SCHEMA.check_match(key) @@ -628,7 +632,7 @@ def create_new_project(project_name, metadata_directory, location_in_repository # Try to create the metadata directory that will hold all of the metadata # files, such as 'root.txt' and 'release.txt'. try: - message = 'Creating '+repr(metadata_directory) + message = 'Creating ' + repr(metadata_directory) logger.info(message) os.makedirs(metadata_directory) @@ -636,33 +640,37 @@ def create_new_project(project_name, metadata_directory, location_in_repository # Check for case where 'repository_directory' has already been created. except OSError as e: if e.errno == errno.EEXIST: - # should check if we have write permissions here + # Should check if we have write permissions here. pass + else: raise # Try to create the targets directory that will hold all of the target files. try: - message = 'Creating '+repr(targets_directory) + message = 'Creating ' + repr(targets_directory) logger.info(message) os.mkdir(targets_directory) + except OSError as e: if e.errno == errno.EEXIST: pass + else: raise - # Create the bare bones project object, where only the top-level roles - # have been set and contain default values (e.g., Root roles has a threshold - # of 1, expires 1 year into the future, etc.) + # Create the bare bones project object, where project role contains default + # values (e.g., threshold of 1, expires 1 year into the future, etc.) project = Project(project_name, metadata_directory, targets_directory, location_in_repository) - # add the key to the project. + # Add 'key' to the project. + # TODO: Add check for expected number of keys for the project (must be 1) and + # its delegated roles (may be greater than one.) if key is not None: project.add_verification_key(key); - # save the layout information + # Save the layout information. project.layout_type = layout_type return project @@ -672,26 +680,26 @@ def create_new_project(project_name, metadata_directory, location_in_repository -def _save_project_configuration(metadata_directory,targets_directory, +def _save_project_configuration(metadata_directory, targets_directory, public_keys, prefix, threshold, layout_type, project_name): """ - Persist the project's information in a file to provide the information - for the load routine + Persist the project's information to a file. The saved project information + can later be loaded with Project.load_project(). metadata_directory: - Where the project's metadata is located + Where the project's metadata is located. targets_directory: - The location of the target files in this project. + The location of the target files for this project. public_keys: A list containing the public keys for the project role. prefix: - The project's prefix (if any) + The project's prefix (if any.) threshold: The threshold value for the project role. @@ -706,39 +714,38 @@ def _save_project_configuration(metadata_directory,targets_directory, matches the one stored in upstream. + tuf.FormatError are also expected if any of the arguments are malformed. + OSError may rise if the metadata_directory/project.cfg file exists and is non-writeable - tuf.FormatError are also expected if any of the arguments are malformed. - - A project.cfg file is created or overwritten + A 'project.cfg' configuration file is created or overwritten. - nothing + None. """ - # schema check for metadata_directory and prefix + # Schema check for the arguments. tuf.formats.PATH_SCHEMA.check_match(metadata_directory) tuf.formats.PATH_SCHEMA.check_match(prefix) tuf.formats.PATH_SCHEMA.check_match(targets_directory) tuf.formats.RELPATH_SCHEMA.check_match(project_name) - # get the absolute filepath to our metadata_directory for consistency - #metadata_directory = os.path.abspath(metadata_directory) cfg_file_directory = metadata_directory - # check wheter if the layout type is "flat" or "repo-like" - # if it is, the .cfg file should be saved in the previous directory. - if(layout_type == "repo-like"): + # Check whether the layout type is 'flat' or 'repo-like'. + # If it is, the .cfg file should be saved in the previous directory. + if layout_type == 'repo-like': cfg_file_directory = os.path.dirname(metadata_directory) absolute_location, targets_directory = os.path.split(targets_directory) absolute_location, metadata_directory = os.path.split(metadata_directory) - # is the file open-able? open for overwriting - project_filename = os.path.join(cfg_file_directory,PROJECT_FILENAME) - # build the data structure + # Can the file be opened? + project_filename = os.path.join(cfg_file_directory, PROJECT_FILENAME) + + # Build the fields of the configuration file. project_config = {} project_config['prefix'] = prefix project_config['public_keys'] = {} @@ -748,17 +755,16 @@ def _save_project_configuration(metadata_directory,targets_directory, project_config['layout_type'] = layout_type project_config['project_name'] = project_name - # build a dictionary containing the actual keys + # Build a dictionary containing the actual keys. for key in public_keys: key_info = tuf.keydb.get_key(key) key_metadata = format_keyval_to_metadata(key_info['keytype'], - key_info['keyval']) + key_info['keyval']) project_config['public_keys'][key] = key_metadata - # save the actual data - with open(project_filename,"wt") as fp: - json.dump(project_config,fp) - + # Save the actual file. + with open(project_filename, 'wt') as fp: + json.dump(project_config, fp) @@ -767,54 +773,55 @@ def _save_project_configuration(metadata_directory,targets_directory, def load_project(project_directory, prefix='', new_targets_location=None): """ - Return a project object initialized with the contents of the metadata - files loaded from the project_directory path + Return a Project object initialized with the contents of the metadata + files loaded from 'project_directory'. project_directory: - The path to the project's folder + The path to the project's metadata and configuration file. prefix: - the prefix for the metadata, if defined, it will replace the current - prefix, by first removing the existing one (Saved) and setting the new - one in the end. + The prefix for the metadata, if defined. It will replace the current + prefix, by first removing the existing one (saved). new_targets_location: - For flat project configurations, you might want to reload the project - with a new location for the target files. This overwrites the previous - path to search for the target files. + For flat project configurations, project owner might want to reload the + project with a new location for the target files. This overwrites the + previous path to search for the target files. tuf.FormatError, if 'project_directory' or any of the metadata files are improperly formatted. - All the metadata files found in the project are loaded and their contents - stored in a libtuf.Repository object. + All the metadata files found in the project are loaded and their contents + stored in a libtuf.Repository object. A tuf.developer_tool.Project object. - """ + # Does 'repository_directory' have the correct format? # Raise 'tuf.FormatError' if there is a mismatch. tuf.formats.PATH_SCHEMA.check_match(project_directory) - # do the same for the prefix + + # Do the same for the prefix tuf.formats.PATH_SCHEMA.check_match(prefix) - # clear the role and keydbs + # Clear the role and key databases since we are loading in a new project. tuf.roledb.clear_roledb() tuf.keydb.clear_keydb() # Locate metadata filepaths and targets filepath. project_directory = os.path.abspath(project_directory) - - # load the cfg file and the project. + # Load the cfg file and the project. config_filename = os.path.join(project_directory,PROJECT_FILENAME) + try: project_configuration = tuf.util.load_json_file(config_filename) tuf.formats.PROJECT_CFG_SCHEMA.check_match(project_configuration) + except (OSError, IOError) as e: raise @@ -836,37 +843,37 @@ def load_project(project_directory, prefix='', new_targets_location=None): prefix = project_configuration['prefix'] - # load the projects filename. + # Load the project's filename. project_name = project_configuration['project_name'] project_filename = project_name + METADATA_EXTENSION - # create a blank project on the target directory + # Create a blank project on the target directory. project = Project(project_name, metadata_directory, targets_directory, prefix) project.threshold = project_configuration['threshold'] - project.prefix = project_configuration['prefix'] + project._prefix = project_configuration['prefix'] project.layout_type = project_configuration['layout_type'] - # traverse the public keys and add them to the project + # Traverse the public keys and add them to the project. keydict = project_configuration['public_keys'] for keyid in keydict: key = format_metadata_to_key(keydict[keyid]) project.add_verification_key(key) - # load the project's metadata + # Load the project's metadata. targets_metadata_path = os.path.join(project_directory, metadata_directory, project_filename) signable = tuf.util.load_json_file(targets_metadata_path) tuf.formats.check_signable_object_format(signable) targets_metadata = signable['signed'] - # remove the prefix from the metadata + # Remove the prefix from the metadata. targets_metadata = _strip_prefix_from_targets_metadata(targets_metadata, prefix) for signature in signable['signatures']: project.add_signature(signature) - # update roledb + # Update roledb.py containing the loaded project attributes. roleinfo = tuf.roledb.get_roleinfo(project_name) roleinfo['signatures'].extend(signable['signatures']) roleinfo['version'] = targets_metadata['version'] @@ -875,15 +882,14 @@ def load_project(project_directory, prefix='', new_targets_location=None): roleinfo['partial_loaded'] = False - # check if the loaded metadata was partially written and update the - # flag in the roledb + # Check if the loaded metadata was partially written and update the + # flag in 'roledb.py'. if _metadata_is_partially_loaded(project_name, signable, roleinfo): roleinfo['partial_loaded'] = True tuf.roledb.update_roleinfo(project_name, roleinfo) - for key_metadata in targets_metadata['delegations']['keys'].values(): key_object = tuf.keys.format_metadata_to_key(key_metadata) tuf.keydb.add_key(key_object) @@ -916,21 +922,24 @@ def load_project(project_directory, prefix='', new_targets_location=None): metadata_name = \ metadata_path[len(metadata_directory):].lstrip(os.path.sep) - # strip the extension + # Strip the extension. The roledb does not include '.json' role + # extensions if metadata_name.endswith(METADATA_EXTENSION): extension_length = len(METADATA_EXTENSION) metadata_name = metadata_name[:-extension_length] + else: continue signable = None try: signable = tuf.util.load_json_file(metadata_path) + except (ValueError, IOError, tuf.Error): raise - # strip the prefix from the local working copy, it will be added again - # at the time of writing. + # Strip the prefix from the local working copy, it will be added again + # when the targets metadata is written to disk. metadata_object = signable['signed'] metadata_object = _strip_prefix_from_targets_metadata(metadata_object, prefix) @@ -946,17 +955,17 @@ def load_project(project_directory, prefix='', new_targets_location=None): if os.path.exists(metadata_path+'.gz'): roleinfo['compressions'].append('gz') - # if the metadata was partially loaded, update the roleinfo flag. + # If the metadata was partially loaded, update the roleinfo flag. if _metadata_is_partially_loaded(metadata_name, signable, roleinfo): roleinfo['partial_loaded'] = True tuf.roledb.update_roleinfo(metadata_name, roleinfo) - # append to list of elements to avoid reloading repeated metadata + # Append to list of elements to avoid reloading repeated metadata. loaded_metadata.append(metadata_name) - # add the delegation + # Add the delegation. new_targets_object = Targets(targets_directory, metadata_name, roleinfo) targets_object = \ targets_objects[tuf.roledb.get_parent_rolename(metadata_name)] @@ -970,6 +979,7 @@ def load_project(project_directory, prefix='', new_targets_location=None): key_object = tuf.keys.format_metadata_to_key(key_metadata) try: tuf.keydb.add_key(key_object) + except tuf.KeyAlreadyExistsError: pass @@ -985,7 +995,8 @@ def load_project(project_directory, prefix='', new_targets_location=None): tuf.roledb.add_role(rolename, roleinfo) if new_prefix: - project.prefix = new_prefix + project._prefix = new_prefix + return project @@ -993,10 +1004,15 @@ def load_project(project_directory, prefix='', new_targets_location=None): def _strip_prefix_from_targets_metadata(targets_metadata, prefix): - """ non-public method that removes the prefix from the targets metadata - so it can be used again in compliance with the local copies """ + Non-public method that removes the prefix from each of the target pahts in + 'targets_metadata' so they can be used again in compliance with the local + copies. The prefix is needed in metadata to match the layout of the remote + repository. + """ + unprefixed_targets_metadata = {} + for targets in targets_metadata['targets'].keys(): unprefixed_target = os.path.relpath(targets, prefix) unprefixed_target = '/' + unprefixed_target @@ -1012,7 +1028,7 @@ def _strip_prefix_from_targets_metadata(targets_metadata, prefix): if __name__ == '__main__': # The interactive sessions of the documentation strings can - # be tested by running libtuf.py as a standalone module: - # $ python libtuf.py. + # be tested by running 'developer_tool.py' as a standalone module: + # $ python developer_tool.py import doctest doctest.testmod() From 2463cc3cc1243f00ada20709812402d915685aa6 Mon Sep 17 00:00:00 2001 From: zanefisher Date: Tue, 1 Jul 2014 10:17:48 -0400 Subject: [PATCH 50/51] Update README-developer-tools.md --- tuf/README-developer-tools.md | 68 +++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/tuf/README-developer-tools.md b/tuf/README-developer-tools.md index 43d9c493..c1b5b60b 100644 --- a/tuf/README-developer-tools.md +++ b/tuf/README-developer-tools.md @@ -1,6 +1,6 @@ -# Developing for a TUF repository # +# The Update Framework Developer Tool: How to Update your Project Securely on a TUF Repository -## Table of Contents ## +## Table of Contents - [Overview](#overview) - [Creating a Simple Project](#creating_a_simple_project) - [Generating a Key](#generating_a_key) @@ -13,29 +13,39 @@ ## Overview -The TUF developer tool is a Python library that enables developers to create -and maintain the required metadata for files hosted in a TUF Repository. The -main concern when generating metadata for a TUF repository is generating -information that matches the future location of the files in the repository. We -use the developer tools to generate valid information so that the project and -its metadata can be applied to the TUF project transparently. +The Update Framework (TUF) is a Python-based security system for software +updates. In order to prevent your users from downloading vulnerable or malicious +code disguised as updates to your software, TUF requires that each update you +release include certain metadata verifying your authorship of the files. -This document has two parts. The first part walks through the creation of a -prototypal TUF project. The second part demonstrates the full capabilities of -the TUF developer tool, which can be used to expand the project from the first -part to meet the developer's needs. +The TUF developer tools are a Python Library that enables you to create and +maintain the required metadata for files hosted on a TUF Repository. (We call +these files “targets,” to distinguish them from the metadata associated with +them. Both of these together comprise a complete “project”.) You will use these +tools to generate the keys and metadata you need to claim and secure your files +on the repository, and to update the metadata and sign it with those keys +whenever you upload a new version of those files. + +This document will teach you how to use these tools in two parts. The first +part walks through the creation of a minimal-complexity TUF project, which is +all you need to get started, and can be expanded later. The second part details +the full functionality of the tools, which offer a finer degree of control in +securing your project. -## Creating a Simple project -The following section describes a very basic example usage of the developer -tools with a one-file project. +## Creating a Simple Project +This section walks through the creation of a small example project with just +one target. Once created, this project will be fully functional, and can be +modified as needed. ### Generating a Key First, we will need to generate a key to sign the metadata. Keys are generated in pairs: one public and the other private. The private key is password-protected and is used to sign metadata. The public key can be shared -freely, and is used to verify signatures made by the private key. +freely, and is used to verify signatures made by the private key. You will need +too share your public key with the repository hosting your project so they can +verify your metadata is signed by the right person. The generate\_and\_write\_rsa\_keypair function will create two key files named "path/to/key.pub", which is the public key and "path/to/key", which @@ -60,11 +70,11 @@ Now we have a key for our project, we can proceed to create our project. ### The Project Class The TUF developer tool is built around the Project class, which is used to -organize groups of targets associated with a single set of metadata. Each -Project instance keeps track of which target files are associated with a single -set of metadata. Each Project instance keeps track of which target files are -signed and which need signing, which keys are used to sign metadata. It also -keeps track of delegated roles, which are covered later. +organize groups of targets associated with a single set of metadata. A single +Project instance is used to keep track of all the target files and metadata +files in one project. The Project also keeps track of the keys and signatures, +so that it can update all the metadata with the correct changes and signatures +on a single command. Before creating a project, you must know where it will be located in the TUF Repository. In the following example, we will create a project to be hosted as @@ -74,7 +84,7 @@ target file, "local/path/to/example\_project/target\_1" locally, and we will secure it with the key generated above. First, we must import the generated keys. We can do that by issuing the -following: +following command: ``` >>> public_key = import_rsa_publickey_from_file("path/to/keys.pub") @@ -110,8 +120,8 @@ To add a target, we issue the following method: >>> project.add_target("target_1") ``` -Have in mind the file "target\_1" should be located in -"local/path/to/example\_project" or else the adding procedure will throw an +Note that the file "target\_1" should be located in +"local/path/to/example\_project", or this method will throw an error. At this point, the metadata is not valid. We have assigned a key to the @@ -132,8 +142,9 @@ Enter password for the RSA key: >>> project.write() ``` -When all changes to a project have been written, the Project instance can -safely be deleted. +When all changes to the project have been written, the metadata is ready to be +uploaded to the repository, and it is safe to exit the Python interpreter, or +to delete the Project instance. The project can be loaded later to update changes to the project. The metadata contains checksums that have to match the actual files or else it won't be @@ -166,6 +177,9 @@ Enter a password for the RSA key: >>> project.write() ``` +If your project does not use any delegations, the five commands above are all +you need to update your project's metadata. + ## Delegations @@ -206,7 +220,7 @@ filepaths to a role by adding them to the list in the first parameter, or by invoking the method again. A role with multiple restricted paths can add targets to any of them. -Note that this method is invoked the parent role (in this case, the Project) +Note that this method is invoked from the parent role (in this case, the Project) and takes the delegated role name as an argument. ### Nested Delegations From 1985c450c910713e5b091aa806598923b4a74df1 Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Wed, 2 Jul 2014 11:10:50 -0400 Subject: [PATCH 51/51] Merge custom info changes, logger messages, and minor edits. --- tests/test_developer_tool.py | 119 +++++++++++++++++------------------ tuf/developer_tool.py | 31 ++++----- tuf/repository_lib.py | 1 + 3 files changed, 77 insertions(+), 74 deletions(-) diff --git a/tests/test_developer_tool.py b/tests/test_developer_tool.py index bedcccf6..3cf70b74 100755 --- a/tests/test_developer_tool.py +++ b/tests/test_developer_tool.py @@ -5,7 +5,7 @@ test_developer_tool.py. - Santiago Torres Arias Zane Fisher @@ -33,7 +33,7 @@ from tuf.developer_tool import METADATA_DIRECTORY_NAME from tuf.developer_tool import TARGETS_DIRECTORY_NAME -logger = logging.getLogger("tuf.test_developer_tool") +logger = logging.getLogger('tuf.test_developer_tool') developer_tool.disable_console_log_messages() @@ -71,7 +71,7 @@ def test_create_new_project(self): # These are the usual values we will be throwing to the function, however # we will swap these for nulls or malformed values every now and then to # test input. - project_name = "test_suite" + project_name = 'test_suite' metadata_directory = local_tmp location_in_repository = '/prefix' targets_directory = None @@ -184,19 +184,19 @@ def test_load_project(self): local_tmp = tempfile.mkdtemp(dir = self.tmp_dir) # Test non-existent project filepath. - nonexistent_path = os.path.join(local_tmp, "nonexistent") + nonexistent_path = os.path.join(local_tmp, 'nonexistent') self.assertRaises(IOError, developer_tool.load_project, nonexistent_path) # Copy the pregenerated metadata. project_data_filepath = os.path.join('repository_data', 'project') target_project_data_filepath = os.path.join(local_tmp, 'project') - shutil.copytree("repository_data/project", target_project_data_filepath) + shutil.copytree('repository_data/project', target_project_data_filepath) # Properly load a project. repo_filepath = os.path.join(local_tmp, 'project', 'test-repo') project = developer_tool.load_project(repo_filepath) - self.assertTrue(project.layout_type == "repo-like") + self.assertTrue(project.layout_type == 'repo-like') repo_filepath = os.path.join(local_tmp, 'project', 'test-flat') new_targets_path = os.path.join(local_tmp, 'project', 'targets') @@ -213,7 +213,7 @@ def test_load_project(self): # Load a project with a file missing. file_to_corrupt = os.path.join(repo_filepath, 'test-flat', 'role1.json') with open(file_to_corrupt, 'wt') as fp: - fp.write("this is not a json file") + fp.write('this is not a json file') self.assertRaises(tuf.Error, developer_tool.load_project, repo_filepath) @@ -221,14 +221,14 @@ def test_load_project(self): def test_add_verification_keys(self): - # create a new project instance - project = developer_tool.Project("test_verification_keys", "somepath", - "someotherpath", "prefix") + # Create a new project instance. + project = developer_tool.Project('test_verification_keys', 'somepath', + 'someotherpath', 'prefix') - # add invalid verification key - self.assertRaises(tuf.FormatError, project.add_verification_key, "invalid") + # Add invalid verification key. + self.assertRaises(tuf.FormatError, project.add_verification_key, 'invalid') - # add verification key + # Add verification key. # - load it first keystore_path = os.path.join('repository_data','keystore') first_verification_key_path = os.path.join(keystore_path,'root_key.pub') @@ -238,7 +238,7 @@ def test_add_verification_keys(self): project.add_verification_key(first_verification_key) - # add another verification key (should expect exception) + # Add another verification key (should expect exception.) second_verification_key_path = os.path.join(keystore_path,'snapshot_key.pub') second_verification_key = \ developer_tool.import_rsa_publickey_from_file(second_verification_key_path) @@ -248,72 +248,73 @@ def test_add_verification_keys(self): - # add a verification key for the delegation - project.delegate("somedelegation", [], []) - project("somedelegation").add_verification_key(first_verification_key) - project("somedelegation").add_verification_key(second_verification_key) + # Add a verification key for the delegation. + project.delegate('somedelegation', [], []) + project('somedelegation').add_verification_key(first_verification_key) + project('somedelegation').add_verification_key(second_verification_key) - # add another delegation of the delegation - project("somedelegation").delegate("somesubdelegation", [], []) - project("somedelegation")("somesubdelegation").add_verification_key( + # Add another delegation of the delegation. + project('somedelegation').delegate('somesubdelegation', [], []) + project('somedelegation')('somesubdelegation').add_verification_key( first_verification_key) - project("somedelegation")("somesubdelegation").add_verification_key( + project('somedelegation')('somesubdelegation').add_verification_key( second_verification_key) def test_write(self): - # create tmp directory + # Create tmp directory. local_tmp = tempfile.mkdtemp(dir=self.tmp_dir) - # create new project inside tmp directory - project = developer_tool.create_new_project("test_write", local_tmp, - "prefix"); + # Create new project inside tmp directory. + project = developer_tool.create_new_project('test_write', local_tmp, + 'prefix'); - # create some target files inside the tmp directory - target_filepath = os.path.join(local_tmp, "targets", "test_target") - with open(target_filepath, "wt") as fp: - fp.write("testing file") + # Create some target files inside the tmp directory. + target_filepath = os.path.join(local_tmp, 'targets', 'test_target') + with open(target_filepath, 'wt') as fp: + fp.write('testing file') - # add the targets + # Add the targets. project.add_target(target_filepath) - # add verification keys - keystore_path = os.path.join('repository_data','keystore') - project_key_path = os.path.join(keystore_path,'root_key.pub') + # Add verification keys. + keystore_path = os.path.join('repository_data', 'keystore') + project_key_path = os.path.join(keystore_path, 'root_key.pub') project_key = \ developer_tool.import_rsa_publickey_from_file(project_key_path) - # call status (for the sake of doing it) + # Call status (for the sake of doing it and to improve test coverage by + # executing its statements.) project.status() project.add_verification_key(project_key) - # add another verification key (should expect exception) - delegation_key_path = os.path.join(keystore_path,'snapshot_key.pub') + # Add another verification key (should expect exception.) + delegation_key_path = os.path.join(keystore_path, 'snapshot_key.pub') delegation_key = \ developer_tool.import_rsa_publickey_from_file(delegation_key_path) - # add a subdelegation - subdelegation_key_path = os.path.join(keystore_path,'timestamp_key.pub') + # Add a subdelegation. + subdelegation_key_path = os.path.join(keystore_path, 'timestamp_key.pub') subdelegation_key = \ developer_tool.import_rsa_publickey_from_file(subdelegation_key_path) - # add a delegation - project.delegate("delegation", [delegation_key], []) - project("delegation").delegate("subdelegation", [subdelegation_key], []) + # Add a delegation. + project.delegate('delegation', [delegation_key], []) + project('delegation').delegate('subdelegation', [subdelegation_key], []) # call write (except) self.assertRaises(tuf.Error, project.write, ()) - # call status (for the sake of doing it) + # Call status (for the sake of doing it and executing its statements.) project.status() - # load private keys + # Load private keys. project_private_key_path = os.path.join(keystore_path, 'root_key') project_private_key = \ developer_tool.import_rsa_privatekey_from_file(project_private_key_path, @@ -330,21 +331,21 @@ def test_write(self): developer_tool.import_rsa_privatekey_from_file(subdelegation_private_key_path, 'password') - # Test partial write + # Test partial write. # backup everything (again) - # + backup targets: + # + backup targets. targets_backup = project.target_files - # + backup delegations + # + backup delegations. delegations_backup = \ tuf.roledb.get_delegated_rolenames(project._project_name) - # + backup layout type + # + backup layout type. layout_type_backup = project.layout_type - # + backup keyids + # + backup keyids. keys_backup = project.keys - delegation_keys_backup = project("delegation").keys + delegation_keys_backup = project('delegation').keys # + backup the prefix. prefix_backup = project._prefix @@ -352,7 +353,7 @@ def test_write(self): # + backup the name. name_backup = project._project_name - # Set the compressions, we will be checking this part here too + # Set the compressions. We will be checking this part here too. project.compressions = ['gz'] project('delegation').compressions = project.compressions @@ -363,12 +364,12 @@ def test_write(self): project = developer_tool.load_project(local_tmp) # Check against backup. - self.assertEqual(project.target_files, list(targets_backup.keys())) + self.assertEqual(list(project.target_files.keys()), list(targets_backup.keys())) new_delegations = tuf.roledb.get_delegated_rolenames(project._project_name) self.assertEqual(new_delegations, delegations_backup) self.assertEqual(project.layout_type, layout_type_backup) self.assertEqual(project.keys, keys_backup) - self.assertEqual(project("delegation").keys, delegation_keys_backup) + self.assertEqual(project('delegation').keys, delegation_keys_backup) self.assertEqual(project._prefix, prefix_backup) self.assertEqual(project._project_name, name_backup) @@ -381,11 +382,11 @@ def test_write(self): # Load_signing_keys. - project("delegation").load_signing_key(delegation_private_key) + project('delegation').load_signing_key(delegation_private_key) project.status() - project("delegation")("subdelegation").load_signing_key( + project('delegation')('subdelegation').load_signing_key( subdelegation_private_key) project.status() @@ -405,7 +406,7 @@ def test_write(self): # + backup keyids keys_backup = project.keys - delegation_keys_backup = project("delegation").keys + delegation_keys_backup = project('delegation').keys # + backup the prefix. prefix_backup = project._prefix @@ -424,19 +425,17 @@ def test_write(self): # Check against backup. - self.assertEqual(project.target_files, targets_backup) + self.assertEqual(list(project.target_files.keys()), list(targets_backup.keys())) new_delegations = tuf.roledb.get_delegated_rolenames(project._project_name) self.assertEqual(new_delegations, delegations_backup) self.assertEqual(project.layout_type, layout_type_backup) self.assertEqual(project.keys, keys_backup) - self.assertEqual(project("delegation").keys, delegation_keys_backup) + self.assertEqual(project('delegation').keys, delegation_keys_backup) self.assertEqual(project._prefix, prefix_backup) self.assertEqual(project._project_name, name_backup) - - if __name__ == '__main__': unittest.main() diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index e94843b9..c9f5cec6 100755 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -46,6 +46,7 @@ import tuf.log import tuf.conf import tuf.repository_tool +import tuf._vendor.six as six # These imports provide the interface for 'developer_tool.py', since the imports # are made there. @@ -397,7 +398,7 @@ def status(self): targets_directory, metadata_directory, False) - self._print_status(delegated_role, signable[0]) + self._log_status(delegated_role, signable[0]) except tuf.Error: insufficient_signatures.append(delegated_role) @@ -405,13 +406,13 @@ def status(self): if len(insufficient_keys): message = 'Delegated roles with insufficient keys: ' +\ repr(insufficient_keys) - print(message) + logger.info(message) return if len(insufficient_signatures): message = 'Delegated roles with insufficient signatures: ' +\ repr(insufficient_signatures) - print(message) + logger.info(message) return # Targets role. @@ -419,7 +420,7 @@ def status(self): _check_role_keys(self.rolename) except tuf.InsufficientKeysError as e: - print(str(e)) + logger.info(str(e)) return try: @@ -428,11 +429,11 @@ def status(self): targets_directory, metadata_directory, False) - self._print_status(self._project_name, signable) + self._log_status(self._project_name, signable) except tuf.Error as e: signable = e[1] - self._print_status(self._project_name, signable) + self._log_status(self._project_name, signable) return finally: @@ -442,7 +443,7 @@ def status(self): - def _print_status(self, rolename, signable): + def _log_status(self, rolename, signable): """ Non-public function prints the number of (good/threshold) signatures of 'rolename'. @@ -453,7 +454,7 @@ def _print_status(self, rolename, signable): message = repr(rolename) + ' role contains ' +\ repr(len(status['good_sigs'])) + ' / ' + repr(status['threshold']) +\ ' signatures.' - print(message) + logger.info(message) @@ -486,8 +487,8 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, roleinfo['delegations'], False) - # Preprend the prefix to the project's filepath to avoid signature errors - # in upstream. + # Prepend the prefix to the project's filepath to avoid signature errors in + # upstream. target_filepaths = metadata['targets'].items() for element in list(metadata['targets']): junk_path, relative_target = os.path.split(element) @@ -877,7 +878,7 @@ def load_project(project_directory, prefix='', new_targets_location=None): roleinfo = tuf.roledb.get_roleinfo(project_name) roleinfo['signatures'].extend(signable['signatures']) roleinfo['version'] = targets_metadata['version'] - roleinfo['paths'] = list(targets_metadata['targets']) + roleinfo['paths'] = targets_metadata['targets'] roleinfo['delegations'] = targets_metadata['delegations'] roleinfo['partial_loaded'] = False @@ -922,8 +923,8 @@ def load_project(project_directory, prefix='', new_targets_location=None): metadata_name = \ metadata_path[len(metadata_directory):].lstrip(os.path.sep) - # Strip the extension. The roledb does not include '.json' role - # extensions + # Strip the extension. The roledb does not include an appended '.json' + # extensions for each role. if metadata_name.endswith(METADATA_EXTENSION): extension_length = len(METADATA_EXTENSION) metadata_name = metadata_name[:-extension_length] @@ -948,7 +949,9 @@ def load_project(project_directory, prefix='', new_targets_location=None): roleinfo['signatures'].extend(signable['signatures']) roleinfo['version'] = metadata_object['version'] roleinfo['expires'] = metadata_object['expires'] - roleinfo['paths'] = list(metadata_object['targets']) + roleinfo['paths'] = {} + for filepath, fileinfo in six.iteritems(metadata_object['targets']): + roleinfo['paths'].update({filepath: fileinfo.get('custom', {})}) roleinfo['delegations'] = metadata_object['delegations'] roleinfo['partial_loaded'] = False diff --git a/tuf/repository_lib.py b/tuf/repository_lib.py index 33cf1b4e..d3ab8995 100755 --- a/tuf/repository_lib.py +++ b/tuf/repository_lib.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python """