diff --git a/tests/repository_data/generate_project_data.py b/tests/repository_data/generate_project_data.py new file mode 100755 index 00000000..6e647487 --- /dev/null +++ b/tests/repository_data/generate_project_data.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python + +""" + + generate_project_data.py + + + Santiago Torres + + + See LICENSE for licensing information. + + + Generate a pre-fabricated set of metadata files for 'test_developer_tool.py' + test cases. +""" + +import shutil +import datetime +import optparse +import os + +import tuf.util +from tuf.developer_tool import * + +parser = optparse.OptionParser() + +parser.add_option("-d","--dry-run", action='store_true', dest="dry_run", + help="Do not write the files, just run", default=False) +(options, args) = parser.parse_args() + + +project_key_file = 'keystore/root_key' +targets_key_file = 'keystore/targets_key' +delegation_key_file = 'keystore/delegation_key' + +# The files we use for signing in the unit tests should exist, if they are not +# populated, run 'generate.py'. +assert os.path.exists(project_key_file) +assert os.path.exists(targets_key_file) +assert os.path.exists(delegation_key_file) + +# Import the public keys. These keys are needed so that metadata roles are +# assigned verification keys, which clients use to verify the signatures created +# by the corresponding private keys. +project_public = import_rsa_publickey_from_file(project_key_file + '.pub') +targets_public = import_rsa_publickey_from_file(targets_key_file + '.pub') +delegation_public = import_rsa_publickey_from_file(delegation_key_file + '.pub') + +# Import the private keys. These private keys are needed to generate the +# signatures included in metadata. +project_private = import_rsa_privatekey_from_file(project_key_file, 'password') +targets_private = import_rsa_privatekey_from_file(targets_key_file, 'password') +delegation_private = import_rsa_privatekey_from_file(delegation_key_file, 'password') + +os.mkdir("project") +os.mkdir("project/targets") + +# Create the target files (downloaded by clients) whose file size and digest +# are specified in the 'targets.json' file. +target1_filepath = 'project/targets/file1.txt' +tuf.util.ensure_parent_dir(target1_filepath) +target2_filepath = 'project/targets/file2.txt' +tuf.util.ensure_parent_dir(target2_filepath) +target3_filepath = 'project/targets/file3.txt' +tuf.util.ensure_parent_dir(target2_filepath) + +if not options.dry_run: + with open(target1_filepath, 'wt') as file_object: + file_object.write('This is an example target file.') + + with open(target2_filepath, 'wt') as file_object: + file_object.write('This is an another example target file.') + + with open(target3_filepath, 'wt') as file_object: + file_object.write('This is role1\'s target file.') + + +project = create_new_project("test-flat", 'project/test-flat', 'prefix', + 'project/targets') + +# Add target files to the top-level projects role. These target files should +# already exist. +project.add_target(target1_filepath) +project.add_target(target2_filepath) + +# Add one key to the project. +project.add_verification_key(project_public) +project.load_signing_key(project_private) + +# Add the delegated role keys. +project.delegate('role1', [delegation_public], [target3_filepath]) +project('role1').load_signing_key(delegation_private) + +# Set the project expiration time far into the future so that its metadata does +# not expire anytime soon, or else the tests fail. Unit tests may modify the +# expiration datetimes (of the copied files), if they wish. +project.expiration = datetime.datetime(2030, 1, 1, 0, 0) +project('role1').expiration = datetime.datetime(2030, 1, 1, 0, 0) + +# Compress the project role metadata so that the unit tests have a pre-generated +# example of compressed metadata. +project.compressions = ['gz'] + +# Create the actual metadata files, which are saved to 'metadata.staged'. +if not options.dry_run: + project.write() diff --git a/tests/repository_data/project/targets/file1.txt b/tests/repository_data/project/targets/file1.txt new file mode 100644 index 00000000..7bf3499f --- /dev/null +++ b/tests/repository_data/project/targets/file1.txt @@ -0,0 +1 @@ +This is an example target file. \ No newline at end of file diff --git a/tests/repository_data/project/targets/file2.txt b/tests/repository_data/project/targets/file2.txt new file mode 100644 index 00000000..606f18ef --- /dev/null +++ b/tests/repository_data/project/targets/file2.txt @@ -0,0 +1 @@ +This is an another example target file. \ No newline at end of file diff --git a/tests/repository_data/project/targets/file3.txt b/tests/repository_data/project/targets/file3.txt new file mode 100644 index 00000000..60464604 --- /dev/null +++ b/tests/repository_data/project/targets/file3.txt @@ -0,0 +1 @@ +This is role1's target file. \ No newline at end of file diff --git a/tests/repository_data/project/test-flat/project.cfg b/tests/repository_data/project/test-flat/project.cfg new file mode 100644 index 00000000..135cbaed --- /dev/null +++ b/tests/repository_data/project/test-flat/project.cfg @@ -0,0 +1 @@ +{"project_name": "test-flat", "targets_location": "/home/santiago/Documents/v2014/TUF/tuf/tests/repository_data/project/targets", "prefix": "prefix", "metadata_location": "test-flat", "threshold": 1, "public_keys": {"6986b667c736a3b37471e030cf4ce7aa6c7e0d530325e64c2660276b77be3754": {"keytype": "rsa", "keyval": {"public": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7J15ZaeDQPrhQsRj29wB\nPhibH+Do59xsT2396L+uCg793gZlar5wZN2eHSh725cNQWyTAa9LwG+lXaKMukQ+\n8176CKR2J5sv3DezrGVu3x8V1qhyJyy79FlNZRVYTVqNaYzvJzxsVnFPpg7f8B7C\nffiqWJr9XkpqwRlCpxooXm4hplZ7uek5Ku21CzQ4OWg7hbuc+ZjCGzpXfm8NuosU\n7TipnKGpEt0Agiph5g6TB2/scoeFar1CKMONIl80maxzAQk+xkWgiJ00+Z2qFCsx\nESfis/YkILS6RMFyZz7oa1WwMtUjYmrsRuz+jlFcbNuxZpIkaISiG9a2YdGcJ1Aj\n3QIDAQAB\n-----END PUBLIC KEY-----"}}}, "layout_type": "flat"} \ No newline at end of file diff --git a/tests/repository_data/project/test-flat/test-flat.json b/tests/repository_data/project/test-flat/test-flat.json new file mode 100644 index 00000000..98c5665e Binary files /dev/null and b/tests/repository_data/project/test-flat/test-flat.json differ diff --git a/tests/repository_data/project/test-flat/test-flat/role1.json b/tests/repository_data/project/test-flat/test-flat/role1.json new file mode 100644 index 00000000..20e9f7cf Binary files /dev/null and b/tests/repository_data/project/test-flat/test-flat/role1.json differ diff --git a/tests/repository_data/project/test-repo/metadata/test-repo-like.json b/tests/repository_data/project/test-repo/metadata/test-repo-like.json new file mode 100644 index 00000000..10caafc6 Binary files /dev/null and b/tests/repository_data/project/test-repo/metadata/test-repo-like.json differ diff --git a/tests/repository_data/project/test-repo/metadata/test-repo-like/role1.json b/tests/repository_data/project/test-repo/metadata/test-repo-like/role1.json new file mode 100644 index 00000000..588d8f6e Binary files /dev/null and b/tests/repository_data/project/test-repo/metadata/test-repo-like/role1.json differ diff --git a/tests/repository_data/project/test-repo/project.cfg b/tests/repository_data/project/test-repo/project.cfg new file mode 100644 index 00000000..7cb30410 --- /dev/null +++ b/tests/repository_data/project/test-repo/project.cfg @@ -0,0 +1 @@ +{"project_name": "test-repo-like", "targets_location": "targets", "prefix": "prefix", "metadata_location": "metadata", "threshold": 1, "public_keys": {"6986b667c736a3b37471e030cf4ce7aa6c7e0d530325e64c2660276b77be3754": {"keytype": "rsa", "keyval": {"public": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7J15ZaeDQPrhQsRj29wB\nPhibH+Do59xsT2396L+uCg793gZlar5wZN2eHSh725cNQWyTAa9LwG+lXaKMukQ+\n8176CKR2J5sv3DezrGVu3x8V1qhyJyy79FlNZRVYTVqNaYzvJzxsVnFPpg7f8B7C\nffiqWJr9XkpqwRlCpxooXm4hplZ7uek5Ku21CzQ4OWg7hbuc+ZjCGzpXfm8NuosU\n7TipnKGpEt0Agiph5g6TB2/scoeFar1CKMONIl80maxzAQk+xkWgiJ00+Z2qFCsx\nESfis/YkILS6RMFyZz7oa1WwMtUjYmrsRuz+jlFcbNuxZpIkaISiG9a2YdGcJ1Aj\n3QIDAQAB\n-----END PUBLIC KEY-----"}}}, "layout_type": "repo-like"} \ No newline at end of file diff --git a/tests/repository_data/project/test-repo/targets/file1.txt b/tests/repository_data/project/test-repo/targets/file1.txt new file mode 100644 index 00000000..7bf3499f --- /dev/null +++ b/tests/repository_data/project/test-repo/targets/file1.txt @@ -0,0 +1 @@ +This is an example target file. \ No newline at end of file diff --git a/tests/repository_data/project/test-repo/targets/file2.txt b/tests/repository_data/project/test-repo/targets/file2.txt new file mode 100644 index 00000000..606f18ef --- /dev/null +++ b/tests/repository_data/project/test-repo/targets/file2.txt @@ -0,0 +1 @@ +This is an another example target file. \ No newline at end of file diff --git a/tests/repository_data/project/test-repo/targets/file3.txt b/tests/repository_data/project/test-repo/targets/file3.txt new file mode 100644 index 00000000..60464604 --- /dev/null +++ b/tests/repository_data/project/test-repo/targets/file3.txt @@ -0,0 +1 @@ +This is role1's target file. \ No newline at end of file diff --git a/tests/test_developer_tool.py b/tests/test_developer_tool.py new file mode 100755 index 00000000..bedcccf6 --- /dev/null +++ b/tests/test_developer_tool.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python + +""" + + test_developer_tool.py. + + + Santiago Torres Arias + + + See LICENSE for licensing inforation. + + + Unit test for the 'developer_tool.py' module. +""" + +import os +import time +import datetime +import unittest +import logging +import tempfile +import shutil + +import tuf +import tuf.log +import tuf.formats +import tuf.roledb +import tuf.keydb +import tuf.developer_tool as developer_tool + +from tuf.developer_tool import METADATA_DIRECTORY_NAME +from tuf.developer_tool import TARGETS_DIRECTORY_NAME + +logger = logging.getLogger("tuf.test_developer_tool") + +developer_tool.disable_console_log_messages() + +class TestProject(unittest.TestCase): + + tmp_dir = None + + @classmethod + def setUpClass(cls): + cls.tmp_dir = tempfile.mkdtemp(dir = os.getcwd()) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmp_dir) + + def setUp(self): + # called before every test case + pass + + def tearDown(self): + # called after every test case + tuf.roledb.clear_roledb() + tuf.keydb.clear_keydb() + pass + + def test_create_new_project(self): + # Test cases for the create_new_project function. In this test we will + # check input, correct file creation and format. We also check + # that a proper object is generated. We will use the normal layout for this + # test suite. + + # Create a local subfolder for this test. + local_tmp = tempfile.mkdtemp(dir = self.tmp_dir) + + # These are the usual values we will be throwing to the function, however + # we will swap these for nulls or malformed values every now and then to + # test input. + project_name = "test_suite" + metadata_directory = local_tmp + location_in_repository = '/prefix' + targets_directory = None + key = None + + # Create a blank project. + project = developer_tool.create_new_project(project_name, metadata_directory, + location_in_repository) + + self.assertTrue(isinstance(project, developer_tool.Project)) + self.assertTrue(project.layout_type == 'repo-like') + self.assertTrue(project._prefix == location_in_repository) + self.assertTrue(project._project_name == project_name) + self.assertTrue(project._metadata_directory == + os.path.join(metadata_directory,METADATA_DIRECTORY_NAME)) + self.assertTrue(project._targets_directory == + os.path.join(metadata_directory,TARGETS_DIRECTORY_NAME)) + + # Create a blank project without a prefix. + project = developer_tool.create_new_project(project_name, metadata_directory) + self.assertTrue(isinstance(project, developer_tool.Project)) + self.assertTrue(project.layout_type == 'repo-like') + self.assertTrue(project._prefix == '') + self.assertTrue(project._project_name == project_name) + self.assertTrue(project._metadata_directory == + os.path.join(metadata_directory,METADATA_DIRECTORY_NAME)) + self.assertTrue(project._targets_directory == + os.path.join(metadata_directory,TARGETS_DIRECTORY_NAME)) + + # Create a blank project without a valid metadata directory. + self.assertRaises(tuf.FormatError, developer_tool.create_new_project, + 0, metadata_directory, location_in_repository) + self.assertRaises(tuf.FormatError, developer_tool.create_new_project, + project_name, 0, location_in_repository) + self.assertRaises(tuf.FormatError, developer_tool.create_new_project, + project_name, metadata_directory, 0) + + + # Create a new project with a flat layout. + targets_directory = tempfile.mkdtemp(dir = local_tmp) + metadata_directory = tempfile.mkdtemp(dir = local_tmp) + project = developer_tool.create_new_project(project_name, metadata_directory, + location_in_repository, targets_directory) + self.assertTrue(isinstance(project, developer_tool.Project)) + self.assertTrue(project.layout_type == 'flat') + self.assertTrue(project._prefix == location_in_repository) + self.assertTrue(project._project_name == project_name) + self.assertTrue(project._metadata_directory == metadata_directory) + self.assertTrue(project._targets_directory == targets_directory) + + # Finally, check that if targets_directory is set, it is valid. + self.assertRaises(tuf.FormatError, developer_tool.create_new_project, + project_name, metadata_directory, location_in_repository, 0) + + # Copy a key to our workspace and create a new project with it. + keystore_path = os.path.join('repository_data','keystore') + + # I will use the same key as the one provided in the repository + # tool tests for the root role, but this is not a root role... + root_key_path = os.path.join(keystore_path,'root_key.pub') + project_key = developer_tool.import_rsa_publickey_from_file(root_key_path) + + # Test create new project with a key added by default. + project = developer_tool.create_new_project(project_name, metadata_directory, + location_in_repository, targets_directory, project_key) + + self.assertTrue(isinstance(project, developer_tool.Project)) + self.assertTrue(project.layout_type == 'flat') + self.assertTrue(project._prefix == location_in_repository) + self.assertTrue(project._project_name == project_name) + self.assertTrue(project._metadata_directory == metadata_directory) + self.assertTrue(project._targets_directory == targets_directory) + self.assertTrue(len(project.keys) == 1) + self.assertTrue(project.keys[0] == project_key['keyid']) + + # Set as readonly and try to write a repo. + shutil.rmtree(targets_directory) + os.chmod(local_tmp, 0o0555) + + tuf.roledb.clear_roledb() + tuf.keydb.clear_keydb() + self.assertRaises(OSError, developer_tool.create_new_project ,project_name, + metadata_directory, location_in_repository, targets_directory, + project_key) + + os.chmod(local_tmp, 0o0777) + + shutil.rmtree(metadata_directory) + os.chmod(local_tmp, 0o0555) + + tuf.roledb.clear_roledb() + tuf.keydb.clear_keydb() + self.assertRaises(OSError, developer_tool.create_new_project ,project_name, + metadata_directory, location_in_repository, targets_directory, + project_key) + + + os.chmod(local_tmp, 0o0777) + shutil.rmtree(local_tmp) + + + + def test_load_project(self): + # This test case will first try to load an existing project and test for + # verify the loaded object. It will next try to load a nonexisting project + # and expect a correct error handler. Finally, it will try to overwrite the + # existing prefix of the loaded project. + + # Create a local subfolder for this test. + local_tmp = tempfile.mkdtemp(dir = self.tmp_dir) + + # Test non-existent project filepath. + nonexistent_path = os.path.join(local_tmp, "nonexistent") + self.assertRaises(IOError, developer_tool.load_project, nonexistent_path) + + # Copy the pregenerated metadata. + project_data_filepath = os.path.join('repository_data', 'project') + target_project_data_filepath = os.path.join(local_tmp, 'project') + shutil.copytree("repository_data/project", target_project_data_filepath) + + # Properly load a project. + repo_filepath = os.path.join(local_tmp, 'project', 'test-repo') + project = developer_tool.load_project(repo_filepath) + + self.assertTrue(project.layout_type == "repo-like") + + repo_filepath = os.path.join(local_tmp, 'project', 'test-flat') + new_targets_path = os.path.join(local_tmp, 'project', 'targets') + project = developer_tool.load_project(repo_filepath, + new_targets_location = new_targets_path) + self.assertTrue(project._targets_directory == new_targets_path) + self.assertTrue(project.layout_type == 'flat') + + + # Load a project overwriting the prefix. + project = developer_tool.load_project(repo_filepath, prefix='new') + self.assertTrue(project._prefix == 'new') + + # Load a project with a file missing. + file_to_corrupt = os.path.join(repo_filepath, 'test-flat', 'role1.json') + with open(file_to_corrupt, 'wt') as fp: + fp.write("this is not a json file") + + self.assertRaises(tuf.Error, developer_tool.load_project, repo_filepath) + + + + + def test_add_verification_keys(self): + # create a new project instance + project = developer_tool.Project("test_verification_keys", "somepath", + "someotherpath", "prefix") + + # add invalid verification key + self.assertRaises(tuf.FormatError, project.add_verification_key, "invalid") + + # add verification key + # - load it first + keystore_path = os.path.join('repository_data','keystore') + first_verification_key_path = os.path.join(keystore_path,'root_key.pub') + first_verification_key = \ + developer_tool.import_rsa_publickey_from_file(first_verification_key_path) + + project.add_verification_key(first_verification_key) + + + # add another verification key (should expect exception) + second_verification_key_path = os.path.join(keystore_path,'snapshot_key.pub') + second_verification_key = \ + developer_tool.import_rsa_publickey_from_file(second_verification_key_path) + + self.assertRaises(tuf.Error, + project.add_verification_key,(second_verification_key)) + + + + # add a verification key for the delegation + project.delegate("somedelegation", [], []) + project("somedelegation").add_verification_key(first_verification_key) + project("somedelegation").add_verification_key(second_verification_key) + + + # add another delegation of the delegation + project("somedelegation").delegate("somesubdelegation", [], []) + project("somedelegation")("somesubdelegation").add_verification_key( + first_verification_key) + project("somedelegation")("somesubdelegation").add_verification_key( + second_verification_key) + + + def test_write(self): + + # create tmp directory + local_tmp = tempfile.mkdtemp(dir=self.tmp_dir) + + # create new project inside tmp directory + project = developer_tool.create_new_project("test_write", local_tmp, + "prefix"); + + # create some target files inside the tmp directory + target_filepath = os.path.join(local_tmp, "targets", "test_target") + with open(target_filepath, "wt") as fp: + fp.write("testing file") + + + # add the targets + project.add_target(target_filepath) + + # add verification keys + keystore_path = os.path.join('repository_data','keystore') + project_key_path = os.path.join(keystore_path,'root_key.pub') + project_key = \ + developer_tool.import_rsa_publickey_from_file(project_key_path) + + + # call status (for the sake of doing it) + project.status() + + project.add_verification_key(project_key) + + + # add another verification key (should expect exception) + delegation_key_path = os.path.join(keystore_path,'snapshot_key.pub') + delegation_key = \ + developer_tool.import_rsa_publickey_from_file(delegation_key_path) + + # add a subdelegation + subdelegation_key_path = os.path.join(keystore_path,'timestamp_key.pub') + subdelegation_key = \ + developer_tool.import_rsa_publickey_from_file(subdelegation_key_path) + + # add a delegation + project.delegate("delegation", [delegation_key], []) + project("delegation").delegate("subdelegation", [subdelegation_key], []) + + # call write (except) + self.assertRaises(tuf.Error, project.write, ()) + + # call status (for the sake of doing it) + project.status() + + # load private keys + project_private_key_path = os.path.join(keystore_path, 'root_key') + project_private_key = \ + developer_tool.import_rsa_privatekey_from_file(project_private_key_path, + 'password') + + delegation_private_key_path = os.path.join(keystore_path, 'snapshot_key') + delegation_private_key = \ + developer_tool.import_rsa_privatekey_from_file(delegation_private_key_path, + 'password') + + subdelegation_private_key_path = \ + os.path.join(keystore_path, 'timestamp_key') + subdelegation_private_key = \ + developer_tool.import_rsa_privatekey_from_file(subdelegation_private_key_path, + 'password') + + # Test partial write + # backup everything (again) + # + backup targets: + targets_backup = project.target_files + + # + backup delegations + delegations_backup = \ + tuf.roledb.get_delegated_rolenames(project._project_name) + + # + backup layout type + layout_type_backup = project.layout_type + + # + backup keyids + keys_backup = project.keys + delegation_keys_backup = project("delegation").keys + + # + backup the prefix. + prefix_backup = project._prefix + + # + backup the name. + name_backup = project._project_name + + # Set the compressions, we will be checking this part here too + project.compressions = ['gz'] + project('delegation').compressions = project.compressions + + # Write and reload. + self.assertRaises(tuf.Error, project.write) + project.write(write_partial=True) + + project = developer_tool.load_project(local_tmp) + + # Check against backup. + self.assertEqual(project.target_files, list(targets_backup.keys())) + new_delegations = tuf.roledb.get_delegated_rolenames(project._project_name) + self.assertEqual(new_delegations, delegations_backup) + self.assertEqual(project.layout_type, layout_type_backup) + self.assertEqual(project.keys, keys_backup) + self.assertEqual(project("delegation").keys, delegation_keys_backup) + self.assertEqual(project._prefix, prefix_backup) + self.assertEqual(project._project_name, name_backup) + + + + roleinfo = tuf.roledb.get_roleinfo(project._project_name) + + self.assertEqual(roleinfo['partial_loaded'], True) + + + + # Load_signing_keys. + project("delegation").load_signing_key(delegation_private_key) + + project.status() + + project("delegation")("subdelegation").load_signing_key( + subdelegation_private_key) + + project.status() + + project.load_signing_key(project_private_key) + + # Backup everything. + # + backup targets. + targets_backup = project.target_files + + # + backup delegations. + delegations_backup = \ + tuf.roledb.get_delegated_rolenames(project._project_name) + + # + backup layout type. + layout_type_backup = project.layout_type + + # + backup keyids + keys_backup = project.keys + delegation_keys_backup = project("delegation").keys + + # + backup the prefix. + prefix_backup = project._prefix + + # + backup the name. + name_backup = project._project_name + + # Call status (for the sake of doing it.) + project.status() + + # Call write. + project.write() + + # Call load. + project = developer_tool.load_project(local_tmp) + + + # Check against backup. + self.assertEqual(project.target_files, targets_backup) + + new_delegations = tuf.roledb.get_delegated_rolenames(project._project_name) + self.assertEqual(new_delegations, delegations_backup) + self.assertEqual(project.layout_type, layout_type_backup) + self.assertEqual(project.keys, keys_backup) + self.assertEqual(project("delegation").keys, delegation_keys_backup) + self.assertEqual(project._prefix, prefix_backup) + self.assertEqual(project._project_name, name_backup) + + + + + +if __name__ == '__main__': + unittest.main() diff --git a/tuf/README-developer-tools.md b/tuf/README-developer-tools.md new file mode 100644 index 00000000..43d9c493 --- /dev/null +++ b/tuf/README-developer-tools.md @@ -0,0 +1,326 @@ +# Developing for a TUF repository # + +## Table of Contents ## +- [Overview](#overview) +- [Creating a Simple Project](#creating_a_simple_project) + - [Generating a Key](#generating_a_key) + - [The Project Class](#the_project_class) + - [Signing and Writing the Metadata](#signing_and_writing_the_metadata) +- [Loading an Existing Project](#loading_an_existing_project) +- [Delegations](#delegations) +- [Managing Keys](#managing_keys) +- [Managing Targets](#managing_targets) + + +## Overview +The TUF developer tool is a Python library that enables developers to create +and maintain the required metadata for files hosted in a TUF Repository. The +main concern when generating metadata for a TUF repository is generating +information that matches the future location of the files in the repository. We +use the developer tools to generate valid information so that the project and +its metadata can be applied to the TUF project transparently. + +This document has two parts. The first part walks through the creation of a +prototypal TUF project. The second part demonstrates the full capabilities of +the TUF developer tool, which can be used to expand the project from the first +part to meet the developer's needs. + + +## Creating a Simple project +The following section describes a very basic example usage of the developer +tools with a one-file project. + + +### Generating a Key +First, we will need to generate a key to sign the metadata. Keys are generated +in pairs: one public and the other private. The private key is +password-protected and is used to sign metadata. The public key can be shared +freely, and is used to verify signatures made by the private key. + +The generate\_and\_write\_rsa\_keypair function will create two key files named +"path/to/key.pub", which is the public key and "path/to/key", which +is the private key. + +``` +>>> from tuf.developer_tool import * +>>> generate_and_write_rsa_keypair("path/to/key") +Enter a password for the RSA key: +Confirm: +>>> +``` + +We can also use the bits parameter to set a different key length (the default +is 3072). We can also provide the password parameter in order to suppress the +password prompt. + +In this example we will be using rsa keys, but ed25519 keys are also supported. + +Now we have a key for our project, we can proceed to create our project. + + +### The Project Class +The TUF developer tool is built around the Project class, which is used to +organize groups of targets associated with a single set of metadata. Each +Project instance keeps track of which target files are associated with a single +set of metadata. Each Project instance keeps track of which target files are +signed and which need signing, which keys are used to sign metadata. It also +keeps track of delegated roles, which are covered later. + +Before creating a project, you must know where it will be located in the TUF +Repository. In the following example, we will create a project to be hosted as +"repo/unclaimed/example_project" within the repository, and store a local copy +of the metadata at "path/to/metadata". The project will comprise a single +target file, "local/path/to/example\_project/target\_1" locally, and we will +secure it with the key generated above. + +First, we must import the generated keys. We can do that by issuing the +following: + +``` +>>> public_key = import_rsa_publickey_from_file("path/to/keys.pub") +``` + +After importing the key, we can generate a new project with the following +command: + +``` +>>> project = create_new_project(name="example_project", +... metadata_directory="local/path/to/metadata/", +... targets_directory="local/path/to/example_project", +... location_in_repository="repo/unclaimed", key=public_key) +``` + +Let's list the arguments and make sense out of this rather long function call: + +- create a project named example_project: the name of the metadata file will match this name +- the metadata will be located in "local/path/to/metadata", this means all of the generated files +for this project will be located here +- the targets are located in local/path/to/example project. If your targets are located in some other +place, you can point the targets directory there. Files must reside under the path local/path/to/example_project or else it won't be possible to add them. +- location\_in\_repository points to repo/unclaimed, this will be prepended to the paths in the generated metadata so the signatures all match. + +Now the project is in memory and we can do different operations on it such as +adding and removing targets, delegating files, changing signatures and keys, +etc. For the moment we are interested in adding our one and only target inside +the project. + +To add a target, we issue the following method: + +``` +>>> project.add_target("target_1") +``` + +Have in mind the file "target\_1" should be located in +"local/path/to/example\_project" or else the adding procedure will throw an +error. + +At this point, the metadata is not valid. We have assigned a key to the +project, but we have not *signed* it with that key. Signing is the process of +generating a signature with our private key so it can be verified with the +public key by the server (upon uploading) and by the clients (when updating). + + +### Signing and Writing the Metadata ### +In order to sign the metadata, we need to import the private key corresponding +to the public key we added to the project. One the key is loaded to the project, +it will automatically be used to sign the metadata whenever it is written. + +``` +>>> private_key = import_rsa_privatekey_from_file("path/to/key") +Enter password for the RSA key: +>>> project.load_signing_key(private_key) +>>> project.write() +``` + +When all changes to a project have been written, the Project instance can +safely be deleted. + +The project can be loaded later to update changes to the project. The metadata +contains checksums that have to match the actual files or else it won't be +accepted by the upstream repository. + +At this point, if you have followed all the steps in this document so far +(substituting appropriate names and filepaths) you will have created a basic +TUF project, which can be expanded as needed. The simplest way to get your +project secured is to add all your files using add\_target() (or see [Managing +Keys](#managing_keys) on how to add whole directories). If your project has +several contributors, you may want to consider adding +[delegations](#delegations) to your project. + + +## Loading an Existing Project +To make changes to existing metadata, we will need the Project again. We can +restore it with the load_project() function. + +``` +>>> from tuf.developer_tool import * +>>> project = load_project("local/path/to/metadata") +``` +Each time the project is loaded anew, the necessary private keys must also be +loaded in order to sign metadata. + +``` +>>> private_key = import_rsa_privatekey_from_file("path/to/key") +Enter a password for the RSA key: +>>> project.load_signing_key(private_key) +>>> project.write() +``` + + +## Delegations + +The project we created above is secured entirely by one key. If you want to +allow someone else to update part of your project independently, you will need +to delegate a new role for them. For example, we can do the following: + +``` +>>> other_key = import_rsa_publickey_from_file(“another_public_key.pub”) +>>> project.delegate(“newrole”, [other_key], targets) +``` + +The new role is now an attribute of the Project instance, and contains the same +methods as Project. For example, we can add targets in the same way as before: + +``` +>>> project(“newrole”).add_target(“delegated_1”) +``` + +Recall that we input the other person’s key as part of a list. That list can +contain any number of public keys. We can also add keys to the role after +creating it using the [add\_verification\_key()](#adding_a_key_to_a_delegation) +method. + +### Restricted Paths + +By default, a delegated role is permitted to add and modify targets anywhere in +the Project's targets directory. We can assign restricted paths to a delegated +role to limit this permission. + +``` +>>> project.add_restricted_paths(["restricted/filepath"], "newrole") +``` + +This will prevent the delegated role from signing targets whose local filepaths +do not begin with "restricted/filepath". We can assign several restricted +filepaths to a role by adding them to the list in the first parameter, or by +invoking the method again. A role with multiple restricted paths can add +targets to any of them. + +Note that this method is invoked the parent role (in this case, the Project) +and takes the delegated role name as an argument. + +### Nested Delegations + +It is possible for a delegated role to have delegations of its own. We can do +this by calling delegate() on a delegated role: + +``` +>>> project("newrole").delegate(“nestedrole”, [key], targets) +``` + +Nested delegations function no differently than first-order delegations. to +demonstrate, adding a target to nested delegation looks like this: + +``` +>>> project("newrole")("nestedrole").add_target("foo") +``` + +### Revoking Delegations +Delegations can be revoked, removing the delegated role from the project. + +``` +>>> project.revoke("newrole") +``` + + +## Managing Keys +This section describes the key-related functions and parameters not covered in +the [Creating a Simple Project](#creating_a_simple_project) section. + +### Additional Parameters for Key Generation +When generating keys, it is possible to specify the length of the key in bits +and its password as parameters: + +``` +>>> generate_and_write_rsa_keypair("path/to/key",bits=2048, password="pw") +``` +The bits parameter defaults to 3072, and values below 2048 will raise an error. +The password parameter is only intended to be used in scripts. + + +### Adding a Key to a Delegation +New verifications keys can be added to an existing delegation using +add\_verification\_key(): + +``` +>>> project("rolename").add_verification_key(pubkey) +``` + +A delegation can have several verification keys at once. By default, a +delegated role with multiple keys can be written using any one of their +corresponding signing keys. To modify this behavior, you can change the +delegated role's [threshold](#delegation_thrsholds). + +### Removing a Key from a Delegation +Verification keys can also be removed, like this: + +``` +>>> project("rolename").remove_verification_key(pubkey) +``` + +Remember that a project can only have one key, so this method will return an +error if there is already a key assigned to it. In order to replace a key we +must first delete the existing one and then add the new one. It is possible to +omit the key parameter in the create\_new\_project() function, and add the key +later. + +### Changing the Project Key +Each Project instance can only have one verification key. This key can be +replaced by removing it and adding a new key, in that order. + +``` +>>> project.remove_verification_key(oldkey) +>>> project.add_verification_key(new) +``` + + +### Delegation Thresholds + +Every delegated role has a threshold, which determines how many of its signing +keys need to be loaded to write the role. The threshold defaults to 1, and +should not exceed the number of verification keys assigned to the role. The +threshold can be accessed as a property of a delegated role. + +``` +>>> project("rolename").threshold = 2 +``` + +The above line will set the "rolename" role's threshold to 2. + + +## Managing Targets +There are supporting functions of the targets library to make the project +maintenance easier. These functions are described in this section. + +### Adding Targets by Directory +This function is especially useful when creating a new project to add all the +files contained in the targets directory. The following code block illustrates +the usage of this function: + +``` +>>> list_of_targets = \ +... project.get_filepaths_in_directory(“path/within/targets/folder”, +... recursive_walk=False, follow_links=False) +>>> project.add_targets(list_of_targets) +``` + +### Deleting Targets from a Project +It is possible that we want to delete existing targets inside our project. To +stop the developer tool from tracking this file we can issue the following +command: + +``` +>>> project.remove_target(“target_1”) +``` + +Now the target file won't be part of the metadata. diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py new file mode 100755 index 00000000..e94843b9 --- /dev/null +++ b/tuf/developer_tool.py @@ -0,0 +1,1034 @@ +#!/usr/bin/env python + +""" + + developer_tool.py + + + Santiago Torres + Zane Fisher + + Based on the work done for 'repository_tool.py' by Vladimir Diaz. + + + January 22, 2014. + + + See LICENSE for licensing information. + + + See 'tuf/README-developer-tools.md' for a complete guide on using + 'developer_tool.py'. +""" + +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +import os +import errno +import sys +import logging +import shutil +import tempfile +import json + +import tuf +import tuf.formats +import tuf.util +import tuf.keydb +import tuf.roledb +import tuf.keys +import tuf.sig +import tuf.log +import tuf.conf +import tuf.repository_tool + +# These imports provide the interface for 'developer_tool.py', since the imports +# are made there. +from tuf.keys import format_keyval_to_metadata +from tuf.keys import format_metadata_to_key + +from tuf.repository_tool import Targets +from tuf.repository_lib import get_metadata_fileinfo +from tuf.repository_lib import get_metadata_filenames +from tuf.repository_tool import generate_and_write_rsa_keypair +from tuf.repository_tool import import_rsa_publickey_from_file +from tuf.repository_tool import import_rsa_privatekey_from_file +from tuf.repository_tool import generate_and_write_ed25519_keypair +from tuf.repository_tool import import_ed25519_publickey_from_file +from tuf.repository_tool import import_ed25519_privatekey_from_file +from tuf.repository_lib import _remove_invalid_and_duplicate_signatures +from tuf.repository_lib import disable_console_log_messages +from tuf.repository_lib import _check_role_keys +from tuf.repository_lib import _delete_obsolete_metadata +from tuf.repository_lib import generate_targets_metadata +from tuf.repository_lib import sign_metadata +from tuf.repository_lib import write_metadata_file +from tuf.repository_lib import _metadata_is_partially_loaded + +# See 'log.py' to learn how logging is handled in TUF. +logger = logging.getLogger('tuf.developer_tool') + +# Recommended RSA key sizes: +# http://www.emc.com/emc-plus/rsa-labs/historical/twirl-and-rsa-key-size.htm#table1 +# According to the document above, revised May 6, 2003, RSA keys of size 3072 +# provide security through 2031 and beyond. 2048-bit keys are the recommended +# minimum and are good from the present through 2030. +from tuf.repository_lib import DEFAULT_RSA_KEY_BITS as DEFAULT_RSA_KEY_BITS + +# The algorithm used by the developer tools to generate the hashes of the +# target filepaths. +from tuf.repository_tool import HASH_FUNCTION as HASH_FUNCTION + +# The extension of TUF metadata. +from tuf.repository_lib import METADATA_EXTENSION as METADATA_EXTENSION + +# The metadata filename for the targets metadata information. +from tuf.repository_lib import TARGETS_FILENAME as TARGETS_FILENAME + +# Project configuration filename. This file is intended to hold all of the +# supporting information about the project that's not contained in a usual +# TUF metadata file. 'project.cfg' consists of the following fields: +# +# targets_location: the location of the targets folder. +# +# prefix: the directory location to prepend to the metadata so it +# matches the metadata signed in the repository. +# +# metadata_location: the location of the metadata files. +# +# threshold: the threshold for this project object, it is fixed to +# one in the current version. +# +# public_keys: a list of the public keys used to verify the metadata +# in this project. +# +# layout_type: a field describing the directory layout: +# +# repo-like: matches the layout of the repository tool. +# the targets and metadata folders are +# located under a common directory for the +# project. +# +# flat: the targets directory and the +# metadata directory are located in different +# paths. +# +# project_name: The name of the current project, this value is used to +# match the resulting filename with the one in upstream. +PROJECT_FILENAME = 'project.cfg' + +# The targets and metadata directory names. Metadata files are written +# to the staged metadata directory instead of the "live" one. +from tuf.repository_tool import METADATA_STAGED_DIRECTORY_NAME +from tuf.repository_tool import METADATA_DIRECTORY_NAME +from tuf.repository_tool import TARGETS_DIRECTORY_NAME + +# The full list of supported TUF metadata extensions. +from tuf.repository_lib import METADATA_EXTENSIONS + +# The recognized compression extensions. +from tuf.repository_lib import SUPPORTED_COMPRESSION_EXTENSIONS + +# Supported key types. +from tuf.repository_lib import SUPPORTED_KEY_TYPES + + +class Project(Targets): + """ + + Simplify the publishing process of third-party projects by handling all of + the bookkeeping, signature handling, and integrity checks of delegated TUF + metadata. 'repository_tool.py' is responsible for publishing and + maintaining metadata of the top-level roles, and 'developer_tool.py' is used + by projects that have been delegated responsibility for a delegated projects + role. Metadata created by this module may then be added to other metadata + available in a TUF repository. + + Project() is the representation of a project's metadata file(s), with the + ability to modify this data in an OOP manner. Project owners do not have to + manually verify that metadata files are properly formatted or that they + contain valid data. + + + project_name: + The name of the metadata file as it should be named in the upstream + repository. + + metadata_directory: + The metadata sub-directory contains the metadata file(s) of this project, + including any of its delegated roles. + + targets_directory: + The targets sub-directory contains the project's target files that are + downloaded by clients and are referenced in its metadata. The hashes and + file lengths are listed in Metadata files so that they are securely + downloaded. Metadata files are similarly referenced in the top-level + metadata. + + file_prefix: + The path string that will be prepended to the generated metadata + (e.g., targets/foo -> targets/prefix/foo) so that it matches the actual + targets location in the upstream repository. + + + tuf.FormatError, if the arguments are improperly formatted. + + + Creates a project Targets role object, with the same object attributes of + the top-level targets role. + + + None. + """ + + def __init__(self, project_name, metadata_directory, targets_directory, + file_prefix): + + # Do the arguments have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if any are improperly formatted. + tuf.formats.NAME_SCHEMA.check_match(project_name) + tuf.formats.PATH_SCHEMA.check_match(metadata_directory) + tuf.formats.PATH_SCHEMA.check_match(targets_directory) + tuf.formats.PATH_SCHEMA.check_match(file_prefix) + + self._metadata_directory = metadata_directory + self._targets_directory = targets_directory + self._project_name = project_name + self._prefix = file_prefix + + # Layout type defaults to "flat" unless explicitly specified in + # create_new_project(). + self.layout_type = 'flat' + + # Set the top-level Targets object. Set the rolename to be the project's + # name. + super(Project, self).__init__(self._targets_directory, project_name) + + + + + + def write(self, write_partial=False): + """ + + Write all the JSON Metadata objects to their corresponding files. + write() raises an exception if any of the role metadata to be written to + disk is invalid, such as an insufficient threshold of signatures, missing + private keys, etc. + + + write_partial: + A boolean indicating whether partial metadata should be written to + disk. Partial metadata may be written to allow multiple maintainters + to independently sign and update role metadata. write() raises an + exception if a metadata role cannot be written due to not having enough + signatures. + + + tuf.Error, if any of the project roles do not have a minimum threshold of + signatures. + + + Creates metadata files in the project's metadata directory. + + + None. + """ + + # Does 'write_partial' have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if any are improperly formatted. + tuf.formats.BOOLEAN_SCHEMA.check_match(write_partial) + + # At this point the tuf.keydb and tuf.roledb stores must be fully + # populated, otherwise write() throwns a 'tuf.Repository' exception if + # any of the project roles are missing signatures, keys, etc. + + # Write the metadata files of all the delegated roles of the project. + delegated_rolenames = \ + tuf.roledb.get_delegated_rolenames(self._project_name) + + for delegated_rolename in delegated_rolenames: + roleinfo = tuf.roledb.get_roleinfo(delegated_rolename) + delegated_filename = os.path.join(self._metadata_directory, + delegated_rolename + METADATA_EXTENSION) + + # Ensure the parent directories of 'metadata_filepath' exist, otherwise an + # IO exception is raised if 'metadata_filepath' is written to a + # sub-directory. + tuf.util.ensure_parent_dir(delegated_filename) + + _generate_and_write_metadata(delegated_rolename, delegated_filename, + write_partial, self._targets_directory, + self._metadata_directory, + prefix=self._prefix) + + + # Generate the 'project_name' metadata file. + targets_filename = self._project_name + METADATA_EXTENSION + targets_filename = os.path.join(self._metadata_directory, targets_filename) + project_signable, targets_filename = \ + _generate_and_write_metadata(self._project_name, targets_filename, + write_partial, self._targets_directory, + self._metadata_directory, prefix=self._prefix) + + # Save configuration information that is not stored in the project's + # metadata + _save_project_configuration(self._metadata_directory, + self._targets_directory, self.keys, self._prefix, + self.threshold, self.layout_type, + self._project_name) + + # Delete the metadata of roles no longer in 'tuf.roledb'. Obsolete roles + # may have been revoked. + _delete_obsolete_metadata(self._metadata_directory, + project_signable['signed'], False) + + + + + + def add_verification_key(self, key): + """ + + Function as a thin wrapper call for the project._targets call + with the same name. This wrapper is only for usability purposes. + + + key: + The role key to be added, conformant to 'tuf.formats.ANYKEY_SCHEMA'. + Adding a public key to a role means that its corresponding private + key must generate and add its signture to the role. + + + tuf.FormatError, if the 'key' argument is improperly formatted. + + tuf.Error, if the project already contains a key. + + + The role's entries in 'tuf.keydb.py' and 'tuf.roledb.py' are updated. + + + None + """ + + # Verify that this role does not already contain a key. The parent project + # role is restricted to one key. Any of its delegated roles may have + # more than one key. + # TODO: Add condition check for the requirement stated above. + if len(self.keys) > 0: + raise tuf.Error("This project already contains a key.") + + try: + super(Project, self).add_verification_key(key) + + except tuf.FormatError: + raise + + + + + + def status(self): + """ + + Determine the status of the project, including its delegated roles. + status() checks if each role provides sufficient public keys, signatures, + and that a valid metadata file is generated if write() were to be called. + Metadata files are temporarily written to check that proper metadata files + is written, where file hashes and lengths are calculated and referenced + by the project. status() does not do a simple check for number of + threshold keys and signatures. + + + None. + + + tuf.Error, if the project, or any of its delegated roles, do not have a + minimum threshold of signatures. + + + Generates and writes temporary metadata files. + + + None. + """ + + temp_project_directory = None + + try: + temp_project_directory = tempfile.mkdtemp() + metadata_directory = os.path.join(temp_project_directory, + self._metadata_directory[1:]) + + targets_directory = self._targets_directory + + os.makedirs(metadata_directory) + + # TODO: We should do the schema check. + filenames = {} + filenames['targets'] = os.path.join(metadata_directory, self._project_name) + + # Delegated roles. + delegated_roles = tuf.roledb.get_delegated_rolenames(self._project_name) + insufficient_keys = [] + insufficient_signatures = [] + + for delegated_role in delegated_roles: + try: + _check_role_keys(delegated_role) + + except tuf.InsufficientKeysError: + insufficient_keys.append(delegated_role) + continue + + roleinfo = tuf.roledb.get_roleinfo(delegated_role) + try: + signable = _generate_and_write_metadata(delegated_role, + filenames['targets'], False, + targets_directory, + metadata_directory, + False) + self._print_status(delegated_role, signable[0]) + + except tuf.Error: + insufficient_signatures.append(delegated_role) + + if len(insufficient_keys): + message = 'Delegated roles with insufficient keys: ' +\ + repr(insufficient_keys) + print(message) + return + + if len(insufficient_signatures): + message = 'Delegated roles with insufficient signatures: ' +\ + repr(insufficient_signatures) + print(message) + return + + # Targets role. + try: + _check_role_keys(self.rolename) + + except tuf.InsufficientKeysError as e: + print(str(e)) + return + + try: + signable, filename = _generate_and_write_metadata(self._project_name, + filenames['targets'], False, + targets_directory, + metadata_directory, + False) + self._print_status(self._project_name, signable) + + except tuf.Error as e: + signable = e[1] + self._print_status(self._project_name, signable) + return + + finally: + shutil.rmtree(temp_project_directory, ignore_errors=True) + + + + + + def _print_status(self, rolename, signable): + """ + Non-public function prints the number of (good/threshold) signatures of + 'rolename'. + """ + + status = tuf.sig.get_signature_status(signable, rolename) + + message = repr(rolename) + ' role contains ' +\ + repr(len(status['good_sigs'])) + ' / ' + repr(status['threshold']) +\ + ' signatures.' + print(message) + + + + + +def _generate_and_write_metadata(rolename, metadata_filename, write_partial, + targets_directory, metadata_directory, + filenames=None, + prefix=''): + """ + Non-public function that can generate and write the metadata of the + specified 'rolename'. It also increments version numbers if: + + 1. write_partial==True and the metadata is the first to be written. + + 2. write_partial=False (i.e., write()), the metadata was not loaded as + partially written, and a write_partial is not needed. + """ + + metadata = None + + # Retrieve the roleinfo of 'rolename' to extract the needed metadata + # attributes, such as version number, expiration, etc. + roleinfo = tuf.roledb.get_roleinfo(rolename) + + metadata = generate_targets_metadata(targets_directory, + roleinfo['paths'], + roleinfo['version'], + roleinfo['expires'], + roleinfo['delegations'], + False) + + # Preprend the prefix to the project's filepath to avoid signature errors + # in upstream. + target_filepaths = metadata['targets'].items() + for element in list(metadata['targets']): + junk_path, relative_target = os.path.split(element) + prefixed_path = os.path.join(prefix,relative_target) + metadata['targets'][prefixed_path] = metadata['targets'][element] + if prefix != '': + del(metadata['targets'][element]) + + signable = sign_metadata(metadata, roleinfo['signing_keyids'], + metadata_filename) + + # Check if the version number of 'rolename' may be automatically incremented, + # depending on whether if partial metadata is loaded or if the metadata is + # written with write() / write_partial(). + # Increment the version number if this is the first partial write. + if write_partial: + temp_signable = sign_metadata(metadata, [], metadata_filename) + temp_signable['signatures'].extend(roleinfo['signatures']) + status = tuf.sig.get_signature_status(temp_signable, rolename) + if len(status['good_sigs']) == 0: + metadata['version'] = metadata['version'] + 1 + signable = sign_metadata(metadata, roleinfo['signing_keyids'], + metadata_filename) + + # non-partial write() + else: + if tuf.sig.verify(signable, rolename): #and not roleinfo['partial_loaded']: + metadata['version'] = metadata['version'] + 1 + signable = sign_metadata(metadata, roleinfo['signing_keyids'], + metadata_filename) + + # Write the metadata to file if contains a threshold of signatures. + signable['signatures'].extend(roleinfo['signatures']) + + if tuf.sig.verify(signable, rolename) or write_partial: + _remove_invalid_and_duplicate_signatures(signable) + compressions = roleinfo['compressions'] + filename = write_metadata_file(signable, metadata_filename, compressions, + False) + + # 'signable' contains an invalid threshold of signatures. + else: + message = 'Not enough signatures for ' + repr(metadata_filename) + raise tuf.Error(message, signable) + + return signable, filename + + + + +def create_new_project(project_name, metadata_directory, + location_in_repository = '', targets_directory=None, + key=None): + """ + + Create a new project object, instantiate barebones metadata for the + targets, and return a blank project object. On disk, create_new_project() + only creates the directories needed to hold the metadata and targets files. + The project object returned can be directly modified to meet the designer's + criteria and then written using the method project.write(). + + The project name provided is the one that will be added to the resulting + metadata file as it should be named in upstream. + + + project_name: + The name of the project as it should be called in upstream. For example + targets/unclaimed/django should have its project_name set to "django" + + metadata_directory: + The directory that will eventually hold the metadata and target files of + the project. + + location_in_repository: + An optional argument to hold the "prefix" or the expected location for + the project files in the "upstream" respository. This value is only + used to sign metadata in a way that it matches the future location + of the files. + + For example, targets/unclaimed/django should have its project name set to + "targets/unclaimed" + + targets_directory: + An optional argument to point the targets directory somewhere else than + the metadata directory if, for example, a project structure already + exists and the user does not want to move it. + + key: + The public key to verify the project's metadata. Projects can only + handle one key with a threshold of one. If a project were to modify it's + key it should be removed and updated. + + + tuf.FormatError, if the arguments are improperly formatted or if the public + key is not a valid one (if it's not none.) + + OSError, if the filepaths provided do not have write permissions. + + + The 'metadata_directory' and 'targets_directory' directories are created + if they do not exist. + + + A 'tuf.developer_tool.Project' object. + """ + + # Does 'metadata_directory' have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(metadata_directory) + + # Do the same for the location in the repo and the project name, we must + # ensure they are valid pathnames. + tuf.formats.NAME_SCHEMA.check_match(project_name) + tuf.formats.PATH_SCHEMA.check_match(location_in_repository) + + # for the targets directory we do the same, but first, let's find out what + # layout the user needs, layout_type is a variable that is usually set to + # 1, which means "flat" (i.e. the cfg file is where the metadata folder is + # located), with a two, the cfg file goes to the "metadata" folder, and a + # new metadata folder is created inside the tree, to separate targets and + # metadata. + layout_type = 'flat' + if targets_directory is None: + targets_directory = os.path.join(metadata_directory, TARGETS_DIRECTORY_NAME) + metadata_directory = \ + os.path.join(metadata_directory, METADATA_DIRECTORY_NAME) + layout_type = 'repo-like' + + if targets_directory is not None: + tuf.formats.PATH_SCHEMA.check_match(targets_directory); + + if key is not None: + tuf.formats.KEY_SCHEMA.check_match(key) + + # Set the metadata and targets directories. These directories + # are created if they do not exist. + metadata_directory = os.path.abspath(metadata_directory) + targets_directory = os.path.abspath(targets_directory) + + # Try to create the metadata directory that will hold all of the metadata + # files, such as 'root.txt' and 'release.txt'. + try: + message = 'Creating ' + repr(metadata_directory) + logger.info(message) + os.makedirs(metadata_directory) + + # 'OSError' raised if the leaf directory already exists or cannot be created. + # Check for case where 'repository_directory' has already been created. + except OSError as e: + if e.errno == errno.EEXIST: + # Should check if we have write permissions here. + pass + + else: + raise + + # Try to create the targets directory that will hold all of the target files. + try: + message = 'Creating ' + repr(targets_directory) + logger.info(message) + os.mkdir(targets_directory) + + except OSError as e: + if e.errno == errno.EEXIST: + pass + + else: + raise + + # Create the bare bones project object, where project role contains default + # values (e.g., threshold of 1, expires 1 year into the future, etc.) + project = Project(project_name, metadata_directory, targets_directory, + location_in_repository) + + # Add 'key' to the project. + # TODO: Add check for expected number of keys for the project (must be 1) and + # its delegated roles (may be greater than one.) + if key is not None: + project.add_verification_key(key); + + # Save the layout information. + project.layout_type = layout_type + + return project + + + + + + +def _save_project_configuration(metadata_directory, targets_directory, + public_keys, prefix, threshold, layout_type, + project_name): + """ + + Persist the project's information to a file. The saved project information + can later be loaded with Project.load_project(). + + + metadata_directory: + Where the project's metadata is located. + + targets_directory: + The location of the target files for this project. + + public_keys: + A list containing the public keys for the project role. + + prefix: + The project's prefix (if any.) + + threshold: + The threshold value for the project role. + + layout_type: + The layout type being used by the project, "flat" stands for separated + targets and metadata directories, "repo-like" emulates the layout used + by the repository tools + + project_name: + The name given to the project, this sets the metadata filename so it + matches the one stored in upstream. + + + tuf.FormatError are also expected if any of the arguments are malformed. + + OSError may rise if the metadata_directory/project.cfg file exists and + is non-writeable + + + A 'project.cfg' configuration file is created or overwritten. + + + None. + """ + + # Schema check for the arguments. + tuf.formats.PATH_SCHEMA.check_match(metadata_directory) + tuf.formats.PATH_SCHEMA.check_match(prefix) + tuf.formats.PATH_SCHEMA.check_match(targets_directory) + tuf.formats.RELPATH_SCHEMA.check_match(project_name) + + cfg_file_directory = metadata_directory + + # Check whether the layout type is 'flat' or 'repo-like'. + # If it is, the .cfg file should be saved in the previous directory. + if layout_type == 'repo-like': + cfg_file_directory = os.path.dirname(metadata_directory) + absolute_location, targets_directory = os.path.split(targets_directory) + + absolute_location, metadata_directory = os.path.split(metadata_directory) + + # Can the file be opened? + project_filename = os.path.join(cfg_file_directory, PROJECT_FILENAME) + + # Build the fields of the configuration file. + project_config = {} + project_config['prefix'] = prefix + project_config['public_keys'] = {} + project_config['metadata_location'] = metadata_directory + project_config['targets_location'] = targets_directory + project_config['threshold'] = threshold + project_config['layout_type'] = layout_type + project_config['project_name'] = project_name + + # Build a dictionary containing the actual keys. + for key in public_keys: + key_info = tuf.keydb.get_key(key) + key_metadata = format_keyval_to_metadata(key_info['keytype'], + key_info['keyval']) + project_config['public_keys'][key] = key_metadata + + # Save the actual file. + with open(project_filename, 'wt') as fp: + json.dump(project_config, fp) + + + + + +def load_project(project_directory, prefix='', new_targets_location=None): + """ + + Return a Project object initialized with the contents of the metadata + files loaded from 'project_directory'. + + + project_directory: + The path to the project's metadata and configuration file. + + prefix: + The prefix for the metadata, if defined. It will replace the current + prefix, by first removing the existing one (saved). + + new_targets_location: + For flat project configurations, project owner might want to reload the + project with a new location for the target files. This overwrites the + previous path to search for the target files. + + + tuf.FormatError, if 'project_directory' or any of the metadata files + are improperly formatted. + + + All the metadata files found in the project are loaded and their contents + stored in a libtuf.Repository object. + + + A tuf.developer_tool.Project object. + """ + + # Does 'repository_directory' have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(project_directory) + + # Do the same for the prefix + tuf.formats.PATH_SCHEMA.check_match(prefix) + + # Clear the role and key databases since we are loading in a new project. + tuf.roledb.clear_roledb() + tuf.keydb.clear_keydb() + + # Locate metadata filepaths and targets filepath. + project_directory = os.path.abspath(project_directory) + + # Load the cfg file and the project. + config_filename = os.path.join(project_directory,PROJECT_FILENAME) + + try: + project_configuration = tuf.util.load_json_file(config_filename) + tuf.formats.PROJECT_CFG_SCHEMA.check_match(project_configuration) + + except (OSError, IOError) as e: + raise + + targets_directory = os.path.join(project_directory, + project_configuration['targets_location']) + + if project_configuration['layout_type'] == 'flat': + project_directory, relative_junk = os.path.split(project_directory) + targets_directory = project_configuration['targets_location'] + if new_targets_location is not None: + targets_directory = new_targets_location + + metadata_directory = os.path.join(project_directory, + project_configuration['metadata_location']) + + new_prefix = None + if prefix != '': + new_prefix = prefix + + prefix = project_configuration['prefix'] + + # Load the project's filename. + project_name = project_configuration['project_name'] + project_filename = project_name + METADATA_EXTENSION + + # Create a blank project on the target directory. + project = Project(project_name, metadata_directory, targets_directory, prefix) + + project.threshold = project_configuration['threshold'] + project._prefix = project_configuration['prefix'] + project.layout_type = project_configuration['layout_type'] + + # Traverse the public keys and add them to the project. + keydict = project_configuration['public_keys'] + for keyid in keydict: + key = format_metadata_to_key(keydict[keyid]) + project.add_verification_key(key) + + # Load the project's metadata. + targets_metadata_path = os.path.join(project_directory, metadata_directory, + project_filename) + signable = tuf.util.load_json_file(targets_metadata_path) + tuf.formats.check_signable_object_format(signable) + targets_metadata = signable['signed'] + + # Remove the prefix from the metadata. + targets_metadata = _strip_prefix_from_targets_metadata(targets_metadata, + prefix) + for signature in signable['signatures']: + project.add_signature(signature) + + # Update roledb.py containing the loaded project attributes. + roleinfo = tuf.roledb.get_roleinfo(project_name) + roleinfo['signatures'].extend(signable['signatures']) + roleinfo['version'] = targets_metadata['version'] + roleinfo['paths'] = list(targets_metadata['targets']) + roleinfo['delegations'] = targets_metadata['delegations'] + roleinfo['partial_loaded'] = False + + + # Check if the loaded metadata was partially written and update the + # flag in 'roledb.py'. + if _metadata_is_partially_loaded(project_name, signable, roleinfo): + roleinfo['partial_loaded'] = True + + tuf.roledb.update_roleinfo(project_name, roleinfo) + + + for key_metadata in targets_metadata['delegations']['keys'].values(): + key_object = tuf.keys.format_metadata_to_key(key_metadata) + tuf.keydb.add_key(key_object) + + for role in targets_metadata['delegations']['roles']: + rolename = role['name'] + roleinfo = {'name': role['name'], 'keyids': role['keyids'], + 'threshold': role['threshold'], 'compressions': [''], + 'signing_keyids': [], 'signatures': [], 'partial_loaded':False, + 'delegations': {'keys':{}, 'roles':[]} + } + tuf.roledb.add_role(rolename, roleinfo) + + # Load delegated targets metadata. + # Walk the 'targets/' directory and generate the fileinfo of all the files + # listed. This information is stored in the 'meta' field of the release + # metadata object. + targets_objects = {} + loaded_metadata = [] + targets_objects[project_name] = project + metadata_directory = os.path.join(project_directory, metadata_directory) + targets_metadata_directory = os.path.join(metadata_directory, project_name) + if os.path.exists(targets_metadata_directory) and \ + os.path.isdir(targets_metadata_directory): + for root, directories, files in os.walk(targets_metadata_directory): + + # 'files' here is a list of target file names. + for basename in files: + metadata_path = os.path.join(root, basename) + metadata_name = \ + metadata_path[len(metadata_directory):].lstrip(os.path.sep) + + # Strip the extension. The roledb does not include '.json' role + # extensions + if metadata_name.endswith(METADATA_EXTENSION): + extension_length = len(METADATA_EXTENSION) + metadata_name = metadata_name[:-extension_length] + + else: + continue + + signable = None + try: + signable = tuf.util.load_json_file(metadata_path) + + except (ValueError, IOError, tuf.Error): + raise + + # Strip the prefix from the local working copy, it will be added again + # when the targets metadata is written to disk. + metadata_object = signable['signed'] + metadata_object = _strip_prefix_from_targets_metadata(metadata_object, + prefix) + + roleinfo = tuf.roledb.get_roleinfo(metadata_name) + roleinfo['signatures'].extend(signable['signatures']) + roleinfo['version'] = metadata_object['version'] + roleinfo['expires'] = metadata_object['expires'] + roleinfo['paths'] = list(metadata_object['targets']) + roleinfo['delegations'] = metadata_object['delegations'] + roleinfo['partial_loaded'] = False + + if os.path.exists(metadata_path+'.gz'): + roleinfo['compressions'].append('gz') + + # If the metadata was partially loaded, update the roleinfo flag. + if _metadata_is_partially_loaded(metadata_name, signable, roleinfo): + roleinfo['partial_loaded'] = True + + + tuf.roledb.update_roleinfo(metadata_name, roleinfo) + + # Append to list of elements to avoid reloading repeated metadata. + loaded_metadata.append(metadata_name) + + # Add the delegation. + new_targets_object = Targets(targets_directory, metadata_name, roleinfo) + targets_object = \ + targets_objects[tuf.roledb.get_parent_rolename(metadata_name)] + targets_objects[metadata_name] = new_targets_object + + targets_object._delegated_roles[(os.path.basename(metadata_name))] = \ + new_targets_object + + # Add the keys specified in the delegations field of the Targets role. + for key_metadata in metadata_object['delegations']['keys'].values(): + key_object = tuf.keys.format_metadata_to_key(key_metadata) + try: + tuf.keydb.add_key(key_object) + + except tuf.KeyAlreadyExistsError: + pass + + for role in metadata_object['delegations']['roles']: + rolename = role['name'] + roleinfo = {'name': role['name'], 'keyids': role['keyids'], + 'threshold': role['threshold'], + 'compressions': [''], 'signing_keyids': [], + 'signatures': [], + 'partial_loaded': False, + 'delegations': {'keys': {}, + 'roles': []}} + tuf.roledb.add_role(rolename, roleinfo) + + if new_prefix: + project._prefix = new_prefix + + return project + + + + + +def _strip_prefix_from_targets_metadata(targets_metadata, prefix): + """ + Non-public method that removes the prefix from each of the target pahts in + 'targets_metadata' so they can be used again in compliance with the local + copies. The prefix is needed in metadata to match the layout of the remote + repository. + """ + + unprefixed_targets_metadata = {} + + for targets in targets_metadata['targets'].keys(): + unprefixed_target = os.path.relpath(targets, prefix) + unprefixed_target = '/' + unprefixed_target + unprefixed_targets_metadata[unprefixed_target] = \ + targets_metadata['targets'][targets] + targets_metadata['targets'] = unprefixed_targets_metadata + + return targets_metadata + + + + + +if __name__ == '__main__': + # The interactive sessions of the documentation strings can + # be tested by running 'developer_tool.py' as a standalone module: + # $ python developer_tool.py + import doctest + doctest.testmod() diff --git a/tuf/formats.py b/tuf/formats.py index c2236ae3..c62675d1 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -457,6 +457,18 @@ expires = ISO8601_DATETIME_SCHEMA, meta = FILEDICT_SCHEMA) +# project.cfg file: stores information about the project in a json dictionary +PROJECT_CFG_SCHEMA = SCHEMA.Object( + object_name = 'PROJECT_CFG_SCHEMA', + project_name = SCHEMA.AnyString(), + layout_type = SCHEMA.OneOf([SCHEMA.String('repo-like'), SCHEMA.String('flat')]), + targets_location = PATH_SCHEMA, + metadata_location = PATH_SCHEMA, + prefix = PATH_SCHEMA, + public_keys = KEYDICT_SCHEMA, + threshold = SCHEMA.Integer(lo = 0, hi = 2) + ) + # A schema containing information a repository mirror may require, # such as a url, the path of the directory metadata files, etc. MIRROR_SCHEMA = SCHEMA.Object(