Merge pull request #231 from vladimir-v-diaz/SantiagoTorres-developer-tools

Review and update pull request #188

tests/repository_data/generate_project_data.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+
+"""
+<Program Name>
+  generate_project_data.py
+
+<Author>
+  Santiago Torres <torresariass@gmail.com>
+
+<Copyright>
+  See LICENSE for licensing information.
+
+<Purpose>
+  Generate a pre-fabricated set of metadata files for 'test_developer_tool.py'
+  test cases.
+"""
+
+import shutil
+import datetime
+import optparse
+import os
+
+import tuf.util
+from tuf.developer_tool import *
+
+parser = optparse.OptionParser()
+
+parser.add_option("-d","--dry-run", action='store_true', dest="dry_run",
+    help="Do not write the files, just run", default=False)
+(options, args) = parser.parse_args()
+
+
+project_key_file = 'keystore/root_key'
+targets_key_file = 'keystore/targets_key' 
+delegation_key_file = 'keystore/delegation_key'
+
+# The files we use for signing in the unit tests should exist, if they are not
+# populated, run 'generate.py'.
+assert os.path.exists(project_key_file)
+assert os.path.exists(targets_key_file)
+assert os.path.exists(delegation_key_file)
+
+# Import the public keys.  These keys are needed so that metadata roles are
+# assigned verification keys, which clients use to verify the signatures created
+# by the corresponding private keys.
+project_public = import_rsa_publickey_from_file(project_key_file + '.pub')
+targets_public = import_rsa_publickey_from_file(targets_key_file + '.pub')
+delegation_public = import_rsa_publickey_from_file(delegation_key_file + '.pub')
+
+# Import the private keys.  These private keys are needed to generate the
+# signatures included in metadata.
+project_private = import_rsa_privatekey_from_file(project_key_file, 'password')
+targets_private = import_rsa_privatekey_from_file(targets_key_file, 'password')
+delegation_private = import_rsa_privatekey_from_file(delegation_key_file, 'password')
+
+os.mkdir("project")
+os.mkdir("project/targets")
+
+# Create the target files (downloaded by clients) whose file size and digest
+# are specified in the 'targets.json' file.
+target1_filepath = 'project/targets/file1.txt'
+tuf.util.ensure_parent_dir(target1_filepath)
+target2_filepath = 'project/targets/file2.txt'
+tuf.util.ensure_parent_dir(target2_filepath)
+target3_filepath = 'project/targets/file3.txt'
+tuf.util.ensure_parent_dir(target2_filepath)
+
+if not options.dry_run:
+  with open(target1_filepath, 'wt') as file_object:
+    file_object.write('This is an example target file.')
+
+  with open(target2_filepath, 'wt') as file_object:
+    file_object.write('This is an another example target file.')
+
+  with open(target3_filepath, 'wt') as file_object:
+    file_object.write('This is role1\'s target file.')
+
+
+project = create_new_project("test-flat", 'project/test-flat', 'prefix', 
+    'project/targets')
+
+# Add target files to the top-level projects role.  These target files should
+# already exist.
+project.add_target(target1_filepath)
+project.add_target(target2_filepath)
+
+# Add one key to the project.
+project.add_verification_key(project_public)
+project.load_signing_key(project_private)
+
+# Add the delegated role keys.
+project.delegate('role1', [delegation_public], [target3_filepath])
+project('role1').load_signing_key(delegation_private)
+
+# Set the project expiration time far into the future so that its metadata does
+# not expire anytime soon, or else the tests fail.  Unit tests may modify the
+# expiration  datetimes (of the copied files), if they wish.
+project.expiration = datetime.datetime(2030, 1, 1, 0, 0)
+project('role1').expiration = datetime.datetime(2030, 1, 1, 0, 0)
+
+# Compress the project role metadata so that the unit tests have a pre-generated
+# example of compressed metadata.
+project.compressions = ['gz']
+
+# Create the actual metadata files, which are saved to 'metadata.staged'. 
+if not options.dry_run:
+  project.write()

tests/repository_data/project/targets/file1.txt

@@ -0,0 +1 @@
+This is an example target file.

tests/repository_data/project/targets/file2.txt

@@ -0,0 +1 @@
+This is an another example target file.

tests/repository_data/project/targets/file3.txt

@@ -0,0 +1 @@
+This is role1's target file.

tests/repository_data/project/test-flat/project.cfg

@@ -0,0 +1 @@
+{"project_name": "test-flat", "targets_location": "/home/santiago/Documents/v2014/TUF/tuf/tests/repository_data/project/targets", "prefix": "prefix", "metadata_location": "test-flat", "threshold": 1, "public_keys": {"6986b667c736a3b37471e030cf4ce7aa6c7e0d530325e64c2660276b77be3754": {"keytype": "rsa", "keyval": {"public": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7J15ZaeDQPrhQsRj29wB\nPhibH+Do59xsT2396L+uCg793gZlar5wZN2eHSh725cNQWyTAa9LwG+lXaKMukQ+\n8176CKR2J5sv3DezrGVu3x8V1qhyJyy79FlNZRVYTVqNaYzvJzxsVnFPpg7f8B7C\nffiqWJr9XkpqwRlCpxooXm4hplZ7uek5Ku21CzQ4OWg7hbuc+ZjCGzpXfm8NuosU\n7TipnKGpEt0Agiph5g6TB2/scoeFar1CKMONIl80maxzAQk+xkWgiJ00+Z2qFCsx\nESfis/YkILS6RMFyZz7oa1WwMtUjYmrsRuz+jlFcbNuxZpIkaISiG9a2YdGcJ1Aj\n3QIDAQAB\n-----END PUBLIC KEY-----"}}}, "layout_type": "flat"}

tests/repository_data/project/test-repo/project.cfg

@@ -0,0 +1 @@
+{"project_name": "test-repo-like", "targets_location": "targets", "prefix": "prefix", "metadata_location": "metadata", "threshold": 1, "public_keys": {"6986b667c736a3b37471e030cf4ce7aa6c7e0d530325e64c2660276b77be3754": {"keytype": "rsa", "keyval": {"public": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7J15ZaeDQPrhQsRj29wB\nPhibH+Do59xsT2396L+uCg793gZlar5wZN2eHSh725cNQWyTAa9LwG+lXaKMukQ+\n8176CKR2J5sv3DezrGVu3x8V1qhyJyy79FlNZRVYTVqNaYzvJzxsVnFPpg7f8B7C\nffiqWJr9XkpqwRlCpxooXm4hplZ7uek5Ku21CzQ4OWg7hbuc+ZjCGzpXfm8NuosU\n7TipnKGpEt0Agiph5g6TB2/scoeFar1CKMONIl80maxzAQk+xkWgiJ00+Z2qFCsx\nESfis/YkILS6RMFyZz7oa1WwMtUjYmrsRuz+jlFcbNuxZpIkaISiG9a2YdGcJ1Aj\n3QIDAQAB\n-----END PUBLIC KEY-----"}}}, "layout_type": "repo-like"}

tests/repository_data/project/test-repo/targets/file1.txt

@@ -0,0 +1 @@
+This is an example target file.

tests/repository_data/project/test-repo/targets/file2.txt

@@ -0,0 +1 @@
+This is an another example target file.

tests/repository_data/project/test-repo/targets/file3.txt

@@ -0,0 +1 @@
+This is role1's target file.

tests/test_developer_tool.py

@@ -0,0 +1,441 @@
+#!/usr/bin/env python
+
+"""
+<Program Name>
+  test_developer_tool.py.
+
+<Authors>
+  Santiago Torres Arias <torresariass@gmail.com>
+  Zane Fisher <zanefisher@gmail.com>
+
+<Copyright>
+  See LICENSE for licensing inforation.
+
+<Purpose>
+  Unit test for the 'developer_tool.py' module.
+"""
+
+import os
+import time
+import datetime
+import unittest
+import logging
+import tempfile
+import shutil
+
+import tuf
+import tuf.log
+import tuf.formats
+import tuf.roledb
+import tuf.keydb
+import tuf.developer_tool as developer_tool
+
+from tuf.developer_tool import METADATA_DIRECTORY_NAME
+from tuf.developer_tool import TARGETS_DIRECTORY_NAME
+
+logger = logging.getLogger('tuf.test_developer_tool')
+
+developer_tool.disable_console_log_messages()
+
+class TestProject(unittest.TestCase):
+
+  tmp_dir = None
+
+  @classmethod
+  def setUpClass(cls):
+    cls.tmp_dir = tempfile.mkdtemp(dir = os.getcwd())
+
+  @classmethod
+  def tearDownClass(cls):
+    shutil.rmtree(cls.tmp_dir)
+
+  def setUp(self):
+    # called before every test case
+    pass
+
+  def tearDown(self):
+    # called after every test case
+    tuf.roledb.clear_roledb()
+    tuf.keydb.clear_keydb()
+    pass
+
+  def test_create_new_project(self):
+    # Test cases for the create_new_project function. In this test we will
+    # check input, correct file creation and format. We also check
+    # that a proper object is generated. We will use the normal layout for this
+    # test suite.
+  
+    # Create a local subfolder for this test.
+    local_tmp = tempfile.mkdtemp(dir = self.tmp_dir)
+
+    # These are the usual values we will be throwing to the function, however
+    # we will swap these for nulls or malformed values every now and then to
+    # test input.
+    project_name = 'test_suite'
+    metadata_directory = local_tmp 
+    location_in_repository = '/prefix'
+    targets_directory = None 
+    key = None
+
+    # Create a blank project.
+    project = developer_tool.create_new_project(project_name, metadata_directory, 
+        location_in_repository)
+
+    self.assertTrue(isinstance(project, developer_tool.Project))
+    self.assertTrue(project.layout_type == 'repo-like')
+    self.assertTrue(project._prefix == location_in_repository)
+    self.assertTrue(project._project_name == project_name)
+    self.assertTrue(project._metadata_directory == 
+        os.path.join(metadata_directory,METADATA_DIRECTORY_NAME))
+    self.assertTrue(project._targets_directory == 
+        os.path.join(metadata_directory,TARGETS_DIRECTORY_NAME))
+
+    # Create a blank project without a prefix.
+    project = developer_tool.create_new_project(project_name, metadata_directory)
+    self.assertTrue(isinstance(project, developer_tool.Project))
+    self.assertTrue(project.layout_type == 'repo-like')
+    self.assertTrue(project._prefix == '')
+    self.assertTrue(project._project_name == project_name)
+    self.assertTrue(project._metadata_directory == 
+        os.path.join(metadata_directory,METADATA_DIRECTORY_NAME))
+    self.assertTrue(project._targets_directory == 
+        os.path.join(metadata_directory,TARGETS_DIRECTORY_NAME))
+
+    # Create a blank project without a valid metadata directory.
+    self.assertRaises(tuf.FormatError, developer_tool.create_new_project,
+       0, metadata_directory, location_in_repository) 
+    self.assertRaises(tuf.FormatError, developer_tool.create_new_project,
+       project_name, 0, location_in_repository) 
+    self.assertRaises(tuf.FormatError, developer_tool.create_new_project,
+       project_name, metadata_directory, 0) 
+
+
+    # Create a new project with a flat layout.
+    targets_directory = tempfile.mkdtemp(dir = local_tmp)
+    metadata_directory = tempfile.mkdtemp(dir = local_tmp)
+    project = developer_tool.create_new_project(project_name, metadata_directory,
+        location_in_repository, targets_directory)
+    self.assertTrue(isinstance(project, developer_tool.Project))
+    self.assertTrue(project.layout_type == 'flat')
+    self.assertTrue(project._prefix == location_in_repository)
+    self.assertTrue(project._project_name == project_name)
+    self.assertTrue(project._metadata_directory == metadata_directory)
+    self.assertTrue(project._targets_directory == targets_directory)
+
+    # Finally, check that if targets_directory is set, it is valid.
+    self.assertRaises(tuf.FormatError, developer_tool.create_new_project,
+        project_name, metadata_directory, location_in_repository, 0)
+   
+    # Copy a key to our workspace and create a new project with it.
+    keystore_path = os.path.join('repository_data','keystore')
+
+    # I will use the same key as the one provided in the repository
+    # tool tests for the root role, but this is not a root role...
+    root_key_path = os.path.join(keystore_path,'root_key.pub')
+    project_key = developer_tool.import_rsa_publickey_from_file(root_key_path)
+
+    # Test create new project with a key added by default.
+    project = developer_tool.create_new_project(project_name, metadata_directory,
+        location_in_repository, targets_directory, project_key)
+    
+    self.assertTrue(isinstance(project, developer_tool.Project))
+    self.assertTrue(project.layout_type == 'flat')
+    self.assertTrue(project._prefix == location_in_repository)
+    self.assertTrue(project._project_name == project_name)
+    self.assertTrue(project._metadata_directory == metadata_directory)
+    self.assertTrue(project._targets_directory == targets_directory)
+    self.assertTrue(len(project.keys) == 1)
+    self.assertTrue(project.keys[0] == project_key['keyid'])
+
+    # Set as readonly and try to write a repo.
+    shutil.rmtree(targets_directory)
+    os.chmod(local_tmp, 0o0555)
+
+    tuf.roledb.clear_roledb()
+    tuf.keydb.clear_keydb()
+    self.assertRaises(OSError, developer_tool.create_new_project ,project_name,
+        metadata_directory, location_in_repository, targets_directory,
+        project_key)
+
+    os.chmod(local_tmp, 0o0777)
+
+    shutil.rmtree(metadata_directory)
+    os.chmod(local_tmp, 0o0555)
+
+    tuf.roledb.clear_roledb()
+    tuf.keydb.clear_keydb()
+    self.assertRaises(OSError, developer_tool.create_new_project ,project_name,
+        metadata_directory, location_in_repository, targets_directory,
+        project_key)
+
+
+    os.chmod(local_tmp, 0o0777)
+    shutil.rmtree(local_tmp)
+    
+
+
+  def test_load_project(self):
+    # This test case will first try to load an existing project and test for
+    # verify the loaded object.  It will next try to load a nonexisting project
+    # and expect a correct error handler.  Finally, it will try to overwrite the
+    # existing prefix of the loaded project.
+
+    # Create a local subfolder for this test.
+    local_tmp = tempfile.mkdtemp(dir = self.tmp_dir)
+
+    # Test non-existent project filepath.
+    nonexistent_path = os.path.join(local_tmp, 'nonexistent')
+    self.assertRaises(IOError, developer_tool.load_project, nonexistent_path)
+
+    # Copy the pregenerated metadata.
+    project_data_filepath = os.path.join('repository_data', 'project')
+    target_project_data_filepath = os.path.join(local_tmp, 'project')
+    shutil.copytree('repository_data/project', target_project_data_filepath)
+
+    # Properly load a project.
+    repo_filepath = os.path.join(local_tmp, 'project', 'test-repo')
+    project = developer_tool.load_project(repo_filepath)
+    
+    self.assertTrue(project.layout_type == 'repo-like')
+
+    repo_filepath = os.path.join(local_tmp, 'project', 'test-flat')
+    new_targets_path = os.path.join(local_tmp, 'project', 'targets')
+    project = developer_tool.load_project(repo_filepath,
+        new_targets_location = new_targets_path)
+    self.assertTrue(project._targets_directory == new_targets_path)
+    self.assertTrue(project.layout_type == 'flat')
+
+    
+    # Load a project overwriting the prefix.
+    project = developer_tool.load_project(repo_filepath, prefix='new')
+    self.assertTrue(project._prefix == 'new')
+
+    # Load a project with a file missing.
+    file_to_corrupt = os.path.join(repo_filepath, 'test-flat', 'role1.json')
+    with open(file_to_corrupt, 'wt') as fp:
+      fp.write('this is not a json file')
+    
+    self.assertRaises(tuf.Error, developer_tool.load_project, repo_filepath)
+
+    
+
+
+  def test_add_verification_keys(self):
+    # Create a new project instance. 
+    project = developer_tool.Project('test_verification_keys', 'somepath',
+        'someotherpath', 'prefix')
+
+    # Add invalid verification key.
+    self.assertRaises(tuf.FormatError, project.add_verification_key, 'invalid')
+
+    # Add verification key.
+    #  - load it first 
+    keystore_path = os.path.join('repository_data','keystore')
+    first_verification_key_path = os.path.join(keystore_path,'root_key.pub')
+    first_verification_key = \
+      developer_tool.import_rsa_publickey_from_file(first_verification_key_path)
+    
+    project.add_verification_key(first_verification_key)
+
+
+    # Add another verification key (should expect exception.)
+    second_verification_key_path = os.path.join(keystore_path,'snapshot_key.pub')
+    second_verification_key = \
+      developer_tool.import_rsa_publickey_from_file(second_verification_key_path)
+    
+    self.assertRaises(tuf.Error,
+        project.add_verification_key,(second_verification_key))
+
+
+    
+    # Add a verification key for the delegation.
+    project.delegate('somedelegation', [], [])
+    project('somedelegation').add_verification_key(first_verification_key)
+    project('somedelegation').add_verification_key(second_verification_key)
+
+
+    # Add another delegation of the delegation.
+    project('somedelegation').delegate('somesubdelegation', [], [])
+    project('somedelegation')('somesubdelegation').add_verification_key(
+        first_verification_key)
+    project('somedelegation')('somesubdelegation').add_verification_key(
+        second_verification_key)
+
+
+  def test_write(self):
+
+    # Create tmp directory.
+    local_tmp = tempfile.mkdtemp(dir=self.tmp_dir)
+
+    # Create new project inside tmp directory.
+    project = developer_tool.create_new_project('test_write', local_tmp, 
+        'prefix');
+
+    # Create some target files inside the tmp directory.
+    target_filepath = os.path.join(local_tmp, 'targets', 'test_target')
+    with open(target_filepath, 'wt') as fp:
+      fp.write('testing file')
+    
+
+    # Add the targets.
+    project.add_target(target_filepath)
+
+    # Add verification keys.
+    keystore_path = os.path.join('repository_data', 'keystore')
+    project_key_path = os.path.join(keystore_path, 'root_key.pub')
+    project_key = \
+      developer_tool.import_rsa_publickey_from_file(project_key_path)
+
+
+    # Call status (for the sake of doing it and to improve test coverage by 
+    # executing its statements.)
+    project.status()
+  
+    project.add_verification_key(project_key)
+
+
+    # Add another verification key (should expect exception.)
+    delegation_key_path = os.path.join(keystore_path, 'snapshot_key.pub')
+    delegation_key = \
+      developer_tool.import_rsa_publickey_from_file(delegation_key_path)
+
+    # Add a subdelegation.
+    subdelegation_key_path = os.path.join(keystore_path, 'timestamp_key.pub')
+    subdelegation_key = \
+        developer_tool.import_rsa_publickey_from_file(subdelegation_key_path)
+    
+    # Add a delegation.
+    project.delegate('delegation', [delegation_key], [])
+    project('delegation').delegate('subdelegation', [subdelegation_key], [])
+
+    # call write (except)
+    self.assertRaises(tuf.Error, project.write, ())
+
+    # Call status (for the sake of doing it and executing its statements.)
+    project.status()
+  
+    # Load private keys.
+    project_private_key_path = os.path.join(keystore_path, 'root_key')
+    project_private_key = \
+        developer_tool.import_rsa_privatekey_from_file(project_private_key_path,
+            'password')
+
+    delegation_private_key_path = os.path.join(keystore_path, 'snapshot_key')
+    delegation_private_key = \
+        developer_tool.import_rsa_privatekey_from_file(delegation_private_key_path,
+            'password')
+
+    subdelegation_private_key_path =  \
+        os.path.join(keystore_path, 'timestamp_key')
+    subdelegation_private_key = \
+        developer_tool.import_rsa_privatekey_from_file(subdelegation_private_key_path,
+            'password')
+
+    # Test partial write.
+    # backup everything (again)
+    # + backup targets.
+    targets_backup = project.target_files
+
+    # + backup delegations.
+    delegations_backup = \
+        tuf.roledb.get_delegated_rolenames(project._project_name)
+
+    # + backup layout type.
+    layout_type_backup = project.layout_type
+
+    # + backup keyids.
+    keys_backup = project.keys
+    delegation_keys_backup = project('delegation').keys
+
+    # + backup the prefix.
+    prefix_backup = project._prefix
+    
+    # + backup the name. 
+    name_backup = project._project_name
+ 
+    # Set the compressions.  We will be checking this part here too.
+    project.compressions = ['gz']
+    project('delegation').compressions = project.compressions
+
+    # Write and reload.
+    self.assertRaises(tuf.Error, project.write)
+    project.write(write_partial=True)
+
+    project = developer_tool.load_project(local_tmp)
+
+    # Check against backup.
+    self.assertEqual(list(project.target_files.keys()), list(targets_backup.keys()))
+    new_delegations = tuf.roledb.get_delegated_rolenames(project._project_name)
+    self.assertEqual(new_delegations, delegations_backup)
+    self.assertEqual(project.layout_type, layout_type_backup)
+    self.assertEqual(project.keys, keys_backup)
+    self.assertEqual(project('delegation').keys, delegation_keys_backup)
+    self.assertEqual(project._prefix, prefix_backup)
+    self.assertEqual(project._project_name, name_backup)
+
+    
+
+    roleinfo = tuf.roledb.get_roleinfo(project._project_name)
+
+    self.assertEqual(roleinfo['partial_loaded'], True)
+
+
+
+    # Load_signing_keys.
+    project('delegation').load_signing_key(delegation_private_key)
+
+    project.status()
+
+    project('delegation')('subdelegation').load_signing_key(
+        subdelegation_private_key)
+
+    project.status()
+
+    project.load_signing_key(project_private_key)
+
+    # Backup everything.
+    # + backup targets.
+    targets_backup = project.target_files
+
+    # + backup delegations.
+    delegations_backup = \
+        tuf.roledb.get_delegated_rolenames(project._project_name)
+
+    # + backup layout type.
+    layout_type_backup = project.layout_type
+
+    # + backup keyids
+    keys_backup = project.keys
+    delegation_keys_backup = project('delegation').keys
+
+    # + backup the prefix.
+    prefix_backup = project._prefix
+    
+    # + backup the name.
+    name_backup = project._project_name
+
+    # Call status (for the sake of doing it.)
+    project.status()
+
+    # Call write.
+    project.write()
+
+    # Call load.
+    project = developer_tool.load_project(local_tmp)
+
+
+    # Check against backup.
+    self.assertEqual(list(project.target_files.keys()), list(targets_backup.keys()))
+
+    new_delegations = tuf.roledb.get_delegated_rolenames(project._project_name)
+    self.assertEqual(new_delegations, delegations_backup)
+    self.assertEqual(project.layout_type, layout_type_backup)
+    self.assertEqual(project.keys, keys_backup)
+    self.assertEqual(project('delegation').keys, delegation_keys_backup)
+    self.assertEqual(project._prefix, prefix_backup)
+    self.assertEqual(project._project_name, name_backup)
+
+
+
+if __name__ == '__main__':
+  unittest.main()

tuf/README-developer-tools.md

@@ -0,0 +1,340 @@
+# The Update Framework Developer Tool: How to Update your Project Securely on a TUF Repository
+
+## Table of Contents
+- [Overview](#overview)
+- [Creating a Simple Project](#creating_a_simple_project)
+  - [Generating a Key](#generating_a_key)
+  - [The Project Class](#the_project_class)
+  - [Signing and Writing the Metadata](#signing_and_writing_the_metadata)
+- [Loading an Existing Project](#loading_an_existing_project)
+- [Delegations](#delegations)
+- [Managing Keys](#managing_keys)
+- [Managing Targets](#managing_targets)
+
+<a name="overview">
+## Overview 
+The Update Framework (TUF) is a Python-based security system for software
+updates. In order to prevent your users from downloading vulnerable or malicious
+code disguised as updates to your software, TUF requires that each update you
+release include certain metadata verifying your authorship of the files.
+
+The TUF developer tools are a Python Library that enables you to create and
+maintain the required metadata for files hosted on a TUF Repository. (We call
+these files “targets,” to distinguish them from the metadata associated with
+them. Both of these together comprise a complete “project”.) You will use these
+tools to generate the keys and metadata you need to claim and secure your files
+on the repository, and to update the metadata and sign it with those keys
+whenever you upload a new version of those files.
+
+This document will teach you how to use these tools in two parts. The first
+part walks through the creation of a minimal-complexity TUF project, which is
+all you need to get started, and can be expanded later. The second part details
+the full functionality of the tools, which offer a finer degree of control in
+securing your project.
+
+<a name="creating_a_simple_project">
+## Creating a Simple Project
+This section walks through the creation of a small example project with just
+one target. Once created, this project will be fully functional, and can be
+modified as needed.
+
+<a name="generating_a_key">
+### Generating a Key
+First, we will need to generate a key to sign the metadata. Keys are generated
+in pairs: one public and the other private. The private key is
+password-protected and is used to sign metadata. The public key can be shared
+freely, and is used to verify signatures made by the private key. You will need
+too share your public key with the repository hosting your project so they can
+verify your metadata is signed by the right person.
+
+The generate\_and\_write\_rsa\_keypair function will create two key files named
+"path/to/key.pub", which is the public key and "path/to/key", which
+is the private key.
+
+```
+>>> from tuf.developer_tool import *
+>>> generate_and_write_rsa_keypair("path/to/key")
+Enter a password for the RSA key:
+Confirm:
+>>> 
+```
+
+We can also use the bits parameter to set a different key length (the default
+is 3072). We can also provide the password parameter in order to suppress the
+password prompt.
+
+In this example we will be using rsa keys, but ed25519 keys are also supported.
+
+Now we have a key for our project, we can proceed to create our project.
+
+<a name="the_project_class">
+### The Project Class
+The TUF developer tool is built around the Project class, which is used to
+organize groups of targets associated with a single set of metadata. A single
+Project instance is used to keep track of all the target files and metadata
+files in one project. The Project also keeps track of the keys and signatures,
+so that it can update all the metadata with the correct changes and signatures
+on a single command.
+
+Before creating a project, you must know where it will be located in the TUF
+Repository. In the following example, we will create a project to be hosted as
+"repo/unclaimed/example_project" within the repository, and store a local copy
+of the metadata at "path/to/metadata". The project will comprise a single
+target file, "local/path/to/example\_project/target\_1" locally, and we will
+secure it with the key generated above.
+
+First, we must import the generated keys. We can do that by issuing the
+following command:
+
+```
+>>> public_key = import_rsa_publickey_from_file("path/to/keys.pub")
+```
+
+After importing the key, we can generate a new project with the following
+command:
+
+```
+>>> project = create_new_project(name="example_project",
+...  metadata_directory="local/path/to/metadata/",
+...  targets_directory="local/path/to/example_project",
+...  location_in_repository="repo/unclaimed", key=public_key)
+```
+
+Let's list the arguments and make sense out of this rather long function call:
+
+- create a project named example_project: the name of the metadata file will match this name
+- the metadata will be located in "local/path/to/metadata", this means all of the generated files
+for this project will be located here
+- the targets are located in local/path/to/example project. If your targets are located in some other
+place, you can point the targets directory there. Files must reside under the path local/path/to/example_project or else it won't be possible to add them.
+- location\_in\_repository points to repo/unclaimed, this will be prepended to the paths in the generated metadata so the signatures all match.
+
+Now the project is in memory and we can do different operations on it such as
+adding and removing targets, delegating files, changing signatures and keys,
+etc. For the moment we are interested in adding our one and only target inside
+the project.
+
+To add a target, we issue the following method:
+
+```
+>>> project.add_target("target_1")
+```
+
+Note that the file "target\_1" should be located in
+"local/path/to/example\_project", or this method will throw an
+error.
+
+At this point, the metadata is not valid. We have assigned a key to the
+project, but we have not *signed* it with that key. Signing is the process of
+generating a signature with our private key so it can be verified with the
+public key by the server (upon uploading) and by the clients (when updating).
+
+<a name="signing_and_writing_the_metadata">
+### Signing and Writing the Metadata ###
+In order to sign the metadata, we need to import the private key corresponding 
+to the public key we added to the project. One the key is loaded to the project,
+it will automatically be used to sign the metadata whenever it is written.
+
+```
+>>> private_key = import_rsa_privatekey_from_file("path/to/key")
+Enter password for the RSA key:
+>>> project.load_signing_key(private_key)
+>>> project.write()
+```
+
+When all changes to the project have been written, the metadata is ready to be
+uploaded to the repository, and it is safe to exit the Python interpreter, or
+to delete the Project instance.
+
+The project can be loaded later to update changes to the project. The metadata
+contains checksums that have to match the actual files or else it won't be
+accepted by the upstream repository.
+
+At this point, if you have followed all the steps in this document so far
+(substituting appropriate names and filepaths) you will have created a basic
+TUF project, which can be expanded as needed. The simplest way to get your
+project secured is to add all your files using add\_target() (or see [Managing
+Keys](#managing_keys) on how to add whole directories). If your project has
+several contributors, you may want to consider adding
+[delegations](#delegations) to your project.
+
+<a name="loading_an_existing_project">
+## Loading an Existing Project
+To make changes to existing metadata, we will need the Project again. We can 
+restore it with the load_project() function.  
+
+```
+>>> from tuf.developer_tool import *
+>>> project = load_project("local/path/to/metadata")
+```
+Each time the project is loaded anew, the necessary private keys must also be 
+loaded in order to sign metadata.
+
+```
+>>> private_key = import_rsa_privatekey_from_file("path/to/key")
+Enter a password for the RSA key:
+>>> project.load_signing_key(private_key)
+>>> project.write()
+```
+
+If your project does not use any delegations, the five commands above are all
+you need to update your project's metadata.
+
+<a name="delegations">
+## Delegations
+
+The project we created above is secured entirely by one key. If you want to
+allow someone else to update part of your project independently, you will need
+to delegate a new role for them. For example, we can do the following:
+
+```
+>>> other_key = import_rsa_publickey_from_file(“another_public_key.pub”)
+>>> project.delegate(“newrole”, [other_key], targets)
+```
+
+The new role is now an attribute of the Project instance, and contains the same
+methods as Project. For example, we can add targets in the same way as before:
+
+```
+>>> project(“newrole”).add_target(“delegated_1”)
+```
+
+Recall that we input the other person’s key as part of a list. That list can
+contain any number of public keys. We can also add keys to the role after
+creating it using the [add\_verification\_key()](#adding_a_key_to_a_delegation)
+method.
+
+### Restricted Paths
+
+By default, a delegated role is permitted to add and modify targets anywhere in
+the Project's targets directory. We can assign restricted paths to a delegated
+role to limit this permission.
+
+```
+>>> project.add_restricted_paths(["restricted/filepath"], "newrole")
+```
+
+This will prevent the delegated role from signing targets whose local filepaths
+do not begin with "restricted/filepath". We can assign several restricted
+filepaths to a role by adding them to the list in the first parameter, or by
+invoking the method again. A role with multiple restricted paths can add
+targets to any of them.
+
+Note that this method is invoked from the parent role (in this case, the Project)
+and takes the delegated role name as an argument.
+
+### Nested Delegations
+
+It is possible for a delegated role to have delegations of its own. We can do
+this by calling delegate() on a delegated role:
+
+```
+>>> project("newrole").delegate(“nestedrole”, [key], targets)
+```
+
+Nested delegations function no differently than first-order delegations. to
+demonstrate, adding a target to nested delegation looks like this:
+
+```
+>>> project("newrole")("nestedrole").add_target("foo")
+```
+
+### Revoking Delegations
+Delegations can be revoked, removing the delegated role from the project.
+
+```
+>>> project.revoke("newrole")
+```
+
+<a name="managing_keys">
+## Managing Keys 
+This section describes the key-related functions and parameters not covered in
+the [Creating a Simple Project](#creating_a_simple_project) section.
+
+### Additional Parameters for Key Generation
+When generating keys, it is possible to specify the length of the key in bits 
+and its password as parameters:
+
+```
+>>> generate_and_write_rsa_keypair("path/to/key",bits=2048, password="pw")
+```
+The bits parameter defaults to 3072, and values below 2048 will raise an error.
+The password parameter is only intended to be used in scripts.
+
+<a name="adding_a_key_to_a_delegation">
+### Adding a Key to a Delegation
+New verifications keys can be added to an existing delegation using
+add\_verification\_key():
+
+```
+>>> project("rolename").add_verification_key(pubkey)
+```
+
+A delegation can have several verification keys at once. By default, a
+delegated role with multiple keys can be written using any one of their
+corresponding signing keys. To modify this behavior, you can change the
+delegated role's [threshold](#delegation_thrsholds).
+
+### Removing a Key from a Delegation
+Verification keys can also be removed, like this:
+
+```
+>>> project("rolename").remove_verification_key(pubkey)
+```
+
+Remember that a project can only have one key, so this method will return an
+error if there is already a key assigned to it. In order to replace a key we
+must first delete the existing one and then add the new one. It is possible to
+omit the key parameter in the create\_new\_project() function, and add the key
+later.
+
+### Changing the Project Key
+Each Project instance can only have one verification key. This key can be
+replaced by removing it and adding a new key, in that order.
+
+```
+>>> project.remove_verification_key(oldkey)
+>>> project.add_verification_key(new)
+```
+
+<a name="delegation_thresholds">
+### Delegation Thresholds
+
+Every delegated role has a threshold, which determines how many of its signing
+keys need to be loaded to write the role. The threshold defaults to 1, and
+should not exceed the number of verification keys assigned to the role. The
+threshold can be accessed as a property of a delegated role.
+
+```
+>>> project("rolename").threshold = 2
+```
+
+The above line will set the "rolename" role's threshold to 2.
+
+<a name="managing_targets">
+## Managing Targets
+There are supporting functions of the targets library to make the project
+maintenance easier. These functions are described in this section.
+
+### Adding Targets by Directory
+This function is especially useful when creating a new project to add all the
+files contained in the targets directory. The following code block illustrates
+the usage of this function:
+
+```
+>>> list_of_targets = \
+...   project.get_filepaths_in_directory(“path/within/targets/folder”,
+...   recursive_walk=False, follow_links=False)
+>>> project.add_targets(list_of_targets)
+```
+
+### Deleting Targets from a Project
+It is possible that we want to delete existing targets inside our project. To
+stop the developer tool from tracking this file we can issue the following
+command:
+
+```
+>>> project.remove_target(“target_1”)
+```
+
+Now the target file won't be part of the metadata.

tuf/README.md

@@ -1,8 +1,7 @@
 # Repository Management #
 
 ## Table of Contents ##
-- [Repository Tool Diagram](#repository-tool-diagram)
-- [TUF Repository](#create-tuf-repository)
+- [The Files of a TUF Repository](#the-files-of-a-tuf-repository)
   - [Purpose](#purpose)
   - [Keys](#keys)
     - [Create RSA Keys](#create-rsa-keys)
@@ -21,19 +20,16 @@
 - [Client Setup and Repository Trial](#client-setup-and-repository-trial)
   - [Using TUF Within an Example Client Updater](#using-tuf-within-an-example-client-updater)
   - [Test TUF Locally](#test-tuf-locally)
+- [Repository Tool Diagram](#repository-tool-diagram)
 
 
-## Repository Tool Diagram ##
-![Repo Tools Diagram 1](../docs/images/repository_tool-diagram.png)
-
-
-## TUF Repository ##
+## The Files of a TUF Repository ##
 
 ### Purpose ###
 
-The **tuf.repository_tool** module can be used to create a TUF repository.
-It may either be imported into a Python module or used with the Python
-interpreter in interactive mode.
+The [tuf.repository_tool](tuf/repository_tool.py) module can be used to create a
+TUF repository.  It may either be imported into a Python module or used with the
+Python interpreter in interactive mode.
 
 ```Bash
 $ python
@@ -43,8 +39,9 @@ Type "help", "copyright", "credits" or "license" for more information.
 >>> from tuf.repository_tool import *
 >>> repository = load_repository("path/to/repository")
 ```
-Note that **tuf.repository_tool.py** is not used in TUF integrations.  The
-[tuf.interposition](/interposition/README.md)** package and
+Note that [tuf.repository_tool.py](tuf/repository_tool.py) is not used in TUF
+integrations.  The
+[tuf.interposition](/interposition/README.md) package and
 [tuf.client.updater](/client/README.md) module assist in integrating TUF with a
 software updater.
 
@@ -440,3 +437,6 @@ django  file1.txt  file2.txt
 targets/django/:
 file4.txt
 ```
+
+## Repository Tool Diagram ##
+![Repo Tools Diagram 1](../docs/images/repository_tool-diagram.png)

tuf/formats.py

@@ -457,6 +457,18 @@
   expires = ISO8601_DATETIME_SCHEMA,
   meta = FILEDICT_SCHEMA)
 
+# project.cfg file: stores information about the project in a json dictionary
+PROJECT_CFG_SCHEMA = SCHEMA.Object(
+    object_name = 'PROJECT_CFG_SCHEMA',
+    project_name = SCHEMA.AnyString(),
+    layout_type = SCHEMA.OneOf([SCHEMA.String('repo-like'), SCHEMA.String('flat')]),
+    targets_location = PATH_SCHEMA,
+    metadata_location = PATH_SCHEMA,
+    prefix = PATH_SCHEMA,
+    public_keys = KEYDICT_SCHEMA,
+    threshold = SCHEMA.Integer(lo = 0, hi = 2)
+    )
+
 # A schema containing information a repository mirror may require,
 # such as a url, the path of the directory metadata files, etc.
 MIRROR_SCHEMA = SCHEMA.Object(

tuf/keys.py

@@ -224,7 +224,7 @@ def generate_rsa_key(bits=_DEFAULT_RSA_KEY_BITS):
     public, private = tuf.pycrypto_keys.generate_rsa_public_and_private(bits)
   
   else: # pragma: no cover
-    message = 'Invalid crypto library: '+repr(_RSA_CRYPTO_LIBRARY)+'.'
+    message = 'Invalid crypto library: ' + repr(_RSA_CRYPTO_LIBRARY) + '.'
     raise tuf.UnsupportedLibraryError(message) 
     
   # Generate the keyid of the RSA key.  'key_value' corresponds to the
@@ -551,9 +551,9 @@ def check_crypto_libraries(required_libraries):
   
   if 'rsa' in required_libraries and _RSA_CRYPTO_LIBRARY not in \
                                    _SUPPORTED_RSA_CRYPTO_LIBRARIES:
-    message = 'The '+repr(_RSA_CRYPTO_LIBRARY)+' crypto library specified'+ \
-      ' in "tuf.conf.RSA_CRYPTO_LIBRARY" is not supported.\n'+ \
-      'Supported crypto libraries: '+repr(_SUPPORTED_RSA_CRYPTO_LIBRARIES)+'.'
+    message = 'The ' + repr(_RSA_CRYPTO_LIBRARY) + ' crypto library specified' +\
+      ' in "tuf.conf.RSA_CRYPTO_LIBRARY" is not supported.\n' +\
+      'Supported crypto libraries: ' + repr(_SUPPORTED_RSA_CRYPTO_LIBRARIES) + '.'
     raise tuf.UnsupportedLibraryError(message)
   
   if 'ed25519' in required_libraries and _ED25519_CRYPTO_LIBRARY not in \
@@ -697,8 +697,8 @@ def create_signature(key_dict, data):
       sig, method = tuf.pycrypto_keys.create_rsa_signature(private, data.encode('utf-8'))
     
     else: # pragma: no cover
-      message = 'Unsupported "tuf.conf.RSA_CRYPTO_LIBRARY": '+\
-        repr(_RSA_CRYPTO_LIBRARY)+'.'
+      message = 'Unsupported "tuf.conf.RSA_CRYPTO_LIBRARY": ' +\
+        repr(_RSA_CRYPTO_LIBRARY) + '.'
       raise tuf.UnsupportedLibraryError(message)
   
   elif keytype == 'ed25519':
@@ -824,7 +824,7 @@ def verify_signature(key_dict, signature, data):
   if keytype == 'rsa':
     if _RSA_CRYPTO_LIBRARY == 'pycrypto':
       if 'pycrypto' not in _available_crypto_libraries: # pragma: no cover
-        message = 'Metadata downloaded from the remote repository specified'+\
+        message = 'Metadata downloaded from the remote repository specified' +\
           ' an RSA signature.  Verifying RSA signatures requires PyCrypto.' +\
           '\n$ pip install PyCrypto, or pip install tuf[tools].'
         raise tuf.UnsupportedLibraryError(message)
@@ -833,7 +833,7 @@ def verify_signature(key_dict, signature, data):
         valid_signature = tuf.pycrypto_keys.verify_rsa_signature(sig, method,
                                                                  public, data) 
     else: # pragma: no cover
-      message = 'Unsupported "tuf.conf.RSA_CRYPTO_LIBRARY": '+\
+      message = 'Unsupported "tuf.conf.RSA_CRYPTO_LIBRARY": ' +\
         repr(_RSA_CRYPTO_LIBRARY)+'.'
       raise tuf.UnsupportedLibraryError(message) 
   
@@ -944,7 +944,7 @@ def import_rsakey_from_encrypted_pem(encrypted_pem, password):
       tuf.pycrypto_keys.create_rsa_public_and_private_from_encrypted_pem(encrypted_pem,
                                                                          password)
   else: #pragma: no cover
-    message = 'Invalid crypto library: '+repr(_RSA_CRYPTO_LIBRARY)+'.'
+    message = 'Invalid crypto library: ' + repr(_RSA_CRYPTO_LIBRARY) + '.'
     raise tuf.UnsupportedLibraryError(message) 
     
   # Generate the keyid of the RSA key.  'key_value' corresponds to the
@@ -1120,7 +1120,7 @@ def encrypt_key(key_object, password):
  
   # check_crypto_libraries() should have fully verified _GENERAL_CRYPTO_LIBRARY.
   else: # pragma: no cover
-    message = 'Invalid crypto library: '+repr(_GENERAL_CRYPTO_LIBRARY)+'.'
+    message = 'Invalid crypto library: ' + repr(_GENERAL_CRYPTO_LIBRARY) + '.'
     raise tuf.UnsupportedLibraryError(message) 
 
   return encrypted_key
@@ -1218,7 +1218,7 @@ def decrypt_key(encrypted_key, passphrase):
   
   # check_crypto_libraries() should have fully verified _GENERAL_CRYPTO_LIBRARY.
   else: # pragma: no cover
-    message = 'Invalid crypto library: '+repr(_GENERAL_CRYPTO_LIBRARY)+'.'
+    message = 'Invalid crypto library: ' + repr(_GENERAL_CRYPTO_LIBRARY) + '.'
     raise tuf.UnsupportedLibraryError(message) 
 
   # The corresponding encrypt_key() encrypts and stores key objects in
@@ -1301,7 +1301,7 @@ def create_rsa_encrypted_pem(private_key, passphrase):
  
   # check_crypto_libraries() should have fully verified _RSA_CRYPTO_LIBRARY.
   else: # pragma: no cover
-    message = 'Invalid crypto library: '+repr(_RSA_CRYPTO_LIBRARY)+'.'
+    message = 'Invalid crypto library: ' + repr(_RSA_CRYPTO_LIBRARY) + '.'
     raise tuf.UnsupportedLibraryError(message)
 
   return encrypted_pem

tuf/pycrypto_keys.py

@@ -295,7 +295,7 @@ def create_rsa_signature(private_key, data):
       rsa_key_object = Crypto.PublicKey.RSA.importKey(private_key)
     
     except (ValueError, IndexError, TypeError) as e:
-      message = 'Invalid private key or hash data: '+str(e)
+      message = 'Invalid private key or hash data: ' + str(e)
       raise tuf.CryptoError(message)
    
     # Generate RSSA-PSS signature.  Raise 'tuf.CryptoError' for the expected
@@ -311,7 +311,7 @@ def create_rsa_signature(private_key, data):
       raise tuf.CryptoError('Missing required RSA private key.')
    
     except IndexError:
-      message = 'An RSA signature cannot be generated: '+str(e)
+      message = 'An RSA signature cannot be generated: ' + str(e)
       raise tuf.CryptoError(message)
   
   else:
@@ -474,7 +474,7 @@ def create_rsa_encrypted_pem(private_key, passphrase):
                                                passphrase=passphrase) 
     
     except (ValueError, IndexError, TypeError) as e:
-      message = 'An encrypted RSA key in PEM format cannot be generated: '+str(e)
+      message = 'An encrypted RSA key in PEM format cannot be generated: ' + str(e)
       raise tuf.CryptoError(message)
   
   else:
@@ -570,8 +570,8 @@ def create_rsa_public_and_private_from_encrypted_pem(encrypted_pem, passphrase):
   # If the passphrase is incorrect, PyCrypto returns: "RSA key format is not
   # supported".
   except (ValueError, IndexError, TypeError) as e:
-    message = 'RSA (public, private) tuple cannot be generated from the'+\
-      ' encrypted PEM string: '+str(e)
+    message = 'RSA (public, private) tuple cannot be generated from the' +\
+      ' encrypted PEM string: ' + str(e)
     # Raise 'tuf.CryptoError' and PyCrypto's exception message.  Avoid
     # propogating PyCrypto's exception trace to avoid revealing sensitive error.
     raise tuf.CryptoError(message)
@@ -692,7 +692,6 @@ def encrypt_key(key_object, password):
 def decrypt_key(encrypted_key, password):
   """
   <Purpose>
-    
     Return a string containing 'encrypted_key' in non-encrypted form.
     The decrypt_key() function can be applied to the encrypted string to restore
     the original key object, a TUF key (e.g., RSAKEY_SCHEMA, ED25519KEY_SCHEMA).
@@ -862,7 +861,7 @@ def _encrypt(key_data, derived_key_information):
   # checking for exceptions.  Avoid propogating the exception trace and only
   # raise 'tuf.CryptoError', along with the cause of encryption failure.
   except (ValueError, IndexError, TypeError) as e:
-    message = 'The key data cannot be encrypted: '+str(e)
+    message = 'The key data cannot be encrypted: ' + str(e)
     raise tuf.CryptoError(message)
 
   # Generate the hmac of the ciphertext to ensure it has not been modified.
@@ -951,7 +950,7 @@ def _decrypt(file_contents, password):
   # Note: decryption failure, due to malicious ciphertext, should not occur here
   # if the hmac check above passed.
   except (ValueError, IndexError, TypeError) as e: # pragma: no cover
-    raise tuf.CryptoError('Decryption failed: '+str(e))
+    raise tuf.CryptoError('Decryption failed: ' + str(e))
 
   return key_plaintext
 

tuf/repository_lib.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 
 """
 <Program Name>
@@ -1941,6 +1942,7 @@ def write_metadata_file(metadata, filename, compressions, consistent_snapshot):
       gzip_object = gzip.GzipFile(fileobj=file_object, mode='wb') 
       try: 
         gzip_object.write(file_content)
+      
       finally:
         gzip_object.close()
 
@@ -2194,8 +2196,8 @@ def create_tuf_client_directory(repository_directory, client_directory):
   
   except OSError as e:
     if e.errno == errno.EEXIST:
-      message = 'Cannot create a fresh client metadata directory: '+ \
-        repr(client_metadata_directory)+'.  Already exists.'
+      message = 'Cannot create a fresh client metadata directory: ' +\
+        repr(client_metadata_directory) + '.  Already exists.'
       raise tuf.RepositoryError(message)
     
     else:

tuf/repository_tool.py

@@ -469,7 +469,7 @@ def get_filepaths_in_directory(files_directory, recursive_walk=False,
 
     # Ensure a valid directory is given.
     if not os.path.isdir(files_directory):
-      message = repr(files_directory)+' is not a directory.'
+      message = repr(files_directory) + ' is not a directory.'
       raise tuf.Error(message)
    
     # A list of the target filepaths found in 'files_directory'.
@@ -2274,8 +2274,8 @@ def delegate_hashed_bins(self, list_of_targets, keys_of_hashed_bins,
     for target_path in list_of_targets:
       target_path = os.path.abspath(target_path)
       if not target_path.startswith(self._targets_directory+os.sep):
-        message = 'A path in the list of targets argument is not '+\
-          'under the repository\'s targets directory: '+repr(target_path) 
+        message = 'A path in the list of targets argument is not ' +\
+          'under the repository\'s targets directory: ' + repr(target_path) 
         raise tuf.Error(message)
       
       # Determine the hash prefix of 'target_path' by computing the digest of